/* * Copyright (C) 2009 Michael Brown . * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as * published by the Free Software Foundation; either version 2 of the * License, or any later version. * * This program is distributed in the hope that it will be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ FILE_LICENCE ( GPL2_OR_LATER ); #include #include #include #include #include #include #include #include #include #include #include #include #include #include "qib7322.h" /** * @file * * QLogic QIB7322 Infiniband HCA * */ /** A QIB7322 send buffer set */ struct qib7322_send_buffers { /** Offset within register space of the first send buffer */ unsigned long base; /** Send buffer size */ unsigned int size; /** Index of first send buffer */ unsigned int start; /** Number of send buffers * * Must be a power of two. */ unsigned int count; /** Send buffer availability producer counter */ unsigned int prod; /** Send buffer availability consumer counter */ unsigned int cons; /** Send buffer availability */ uint16_t avail[0]; }; /** A QIB7322 send work queue */ struct qib7322_send_work_queue { /** Send buffer set */ struct qib7322_send_buffers *send_bufs; /** Send buffer usage */ uint16_t *used; /** Producer index */ unsigned int prod; /** Consumer index */ unsigned int cons; }; /** A QIB7322 receive work queue */ struct qib7322_recv_work_queue { /** Receive header ring */ void *header; /** Receive header producer offset (written by hardware) */ struct QIB_7322_scalar header_prod; /** Receive header consumer offset */ unsigned int header_cons; /** Offset within register space of the eager array */ unsigned long eager_array; /** Number of entries in eager array */ unsigned int eager_entries; /** Eager array producer index */ unsigned int eager_prod; /** Eager array consumer index */ unsigned int eager_cons; }; /** A QIB7322 HCA */ struct qib7322 { /** Registers */ void *regs; /** In-use contexts */ uint8_t used_ctx[QIB7322_NUM_CONTEXTS]; /** Send work queues */ struct qib7322_send_work_queue send_wq[QIB7322_NUM_CONTEXTS]; /** Receive work queues */ struct qib7322_recv_work_queue recv_wq[QIB7322_NUM_CONTEXTS]; /** Send buffer availability (reported by hardware) */ struct QIB_7322_SendBufAvail *sendbufavail; /** Small send buffers */ struct qib7322_send_buffers *send_bufs_small; /** VL15 port 0 send buffers */ struct qib7322_send_buffers *send_bufs_vl15_port0; /** VL15 port 1 send buffers */ struct qib7322_send_buffers *send_bufs_vl15_port1; /** I2C bit-bashing interface */ struct i2c_bit_basher i2c; /** I2C serial EEPROM */ struct i2c_device eeprom; /** Base GUID */ struct ib_gid_half guid; /** Infiniband devices */ struct ib_device *ibdev[QIB7322_MAX_PORTS]; }; /*************************************************************************** * * QIB7322 register access * *************************************************************************** * * This card requires atomic 64-bit accesses. Strange things happen * if you try to use 32-bit accesses; sometimes they work, sometimes * they don't, sometimes you get random data. * * These accessors use the "movq" MMX instruction, and so won't work * on really old Pentiums (which won't have PCIe anyway, so this is * something of a moot point). */ /** * Read QIB7322 qword register * * @v qib7322 QIB7322 device * @v dwords Register buffer to read into * @v offset Register offset */ static void qib7322_readq ( struct qib7322 *qib7322, uint32_t *dwords, unsigned long offset ) { void *addr = ( qib7322->regs + offset ); __asm__ __volatile__ ( "movq (%1), %%mm0\n\t" "movq %%mm0, (%0)\n\t" : : "r" ( dwords ), "r" ( addr ) : "memory" ); DBGIO ( "[%08lx] => %08x%08x\n", virt_to_phys ( addr ), dwords[1], dwords[0] ); } #define qib7322_readq( _qib7322, _ptr, _offset ) \ qib7322_readq ( (_qib7322), (_ptr)->u.dwords, (_offset) ) #define qib7322_readq_array8b( _qib7322, _ptr, _offset, _idx ) \ qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) ) #define qib7322_readq_array64k( _qib7322, _ptr, _offset, _idx ) \ qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) ) ) #define qib7322_readq_port( _qib7322, _ptr, _offset, _port ) \ qib7322_readq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) ) ) /** * Write QIB7322 qword register * * @v qib7322 QIB7322 device * @v dwords Register buffer to write * @v offset Register offset */ static void qib7322_writeq ( struct qib7322 *qib7322, const uint32_t *dwords, unsigned long offset ) { void *addr = ( qib7322->regs + offset ); DBGIO ( "[%08lx] <= %08x%08x\n", virt_to_phys ( addr ), dwords[1], dwords[0] ); __asm__ __volatile__ ( "movq (%0), %%mm0\n\t" "movq %%mm0, (%1)\n\t" : : "r" ( dwords ), "r" ( addr ) : "memory" ); } #define qib7322_writeq( _qib7322, _ptr, _offset ) \ qib7322_writeq ( (_qib7322), (_ptr)->u.dwords, (_offset) ) #define qib7322_writeq_array8b( _qib7322, _ptr, _offset, _idx ) \ qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 8 ) ) ) #define qib7322_writeq_array64k( _qib7322, _ptr, _offset, _idx ) \ qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_idx) * 65536 ) )) #define qib7322_writeq_port( _qib7322, _ptr, _offset, _port ) \ qib7322_writeq ( (_qib7322), (_ptr), ( (_offset) + ( (_port) * 4096 ) )) /** * Write QIB7322 dword register * * @v qib7322 QIB7322 device * @v dword Value to write * @v offset Register offset */ static void qib7322_writel ( struct qib7322 *qib7322, uint32_t dword, unsigned long offset ) { writel ( dword, ( qib7322->regs + offset ) ); } /*************************************************************************** * * Link state management * *************************************************************************** */ /** * Textual representation of link state * * @v link_state Link state * @ret link_text Link state text */ static const char * qib7322_link_state_text ( unsigned int link_state ) { switch ( link_state ) { case QIB7322_LINK_STATE_DOWN: return "DOWN"; case QIB7322_LINK_STATE_INIT: return "INIT"; case QIB7322_LINK_STATE_ARM: return "ARM"; case QIB7322_LINK_STATE_ACTIVE: return "ACTIVE"; case QIB7322_LINK_STATE_ACT_DEFER: return "ACT_DEFER"; default: return "UNKNOWN"; } } /** * Handle link state change * * @v qib7322 QIB7322 device */ static void qib7322_link_state_changed ( struct ib_device *ibdev ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct QIB_7322_IBCStatusA_0 ibcstatusa; struct QIB_7322_EXTCtrl extctrl; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int link_training_state; unsigned int link_state; unsigned int link_width; unsigned int link_speed; unsigned int link_speed_qdr; unsigned int green; unsigned int yellow; /* Read link state */ qib7322_readq_port ( qib7322, &ibcstatusa, QIB_7322_IBCStatusA_0_offset, port ); link_training_state = BIT_GET ( &ibcstatusa, LinkTrainingState ); link_state = BIT_GET ( &ibcstatusa, LinkState ); link_width = BIT_GET ( &ibcstatusa, LinkWidthActive ); link_speed = BIT_GET ( &ibcstatusa, LinkSpeedActive ); link_speed_qdr = BIT_GET ( &ibcstatusa, LinkSpeedQDR ); DBGC ( qib7322, "QIB7322 %p port %d training state %#x link state %s " "(%s %s)\n", qib7322, port, link_training_state, qib7322_link_state_text ( link_state ), ( link_speed_qdr ? "QDR" : ( link_speed ? "DDR" : "SDR" ) ), ( link_width ? "x4" : "x1" ) ); /* Set LEDs according to link state */ qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset ); green = ( ( link_state >= QIB7322_LINK_STATE_INIT ) ? 1 : 0 ); yellow = ( ( link_state >= QIB7322_LINK_STATE_ACTIVE ) ? 1 : 0 ); if ( port == 0 ) { BIT_SET ( &extctrl, LEDPort0GreenOn, green ); BIT_SET ( &extctrl, LEDPort0YellowOn, yellow ); } else { BIT_SET ( &extctrl, LEDPort1GreenOn, green ); BIT_SET ( &extctrl, LEDPort1YellowOn, yellow ); } qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset ); /* Notify Infiniband core of link state change */ ibdev->port_state = ( link_state + 1 ); ibdev->link_width_active = ( link_width ? IB_LINK_WIDTH_4X : IB_LINK_WIDTH_1X ); ibdev->link_speed_active = ( link_speed ? IB_LINK_SPEED_DDR : IB_LINK_SPEED_SDR ); ib_link_state_changed ( ibdev ); } /** * Wait for link state change to take effect * * @v ibdev Infiniband device * @v new_link_state Expected link state * @ret rc Return status code */ static int qib7322_link_state_check ( struct ib_device *ibdev, unsigned int new_link_state ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct QIB_7322_IBCStatusA_0 ibcstatusa; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int link_state; unsigned int i; for ( i = 0 ; i < QIB7322_LINK_STATE_MAX_WAIT_US ; i++ ) { qib7322_readq_port ( qib7322, &ibcstatusa, QIB_7322_IBCStatusA_0_offset, port ); link_state = BIT_GET ( &ibcstatusa, LinkState ); if ( link_state == new_link_state ) return 0; udelay ( 1 ); } DBGC ( qib7322, "QIB7322 %p port %d timed out waiting for link state " "%s\n", qib7322, port, qib7322_link_state_text ( link_state ) ); return -ETIMEDOUT; } /** * Set port information * * @v ibdev Infiniband device * @v mad Set port information MAD */ static int qib7322_set_port_info ( struct ib_device *ibdev, union ib_mad *mad ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_port_info *port_info = &mad->smp.smp_data.port_info; struct QIB_7322_IBCCtrlA_0 ibcctrla; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int port_state; unsigned int link_state; /* Set new link state */ port_state = ( port_info->link_speed_supported__port_state & 0xf ); if ( port_state ) { link_state = ( port_state - 1 ); DBGC ( qib7322, "QIB7322 %p set link state to %s (%x)\n", qib7322, qib7322_link_state_text ( link_state ), link_state ); qib7322_readq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); BIT_SET ( &ibcctrla, LinkCmd, link_state ); qib7322_writeq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); /* Wait for link state change to take effect. Ignore * errors; the current link state will be returned via * the GetResponse MAD. */ qib7322_link_state_check ( ibdev, link_state ); } /* Detect and report link state change */ qib7322_link_state_changed ( ibdev ); return 0; } /** * Set partition key table * * @v ibdev Infiniband device * @v mad Set partition key table MAD */ static int qib7322_set_pkey_table ( struct ib_device *ibdev __unused, union ib_mad *mad __unused ) { /* Nothing to do */ return 0; } /*************************************************************************** * * Context allocation * *************************************************************************** */ /** * Allocate a context and set queue pair number * * @v ibdev Infiniband device * @v qp Queue pair * @ret rc Return status code */ static int qib7322_alloc_ctx ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int ctx; for ( ctx = port ; ctx < QIB7322_NUM_CONTEXTS ; ctx += 2 ) { if ( ! qib7322->used_ctx[ctx] ) { qib7322->used_ctx[ctx] = 1; qp->qpn = ( ctx & ~0x01 ); DBGC2 ( qib7322, "QIB7322 %p port %d QPN %ld is CTX " "%d\n", qib7322, port, qp->qpn, ctx ); return 0; } } DBGC ( qib7322, "QIB7322 %p port %d out of available contexts\n", qib7322, port ); return -ENOENT; } /** * Get queue pair context number * * @v ibdev Infiniband device * @v qp Queue pair * @ret ctx Context index */ static unsigned int qib7322_ctx ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { return ( qp->qpn + ( ibdev->port - QIB7322_PORT_BASE ) ); } /** * Free a context * * @v qib7322 QIB7322 device * @v ctx Context index */ static void qib7322_free_ctx ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int ctx = qib7322_ctx ( ibdev, qp ); qib7322->used_ctx[ctx] = 0; DBGC2 ( qib7322, "QIB7322 %p port %d CTX %d freed\n", qib7322, port, ctx ); } /*************************************************************************** * * Send datapath * *************************************************************************** */ /** Send buffer toggle bit * * We encode send buffers as 15 bits of send buffer index plus a * single bit which should match the "check" bit in the SendBufAvail * array. */ #define QIB7322_SEND_BUF_TOGGLE 0x8000 /** * Create send buffer set * * @v qib7322 QIB7322 device * @v base Send buffer base offset * @v size Send buffer size * @v start Index of first send buffer * @v count Number of send buffers * @ret send_bufs Send buffer set */ static struct qib7322_send_buffers * qib7322_create_send_bufs ( struct qib7322 *qib7322, unsigned long base, unsigned int size, unsigned int start, unsigned int count ) { struct qib7322_send_buffers *send_bufs; unsigned int i; /* Allocate send buffer set */ send_bufs = zalloc ( sizeof ( *send_bufs ) + ( count * sizeof ( send_bufs->avail[0] ) ) ); if ( ! send_bufs ) return NULL; /* Populate send buffer set */ send_bufs->base = base; send_bufs->size = size; send_bufs->start = start; send_bufs->count = count; for ( i = 0 ; i < count ; i++ ) send_bufs->avail[i] = ( start + i ); DBGC2 ( qib7322, "QIB7322 %p send buffer set %p [%d,%d] at %lx\n", qib7322, send_bufs, start, ( start + count - 1 ), send_bufs->base ); return send_bufs; } /** * Destroy send buffer set * * @v qib7322 QIB7322 device * @v send_bufs Send buffer set */ static void qib7322_destroy_send_bufs ( struct qib7322 *qib7322 __unused, struct qib7322_send_buffers *send_bufs ) { free ( send_bufs ); } /** * Allocate a send buffer * * @v qib7322 QIB7322 device * @v send_bufs Send buffer set * @ret send_buf Send buffer, or negative error */ static int qib7322_alloc_send_buf ( struct qib7322 *qib7322, struct qib7322_send_buffers *send_bufs ) { unsigned int used; unsigned int mask; unsigned int send_buf; used = ( send_bufs->cons - send_bufs->prod ); if ( used >= send_bufs->count ) { DBGC ( qib7322, "QIB7322 %p send buffer set %p out of " "buffers\n", qib7322, send_bufs ); return -ENOBUFS; } mask = ( send_bufs->count - 1 ); send_buf = send_bufs->avail[ send_bufs->cons++ & mask ]; send_buf ^= QIB7322_SEND_BUF_TOGGLE; return send_buf; } /** * Free a send buffer * * @v qib7322 QIB7322 device * @v send_bufs Send buffer set * @v send_buf Send buffer */ static void qib7322_free_send_buf ( struct qib7322 *qib7322 __unused, struct qib7322_send_buffers *send_bufs, unsigned int send_buf ) { unsigned int mask; mask = ( send_bufs->count - 1 ); send_bufs->avail[ send_bufs->prod++ & mask ] = send_buf; } /** * Check to see if send buffer is in use * * @v qib7322 QIB7322 device * @v send_buf Send buffer * @ret in_use Send buffer is in use */ static int qib7322_send_buf_in_use ( struct qib7322 *qib7322, unsigned int send_buf ) { unsigned int send_idx; unsigned int send_check; unsigned int inusecheck; unsigned int inuse; unsigned int check; send_idx = ( send_buf & ~QIB7322_SEND_BUF_TOGGLE ); send_check = ( !! ( send_buf & QIB7322_SEND_BUF_TOGGLE ) ); inusecheck = BIT_GET ( qib7322->sendbufavail, InUseCheck[send_idx] ); inuse = ( !! ( inusecheck & 0x02 ) ); check = ( !! ( inusecheck & 0x01 ) ); return ( inuse || ( check != send_check ) ); } /** * Calculate starting offset for send buffer * * @v qib7322 QIB7322 device * @v send_buf Send buffer * @ret offset Starting offset */ static unsigned long qib7322_send_buffer_offset ( struct qib7322 *qib7322 __unused, struct qib7322_send_buffers *send_bufs, unsigned int send_buf ) { unsigned int index; index = ( ( send_buf & ~QIB7322_SEND_BUF_TOGGLE ) - send_bufs->start ); return ( send_bufs->base + ( index * send_bufs->size ) ); } /** * Create send work queue * * @v ibdev Infiniband device * @v qp Queue pair */ static int qib7322_create_send_wq ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->send; struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); /* Select send buffer set */ if ( qp->type == IB_QPT_SMI ) { if ( port == 0 ) { qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port0; } else { qib7322_wq->send_bufs = qib7322->send_bufs_vl15_port1; } } else { qib7322_wq->send_bufs = qib7322->send_bufs_small; } /* Allocate space for send buffer usage list */ qib7322_wq->used = zalloc ( qp->send.num_wqes * sizeof ( qib7322_wq->used[0] ) ); if ( ! qib7322_wq->used ) return -ENOMEM; /* Reset work queue */ qib7322_wq->prod = 0; qib7322_wq->cons = 0; return 0; } /** * Destroy send work queue * * @v ibdev Infiniband device * @v qp Queue pair */ static void qib7322_destroy_send_wq ( struct ib_device *ibdev __unused, struct ib_queue_pair *qp ) { struct ib_work_queue *wq = &qp->send; struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); free ( qib7322_wq->used ); } /** * Initialise send datapath * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_init_send ( struct qib7322 *qib7322 ) { struct QIB_7322_SendBufBase sendbufbase; struct QIB_7322_SendBufAvailAddr sendbufavailaddr; struct QIB_7322_SendCtrl sendctrl; struct QIB_7322_SendCtrl_0 sendctrlp; unsigned long baseaddr_smallpio; unsigned long baseaddr_largepio; unsigned long baseaddr_vl15_port0; unsigned long baseaddr_vl15_port1; int rc; /* Create send buffer sets */ qib7322_readq ( qib7322, &sendbufbase, QIB_7322_SendBufBase_offset ); baseaddr_smallpio = BIT_GET ( &sendbufbase, BaseAddr_SmallPIO ); baseaddr_largepio = BIT_GET ( &sendbufbase, BaseAddr_LargePIO ); baseaddr_vl15_port0 = ( baseaddr_largepio + ( QIB7322_LARGE_SEND_BUF_SIZE * QIB7322_LARGE_SEND_BUF_COUNT ) ); baseaddr_vl15_port1 = ( baseaddr_vl15_port0 + QIB7322_VL15_PORT0_SEND_BUF_SIZE ); qib7322->send_bufs_small = qib7322_create_send_bufs ( qib7322, baseaddr_smallpio, QIB7322_SMALL_SEND_BUF_SIZE, QIB7322_SMALL_SEND_BUF_START, QIB7322_SMALL_SEND_BUF_USED ); if ( ! qib7322->send_bufs_small ) { rc = -ENOMEM; goto err_create_send_bufs_small; } qib7322->send_bufs_vl15_port0 = qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port0, QIB7322_VL15_PORT0_SEND_BUF_SIZE, QIB7322_VL15_PORT0_SEND_BUF_START, QIB7322_VL15_PORT0_SEND_BUF_COUNT ); if ( ! qib7322->send_bufs_vl15_port0 ) { rc = -ENOMEM; goto err_create_send_bufs_vl15_port0; } qib7322->send_bufs_vl15_port1 = qib7322_create_send_bufs ( qib7322, baseaddr_vl15_port1, QIB7322_VL15_PORT1_SEND_BUF_SIZE, QIB7322_VL15_PORT1_SEND_BUF_START, QIB7322_VL15_PORT1_SEND_BUF_COUNT ); if ( ! qib7322->send_bufs_vl15_port1 ) { rc = -ENOMEM; goto err_create_send_bufs_vl15_port1; } /* Allocate space for the SendBufAvail array */ qib7322->sendbufavail = malloc_dma ( sizeof ( *qib7322->sendbufavail ), QIB7322_SENDBUFAVAIL_ALIGN ); if ( ! qib7322->sendbufavail ) { rc = -ENOMEM; goto err_alloc_sendbufavail; } memset ( qib7322->sendbufavail, 0, sizeof ( qib7322->sendbufavail ) ); /* Program SendBufAvailAddr into the hardware */ memset ( &sendbufavailaddr, 0, sizeof ( sendbufavailaddr ) ); BIT_FILL_1 ( &sendbufavailaddr, SendBufAvailAddr, ( virt_to_bus ( qib7322->sendbufavail ) >> 6 ) ); qib7322_writeq ( qib7322, &sendbufavailaddr, QIB_7322_SendBufAvailAddr_offset ); /* Enable sending */ memset ( &sendctrlp, 0, sizeof ( sendctrlp ) ); BIT_FILL_1 ( &sendctrlp, SendEnable, 1 ); qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_0_offset ); qib7322_writeq ( qib7322, &sendctrlp, QIB_7322_SendCtrl_1_offset ); /* Enable DMA of SendBufAvail */ memset ( &sendctrl, 0, sizeof ( sendctrl ) ); BIT_FILL_1 ( &sendctrl, SendBufAvailUpd, 1 ); qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset ); return 0; free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) ); err_alloc_sendbufavail: qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 ); err_create_send_bufs_vl15_port1: qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 ); err_create_send_bufs_vl15_port0: qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small ); err_create_send_bufs_small: return rc; } /** * Shut down send datapath * * @v qib7322 QIB7322 device */ static void qib7322_fini_send ( struct qib7322 *qib7322 ) { struct QIB_7322_SendCtrl sendctrl; /* Disable sending and DMA of SendBufAvail */ memset ( &sendctrl, 0, sizeof ( sendctrl ) ); qib7322_writeq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset ); mb(); /* Ensure hardware has seen this disable */ qib7322_readq ( qib7322, &sendctrl, QIB_7322_SendCtrl_offset ); free_dma ( qib7322->sendbufavail, sizeof ( *qib7322->sendbufavail ) ); qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port1 ); qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_vl15_port0 ); qib7322_destroy_send_bufs ( qib7322, qib7322->send_bufs_small ); } /*************************************************************************** * * Receive datapath * *************************************************************************** */ /** * Create receive work queue * * @v ibdev Infiniband device * @v qp Queue pair * @ret rc Return status code */ static int qib7322_create_recv_wq ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->recv; struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_RcvHdrAddr0 rcvhdraddr; struct QIB_7322_RcvHdrTailAddr0 rcvhdrtailaddr; struct QIB_7322_RcvHdrHead0 rcvhdrhead; struct QIB_7322_scalar rcvegrindexhead; struct QIB_7322_RcvCtrl rcvctrl; struct QIB_7322_RcvCtrl_P rcvctrlp; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int ctx = qib7322_ctx ( ibdev, qp ); int rc; /* Reset context information */ memset ( &qib7322_wq->header_prod, 0, sizeof ( qib7322_wq->header_prod ) ); qib7322_wq->header_cons = 0; qib7322_wq->eager_prod = 0; qib7322_wq->eager_cons = 0; /* Allocate receive header buffer */ qib7322_wq->header = malloc_dma ( QIB7322_RECV_HEADERS_SIZE, QIB7322_RECV_HEADERS_ALIGN ); if ( ! qib7322_wq->header ) { rc = -ENOMEM; goto err_alloc_header; } /* Enable context in hardware */ memset ( &rcvhdraddr, 0, sizeof ( rcvhdraddr ) ); BIT_FILL_1 ( &rcvhdraddr, RcvHdrAddr, ( virt_to_bus ( qib7322_wq->header ) >> 2 ) ); qib7322_writeq_array8b ( qib7322, &rcvhdraddr, QIB_7322_RcvHdrAddr0_offset, ctx ); memset ( &rcvhdrtailaddr, 0, sizeof ( rcvhdrtailaddr ) ); BIT_FILL_1 ( &rcvhdrtailaddr, RcvHdrTailAddr, ( virt_to_bus ( &qib7322_wq->header_prod ) >> 2 ) ); qib7322_writeq_array8b ( qib7322, &rcvhdrtailaddr, QIB_7322_RcvHdrTailAddr0_offset, ctx ); memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) ); BIT_FILL_1 ( &rcvhdrhead, counter, 1 ); qib7322_writeq_array64k ( qib7322, &rcvhdrhead, QIB_7322_RcvHdrHead0_offset, ctx ); memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) ); BIT_FILL_1 ( &rcvegrindexhead, Value, 1 ); qib7322_writeq_array64k ( qib7322, &rcvegrindexhead, QIB_7322_RcvEgrIndexHead0_offset, ctx ); qib7322_readq_port ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset, port ); BIT_SET ( &rcvctrlp, ContextEnable[ctx], 1 ); qib7322_writeq_port ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset, port ); qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); BIT_SET ( &rcvctrl, IntrAvail[ctx], 1 ); qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); DBGC ( qib7322, "QIB7322 %p port %d QPN %ld CTX %d hdrs [%lx,%lx) prod " "%lx\n", qib7322, port, qp->qpn, ctx, virt_to_bus ( qib7322_wq->header ), ( virt_to_bus ( qib7322_wq->header ) + QIB7322_RECV_HEADERS_SIZE ), virt_to_bus ( &qib7322_wq->header_prod ) ); return 0; free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE ); err_alloc_header: return rc; } /** * Destroy receive work queue * * @v ibdev Infiniband device * @v qp Queue pair */ static void qib7322_destroy_recv_wq ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->recv; struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_RcvCtrl rcvctrl; struct QIB_7322_RcvCtrl_P rcvctrlp; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); unsigned int ctx = qib7322_ctx ( ibdev, qp ); /* Disable context in hardware */ qib7322_readq_port ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset, port ); BIT_SET ( &rcvctrlp, ContextEnable[ctx], 0 ); qib7322_writeq_port ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset, port ); qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); BIT_SET ( &rcvctrl, IntrAvail[ctx], 0 ); qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); /* Make sure the hardware has seen that the context is disabled */ qib7322_readq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); mb(); /* Free headers ring */ free_dma ( qib7322_wq->header, QIB7322_RECV_HEADERS_SIZE ); } /** * Initialise receive datapath * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_init_recv ( struct qib7322 *qib7322 ) { struct QIB_7322_RcvCtrl rcvctrl; struct QIB_7322_RcvCtrl_0 rcvctrlp; struct QIB_7322_RcvQPMapTableA_0 rcvqpmaptablea0; struct QIB_7322_RcvQPMapTableB_0 rcvqpmaptableb0; struct QIB_7322_RcvQPMapTableA_1 rcvqpmaptablea1; struct QIB_7322_RcvQPMapTableB_1 rcvqpmaptableb1; struct QIB_7322_RcvQPMulticastContext_0 rcvqpmcastctx0; struct QIB_7322_RcvQPMulticastContext_1 rcvqpmcastctx1; struct QIB_7322_scalar rcvegrbase; struct QIB_7322_scalar rcvhdrentsize; struct QIB_7322_scalar rcvhdrcnt; struct QIB_7322_RcvBTHQP_0 rcvbthqp; struct QIB_7322_RxCreditVL0_0 rxcreditvl; unsigned int contextcfg; unsigned long egrbase; unsigned int eager_array_size_kernel; unsigned int eager_array_size_user; unsigned int user_context_mask; unsigned int ctx; /* Select configuration based on number of contexts */ switch ( QIB7322_NUM_CONTEXTS ) { case 6: contextcfg = QIB7322_CONTEXTCFG_6CTX; eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_6CTX_KERNEL; eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_6CTX_USER; user_context_mask = 0x000f; break; case 10: contextcfg = QIB7322_CONTEXTCFG_10CTX; eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_10CTX_KERNEL; eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_10CTX_USER; user_context_mask = 0x00ff; break; case 18: contextcfg = QIB7322_CONTEXTCFG_18CTX; eager_array_size_kernel = QIB7322_EAGER_ARRAY_SIZE_18CTX_KERNEL; eager_array_size_user = QIB7322_EAGER_ARRAY_SIZE_18CTX_USER; user_context_mask = 0xffff; break; default: linker_assert ( 0, invalid_QIB7322_NUM_CONTEXTS ); return -EINVAL; } /* Configure number of contexts */ memset ( &rcvctrl, 0, sizeof ( rcvctrl ) ); BIT_FILL_2 ( &rcvctrl, TailUpd, 1, ContextCfg, contextcfg ); qib7322_writeq ( qib7322, &rcvctrl, QIB_7322_RcvCtrl_offset ); /* Map QPNs to contexts */ memset ( &rcvctrlp, 0, sizeof ( rcvctrlp ) ); BIT_FILL_3 ( &rcvctrlp, RcvIBPortEnable, 1, RcvQPMapEnable, 1, RcvPartitionKeyDisable, 1 ); qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_0_offset ); qib7322_writeq ( qib7322, &rcvctrlp, QIB_7322_RcvCtrl_1_offset ); memset ( &rcvqpmaptablea0, 0, sizeof ( rcvqpmaptablea0 ) ); BIT_FILL_6 ( &rcvqpmaptablea0, RcvQPMapContext0, 0, RcvQPMapContext1, 2, RcvQPMapContext2, 4, RcvQPMapContext3, 6, RcvQPMapContext4, 8, RcvQPMapContext5, 10 ); qib7322_writeq ( qib7322, &rcvqpmaptablea0, QIB_7322_RcvQPMapTableA_0_offset ); memset ( &rcvqpmaptableb0, 0, sizeof ( rcvqpmaptableb0 ) ); BIT_FILL_3 ( &rcvqpmaptableb0, RcvQPMapContext6, 12, RcvQPMapContext7, 14, RcvQPMapContext8, 16 ); qib7322_writeq ( qib7322, &rcvqpmaptableb0, QIB_7322_RcvQPMapTableB_0_offset ); memset ( &rcvqpmaptablea1, 0, sizeof ( rcvqpmaptablea1 ) ); BIT_FILL_6 ( &rcvqpmaptablea1, RcvQPMapContext0, 1, RcvQPMapContext1, 3, RcvQPMapContext2, 5, RcvQPMapContext3, 7, RcvQPMapContext4, 9, RcvQPMapContext5, 11 ); qib7322_writeq ( qib7322, &rcvqpmaptablea1, QIB_7322_RcvQPMapTableA_1_offset ); memset ( &rcvqpmaptableb1, 0, sizeof ( rcvqpmaptableb1 ) ); BIT_FILL_3 ( &rcvqpmaptableb1, RcvQPMapContext6, 13, RcvQPMapContext7, 15, RcvQPMapContext8, 17 ); qib7322_writeq ( qib7322, &rcvqpmaptableb1, QIB_7322_RcvQPMapTableB_1_offset ); /* Map multicast QPNs to contexts */ memset ( &rcvqpmcastctx0, 0, sizeof ( rcvqpmcastctx0 ) ); BIT_FILL_1 ( &rcvqpmcastctx0, RcvQpMcContext, 0 ); qib7322_writeq ( qib7322, &rcvqpmcastctx0, QIB_7322_RcvQPMulticastContext_0_offset ); memset ( &rcvqpmcastctx1, 0, sizeof ( rcvqpmcastctx1 ) ); BIT_FILL_1 ( &rcvqpmcastctx1, RcvQpMcContext, 1 ); qib7322_writeq ( qib7322, &rcvqpmcastctx1, QIB_7322_RcvQPMulticastContext_1_offset ); /* Configure receive header buffer sizes */ memset ( &rcvhdrcnt, 0, sizeof ( rcvhdrcnt ) ); BIT_FILL_1 ( &rcvhdrcnt, Value, QIB7322_RECV_HEADER_COUNT ); qib7322_writeq ( qib7322, &rcvhdrcnt, QIB_7322_RcvHdrCnt_offset ); memset ( &rcvhdrentsize, 0, sizeof ( rcvhdrentsize ) ); BIT_FILL_1 ( &rcvhdrentsize, Value, ( QIB7322_RECV_HEADER_SIZE >> 2 ) ); qib7322_writeq ( qib7322, &rcvhdrentsize, QIB_7322_RcvHdrEntSize_offset ); /* Calculate eager array start addresses for each context */ qib7322_readq ( qib7322, &rcvegrbase, QIB_7322_RcvEgrBase_offset ); egrbase = BIT_GET ( &rcvegrbase, Value ); for ( ctx = 0 ; ctx < QIB7322_MAX_PORTS ; ctx++ ) { qib7322->recv_wq[ctx].eager_array = egrbase; qib7322->recv_wq[ctx].eager_entries = eager_array_size_kernel; egrbase += ( eager_array_size_kernel * sizeof ( struct QIB_7322_RcvEgr ) ); } for ( ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) { qib7322->recv_wq[ctx].eager_array = egrbase; qib7322->recv_wq[ctx].eager_entries = eager_array_size_user; egrbase += ( eager_array_size_user * sizeof ( struct QIB_7322_RcvEgr ) ); } for ( ctx = 0 ; ctx < QIB7322_NUM_CONTEXTS ; ctx++ ) { DBGC ( qib7322, "QIB7322 %p CTX %d eager array at %lx (%d " "entries)\n", qib7322, ctx, qib7322->recv_wq[ctx].eager_array, qib7322->recv_wq[ctx].eager_entries ); } /* Set the BTH QP for Infinipath packets to an unused value */ memset ( &rcvbthqp, 0, sizeof ( rcvbthqp ) ); BIT_FILL_1 ( &rcvbthqp, RcvBTHQP, QIB7322_QP_IDETH ); qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_0_offset ); qib7322_writeq ( qib7322, &rcvbthqp, QIB_7322_RcvBTHQP_1_offset ); /* Assign initial credits */ memset ( &rxcreditvl, 0, sizeof ( rxcreditvl ) ); BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL0 ); qib7322_writeq_array8b ( qib7322, &rxcreditvl, QIB_7322_RxCreditVL0_0_offset, 0 ); qib7322_writeq_array8b ( qib7322, &rxcreditvl, QIB_7322_RxCreditVL0_1_offset, 0 ); BIT_FILL_1 ( &rxcreditvl, RxMaxCreditVL, QIB7322_MAX_CREDITS_VL15 ); qib7322_writeq_array8b ( qib7322, &rxcreditvl, QIB_7322_RxCreditVL0_0_offset, 15 ); qib7322_writeq_array8b ( qib7322, &rxcreditvl, QIB_7322_RxCreditVL0_1_offset, 15 ); return 0; } /** * Shut down receive datapath * * @v qib7322 QIB7322 device */ static void qib7322_fini_recv ( struct qib7322 *qib7322 __unused ) { /* Nothing to do; all contexts were already disabled when the * queue pairs were destroyed */ } /*************************************************************************** * * Completion queue operations * *************************************************************************** */ /** * Create completion queue * * @v ibdev Infiniband device * @v cq Completion queue * @ret rc Return status code */ static int qib7322_create_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); static int cqn; /* The hardware has no concept of completion queues. We * simply use the association between CQs and WQs (already * handled by the IB core) to decide which WQs to poll. * * We do set a CQN, just to avoid confusing debug messages * from the IB core. */ cq->cqn = ++cqn; DBGC ( qib7322, "QIB7322 %p CQN %ld created\n", qib7322, cq->cqn ); return 0; } /** * Destroy completion queue * * @v ibdev Infiniband device * @v cq Completion queue */ static void qib7322_destroy_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); /* Nothing to do */ DBGC ( qib7322, "QIB7322 %p CQN %ld destroyed\n", qib7322, cq->cqn ); } /*************************************************************************** * * Queue pair operations * *************************************************************************** */ /** * Create queue pair * * @v ibdev Infiniband device * @v qp Queue pair * @ret rc Return status code */ static int qib7322_create_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); unsigned int ctx; int rc; /* Allocate a context and QPN */ if ( ( rc = qib7322_alloc_ctx ( ibdev, qp ) ) != 0 ) goto err_alloc_ctx; ctx = qib7322_ctx ( ibdev, qp ); /* Set work-queue private data pointers */ ib_wq_set_drvdata ( &qp->send, &qib7322->send_wq[ctx] ); ib_wq_set_drvdata ( &qp->recv, &qib7322->recv_wq[ctx] ); /* Create receive work queue */ if ( ( rc = qib7322_create_recv_wq ( ibdev, qp ) ) != 0 ) goto err_create_recv_wq; /* Create send work queue */ if ( ( rc = qib7322_create_send_wq ( ibdev, qp ) ) != 0 ) goto err_create_send_wq; return 0; qib7322_destroy_send_wq ( ibdev, qp ); err_create_send_wq: qib7322_destroy_recv_wq ( ibdev, qp ); err_create_recv_wq: qib7322_free_ctx ( ibdev, qp ); err_alloc_ctx: return rc; } /** * Modify queue pair * * @v ibdev Infiniband device * @v qp Queue pair * @ret rc Return status code */ static int qib7322_modify_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); /* Nothing to do; the hardware doesn't have a notion of queue * keys */ DBGC2 ( qib7322, "QIB7322 %p QPN %ld modified\n", qib7322, qp->qpn ); return 0; } /** * Destroy queue pair * * @v ibdev Infiniband device * @v qp Queue pair */ static void qib7322_destroy_qp ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { qib7322_destroy_send_wq ( ibdev, qp ); qib7322_destroy_recv_wq ( ibdev, qp ); qib7322_free_ctx ( ibdev, qp ); } /*************************************************************************** * * Work request operations * *************************************************************************** */ /** * Post send work queue entry * * @v ibdev Infiniband device * @v qp Queue pair * @v av Address vector * @v iobuf I/O buffer * @ret rc Return status code */ static int qib7322_post_send ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_address_vector *av, struct io_buffer *iobuf ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->send; struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_SendPbc sendpbc; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); uint8_t header_buf[IB_MAX_HEADER_SIZE]; struct io_buffer headers; int send_buf; unsigned long start_offset; unsigned long offset; size_t len; ssize_t frag_len; uint32_t *data; /* Allocate send buffer and calculate offset */ send_buf = qib7322_alloc_send_buf ( qib7322, qib7322_wq->send_bufs ); if ( send_buf < 0 ) return send_buf; start_offset = offset = qib7322_send_buffer_offset ( qib7322, qib7322_wq->send_bufs, send_buf ); /* Store I/O buffer and send buffer index */ assert ( wq->iobufs[qib7322_wq->prod] == NULL ); wq->iobufs[qib7322_wq->prod] = iobuf; qib7322_wq->used[qib7322_wq->prod] = send_buf; /* Construct headers */ iob_populate ( &headers, header_buf, 0, sizeof ( header_buf ) ); iob_reserve ( &headers, sizeof ( header_buf ) ); ib_push ( ibdev, &headers, qp, iob_len ( iobuf ), av ); /* Calculate packet length */ len = ( ( sizeof ( sendpbc ) + iob_len ( &headers ) + iob_len ( iobuf ) + 3 ) & ~3 ); /* Construct send per-buffer control word */ memset ( &sendpbc, 0, sizeof ( sendpbc ) ); BIT_FILL_3 ( &sendpbc, LengthP1_toibc, ( ( len >> 2 ) - 1 ), Port, port, VL15, ( ( qp->type == IB_QPT_SMI ) ? 1 : 0 ) ); /* Write SendPbc */ DBG_DISABLE ( DBGLVL_IO ); qib7322_writeq ( qib7322, &sendpbc, offset ); offset += sizeof ( sendpbc ); /* Write headers */ for ( data = headers.data, frag_len = iob_len ( &headers ) ; frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) { qib7322_writel ( qib7322, *data, offset ); } /* Write data */ for ( data = iobuf->data, frag_len = iob_len ( iobuf ) ; frag_len > 0 ; data++, offset += 4, frag_len -= 4 ) { qib7322_writel ( qib7322, *data, offset ); } DBG_ENABLE ( DBGLVL_IO ); assert ( ( start_offset + len ) == offset ); DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) posted [%lx,%lx)\n", qib7322, qp->qpn, send_buf, qib7322_wq->prod, start_offset, offset ); /* Increment producer counter */ qib7322_wq->prod = ( ( qib7322_wq->prod + 1 ) & ( wq->num_wqes - 1 ) ); return 0; } /** * Complete send work queue entry * * @v ibdev Infiniband device * @v qp Queue pair * @v wqe_idx Work queue entry index */ static void qib7322_complete_send ( struct ib_device *ibdev, struct ib_queue_pair *qp, unsigned int wqe_idx ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->send; struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct io_buffer *iobuf; unsigned int send_buf; /* Parse completion */ send_buf = qib7322_wq->used[wqe_idx]; DBGC2 ( qib7322, "QIB7322 %p QPN %ld TX %04x(%04x) complete\n", qib7322, qp->qpn, send_buf, wqe_idx ); /* Complete work queue entry */ iobuf = wq->iobufs[wqe_idx]; assert ( iobuf != NULL ); ib_complete_send ( ibdev, qp, iobuf, 0 ); wq->iobufs[wqe_idx] = NULL; /* Free send buffer */ qib7322_free_send_buf ( qib7322, qib7322_wq->send_bufs, send_buf ); } /** * Poll send work queue * * @v ibdev Infiniband device * @v qp Queue pair */ static void qib7322_poll_send_wq ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->send; struct qib7322_send_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); unsigned int send_buf; /* Look for completions */ while ( wq->fill ) { /* Check to see if send buffer has completed */ send_buf = qib7322_wq->used[qib7322_wq->cons]; if ( qib7322_send_buf_in_use ( qib7322, send_buf ) ) break; /* Complete this buffer */ qib7322_complete_send ( ibdev, qp, qib7322_wq->cons ); /* Increment consumer counter */ qib7322_wq->cons = ( ( qib7322_wq->cons + 1 ) & ( wq->num_wqes - 1 ) ); } } /** * Post receive work queue entry * * @v ibdev Infiniband device * @v qp Queue pair * @v iobuf I/O buffer * @ret rc Return status code */ static int qib7322_post_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct io_buffer *iobuf ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->recv; struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_RcvEgr rcvegr; struct QIB_7322_scalar rcvegrindexhead; unsigned int ctx = qib7322_ctx ( ibdev, qp ); physaddr_t addr; size_t len; unsigned int wqe_idx; unsigned int bufsize; /* Sanity checks */ addr = virt_to_bus ( iobuf->data ); len = iob_tailroom ( iobuf ); if ( addr & ( QIB7322_EAGER_BUFFER_ALIGN - 1 ) ) { DBGC ( qib7322, "QIB7322 %p QPN %ld misaligned RX buffer " "(%08lx)\n", qib7322, qp->qpn, addr ); return -EINVAL; } if ( len != QIB7322_RECV_PAYLOAD_SIZE ) { DBGC ( qib7322, "QIB7322 %p QPN %ld wrong RX buffer size " "(%zd)\n", qib7322, qp->qpn, len ); return -EINVAL; } /* Calculate eager producer index and WQE index */ wqe_idx = ( qib7322_wq->eager_prod & ( wq->num_wqes - 1 ) ); assert ( wq->iobufs[wqe_idx] == NULL ); /* Store I/O buffer */ wq->iobufs[wqe_idx] = iobuf; /* Calculate buffer size */ switch ( QIB7322_RECV_PAYLOAD_SIZE ) { case 2048: bufsize = QIB7322_EAGER_BUFFER_2K; break; case 4096: bufsize = QIB7322_EAGER_BUFFER_4K; break; case 8192: bufsize = QIB7322_EAGER_BUFFER_8K; break; case 16384: bufsize = QIB7322_EAGER_BUFFER_16K; break; case 32768: bufsize = QIB7322_EAGER_BUFFER_32K; break; case 65536: bufsize = QIB7322_EAGER_BUFFER_64K; break; default: linker_assert ( 0, invalid_rx_payload_size ); bufsize = QIB7322_EAGER_BUFFER_NONE; } /* Post eager buffer */ memset ( &rcvegr, 0, sizeof ( rcvegr ) ); BIT_FILL_2 ( &rcvegr, Addr, ( addr >> 11 ), BufSize, bufsize ); qib7322_writeq_array8b ( qib7322, &rcvegr, qib7322_wq->eager_array, qib7322_wq->eager_prod ); DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x(%04x) posted " "[%lx,%lx)\n", qib7322, qp->qpn, qib7322_wq->eager_prod, wqe_idx, addr, ( addr + len ) ); /* Increment producer index */ qib7322_wq->eager_prod = ( ( qib7322_wq->eager_prod + 1 ) & ( qib7322_wq->eager_entries - 1 ) ); /* Update head index */ memset ( &rcvegrindexhead, 0, sizeof ( rcvegrindexhead ) ); BIT_FILL_1 ( &rcvegrindexhead, Value, ( ( qib7322_wq->eager_prod + 1 ) & ( qib7322_wq->eager_entries - 1 ) ) ); qib7322_writeq_array64k ( qib7322, &rcvegrindexhead, QIB_7322_RcvEgrIndexHead0_offset, ctx ); return 0; } /** * Complete receive work queue entry * * @v ibdev Infiniband device * @v qp Queue pair * @v header_offs Header offset */ static void qib7322_complete_recv ( struct ib_device *ibdev, struct ib_queue_pair *qp, unsigned int header_offs ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->recv; struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_RcvHdrFlags *rcvhdrflags; struct QIB_7322_RcvEgr rcvegr; struct io_buffer headers; struct io_buffer *iobuf; struct ib_queue_pair *intended_qp; struct ib_address_vector av; unsigned int rcvtype; unsigned int pktlen; unsigned int egrindex; unsigned int useegrbfr; unsigned int iberr, mkerr, tiderr, khdrerr, mtuerr; unsigned int lenerr, parityerr, vcrcerr, icrcerr; unsigned int err; unsigned int hdrqoffset; unsigned int header_len; unsigned int padded_payload_len; unsigned int wqe_idx; size_t payload_len; int qp0; int rc; /* RcvHdrFlags are at the end of the header entry */ rcvhdrflags = ( qib7322_wq->header + header_offs + QIB7322_RECV_HEADER_SIZE - sizeof ( *rcvhdrflags ) ); rcvtype = BIT_GET ( rcvhdrflags, RcvType ); pktlen = ( BIT_GET ( rcvhdrflags, PktLen ) << 2 ); egrindex = BIT_GET ( rcvhdrflags, EgrIndex ); useegrbfr = BIT_GET ( rcvhdrflags, UseEgrBfr ); hdrqoffset = ( BIT_GET ( rcvhdrflags, HdrqOffset ) << 2 ); iberr = BIT_GET ( rcvhdrflags, IBErr ); mkerr = BIT_GET ( rcvhdrflags, MKErr ); tiderr = BIT_GET ( rcvhdrflags, TIDErr ); khdrerr = BIT_GET ( rcvhdrflags, KHdrErr ); mtuerr = BIT_GET ( rcvhdrflags, MTUErr ); lenerr = BIT_GET ( rcvhdrflags, LenErr ); parityerr = BIT_GET ( rcvhdrflags, ParityErr ); vcrcerr = BIT_GET ( rcvhdrflags, VCRCErr ); icrcerr = BIT_GET ( rcvhdrflags, ICRCErr ); header_len = ( QIB7322_RECV_HEADER_SIZE - hdrqoffset - sizeof ( *rcvhdrflags ) ); padded_payload_len = ( pktlen - header_len - 4 /* ICRC */ ); err = ( iberr | mkerr | tiderr | khdrerr | mtuerr | lenerr | parityerr | vcrcerr | icrcerr ); /* IB header is placed immediately before RcvHdrFlags */ iob_populate ( &headers, ( ( ( void * ) rcvhdrflags ) - header_len ), header_len, header_len ); /* Dump diagnostic information */ DBGC2 ( qib7322, "QIB7322 %p QPN %ld RX egr %04x%s hdr %d type %d len " "%d(%d+%d+4)%s%s%s%s%s%s%s%s%s%s%s\n", qib7322, qp->qpn, egrindex, ( useegrbfr ? "" : "(unused)" ), ( header_offs / QIB7322_RECV_HEADER_SIZE ), rcvtype, pktlen, header_len, padded_payload_len, ( err ? " [Err" : "" ), ( iberr ? " IB" : "" ), ( mkerr ? " MK" : "" ), ( tiderr ? " TID" : "" ), ( khdrerr ? " KHdr" : "" ), ( mtuerr ? " MTU" : "" ), ( lenerr ? " Len" : "" ), ( parityerr ? " Parity" : ""), ( vcrcerr ? " VCRC" : "" ), ( icrcerr ? " ICRC" : "" ), ( err ? "]" : "" ) ); DBGCP_HDA ( qib7322, hdrqoffset, headers.data, ( header_len + sizeof ( *rcvhdrflags ) ) ); /* Parse header to generate address vector */ qp0 = ( qp->qpn == 0 ); intended_qp = NULL; if ( ( rc = ib_pull ( ibdev, &headers, ( qp0 ? &intended_qp : NULL ), &payload_len, &av ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p could not parse headers: %s\n", qib7322, strerror ( rc ) ); err = 1; } if ( ! intended_qp ) intended_qp = qp; /* Complete this buffer and any skipped buffers. Note that * when the hardware runs out of buffers, it will repeatedly * report the same buffer (the tail) as a TID error, and that * it also has a habit of sometimes skipping over several * buffers at once. */ while ( 1 ) { /* If we have caught up to the producer counter, stop. * This will happen when the hardware first runs out * of buffers and starts reporting TID errors against * the eager buffer it wants to use next. */ if ( qib7322_wq->eager_cons == qib7322_wq->eager_prod ) break; /* If we have caught up to where we should be after * completing this egrindex, stop. We phrase the test * this way to avoid completing the entire ring when * we receive the same egrindex twice in a row. */ if ( ( qib7322_wq->eager_cons == ( ( egrindex + 1 ) & ( qib7322_wq->eager_entries - 1 )))) break; /* Identify work queue entry and corresponding I/O * buffer. */ wqe_idx = ( qib7322_wq->eager_cons & ( wq->num_wqes - 1 ) ); iobuf = wq->iobufs[wqe_idx]; assert ( iobuf != NULL ); wq->iobufs[wqe_idx] = NULL; /* Complete the eager buffer */ if ( qib7322_wq->eager_cons == egrindex ) { /* Completing the eager buffer described in * this header entry. */ iob_put ( iobuf, payload_len ); rc = ( err ? -EIO : ( useegrbfr ? 0 : -ECANCELED ) ); /* Redirect to target QP if necessary */ if ( qp != intended_qp ) { DBGC2 ( qib7322, "QIB7322 %p redirecting QPN " "%ld => %ld\n", qib7322, qp->qpn, intended_qp->qpn ); /* Compensate for incorrect fill levels */ qp->recv.fill--; intended_qp->recv.fill++; } ib_complete_recv ( ibdev, intended_qp, &av, iobuf, rc); } else { /* Completing on a skipped-over eager buffer */ ib_complete_recv ( ibdev, qp, &av, iobuf, -ECANCELED ); } /* Clear eager buffer */ memset ( &rcvegr, 0, sizeof ( rcvegr ) ); qib7322_writeq_array8b ( qib7322, &rcvegr, qib7322_wq->eager_array, qib7322_wq->eager_cons ); /* Increment consumer index */ qib7322_wq->eager_cons = ( ( qib7322_wq->eager_cons + 1 ) & ( qib7322_wq->eager_entries - 1 ) ); } } /** * Poll receive work queue * * @v ibdev Infiniband device * @v qp Queue pair */ static void qib7322_poll_recv_wq ( struct ib_device *ibdev, struct ib_queue_pair *qp ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct ib_work_queue *wq = &qp->recv; struct qib7322_recv_work_queue *qib7322_wq = ib_wq_get_drvdata ( wq ); struct QIB_7322_RcvHdrHead0 rcvhdrhead; unsigned int ctx = qib7322_ctx ( ibdev, qp ); unsigned int header_prod; /* Check for received packets */ header_prod = ( BIT_GET ( &qib7322_wq->header_prod, Value ) << 2 ); if ( header_prod == qib7322_wq->header_cons ) return; /* Process all received packets */ while ( qib7322_wq->header_cons != header_prod ) { /* Complete the receive */ qib7322_complete_recv ( ibdev, qp, qib7322_wq->header_cons ); /* Increment the consumer offset */ qib7322_wq->header_cons += QIB7322_RECV_HEADER_SIZE; qib7322_wq->header_cons %= QIB7322_RECV_HEADERS_SIZE; /* QIB7322 has only one send buffer per port for VL15, * which almost always leads to send buffer exhaustion * and dropped MADs. Mitigate this by refusing to * process more than one VL15 MAD per poll, which will * enforce interleaved TX/RX polls. */ if ( qp->type == IB_QPT_SMI ) break; } /* Update consumer offset */ memset ( &rcvhdrhead, 0, sizeof ( rcvhdrhead ) ); BIT_FILL_2 ( &rcvhdrhead, RcvHeadPointer, ( qib7322_wq->header_cons >> 2 ), counter, 1 ); qib7322_writeq_array64k ( qib7322, &rcvhdrhead, QIB_7322_RcvHdrHead0_offset, ctx ); } /** * Poll completion queue * * @v ibdev Infiniband device * @v cq Completion queue */ static void qib7322_poll_cq ( struct ib_device *ibdev, struct ib_completion_queue *cq ) { struct ib_work_queue *wq; /* Poll associated send and receive queues */ list_for_each_entry ( wq, &cq->work_queues, list ) { if ( wq->is_send ) { qib7322_poll_send_wq ( ibdev, wq->qp ); } else { qib7322_poll_recv_wq ( ibdev, wq->qp ); } } } /*************************************************************************** * * Event queues * *************************************************************************** */ /** * Poll event queue * * @v ibdev Infiniband device */ static void qib7322_poll_eq ( struct ib_device *ibdev ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct QIB_7322_ErrStatus_0 errstatus; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); /* Check for and clear status bits */ DBG_DISABLE ( DBGLVL_IO ); qib7322_readq_port ( qib7322, &errstatus, QIB_7322_ErrStatus_0_offset, port ); if ( errstatus.u.qwords[0] ) { DBGC ( qib7322, "QIB7322 %p port %d status %08x%08x\n", qib7322, port, errstatus.u.dwords[1], errstatus.u.dwords[0] ); qib7322_writeq_port ( qib7322, &errstatus, QIB_7322_ErrClear_0_offset, port ); } DBG_ENABLE ( DBGLVL_IO ); /* Check for link status changes */ if ( BIT_GET ( &errstatus, IBStatusChanged ) ) qib7322_link_state_changed ( ibdev ); } /*************************************************************************** * * Infiniband link-layer operations * *************************************************************************** */ /** * Determine supported link speeds * * @v qib7322 QIB7322 device * @ret supported Supported link speeds */ static unsigned int qib7322_link_speed_supported ( struct qib7322 *qib7322, unsigned int port ) { struct QIB_7322_feature_mask features; struct QIB_7322_Revision revision; unsigned int supported; unsigned int boardid; /* Read the active feature mask */ qib7322_readq ( qib7322, &features, QIB_7322_active_feature_mask_offset ); switch ( port ) { case 0 : supported = BIT_GET ( &features, Port0_Link_Speed_Supported ); break; case 1 : supported = BIT_GET ( &features, Port1_Link_Speed_Supported ); break; default: DBGC ( qib7322, "QIB7322 %p port %d is invalid\n", qib7322, port ); supported = 0; break; } /* Apply hacks for specific board IDs */ qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset ); boardid = BIT_GET ( &revision, BoardID ); switch ( boardid ) { case QIB7322_BOARD_QMH7342 : DBGC2 ( qib7322, "QIB7322 %p is a QMH7342; forcing QDR-only\n", qib7322 ); supported = IB_LINK_SPEED_QDR; break; default: /* Do nothing */ break; } DBGC2 ( qib7322, "QIB7322 %p port %d %s%s%s%s\n", qib7322, port, ( supported ? "supports" : "disabled" ), ( ( supported & IB_LINK_SPEED_SDR ) ? " SDR" : "" ), ( ( supported & IB_LINK_SPEED_DDR ) ? " DDR" : "" ), ( ( supported & IB_LINK_SPEED_QDR ) ? " QDR" : "" ) ); return supported; } /** * Initialise Infiniband link * * @v ibdev Infiniband device * @ret rc Return status code */ static int qib7322_open ( struct ib_device *ibdev ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct QIB_7322_IBCCtrlA_0 ibcctrla; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); /* Enable link */ qib7322_readq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); BIT_SET ( &ibcctrla, IBLinkEn, 1 ); qib7322_writeq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); return 0; } /** * Close Infiniband link * * @v ibdev Infiniband device */ static void qib7322_close ( struct ib_device *ibdev ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); struct QIB_7322_IBCCtrlA_0 ibcctrla; unsigned int port = ( ibdev->port - QIB7322_PORT_BASE ); /* Disable link */ qib7322_readq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); BIT_SET ( &ibcctrla, IBLinkEn, 0 ); qib7322_writeq_port ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset, port ); } /*************************************************************************** * * Multicast group operations * *************************************************************************** */ /** * Attach to multicast group * * @v ibdev Infiniband device * @v qp Queue pair * @v gid Multicast GID * @ret rc Return status code */ static int qib7322_mcast_attach ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); ( void ) qib7322; ( void ) qp; ( void ) gid; return 0; } /** * Detach from multicast group * * @v ibdev Infiniband device * @v qp Queue pair * @v gid Multicast GID */ static void qib7322_mcast_detach ( struct ib_device *ibdev, struct ib_queue_pair *qp, struct ib_gid *gid ) { struct qib7322 *qib7322 = ib_get_drvdata ( ibdev ); ( void ) qib7322; ( void ) qp; ( void ) gid; } /** QIB7322 Infiniband operations */ static struct ib_device_operations qib7322_ib_operations = { .create_cq = qib7322_create_cq, .destroy_cq = qib7322_destroy_cq, .create_qp = qib7322_create_qp, .modify_qp = qib7322_modify_qp, .destroy_qp = qib7322_destroy_qp, .post_send = qib7322_post_send, .post_recv = qib7322_post_recv, .poll_cq = qib7322_poll_cq, .poll_eq = qib7322_poll_eq, .open = qib7322_open, .close = qib7322_close, .mcast_attach = qib7322_mcast_attach, .mcast_detach = qib7322_mcast_detach, .set_port_info = qib7322_set_port_info, .set_pkey_table = qib7322_set_pkey_table, }; /*************************************************************************** * * I2C bus operations * *************************************************************************** */ /** QIB7322 I2C bit to GPIO mappings */ static unsigned int qib7322_i2c_bits[] = { [I2C_BIT_SCL] = ( 1 << QIB7322_GPIO_SCL ), [I2C_BIT_SDA] = ( 1 << QIB7322_GPIO_SDA ), }; /** * Read QIB7322 I2C line status * * @v basher Bit-bashing interface * @v bit_id Bit number * @ret zero Input is a logic 0 * @ret non-zero Input is a logic 1 */ static int qib7322_i2c_read_bit ( struct bit_basher *basher, unsigned int bit_id ) { struct qib7322 *qib7322 = container_of ( basher, struct qib7322, i2c.basher ); struct QIB_7322_EXTStatus extstatus; unsigned int status; DBG_DISABLE ( DBGLVL_IO ); qib7322_readq ( qib7322, &extstatus, QIB_7322_EXTStatus_offset ); status = ( BIT_GET ( &extstatus, GPIOIn ) & qib7322_i2c_bits[bit_id] ); DBG_ENABLE ( DBGLVL_IO ); return status; } /** * Write QIB7322 I2C line status * * @v basher Bit-bashing interface * @v bit_id Bit number * @v data Value to write */ static void qib7322_i2c_write_bit ( struct bit_basher *basher, unsigned int bit_id, unsigned long data ) { struct qib7322 *qib7322 = container_of ( basher, struct qib7322, i2c.basher ); struct QIB_7322_EXTCtrl extctrl; struct QIB_7322_GPIO gpioout; unsigned int bit = qib7322_i2c_bits[bit_id]; unsigned int outputs = 0; unsigned int output_enables = 0; DBG_DISABLE ( DBGLVL_IO ); /* Read current GPIO mask and outputs */ qib7322_readq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset ); qib7322_readq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset ); /* Update outputs and output enables. I2C lines are tied * high, so we always set the output to 0 and use the output * enable to control the line. */ output_enables = BIT_GET ( &extctrl, GPIOOe ); output_enables = ( ( output_enables & ~bit ) | ( ~data & bit ) ); outputs = BIT_GET ( &gpioout, GPIO ); outputs = ( outputs & ~bit ); BIT_SET ( &extctrl, GPIOOe, output_enables ); BIT_SET ( &gpioout, GPIO, outputs ); /* Write the output enable first; that way we avoid logic * hazards. */ qib7322_writeq ( qib7322, &extctrl, QIB_7322_EXTCtrl_offset ); qib7322_writeq ( qib7322, &gpioout, QIB_7322_GPIOOut_offset ); mb(); DBG_ENABLE ( DBGLVL_IO ); } /** QIB7322 I2C bit-bashing interface operations */ static struct bit_basher_operations qib7322_i2c_basher_ops = { .read = qib7322_i2c_read_bit, .write = qib7322_i2c_write_bit, }; /** * Initialise QIB7322 I2C subsystem * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_init_i2c ( struct qib7322 *qib7322 ) { static int try_eeprom_address[] = { 0x51, 0x50 }; unsigned int i; int rc; /* Initialise bus */ if ( ( rc = init_i2c_bit_basher ( &qib7322->i2c, &qib7322_i2c_basher_ops ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p could not initialise I2C bus: %s\n", qib7322, strerror ( rc ) ); return rc; } /* Probe for devices */ for ( i = 0 ; i < ( sizeof ( try_eeprom_address ) / sizeof ( try_eeprom_address[0] ) ) ; i++ ) { init_i2c_eeprom ( &qib7322->eeprom, try_eeprom_address[i] ); if ( ( rc = i2c_check_presence ( &qib7322->i2c.i2c, &qib7322->eeprom ) ) == 0 ) { DBGC2 ( qib7322, "QIB7322 %p found EEPROM at %02x\n", qib7322, try_eeprom_address[i] ); return 0; } } DBGC ( qib7322, "QIB7322 %p could not find EEPROM\n", qib7322 ); return -ENODEV; } /** * Read EEPROM parameters * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_read_eeprom ( struct qib7322 *qib7322 ) { struct i2c_interface *i2c = &qib7322->i2c.i2c; struct ib_gid_half *guid = &qib7322->guid; int rc; /* Read GUID */ if ( ( rc = i2c->read ( i2c, &qib7322->eeprom, QIB7322_EEPROM_GUID_OFFSET, guid->u.bytes, sizeof ( *guid ) ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p could not read GUID: %s\n", qib7322, strerror ( rc ) ); return rc; } DBGC2 ( qib7322, "QIB7322 %p has GUID %02x:%02x:%02x:%02x:%02x:%02x:" "%02x:%02x\n", qib7322, guid->u.bytes[0], guid->u.bytes[1], guid->u.bytes[2], guid->u.bytes[3], guid->u.bytes[4], guid->u.bytes[5], guid->u.bytes[6], guid->u.bytes[7] ); /* Read serial number (debug only) */ if ( DBG_LOG ) { uint8_t serial[QIB7322_EEPROM_SERIAL_SIZE + 1]; serial[ sizeof ( serial ) - 1 ] = '\0'; if ( ( rc = i2c->read ( i2c, &qib7322->eeprom, QIB7322_EEPROM_SERIAL_OFFSET, serial, ( sizeof ( serial ) - 1 ) ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p could not read serial: " "%s\n", qib7322, strerror ( rc ) ); return rc; } DBGC2 ( qib7322, "QIB7322 %p has serial number \"%s\"\n", qib7322, serial ); } return 0; } /*************************************************************************** * * Advanced High-performance Bus (AHB) access * *************************************************************************** */ /** * Wait for AHB transaction to complete * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_ahb_wait ( struct qib7322 *qib7322 ) { struct QIB_7322_ahb_transaction_reg transaction; unsigned int i; /* Wait for Ready bit to be asserted */ for ( i = 0 ; i < QIB7322_AHB_MAX_WAIT_US ; i++ ) { qib7322_readq ( qib7322, &transaction, QIB_7322_ahb_transaction_reg_offset ); if ( BIT_GET ( &transaction, ahb_rdy ) ) return 0; udelay ( 1 ); } DBGC ( qib7322, "QIB7322 %p timed out waiting for AHB transaction\n", qib7322 ); return -ETIMEDOUT; } /** * Request ownership of the AHB * * @v qib7322 QIB7322 device * @v location AHB location * @ret rc Return status code */ static int qib7322_ahb_request ( struct qib7322 *qib7322, unsigned int location ) { struct QIB_7322_ahb_access_ctrl access; int rc; /* Request ownership */ memset ( &access, 0, sizeof ( access ) ); BIT_FILL_2 ( &access, sw_ahb_sel, 1, sw_sel_ahb_trgt, QIB7322_AHB_LOC_TARGET ( location ) ); qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset ); /* Wait for ownership to be granted */ if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p could not obtain AHB ownership: " "%s\n", qib7322, strerror ( rc ) ); return rc; } return 0; } /** * Release ownership of the AHB * * @v qib7322 QIB7322 device */ static void qib7322_ahb_release ( struct qib7322 *qib7322 ) { struct QIB_7322_ahb_access_ctrl access; memset ( &access, 0, sizeof ( access ) ); qib7322_writeq ( qib7322, &access, QIB_7322_ahb_access_ctrl_offset ); } /** * Read data via AHB * * @v qib7322 QIB7322 device * @v location AHB location * @v data Data to read * @ret rc Return status code * * You must have already acquired ownership of the AHB. */ static int qib7322_ahb_read ( struct qib7322 *qib7322, unsigned int location, uint32_t *data ) { struct QIB_7322_ahb_transaction_reg xact; int rc; /* Initiate transaction */ memset ( &xact, 0, sizeof ( xact ) ); BIT_FILL_2 ( &xact, ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ), write_not_read, 0 ); qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset ); /* Wait for transaction to complete */ if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 ) return rc; /* Read transaction data */ qib7322_readq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset ); *data = BIT_GET ( &xact, ahb_data ); return 0; } /** * Write data via AHB * * @v qib7322 QIB7322 device * @v location AHB location * @v data Data to write * @ret rc Return status code * * You must have already acquired ownership of the AHB. */ static int qib7322_ahb_write ( struct qib7322 *qib7322, unsigned int location, uint32_t data ) { struct QIB_7322_ahb_transaction_reg xact; int rc; /* Initiate transaction */ memset ( &xact, 0, sizeof ( xact ) ); BIT_FILL_3 ( &xact, ahb_address, QIB7322_AHB_LOC_ADDRESS ( location ), write_not_read, 1, ahb_data, data ); qib7322_writeq ( qib7322, &xact, QIB_7322_ahb_transaction_reg_offset ); /* Wait for transaction to complete */ if ( ( rc = qib7322_ahb_wait ( qib7322 ) ) != 0 ) return rc; return 0; } /** * Read/modify/write AHB register * * @v qib7322 QIB7322 device * @v location AHB location * @v value Value to set * @v mask Mask to apply to old value * @ret rc Return status code */ static int qib7322_ahb_mod_reg ( struct qib7322 *qib7322, unsigned int location, uint32_t value, uint32_t mask ) { uint32_t old_value; uint32_t new_value; int rc; DBG_DISABLE ( DBGLVL_IO ); /* Sanity check */ assert ( ( value & mask ) == value ); /* Acquire bus ownership */ if ( ( rc = qib7322_ahb_request ( qib7322, location ) ) != 0 ) goto out; /* Read existing value */ if ( ( rc = qib7322_ahb_read ( qib7322, location, &old_value ) ) != 0 ) goto out_release; /* Update value */ new_value = ( ( old_value & ~mask ) | value ); DBGCP ( qib7322, "QIB7322 %p AHB %x %#08x => %#08x\n", qib7322, location, old_value, new_value ); if ( ( rc = qib7322_ahb_write ( qib7322, location, new_value ) ) != 0 ) goto out_release; out_release: /* Release bus */ qib7322_ahb_release ( qib7322 ); out: DBG_ENABLE ( DBGLVL_IO ); return rc; } /** * Read/modify/write AHB register across all ports and channels * * @v qib7322 QIB7322 device * @v reg AHB register * @v value Value to set * @v mask Mask to apply to old value * @ret rc Return status code */ static int qib7322_ahb_mod_reg_all ( struct qib7322 *qib7322, unsigned int reg, uint32_t value, uint32_t mask ) { unsigned int port; unsigned int channel; unsigned int location; int rc; for ( port = 0 ; port < QIB7322_MAX_PORTS ; port++ ) { for ( channel = 0 ; channel < QIB7322_MAX_WIDTH ; channel++ ) { location = QIB7322_AHB_LOCATION ( port, channel, reg ); if ( ( rc = qib7322_ahb_mod_reg ( qib7322, location, value, mask ) ) != 0 ) return rc; } } return 0; } /*************************************************************************** * * Infiniband SerDes initialisation * *************************************************************************** */ /** * Initialise the IB SerDes * * @v qib7322 QIB7322 device * @ret rc Return status code */ static int qib7322_init_ib_serdes ( struct qib7322 *qib7322 ) { struct QIB_7322_IBCCtrlA_0 ibcctrla; struct QIB_7322_IBCCtrlB_0 ibcctrlb; struct QIB_7322_IBPCSConfig_0 ibpcsconfig; /* Configure sensible defaults for IBC */ memset ( &ibcctrla, 0, sizeof ( ibcctrla ) ); BIT_FILL_5 ( &ibcctrla, /* Tuning values taken from Linux driver */ FlowCtrlPeriod, 0x03, FlowCtrlWaterMark, 0x05, MaxPktLen, ( ( QIB7322_RECV_HEADER_SIZE + QIB7322_RECV_PAYLOAD_SIZE + 4 /* ICRC */ ) >> 2 ), PhyerrThreshold, 0xf, OverrunThreshold, 0xf ); qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_0_offset ); qib7322_writeq ( qib7322, &ibcctrla, QIB_7322_IBCCtrlA_1_offset ); /* Force SDR only to avoid needing all the DDR tuning, * Mellanox compatibility hacks etc. SDR is plenty for * boot-time operation. */ qib7322_readq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset ); BIT_SET ( &ibcctrlb, IB_ENHANCED_MODE, 0 ); BIT_SET ( &ibcctrlb, SD_SPEED_SDR, 1 ); BIT_SET ( &ibcctrlb, SD_SPEED_DDR, 0 ); BIT_SET ( &ibcctrlb, SD_SPEED_QDR, 0 ); BIT_SET ( &ibcctrlb, IB_NUM_CHANNELS, 1 ); /* 4X only */ BIT_SET ( &ibcctrlb, IB_LANE_REV_SUPPORTED, 0 ); BIT_SET ( &ibcctrlb, HRTBT_ENB, 0 ); BIT_SET ( &ibcctrlb, HRTBT_AUTO, 0 ); qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_0_offset ); qib7322_writeq ( qib7322, &ibcctrlb, QIB_7322_IBCCtrlB_1_offset ); /* Tune SerDes */ qib7322_ahb_mod_reg_all ( qib7322, 2, 0, 0x00000e00UL ); /* Bring XGXS out of reset */ memset ( &ibpcsconfig, 0, sizeof ( ibpcsconfig ) ); qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_0_offset ); qib7322_writeq ( qib7322, &ibpcsconfig, QIB_7322_IBPCSConfig_1_offset ); return 0; } /*************************************************************************** * * PCI layer interface * *************************************************************************** */ /** * Reset QIB7322 * * @v qib7322 QIB7322 device * @v pci PCI device * @ret rc Return status code */ static void qib7322_reset ( struct qib7322 *qib7322, struct pci_device *pci ) { struct QIB_7322_Control control; struct pci_config_backup backup; /* Back up PCI configuration space */ pci_backup ( pci, &backup, NULL ); /* Assert reset */ memset ( &control, 0, sizeof ( control ) ); BIT_FILL_1 ( &control, SyncReset, 1 ); qib7322_writeq ( qib7322, &control, QIB_7322_Control_offset ); /* Wait for reset to complete */ mdelay ( 1000 ); /* Restore PCI configuration space */ pci_restore ( pci, &backup, NULL ); } /** * Probe PCI device * * @v pci PCI device * @v id PCI ID * @ret rc Return status code */ static int qib7322_probe ( struct pci_device *pci, const struct pci_device_id *id __unused ) { struct qib7322 *qib7322; struct QIB_7322_Revision revision; struct ib_device *ibdev; unsigned int link_speed_supported; int i; int rc; /* Allocate QIB7322 device */ qib7322 = zalloc ( sizeof ( *qib7322 ) ); if ( ! qib7322 ) { rc = -ENOMEM; goto err_alloc_qib7322; } pci_set_drvdata ( pci, qib7322 ); /* Fix up PCI device */ adjust_pci_device ( pci ); /* Get PCI BARs */ qib7322->regs = ioremap ( pci->membase, QIB7322_BAR0_SIZE ); DBGC2 ( qib7322, "QIB7322 %p has BAR at %08lx\n", qib7322, pci->membase ); /* Reset device */ qib7322_reset ( qib7322, pci ); /* Print some general data */ qib7322_readq ( qib7322, &revision, QIB_7322_Revision_offset ); DBGC2 ( qib7322, "QIB7322 %p board %02lx v%ld.%ld.%ld.%ld\n", qib7322, BIT_GET ( &revision, BoardID ), BIT_GET ( &revision, R_SW ), BIT_GET ( &revision, R_Arch ), BIT_GET ( &revision, R_ChipRevMajor ), BIT_GET ( &revision, R_ChipRevMinor ) ); /* Initialise I2C subsystem */ if ( ( rc = qib7322_init_i2c ( qib7322 ) ) != 0 ) goto err_init_i2c; /* Read EEPROM parameters */ if ( ( rc = qib7322_read_eeprom ( qib7322 ) ) != 0 ) goto err_read_eeprom; /* Initialise send datapath */ if ( ( rc = qib7322_init_send ( qib7322 ) ) != 0 ) goto err_init_send; /* Initialise receive datapath */ if ( ( rc = qib7322_init_recv ( qib7322 ) ) != 0 ) goto err_init_recv; /* Initialise the IB SerDes */ if ( ( rc = qib7322_init_ib_serdes ( qib7322 ) ) != 0 ) goto err_init_ib_serdes; /* Allocate Infiniband devices */ for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) { link_speed_supported = qib7322_link_speed_supported ( qib7322, i ); if ( ! link_speed_supported ) continue; ibdev = alloc_ibdev ( 0 ); if ( ! ibdev ) { rc = -ENOMEM; goto err_alloc_ibdev; } qib7322->ibdev[i] = ibdev; ibdev->dev = &pci->dev; ibdev->op = &qib7322_ib_operations; ibdev->port = ( QIB7322_PORT_BASE + i ); ibdev->link_width_enabled = ibdev->link_width_supported = IB_LINK_WIDTH_4X; /* 1x does not work */ ibdev->link_speed_enabled = ibdev->link_speed_supported = IB_LINK_SPEED_SDR; /* to avoid need for link tuning */ memcpy ( &ibdev->gid.u.half[1], &qib7322->guid, sizeof ( ibdev->gid.u.half[1] ) ); assert ( ( ibdev->gid.u.half[1].u.bytes[7] & i ) == 0 ); ibdev->gid.u.half[1].u.bytes[7] |= i; ib_set_drvdata ( ibdev, qib7322 ); } /* Register Infiniband devices */ for ( i = 0 ; i < QIB7322_MAX_PORTS ; i++ ) { if ( ! qib7322->ibdev[i] ) continue; if ( ( rc = register_ibdev ( qib7322->ibdev[i] ) ) != 0 ) { DBGC ( qib7322, "QIB7322 %p port %d could not register " "IB device: %s\n", qib7322, i, strerror ( rc ) ); goto err_register_ibdev; } } return 0; i = QIB7322_MAX_PORTS; err_register_ibdev: for ( i-- ; i >= 0 ; i-- ) { if ( qib7322->ibdev[i] ) unregister_ibdev ( qib7322->ibdev[i] ); } i = QIB7322_MAX_PORTS; err_alloc_ibdev: for ( i-- ; i >= 0 ; i-- ) ibdev_put ( qib7322->ibdev[i] ); err_init_ib_serdes: qib7322_fini_send ( qib7322 ); err_init_send: qib7322_fini_recv ( qib7322 ); err_init_recv: err_read_eeprom: err_init_i2c: free ( qib7322 ); err_alloc_qib7322: return rc; } /** * Remove PCI device * * @v pci PCI device */ static void qib7322_remove ( struct pci_device *pci ) { struct qib7322 *qib7322 = pci_get_drvdata ( pci ); int i; for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- ) { if ( qib7322->ibdev[i] ) unregister_ibdev ( qib7322->ibdev[i] ); } for ( i = ( QIB7322_MAX_PORTS - 1 ) ; i >= 0 ; i-- ) ibdev_put ( qib7322->ibdev[i] ); qib7322_fini_send ( qib7322 ); qib7322_fini_recv ( qib7322 ); free ( qib7322 ); } static struct pci_device_id qib7322_nics[] = { PCI_ROM ( 0x1077, 0x7322, "iba7322", "IBA7322 QDR InfiniBand HCA", 0 ), }; struct pci_driver qib7322_driver __pci_driver = { .ids = qib7322_nics, .id_count = ( sizeof ( qib7322_nics ) / sizeof ( qib7322_nics[0] ) ), .probe = qib7322_probe, .remove = qib7322_remove, };