diff options
Diffstat (limited to 'drivers/infiniband/hw/mthca')
28 files changed, 1594 insertions, 1615 deletions
diff --git a/drivers/infiniband/hw/mthca/Kconfig b/drivers/infiniband/hw/mthca/Kconfig index 9aa5a4468a7..da314c3fec2 100644 --- a/drivers/infiniband/hw/mthca/Kconfig +++ b/drivers/infiniband/hw/mthca/Kconfig @@ -1,13 +1,13 @@ config INFINIBAND_MTHCA tristate "Mellanox HCA support" - depends on PCI && INFINIBAND + depends on PCI ---help--- This is a low-level driver for Mellanox InfiniHost host channel adapters (HCAs), including the MT23108 PCI-X HCA ("Tavor") and the MT25208 PCI Express HCA ("Arbel"). config INFINIBAND_MTHCA_DEBUG - bool "Verbose debugging output" if EMBEDDED + bool "Verbose debugging output" if EXPERT depends on INFINIBAND_MTHCA default y ---help--- diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c index f930e55b58f..b4e0cf4e95c 100644 --- a/drivers/infiniband/hw/mthca/mthca_allocator.c +++ b/drivers/infiniband/hw/mthca/mthca_allocator.c @@ -28,8 +28,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_allocator.c 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/errno.h> @@ -213,7 +211,7 @@ int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, if (!buf->direct.buf) return -ENOMEM; - pci_unmap_addr_set(&buf->direct, mapping, t); + dma_unmap_addr_set(&buf->direct, mapping, t); memset(buf->direct.buf, 0, size); @@ -253,9 +251,9 @@ int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, goto err_free; dma_list[i] = t; - pci_unmap_addr_set(&buf->page_list[i], mapping, t); + dma_unmap_addr_set(&buf->page_list[i], mapping, t); - memset(buf->page_list[i].buf, 0, PAGE_SIZE); + clear_page(buf->page_list[i].buf); } } @@ -291,12 +289,12 @@ void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, if (is_direct) dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, - pci_unmap_addr(&buf->direct, mapping)); + dma_unmap_addr(&buf->direct, mapping)); else { for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, buf->page_list[i].buf, - pci_unmap_addr(&buf->page_list[i], + dma_unmap_addr(&buf->page_list[i], mapping)); kfree(buf->page_list); } diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c index 69599455aca..32f6c631545 100644 --- a/drivers/infiniband/hw/mthca/mthca_av.c +++ b/drivers/infiniband/hw/mthca/mthca_av.c @@ -29,11 +29,8 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_av.c 1349 2004-12-16 21:09:43Z roland $ */ -#include <linux/init.h> #include <linux/string.h> #include <linux/slab.h> @@ -190,7 +187,7 @@ int mthca_create_ah(struct mthca_dev *dev, on_hca_fail: if (ah->type == MTHCA_AH_PCI_POOL) { ah->av = pci_pool_alloc(dev->av_table.pool, - SLAB_ATOMIC, &ah->avdma); + GFP_ATOMIC, &ah->avdma); if (!ah->av) return -ENOMEM; @@ -280,6 +277,7 @@ int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah, (be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20) & 0xff; header->grh.flow_label = ah->av->sl_tclass_flowlabel & cpu_to_be32(0xfffff); + header->grh.hop_limit = ah->av->hop_limit; ib_get_cached_gid(&dev->ib_dev, be32_to_cpu(ah->av->port_pd) >> 24, ah->av->gid_index % dev->limits.gid_table_len, @@ -323,7 +321,7 @@ int mthca_ah_query(struct ib_ah *ibah, struct ib_ah_attr *attr) return 0; } -int __devinit mthca_init_av_table(struct mthca_dev *dev) +int mthca_init_av_table(struct mthca_dev *dev) { int err; diff --git a/drivers/infiniband/hw/mthca/mthca_catas.c b/drivers/infiniband/hw/mthca/mthca_catas.c index cd044ea2dfa..712d2a30fbe 100644 --- a/drivers/infiniband/hw/mthca/mthca_catas.c +++ b/drivers/infiniband/hw/mthca/mthca_catas.c @@ -28,11 +28,10 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id$ */ #include <linux/jiffies.h> +#include <linux/module.h> #include <linux/timer.h> #include <linux/workqueue.h> @@ -57,7 +56,7 @@ static int catas_reset_disable; module_param_named(catas_reset_disable, catas_reset_disable, int, 0644); MODULE_PARM_DESC(catas_reset_disable, "disable reset on catastrophic event if nonzero"); -static void catas_reset(void *work_ptr) +static void catas_reset(struct work_struct *work) { struct mthca_dev *dev, *tmpdev; LIST_HEAD(tlist); @@ -70,11 +69,16 @@ static void catas_reset(void *work_ptr) spin_unlock_irq(&catas_lock); list_for_each_entry_safe(dev, tmpdev, &tlist, catas_err.list) { + struct pci_dev *pdev = dev->pdev; ret = __mthca_restart_one(dev->pdev); + /* 'dev' now is not valid */ if (ret) - mthca_err(dev, "Reset failed (%d)\n", ret); - else - mthca_dbg(dev, "Reset succeeded\n"); + printk(KERN_ERR "mthca %s: Reset failed (%d)\n", + pci_name(pdev), ret); + else { + struct mthca_dev *d = pci_get_drvdata(pdev); + mthca_dbg(d, "Reset succeeded\n"); + } } mutex_unlock(&mthca_device_mutex); @@ -90,6 +94,7 @@ static void handle_catas(struct mthca_dev *dev) event.device = &dev->ib_dev; event.event = IB_EVENT_DEVICE_FATAL; event.element.port_num = 0; + dev->active = false; ib_dispatch_event(&event); @@ -128,7 +133,6 @@ static void handle_catas(struct mthca_dev *dev) static void poll_catas(unsigned long dev_ptr) { struct mthca_dev *dev = (struct mthca_dev *) dev_ptr; - unsigned long flags; int i; for (i = 0; i < dev->catas_err.size; ++i) @@ -137,39 +141,26 @@ static void poll_catas(unsigned long dev_ptr) return; } - spin_lock_irqsave(&catas_lock, flags); - if (!dev->catas_err.stop) - mod_timer(&dev->catas_err.timer, - jiffies + MTHCA_CATAS_POLL_INTERVAL); - spin_unlock_irqrestore(&catas_lock, flags); - - return; + mod_timer(&dev->catas_err.timer, + round_jiffies(jiffies + MTHCA_CATAS_POLL_INTERVAL)); } void mthca_start_catas_poll(struct mthca_dev *dev) { - unsigned long addr; + phys_addr_t addr; init_timer(&dev->catas_err.timer); - dev->catas_err.stop = 0; dev->catas_err.map = NULL; addr = pci_resource_start(dev->pdev, 0) + ((pci_resource_len(dev->pdev, 0) - 1) & dev->catas_err.addr); - if (!request_mem_region(addr, dev->catas_err.size * 4, - DRV_NAME)) { - mthca_warn(dev, "couldn't request catastrophic error region " - "at 0x%lx/0x%x\n", addr, dev->catas_err.size * 4); - return; - } - dev->catas_err.map = ioremap(addr, dev->catas_err.size * 4); if (!dev->catas_err.map) { mthca_warn(dev, "couldn't map catastrophic error region " - "at 0x%lx/0x%x\n", addr, dev->catas_err.size * 4); - release_mem_region(addr, dev->catas_err.size * 4); + "at 0x%llx/0x%x\n", (unsigned long long) addr, + dev->catas_err.size * 4); return; } @@ -182,19 +173,10 @@ void mthca_start_catas_poll(struct mthca_dev *dev) void mthca_stop_catas_poll(struct mthca_dev *dev) { - spin_lock_irq(&catas_lock); - dev->catas_err.stop = 1; - spin_unlock_irq(&catas_lock); - del_timer_sync(&dev->catas_err.timer); - if (dev->catas_err.map) { + if (dev->catas_err.map) iounmap(dev->catas_err.map); - release_mem_region(pci_resource_start(dev->pdev, 0) + - ((pci_resource_len(dev->pdev, 0) - 1) & - dev->catas_err.addr), - dev->catas_err.size * 4); - } spin_lock_irq(&catas_lock); list_del(&dev->catas_err.list); @@ -203,7 +185,7 @@ void mthca_stop_catas_poll(struct mthca_dev *dev) int __init mthca_catas_init(void) { - INIT_WORK(&catas_work, catas_reset, NULL); + INIT_WORK(&catas_work, catas_reset); catas_wq = create_singlethread_workqueue("mthca_catas"); if (!catas_wq) diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c index 99a94d71093..9d3e5c1ac60 100644 --- a/drivers/infiniband/hw/mthca/mthca_cmd.c +++ b/drivers/infiniband/hw/mthca/mthca_cmd.c @@ -30,13 +30,14 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_cmd.c 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/completion.h> #include <linux/pci.h> #include <linux/errno.h> +#include <linux/sched.h> +#include <linux/module.h> +#include <linux/slab.h> #include <asm/io.h> #include <rdma/ib_mad.h> @@ -158,13 +159,15 @@ enum { enum { CMD_TIME_CLASS_A = (HZ + 999) / 1000 + 1, CMD_TIME_CLASS_B = (HZ + 99) / 100 + 1, - CMD_TIME_CLASS_C = (HZ + 9) / 10 + 1 + CMD_TIME_CLASS_C = (HZ + 9) / 10 + 1, + CMD_TIME_CLASS_D = 60 * HZ }; #else enum { CMD_TIME_CLASS_A = 60 * HZ, CMD_TIME_CLASS_B = 60 * HZ, - CMD_TIME_CLASS_C = 60 * HZ + CMD_TIME_CLASS_C = 60 * HZ, + CMD_TIME_CLASS_D = 60 * HZ }; #endif @@ -218,7 +221,7 @@ static void mthca_cmd_post_dbell(struct mthca_dev *dev, __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | (1 << HCA_E_BIT) | (op_modifier << HCR_OPMOD_SHIFT) | - op), ptr + offs[6]); + op), ptr + offs[6]); wmb(); __raw_writel((__force u32) 0, ptr + offs[7]); wmb(); @@ -289,10 +292,48 @@ static int mthca_cmd_post(struct mthca_dev *dev, err = mthca_cmd_post_hcr(dev, in_param, out_param, in_modifier, op_modifier, op, token, event); + /* + * Make sure that our HCR writes don't get mixed in with + * writes from another CPU starting a FW command. + */ + mmiowb(); + mutex_unlock(&dev->cmd.hcr_mutex); return err; } + +static int mthca_status_to_errno(u8 status) +{ + static const int trans_table[] = { + [MTHCA_CMD_STAT_INTERNAL_ERR] = -EIO, + [MTHCA_CMD_STAT_BAD_OP] = -EPERM, + [MTHCA_CMD_STAT_BAD_PARAM] = -EINVAL, + [MTHCA_CMD_STAT_BAD_SYS_STATE] = -ENXIO, + [MTHCA_CMD_STAT_BAD_RESOURCE] = -EBADF, + [MTHCA_CMD_STAT_RESOURCE_BUSY] = -EBUSY, + [MTHCA_CMD_STAT_DDR_MEM_ERR] = -ENOMEM, + [MTHCA_CMD_STAT_EXCEED_LIM] = -ENOMEM, + [MTHCA_CMD_STAT_BAD_RES_STATE] = -EBADF, + [MTHCA_CMD_STAT_BAD_INDEX] = -EBADF, + [MTHCA_CMD_STAT_BAD_NVMEM] = -EFAULT, + [MTHCA_CMD_STAT_BAD_QPEE_STATE] = -EINVAL, + [MTHCA_CMD_STAT_BAD_SEG_PARAM] = -EFAULT, + [MTHCA_CMD_STAT_REG_BOUND] = -EBUSY, + [MTHCA_CMD_STAT_LAM_NOT_PRE] = -EAGAIN, + [MTHCA_CMD_STAT_BAD_PKT] = -EBADMSG, + [MTHCA_CMD_STAT_BAD_SIZE] = -ENOMEM, + }; + + if (status >= ARRAY_SIZE(trans_table) || + (status != MTHCA_CMD_STAT_OK + && trans_table[status] == 0)) + return -EINVAL; + + return trans_table[status]; +} + + static int mthca_cmd_poll(struct mthca_dev *dev, u64 in_param, u64 *out_param, @@ -300,11 +341,11 @@ static int mthca_cmd_poll(struct mthca_dev *dev, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, - u8 *status) + unsigned long timeout) { int err = 0; unsigned long end; + u8 status; down(&dev->cmd.poll_sem); @@ -333,7 +374,12 @@ static int mthca_cmd_poll(struct mthca_dev *dev, (u64) be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_OUT_PARAM_OFFSET + 4)); - *status = be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_STATUS_OFFSET)) >> 24; + status = be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_STATUS_OFFSET)) >> 24; + if (status) { + mthca_dbg(dev, "Command %02x completed with status %02x\n", + op, status); + err = mthca_status_to_errno(status); + } out: up(&dev->cmd.poll_sem); @@ -356,8 +402,6 @@ void mthca_cmd_event(struct mthca_dev *dev, context->status = status; context->out_param = out_param; - context->token += dev->cmd.token_mask + 1; - complete(&context->done); } @@ -368,8 +412,7 @@ static int mthca_cmd_wait(struct mthca_dev *dev, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, - u8 *status) + unsigned long timeout) { int err = 0; struct mthca_cmd_context *context; @@ -379,6 +422,7 @@ static int mthca_cmd_wait(struct mthca_dev *dev, spin_lock(&dev->cmd.context_lock); BUG_ON(dev->cmd.free_head < 0); context = &dev->cmd.context[dev->cmd.free_head]; + context->token += dev->cmd.token_mask + 1; dev->cmd.free_head = context->next; spin_unlock(&dev->cmd.context_lock); @@ -400,10 +444,11 @@ static int mthca_cmd_wait(struct mthca_dev *dev, if (err) goto out; - *status = context->status; - if (*status) + if (context->status) { mthca_dbg(dev, "Command %02x completed with status %02x\n", - op, *status); + op, context->status); + err = mthca_status_to_errno(context->status); + } if (out_is_imm) *out_param = context->out_param; @@ -425,17 +470,16 @@ static int mthca_cmd_box(struct mthca_dev *dev, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, - u8 *status) + unsigned long timeout) { if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) return mthca_cmd_wait(dev, in_param, &out_param, 0, in_modifier, op_modifier, op, - timeout, status); + timeout); else return mthca_cmd_poll(dev, in_param, &out_param, 0, in_modifier, op_modifier, op, - timeout, status); + timeout); } /* Invoke a command with no output parameter */ @@ -444,11 +488,10 @@ static int mthca_cmd(struct mthca_dev *dev, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, - u8 *status) + unsigned long timeout) { return mthca_cmd_box(dev, in_param, 0, in_modifier, - op_modifier, op, timeout, status); + op_modifier, op, timeout); } /* @@ -462,17 +505,16 @@ static int mthca_cmd_imm(struct mthca_dev *dev, u32 in_modifier, u8 op_modifier, u16 op, - unsigned long timeout, - u8 *status) + unsigned long timeout) { if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) return mthca_cmd_wait(dev, in_param, out_param, 1, in_modifier, op_modifier, op, - timeout, status); + timeout); else return mthca_cmd_poll(dev, in_param, out_param, 1, in_modifier, op_modifier, op, - timeout, status); + timeout); } int mthca_cmd_init(struct mthca_dev *dev) @@ -589,14 +631,14 @@ void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox) kfree(mailbox); } -int mthca_SYS_EN(struct mthca_dev *dev, u8 *status) +int mthca_SYS_EN(struct mthca_dev *dev) { u64 out; int ret; - ret = mthca_cmd_imm(dev, 0, &out, 0, 0, CMD_SYS_EN, HZ, status); + ret = mthca_cmd_imm(dev, 0, &out, 0, 0, CMD_SYS_EN, CMD_TIME_CLASS_D); - if (*status == MTHCA_CMD_STAT_DDR_MEM_ERR) + if (ret == -ENOMEM) mthca_warn(dev, "SYS_EN DDR error: syn=%x, sock=%d, " "sladdr=%d, SPD source=%s\n", (int) (out >> 6) & 0xf, (int) (out >> 4) & 3, @@ -605,13 +647,13 @@ int mthca_SYS_EN(struct mthca_dev *dev, u8 *status) return ret; } -int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status) +int mthca_SYS_DIS(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, HZ, status); + return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C); } static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm, - u64 virt, u8 *status) + u64 virt) { struct mthca_mailbox *mailbox; struct mthca_icm_iter iter; @@ -659,8 +701,8 @@ static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm, if (++nent == MTHCA_MAILBOX_SIZE / 16) { err = mthca_cmd(dev, mailbox->dma, nent, 0, op, - CMD_TIME_CLASS_B, status); - if (err || *status) + CMD_TIME_CLASS_B); + if (err) goto out; nent = 0; } @@ -669,7 +711,7 @@ static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm, if (nent) err = mthca_cmd(dev, mailbox->dma, nent, 0, op, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); switch (op) { case CMD_MAP_FA: @@ -689,24 +731,24 @@ out: return err; } -int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) +int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm) { - return mthca_map_cmd(dev, CMD_MAP_FA, icm, -1, status); + return mthca_map_cmd(dev, CMD_MAP_FA, icm, -1); } -int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status) +int mthca_UNMAP_FA(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_FA, CMD_TIME_CLASS_B, status); + return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_FA, CMD_TIME_CLASS_B); } -int mthca_RUN_FW(struct mthca_dev *dev, u8 *status) +int mthca_RUN_FW(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_RUN_FW, CMD_TIME_CLASS_A, status); + return mthca_cmd(dev, 0, 0, 0, CMD_RUN_FW, CMD_TIME_CLASS_A); } static void mthca_setup_cmd_doorbells(struct mthca_dev *dev, u64 base) { - unsigned long addr; + phys_addr_t addr; u16 max_off = 0; int i; @@ -730,7 +772,7 @@ static void mthca_setup_cmd_doorbells(struct mthca_dev *dev, u64 base) mthca_dbg(dev, "Mapped doorbell page for posting FW commands\n"); } -int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status) +int mthca_QUERY_FW(struct mthca_dev *dev) { struct mthca_mailbox *mailbox; u32 *outbox; @@ -764,14 +806,14 @@ int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status) outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_FW, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); if (err) goto out; MTHCA_GET(dev->fw_ver, outbox, QUERY_FW_VER_OFFSET); /* - * FW subminor version is at more signifant bits than minor + * FW subminor version is at more significant bits than minor * version, so swap here. */ dev->fw_ver = (dev->fw_ver & 0xffff00000000ull) | @@ -836,7 +878,7 @@ out: return err; } -int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status) +int mthca_ENABLE_LAM(struct mthca_dev *dev) { struct mthca_mailbox *mailbox; u8 info; @@ -857,14 +899,11 @@ int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status) outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_ENABLE_LAM, - CMD_TIME_CLASS_C, status); + CMD_TIME_CLASS_C); if (err) goto out; - if (*status == MTHCA_CMD_STAT_LAM_NOT_PRE) - goto out; - MTHCA_GET(dev->ddr_start, outbox, ENABLE_LAM_START_OFFSET); MTHCA_GET(dev->ddr_end, outbox, ENABLE_LAM_END_OFFSET); MTHCA_GET(info, outbox, ENABLE_LAM_INFO_OFFSET); @@ -889,12 +928,12 @@ out: return err; } -int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status) +int mthca_DISABLE_LAM(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status); + return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C); } -int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status) +int mthca_QUERY_DDR(struct mthca_dev *dev) { struct mthca_mailbox *mailbox; u8 info; @@ -915,7 +954,7 @@ int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status) outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DDR, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); if (err) goto out; @@ -945,7 +984,7 @@ out: } int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, - struct mthca_dev_lim *dev_lim, u8 *status) + struct mthca_dev_lim *dev_lim) { struct mthca_mailbox *mailbox; u32 *outbox; @@ -1021,7 +1060,7 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DEV_LIM, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); if (err) goto out; @@ -1051,7 +1090,11 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EQ_OFFSET); dev_lim->max_eqs = 1 << (field & 0x7); MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MTT_OFFSET); - dev_lim->reserved_mtts = 1 << (field >> 4); + if (mthca_is_memfree(dev)) + dev_lim->reserved_mtts = ALIGN((1 << (field >> 4)) * sizeof(u64), + dev->limits.mtt_seg_size) / dev->limits.mtt_seg_size; + else + dev_lim->reserved_mtts = 1 << (field >> 4); MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET); dev_lim->max_mrw_sz = 1 << field; MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MRW_OFFSET); @@ -1221,7 +1264,7 @@ static void get_board_id(void *vsd, char *board_id) } int mthca_QUERY_ADAPTER(struct mthca_dev *dev, - struct mthca_adapter *adapter, u8 *status) + struct mthca_adapter *adapter) { struct mthca_mailbox *mailbox; u32 *outbox; @@ -1240,14 +1283,19 @@ int mthca_QUERY_ADAPTER(struct mthca_dev *dev, outbox = mailbox->buf; err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_ADAPTER, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); if (err) goto out; - MTHCA_GET(adapter->vendor_id, outbox, QUERY_ADAPTER_VENDOR_ID_OFFSET); - MTHCA_GET(adapter->device_id, outbox, QUERY_ADAPTER_DEVICE_ID_OFFSET); - MTHCA_GET(adapter->revision_id, outbox, QUERY_ADAPTER_REVISION_ID_OFFSET); + if (!mthca_is_memfree(dev)) { + MTHCA_GET(adapter->vendor_id, outbox, + QUERY_ADAPTER_VENDOR_ID_OFFSET); + MTHCA_GET(adapter->device_id, outbox, + QUERY_ADAPTER_DEVICE_ID_OFFSET); + MTHCA_GET(adapter->revision_id, outbox, + QUERY_ADAPTER_REVISION_ID_OFFSET); + } MTHCA_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, @@ -1259,8 +1307,7 @@ out: } int mthca_INIT_HCA(struct mthca_dev *dev, - struct mthca_init_hca_param *param, - u8 *status) + struct mthca_init_hca_param *param) { struct mthca_mailbox *mailbox; __be32 *inbox; @@ -1324,6 +1371,10 @@ int mthca_INIT_HCA(struct mthca_dev *dev, /* Check port for UD address vector: */ *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(1); + /* Enable IPoIB checksumming if we can: */ + if (dev->device_cap_flags & IB_DEVICE_UD_IP_CSUM) + *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(7 << 3); + /* We leave wqe_quota, responder_exu, etc as 0 (default) */ /* QPC/EEC/CQC/EQC/RDB attributes */ @@ -1373,7 +1424,8 @@ int mthca_INIT_HCA(struct mthca_dev *dev, MTHCA_PUT(inbox, param->uarc_base, INIT_HCA_UAR_CTX_BASE_OFFSET); } - err = mthca_cmd(dev, mailbox->dma, 0, 0, CMD_INIT_HCA, HZ, status); + err = mthca_cmd(dev, mailbox->dma, 0, 0, + CMD_INIT_HCA, CMD_TIME_CLASS_D); mthca_free_mailbox(dev, mailbox); return err; @@ -1381,7 +1433,7 @@ int mthca_INIT_HCA(struct mthca_dev *dev, int mthca_INIT_IB(struct mthca_dev *dev, struct mthca_init_ib_param *param, - int port, u8 *status) + int port) { struct mthca_mailbox *mailbox; u32 *inbox; @@ -1425,24 +1477,24 @@ int mthca_INIT_IB(struct mthca_dev *dev, MTHCA_PUT(inbox, param->si_guid, INIT_IB_SI_GUID_OFFSET); err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_INIT_IB, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); mthca_free_mailbox(dev, mailbox); return err; } -int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status) +int mthca_CLOSE_IB(struct mthca_dev *dev, int port) { - return mthca_cmd(dev, 0, port, 0, CMD_CLOSE_IB, HZ, status); + return mthca_cmd(dev, 0, port, 0, CMD_CLOSE_IB, CMD_TIME_CLASS_A); } -int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status) +int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic) { - return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, HZ, status); + return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, CMD_TIME_CLASS_C); } int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, - int port, u8 *status) + int port) { struct mthca_mailbox *mailbox; u32 *inbox; @@ -1471,18 +1523,18 @@ int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, MTHCA_PUT(inbox, param->si_guid, SET_IB_SI_GUID_OFFSET); err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_SET_IB, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); mthca_free_mailbox(dev, mailbox); return err; } -int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status) +int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt) { - return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt, status); + return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt); } -int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status) +int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt) { struct mthca_mailbox *mailbox; __be64 *inbox; @@ -1497,7 +1549,7 @@ int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status inbox[1] = cpu_to_be64(dma_addr); err = mthca_cmd(dev, mailbox->dma, 1, 0, CMD_MAP_ICM, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); mthca_free_mailbox(dev, mailbox); @@ -1508,31 +1560,31 @@ int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status return err; } -int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status) +int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count) { mthca_dbg(dev, "Unmapping %d pages at %llx from ICM.\n", page_count, (unsigned long long) virt); - return mthca_cmd(dev, virt, page_count, 0, CMD_UNMAP_ICM, CMD_TIME_CLASS_B, status); + return mthca_cmd(dev, virt, page_count, 0, + CMD_UNMAP_ICM, CMD_TIME_CLASS_B); } -int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) +int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm) { - return mthca_map_cmd(dev, CMD_MAP_ICM_AUX, icm, -1, status); + return mthca_map_cmd(dev, CMD_MAP_ICM_AUX, icm, -1); } -int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status) +int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_ICM_AUX, CMD_TIME_CLASS_B, status); + return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_ICM_AUX, CMD_TIME_CLASS_B); } -int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, - u8 *status) +int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages) { - int ret = mthca_cmd_imm(dev, icm_size, aux_pages, 0, 0, CMD_SET_ICM_SIZE, - CMD_TIME_CLASS_A, status); + int ret = mthca_cmd_imm(dev, icm_size, aux_pages, 0, + 0, CMD_SET_ICM_SIZE, CMD_TIME_CLASS_A); - if (ret || status) + if (ret) return ret; /* @@ -1546,74 +1598,73 @@ int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, } int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status) + int mpt_index) { return mthca_cmd(dev, mailbox->dma, mpt_index, 0, CMD_SW2HW_MPT, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); } int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status) + int mpt_index) { return mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, !mailbox, CMD_HW2SW_MPT, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); } int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int num_mtt, u8 *status) + int num_mtt) { return mthca_cmd(dev, mailbox->dma, num_mtt, 0, CMD_WRITE_MTT, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); } -int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status) +int mthca_SYNC_TPT(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0, 0, CMD_SYNC_TPT, CMD_TIME_CLASS_B, status); + return mthca_cmd(dev, 0, 0, 0, CMD_SYNC_TPT, CMD_TIME_CLASS_B); } int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap, - int eq_num, u8 *status) + int eq_num) { mthca_dbg(dev, "%s mask %016llx for eqn %d\n", unmap ? "Clearing" : "Setting", (unsigned long long) event_mask, eq_num); return mthca_cmd(dev, event_mask, (unmap << 31) | eq_num, - 0, CMD_MAP_EQ, CMD_TIME_CLASS_B, status); + 0, CMD_MAP_EQ, CMD_TIME_CLASS_B); } int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status) + int eq_num) { return mthca_cmd(dev, mailbox->dma, eq_num, 0, CMD_SW2HW_EQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status) + int eq_num) { return mthca_cmd_box(dev, 0, mailbox->dma, eq_num, 0, CMD_HW2SW_EQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status) + int cq_num) { return mthca_cmd(dev, mailbox->dma, cq_num, 0, CMD_SW2HW_CQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status) + int cq_num) { return mthca_cmd_box(dev, 0, mailbox->dma, cq_num, 0, CMD_HW2SW_CQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } -int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, - u8 *status) +int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size) { struct mthca_mailbox *mailbox; __be32 *inbox; @@ -1637,44 +1688,43 @@ int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, MTHCA_PUT(inbox, lkey, RESIZE_CQ_LKEY_OFFSET); err = mthca_cmd(dev, mailbox->dma, cq_num, 1, CMD_RESIZE_CQ, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); mthca_free_mailbox(dev, mailbox); return err; } int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status) + int srq_num) { return mthca_cmd(dev, mailbox->dma, srq_num, 0, CMD_SW2HW_SRQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status) + int srq_num) { return mthca_cmd_box(dev, 0, mailbox->dma, srq_num, 0, CMD_HW2SW_SRQ, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num, - struct mthca_mailbox *mailbox, u8 *status) + struct mthca_mailbox *mailbox) { return mthca_cmd_box(dev, 0, mailbox->dma, num, 0, - CMD_QUERY_SRQ, CMD_TIME_CLASS_A, status); + CMD_QUERY_SRQ, CMD_TIME_CLASS_A); } -int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status) +int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit) { return mthca_cmd(dev, limit, srq_num, 0, CMD_ARM_SRQ, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); } int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, enum ib_qp_state next, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u32 optmask, - u8 *status) + struct mthca_mailbox *mailbox, u32 optmask) { static const u16 op[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { [IB_QPS_RESET] = { @@ -1735,7 +1785,7 @@ int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, err = mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, (!!is_ee << 24) | num, op_mod, - op[cur][next], CMD_TIME_CLASS_C, status); + op[cur][next], CMD_TIME_CLASS_C); if (0 && mailbox) { int i; @@ -1769,21 +1819,20 @@ int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, } err = mthca_cmd(dev, mailbox->dma, optmask | (!!is_ee << 24) | num, - op_mod, op[cur][next], CMD_TIME_CLASS_C, status); + op_mod, op[cur][next], CMD_TIME_CLASS_C); } return err; } int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u8 *status) + struct mthca_mailbox *mailbox) { return mthca_cmd_box(dev, 0, mailbox->dma, (!!is_ee << 24) | num, 0, - CMD_QUERY_QPEE, CMD_TIME_CLASS_A, status); + CMD_QUERY_QPEE, CMD_TIME_CLASS_A); } -int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, - u8 *status) +int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn) { u8 op_mod; @@ -1797,7 +1846,7 @@ int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, case IB_QPT_RAW_IPV6: op_mod = 2; break; - case IB_QPT_RAW_ETY: + case IB_QPT_RAW_ETHERTYPE: op_mod = 3; break; default: @@ -1805,12 +1854,12 @@ int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, } return mthca_cmd(dev, 0, qpn, op_mod, CMD_CONF_SPECIAL_QP, - CMD_TIME_CLASS_B, status); + CMD_TIME_CLASS_B); } int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad, u8 *status) + void *in_mad, void *response_mad) { struct mthca_mailbox *inmailbox, *outmailbox; void *inbox; @@ -1820,11 +1869,11 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, #define MAD_IFC_BOX_SIZE 0x400 #define MAD_IFC_MY_QPN_OFFSET 0x100 -#define MAD_IFC_RQPN_OFFSET 0x104 -#define MAD_IFC_SL_OFFSET 0x108 -#define MAD_IFC_G_PATH_OFFSET 0x109 -#define MAD_IFC_RLID_OFFSET 0x10a -#define MAD_IFC_PKEY_OFFSET 0x10e +#define MAD_IFC_RQPN_OFFSET 0x108 +#define MAD_IFC_SL_OFFSET 0x10c +#define MAD_IFC_G_PATH_OFFSET 0x10d +#define MAD_IFC_RLID_OFFSET 0x10e +#define MAD_IFC_PKEY_OFFSET 0x112 #define MAD_IFC_GRH_OFFSET 0x140 inmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); @@ -1854,7 +1903,7 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, memset(inbox + 256, 0, 256); - MTHCA_PUT(inbox, in_wc->qp_num, MAD_IFC_MY_QPN_OFFSET); + MTHCA_PUT(inbox, in_wc->qp->qp_num, MAD_IFC_MY_QPN_OFFSET); MTHCA_PUT(inbox, in_wc->src_qp, MAD_IFC_RQPN_OFFSET); val = in_wc->sl << 4; @@ -1862,7 +1911,7 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, val = in_wc->dlid_path_bits | (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); - MTHCA_PUT(inbox, val, MAD_IFC_GRH_OFFSET); + MTHCA_PUT(inbox, val, MAD_IFC_G_PATH_OFFSET); MTHCA_PUT(inbox, in_wc->slid, MAD_IFC_RLID_OFFSET); MTHCA_PUT(inbox, in_wc->pkey_index, MAD_IFC_PKEY_OFFSET); @@ -1870,16 +1919,16 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, if (in_grh) memcpy(inbox + MAD_IFC_GRH_OFFSET, in_grh, 40); - op_modifier |= 0x10; + op_modifier |= 0x4; in_modifier |= in_wc->slid << 16; } err = mthca_cmd_box(dev, inmailbox->dma, outmailbox->dma, in_modifier, op_modifier, - CMD_MAD_IFC, CMD_TIME_CLASS_C, status); + CMD_MAD_IFC, CMD_TIME_CLASS_C); - if (!err && !*status) + if (!err) memcpy(response_mad, outmailbox->buf, 256); mthca_free_mailbox(dev, inmailbox); @@ -1888,33 +1937,33 @@ int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, } int mthca_READ_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status) + struct mthca_mailbox *mailbox) { return mthca_cmd_box(dev, 0, mailbox->dma, index, 0, - CMD_READ_MGM, CMD_TIME_CLASS_A, status); + CMD_READ_MGM, CMD_TIME_CLASS_A); } int mthca_WRITE_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status) + struct mthca_mailbox *mailbox) { return mthca_cmd(dev, mailbox->dma, index, 0, CMD_WRITE_MGM, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); } int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - u16 *hash, u8 *status) + u16 *hash) { u64 imm; int err; err = mthca_cmd_imm(dev, mailbox->dma, &imm, 0, 0, CMD_MGID_HASH, - CMD_TIME_CLASS_A, status); + CMD_TIME_CLASS_A); *hash = imm; return err; } -int mthca_NOP(struct mthca_dev *dev, u8 *status) +int mthca_NOP(struct mthca_dev *dev) { - return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100), status); + return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100)); } diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.h b/drivers/infiniband/hw/mthca/mthca_cmd.h index 2f976f2051d..f952244c54d 100644 --- a/drivers/infiniband/hw/mthca/mthca_cmd.h +++ b/drivers/infiniband/hw/mthca/mthca_cmd.h @@ -30,8 +30,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_cmd.h 1349 2004-12-16 21:09:43Z roland $ */ #ifndef MTHCA_CMD_H @@ -103,6 +101,7 @@ enum { DEV_LIM_FLAG_RAW_IPV6 = 1 << 4, DEV_LIM_FLAG_RAW_ETHER = 1 << 5, DEV_LIM_FLAG_SRQ = 1 << 6, + DEV_LIM_FLAG_IPOIB_CSUM = 1 << 7, DEV_LIM_FLAG_BAD_PKEY_CNTR = 1 << 8, DEV_LIM_FLAG_BAD_QKEY_CNTR = 1 << 9, DEV_LIM_FLAG_MW = 1 << 16, @@ -253,79 +252,74 @@ struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev, gfp_t gfp_mask); void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox); -int mthca_SYS_EN(struct mthca_dev *dev, u8 *status); -int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status); -int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status); -int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status); -int mthca_RUN_FW(struct mthca_dev *dev, u8 *status); -int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status); -int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status); -int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status); -int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status); +int mthca_SYS_EN(struct mthca_dev *dev); +int mthca_SYS_DIS(struct mthca_dev *dev); +int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm); +int mthca_UNMAP_FA(struct mthca_dev *dev); +int mthca_RUN_FW(struct mthca_dev *dev); +int mthca_QUERY_FW(struct mthca_dev *dev); +int mthca_ENABLE_LAM(struct mthca_dev *dev); +int mthca_DISABLE_LAM(struct mthca_dev *dev); +int mthca_QUERY_DDR(struct mthca_dev *dev); int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, - struct mthca_dev_lim *dev_lim, u8 *status); + struct mthca_dev_lim *dev_lim); int mthca_QUERY_ADAPTER(struct mthca_dev *dev, - struct mthca_adapter *adapter, u8 *status); + struct mthca_adapter *adapter); int mthca_INIT_HCA(struct mthca_dev *dev, - struct mthca_init_hca_param *param, - u8 *status); + struct mthca_init_hca_param *param); int mthca_INIT_IB(struct mthca_dev *dev, struct mthca_init_ib_param *param, - int port, u8 *status); -int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status); -int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status); + int port); +int mthca_CLOSE_IB(struct mthca_dev *dev, int port); +int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic); int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, - int port, u8 *status); -int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status); -int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status); -int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status); -int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status); -int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status); -int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, - u8 *status); + int port); +int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt); +int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt); +int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count); +int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm); +int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev); +int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages); int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status); + int mpt_index); int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status); + int mpt_index); int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int num_mtt, u8 *status); -int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status); + int num_mtt); +int mthca_SYNC_TPT(struct mthca_dev *dev); int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap, - int eq_num, u8 *status); + int eq_num); int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status); + int eq_num); int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status); + int eq_num); int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status); + int cq_num); int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status); -int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, - u8 *status); + int cq_num); +int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size); int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status); + int srq_num); int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status); + int srq_num); int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status); + struct mthca_mailbox *mailbox); +int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit); int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, enum ib_qp_state next, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u32 optmask, - u8 *status); + struct mthca_mailbox *mailbox, u32 optmask); int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, - u8 *status); + struct mthca_mailbox *mailbox); +int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn); int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad, u8 *status); + void *in_mad, void *response_mad); int mthca_READ_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status); + struct mthca_mailbox *mailbox); int mthca_WRITE_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status); + struct mthca_mailbox *mailbox); int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - u16 *hash, u8 *status); -int mthca_NOP(struct mthca_dev *dev, u8 *status); + u16 *hash); +int mthca_NOP(struct mthca_dev *dev); #endif /* MTHCA_CMD_H */ diff --git a/drivers/infiniband/hw/mthca/mthca_config_reg.h b/drivers/infiniband/hw/mthca/mthca_config_reg.h index afa56bfaab2..155bc66395b 100644 --- a/drivers/infiniband/hw/mthca/mthca_config_reg.h +++ b/drivers/infiniband/hw/mthca/mthca_config_reg.h @@ -29,15 +29,11 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_config_reg.h 1349 2004-12-16 21:09:43Z roland $ */ #ifndef MTHCA_CONFIG_REG_H #define MTHCA_CONFIG_REG_H -#include <asm/page.h> - #define MTHCA_HCR_BASE 0x80680 #define MTHCA_HCR_SIZE 0x0001c #define MTHCA_ECR_BASE 0x80700 diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c index e393681ba7d..40ba8333815 100644 --- a/drivers/infiniband/hw/mthca/mthca_cq.c +++ b/drivers/infiniband/hw/mthca/mthca_cq.c @@ -32,12 +32,13 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_cq.c 1369 2004-12-20 16:17:07Z roland $ */ -#include <linux/init.h> +#include <linux/gfp.h> #include <linux/hardirq.h> +#include <linux/sched.h> + +#include <asm/io.h> #include <rdma/ib_pack.h> @@ -53,6 +54,10 @@ enum { MTHCA_CQ_ENTRY_SIZE = 0x20 }; +enum { + MTHCA_ATOMIC_BYTE_LEN = 8 +}; + /* * Must be packed because start is 64 bits but only aligned to 32 bits. */ @@ -113,7 +118,8 @@ struct mthca_cqe { __be32 my_qpn; __be32 my_ee; __be32 rqpn; - __be16 sl_g_mlpath; + u8 sl_ipok; + u8 g_mlpath; __be16 rlid; __be32 imm_etype_pkey_eec; __be32 byte_cnt; @@ -198,18 +204,18 @@ static void dump_cqe(struct mthca_dev *dev, void *cqe_ptr) static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq, int incr) { - __be32 doorbell[2]; - if (mthca_is_memfree(dev)) { *cq->set_ci_db = cpu_to_be32(cq->cons_index); wmb(); } else { - doorbell[0] = cpu_to_be32(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn); - doorbell[1] = cpu_to_be32(incr - 1); - - mthca_write64(doorbell, + mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1, dev->kar + MTHCA_CQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); + /* + * Make sure doorbells don't leak out of CQ spinlock + * and reach the HCA out of order: + */ + mmiowb(); } } @@ -274,7 +280,7 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, { struct mthca_cqe *cqe; u32 prod_index; - int nfreed = 0; + int i, nfreed = 0; spin_lock_irq(&cq->lock); @@ -311,6 +317,8 @@ void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, } if (nfreed) { + for (i = 0; i < nfreed; ++i) + set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe)); wmb(); cq->cons_index += nfreed; update_cons_index(dev, cq, nfreed); @@ -465,7 +473,7 @@ static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd)) return; - cqe->db_cnt = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd); + be16_add_cpu(&cqe->db_cnt, -dbd); cqe->wqe = new_wqe; cqe->syndrome = SYNDROME_WR_FLUSH_ERR; @@ -485,6 +493,7 @@ static inline int mthca_poll_one(struct mthca_dev *dev, int is_send; int free_cqe = 1; int err = 0; + u16 checksum; cqe = next_cqe_sw(cq); if (!cqe) @@ -524,7 +533,7 @@ static inline int mthca_poll_one(struct mthca_dev *dev, } } - entry->qp_num = (*cur_qp)->qpn; + entry->qp = &(*cur_qp)->ibqp; if (is_send) { wq = &(*cur_qp)->sq; @@ -593,11 +602,11 @@ static inline int mthca_poll_one(struct mthca_dev *dev, break; case MTHCA_OPCODE_ATOMIC_CS: entry->opcode = IB_WC_COMP_SWAP; - entry->byte_len = be32_to_cpu(cqe->byte_cnt); + entry->byte_len = MTHCA_ATOMIC_BYTE_LEN; break; case MTHCA_OPCODE_ATOMIC_FA: entry->opcode = IB_WC_FETCH_ADD; - entry->byte_len = be32_to_cpu(cqe->byte_cnt); + entry->byte_len = MTHCA_ATOMIC_BYTE_LEN; break; case MTHCA_OPCODE_BIND_MW: entry->opcode = IB_WC_BIND_MW; @@ -612,13 +621,13 @@ static inline int mthca_poll_one(struct mthca_dev *dev, case IB_OPCODE_SEND_LAST_WITH_IMMEDIATE: case IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE: entry->wc_flags = IB_WC_WITH_IMM; - entry->imm_data = cqe->imm_etype_pkey_eec; + entry->ex.imm_data = cqe->imm_etype_pkey_eec; entry->opcode = IB_WC_RECV; break; case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: entry->wc_flags = IB_WC_WITH_IMM; - entry->imm_data = cqe->imm_etype_pkey_eec; + entry->ex.imm_data = cqe->imm_etype_pkey_eec; entry->opcode = IB_WC_RECV_RDMA_WITH_IMM; break; default: @@ -627,12 +636,15 @@ static inline int mthca_poll_one(struct mthca_dev *dev, break; } entry->slid = be16_to_cpu(cqe->rlid); - entry->sl = be16_to_cpu(cqe->sl_g_mlpath) >> 12; + entry->sl = cqe->sl_ipok >> 4; entry->src_qp = be32_to_cpu(cqe->rqpn) & 0xffffff; - entry->dlid_path_bits = be16_to_cpu(cqe->sl_g_mlpath) & 0x7f; + entry->dlid_path_bits = cqe->g_mlpath & 0x7f; entry->pkey_index = be32_to_cpu(cqe->imm_etype_pkey_eec) >> 16; - entry->wc_flags |= be16_to_cpu(cqe->sl_g_mlpath) & 0x80 ? - IB_WC_GRH : 0; + entry->wc_flags |= cqe->g_mlpath & 0x80 ? IB_WC_GRH : 0; + checksum = (be32_to_cpu(cqe->rqpn) >> 24) | + ((be32_to_cpu(cqe->my_ee) >> 16) & 0xff00); + entry->wc_flags |= (cqe->sl_ipok & 1 && checksum == 0xffff) ? + IB_WC_IP_CSUM_OK : 0; } entry->status = IB_WC_SUCCESS; @@ -716,38 +728,32 @@ repoll: return err == 0 || err == -EAGAIN ? npolled : err; } -int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify) +int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) { - __be32 doorbell[2]; - - doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ? - MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL : - MTHCA_TAVOR_CQ_DB_REQ_NOT) | - to_mcq(cq)->cqn); - doorbell[1] = (__force __be32) 0xffffffff; + u32 dbhi = ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL : + MTHCA_TAVOR_CQ_DB_REQ_NOT) | + to_mcq(cq)->cqn; - mthca_write64(doorbell, - to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, + mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock)); return 0; } -int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify) +int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) { struct mthca_cq *cq = to_mcq(ibcq); - __be32 doorbell[2]; - u32 sn; - __be32 ci; + __be32 db_rec[2]; + u32 dbhi; + u32 sn = cq->arm_sn & 3; - sn = cq->arm_sn & 3; - ci = cpu_to_be32(cq->cons_index); + db_rec[0] = cpu_to_be32(cq->cons_index); + db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | + ((flags & IB_CQ_SOLICITED_MASK) == + IB_CQ_SOLICITED ? 1 : 2)); - doorbell[0] = ci; - doorbell[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | - (notify == IB_CQ_SOLICITED ? 1 : 2)); - - mthca_write_db_rec(doorbell, cq->arm_db); + mthca_write_db_rec(db_rec, cq->arm_db); /* * Make sure that the doorbell record in host memory is @@ -755,14 +761,12 @@ int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify) */ wmb(); - doorbell[0] = cpu_to_be32((sn << 28) | - (notify == IB_CQ_SOLICITED ? - MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL : - MTHCA_ARBEL_CQ_DB_REQ_NOT) | - cq->cqn); - doorbell[1] = ci; + dbhi = (sn << 28) | + ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? + MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL : + MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn; - mthca_write64(doorbell, + mthca_write64(dbhi, cq->cons_index, to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->doorbell_lock)); @@ -776,7 +780,6 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, struct mthca_mailbox *mailbox; struct mthca_cq_context *cq_context; int err = -ENOMEM; - u8 status; cq->ibcq.cqe = nent - 1; cq->is_kernel = !ctx; @@ -844,19 +847,12 @@ int mthca_init_cq(struct mthca_dev *dev, int nent, cq_context->state_db = cpu_to_be32(cq->arm_db_index); } - err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn, &status); + err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn); if (err) { mthca_warn(dev, "SW2HW_CQ failed (%d)\n", err); goto err_out_free_mr; } - if (status) { - mthca_warn(dev, "SW2HW_CQ returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_free_mr; - } - spin_lock_irq(&dev->cq_table.lock); if (mthca_array_set(&dev->cq_table.cq, cq->cqn & (dev->limits.num_cqs - 1), @@ -912,7 +908,6 @@ void mthca_free_cq(struct mthca_dev *dev, { struct mthca_mailbox *mailbox; int err; - u8 status; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) { @@ -920,11 +915,9 @@ void mthca_free_cq(struct mthca_dev *dev, return; } - err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn, &status); + err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn); if (err) mthca_warn(dev, "HW2SW_CQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n", status); if (0) { __be32 *ctx = mailbox->buf; @@ -963,7 +956,7 @@ void mthca_free_cq(struct mthca_dev *dev, mthca_free_mailbox(dev, mailbox); } -int __devinit mthca_init_cq_table(struct mthca_dev *dev) +int mthca_init_cq_table(struct mthca_dev *dev) { int err; diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h index fe5cecf70fe..7e6a6d64ad4 100644 --- a/drivers/infiniband/hw/mthca/mthca_dev.h +++ b/drivers/infiniband/hw/mthca/mthca_dev.h @@ -32,8 +32,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_dev.h 1349 2004-12-16 21:09:43Z roland $ */ #ifndef MTHCA_DEV_H @@ -46,27 +44,25 @@ #include <linux/timer.h> #include <linux/mutex.h> #include <linux/list.h> - -#include <asm/semaphore.h> +#include <linux/semaphore.h> #include "mthca_provider.h" #include "mthca_doorbell.h" #define DRV_NAME "ib_mthca" #define PFX DRV_NAME ": " -#define DRV_VERSION "0.08" -#define DRV_RELDATE "February 14, 2006" +#define DRV_VERSION "1.0" +#define DRV_RELDATE "April 4, 2008" enum { MTHCA_FLAG_DDR_HIDDEN = 1 << 1, MTHCA_FLAG_SRQ = 1 << 2, - MTHCA_FLAG_MSI = 1 << 3, - MTHCA_FLAG_MSI_X = 1 << 4, - MTHCA_FLAG_NO_LAM = 1 << 5, - MTHCA_FLAG_FMR = 1 << 6, - MTHCA_FLAG_MEMFREE = 1 << 7, - MTHCA_FLAG_PCIE = 1 << 8, - MTHCA_FLAG_SINAI_OPT = 1 << 9 + MTHCA_FLAG_MSI_X = 1 << 3, + MTHCA_FLAG_NO_LAM = 1 << 4, + MTHCA_FLAG_FMR = 1 << 5, + MTHCA_FLAG_MEMFREE = 1 << 6, + MTHCA_FLAG_PCIE = 1 << 7, + MTHCA_FLAG_SINAI_OPT = 1 << 8 }; enum { @@ -83,7 +79,7 @@ enum { MTHCA_QP_CONTEXT_SIZE = 0x200, MTHCA_RDB_ENTRY_SIZE = 0x20, MTHCA_AV_SIZE = 0x20, - MTHCA_MGM_ENTRY_SIZE = 0x40, + MTHCA_MGM_ENTRY_SIZE = 0x100, /* Arbel FW gives us these, but we need them for Tavor */ MTHCA_MPT_ENTRY_SIZE = 0x40, @@ -163,6 +159,7 @@ struct mthca_limits { int reserved_eqs; int num_mpts; int num_mtt_segs; + int mtt_seg_size; int fmr_reserved_mtts; int reserved_mtts; int reserved_mrws; @@ -206,6 +203,7 @@ struct mthca_pd_table { struct mthca_buddy { unsigned long **bits; + int *num_free; int max_order; spinlock_t lock; }; @@ -281,7 +279,6 @@ struct mthca_mcg_table { struct mthca_catas_err { u64 addr; u32 __iomem *map; - unsigned long stop; u32 size; struct timer_list timer; struct list_head list; @@ -360,6 +357,7 @@ struct mthca_dev { struct ib_ah *sm_ah[MTHCA_MAX_PORTS]; spinlock_t sm_lock; u8 rate[MTHCA_MAX_PORTS]; + bool active; }; #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG @@ -391,11 +389,11 @@ extern void __buggy_use_of_MTHCA_PUT(void); do { \ void *__p = (char *) (source) + (offset); \ switch (sizeof (dest)) { \ - case 1: (dest) = *(u8 *) __p; break; \ - case 2: (dest) = be16_to_cpup(__p); break; \ - case 4: (dest) = be32_to_cpup(__p); break; \ - case 8: (dest) = be64_to_cpup(__p); break; \ - default: __buggy_use_of_MTHCA_GET(); \ + case 1: (dest) = *(u8 *) __p; break; \ + case 2: (dest) = be16_to_cpup(__p); break; \ + case 4: (dest) = be32_to_cpup(__p); break; \ + case 8: (dest) = be64_to_cpup(__p); break; \ + default: __buggy_use_of_MTHCA_GET(); \ } \ } while (0) @@ -464,6 +462,8 @@ void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar); int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd); void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd); +int mthca_write_mtt_size(struct mthca_dev *dev); + struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size); void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt); int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, @@ -493,8 +493,8 @@ void mthca_unmap_eq_icm(struct mthca_dev *dev); int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry); -int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify); -int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify); +int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); +int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); int mthca_init_cq(struct mthca_dev *dev, int nent, struct mthca_ucontext *ctx, u32 pdn, struct mthca_cq *cq); diff --git a/drivers/infiniband/hw/mthca/mthca_doorbell.h b/drivers/infiniband/hw/mthca/mthca_doorbell.h index dd9a44d170c..14f51ef97d7 100644 --- a/drivers/infiniband/hw/mthca/mthca_doorbell.h +++ b/drivers/infiniband/hw/mthca/mthca_doorbell.h @@ -30,8 +30,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_doorbell.h 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/types.h> @@ -58,10 +56,10 @@ static inline void mthca_write64_raw(__be64 val, void __iomem *dest) __raw_writeq((__force u64) val, dest); } -static inline void mthca_write64(__be32 val[2], void __iomem *dest, +static inline void mthca_write64(u32 hi, u32 lo, void __iomem *dest, spinlock_t *doorbell_lock) { - __raw_writeq(*(u64 *) val, dest); + __raw_writeq((__force u64) cpu_to_be64((u64) hi << 32 | lo), dest); } static inline void mthca_write_db_rec(__be32 val[2], __be32 *db) @@ -87,14 +85,17 @@ static inline void mthca_write64_raw(__be64 val, void __iomem *dest) __raw_writel(((__force u32 *) &val)[1], dest + 4); } -static inline void mthca_write64(__be32 val[2], void __iomem *dest, +static inline void mthca_write64(u32 hi, u32 lo, void __iomem *dest, spinlock_t *doorbell_lock) { unsigned long flags; + hi = (__force u32) cpu_to_be32(hi); + lo = (__force u32) cpu_to_be32(lo); + spin_lock_irqsave(doorbell_lock, flags); - __raw_writel((__force u32) val[0], dest); - __raw_writel((__force u32) val[1], dest + 4); + __raw_writel(hi, dest); + __raw_writel(lo, dest + 4); spin_unlock_irqrestore(doorbell_lock, flags); } diff --git a/drivers/infiniband/hw/mthca/mthca_eq.c b/drivers/infiniband/hw/mthca/mthca_eq.c index a29b1b6d82b..69020173899 100644 --- a/drivers/infiniband/hw/mthca/mthca_eq.c +++ b/drivers/infiniband/hw/mthca/mthca_eq.c @@ -29,14 +29,12 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_eq.c 1382 2004-12-24 02:21:02Z roland $ */ -#include <linux/init.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/pci.h> +#include <linux/slab.h> #include "mthca_dev.h" #include "mthca_cmd.h" @@ -174,11 +172,6 @@ static inline u64 async_mask(struct mthca_dev *dev) static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci) { - __be32 doorbell[2]; - - doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_SET_CI | eq->eqn); - doorbell[1] = cpu_to_be32(ci & (eq->nent - 1)); - /* * This barrier makes sure that all updates to ownership bits * done by set_eqe_hw() hit memory before the consumer index @@ -188,7 +181,7 @@ static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u * having set_eqe_hw() overwrite the owner field. */ wmb(); - mthca_write64(doorbell, + mthca_write64(MTHCA_EQ_DB_SET_CI | eq->eqn, ci & (eq->nent - 1), dev->kar + MTHCA_EQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } @@ -213,12 +206,7 @@ static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci) static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn) { - __be32 doorbell[2]; - - doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_REQ_NOT | eqn); - doorbell[1] = 0; - - mthca_write64(doorbell, + mthca_write64(MTHCA_EQ_DB_REQ_NOT | eqn, 0, dev->kar + MTHCA_EQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } @@ -231,12 +219,7 @@ static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask) static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn) { if (!mthca_is_memfree(dev)) { - __be32 doorbell[2]; - - doorbell[0] = cpu_to_be32(MTHCA_EQ_DB_DISARM_CQ | eqn); - doorbell[1] = cpu_to_be32(cqn); - - mthca_write64(doorbell, + mthca_write64(MTHCA_EQ_DB_DISARM_CQ | eqn, cqn, dev->kar + MTHCA_EQ_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } @@ -248,9 +231,9 @@ static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry) return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE; } -static inline struct mthca_eqe* next_eqe_sw(struct mthca_eq *eq) +static inline struct mthca_eqe *next_eqe_sw(struct mthca_eq *eq) { - struct mthca_eqe* eqe; + struct mthca_eqe *eqe; eqe = get_eqe(eq, eq->cons_index); return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe; } @@ -374,7 +357,7 @@ static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq) mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n", eqe->type, eqe->subtype, eq->eqn); break; - }; + } set_eqe_hw(eqe); ++eq->cons_index; @@ -405,7 +388,7 @@ static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq) return eqes_found; } -static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr, struct pt_regs *regs) +static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr) { struct mthca_dev *dev = dev_ptr; u32 ecr; @@ -432,8 +415,7 @@ static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr, struct pt_regs return IRQ_HANDLED; } -static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr, - struct pt_regs *regs) +static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr) { struct mthca_eq *eq = eq_ptr; struct mthca_dev *dev = eq->dev; @@ -446,7 +428,7 @@ static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr, return IRQ_HANDLED; } -static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr, struct pt_regs *regs) +static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr) { struct mthca_dev *dev = dev_ptr; int work = 0; @@ -467,8 +449,7 @@ static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr, struct pt_regs return IRQ_RETVAL(work); } -static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr, - struct pt_regs *regs) +static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr) { struct mthca_eq *eq = eq_ptr; struct mthca_dev *dev = eq->dev; @@ -481,10 +462,10 @@ static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr, return IRQ_HANDLED; } -static int __devinit mthca_create_eq(struct mthca_dev *dev, - int nent, - u8 intr, - struct mthca_eq *eq) +static int mthca_create_eq(struct mthca_dev *dev, + int nent, + u8 intr, + struct mthca_eq *eq) { int npages; u64 *dma_list = NULL; @@ -493,7 +474,6 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev, struct mthca_eq_context *eq_context; int err = -ENOMEM; int i; - u8 status; eq->dev = dev; eq->nent = roundup_pow_of_two(max(nent, 2)); @@ -523,9 +503,9 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev, goto err_out_free_pages; dma_list[i] = t; - pci_unmap_addr_set(&eq->page_list[i], mapping, t); + dma_unmap_addr_set(&eq->page_list[i], mapping, t); - memset(eq->page_list[i].buf, 0, PAGE_SIZE); + clear_page(eq->page_list[i].buf); } for (i = 0; i < eq->nent; ++i) @@ -562,15 +542,9 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev, eq_context->intr = intr; eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey); - err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn, &status); + err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn); if (err) { - mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err); - goto err_out_free_mr; - } - if (status) { - mthca_warn(dev, "SW2HW_EQ returned status 0x%02x\n", - status); - err = -EINVAL; + mthca_warn(dev, "SW2HW_EQ returned %d\n", err); goto err_out_free_mr; } @@ -598,7 +572,7 @@ static int __devinit mthca_create_eq(struct mthca_dev *dev, if (eq->page_list[i].buf) dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, eq->page_list[i].buf, - pci_unmap_addr(&eq->page_list[i], + dma_unmap_addr(&eq->page_list[i], mapping)); mthca_free_mailbox(dev, mailbox); @@ -616,7 +590,6 @@ static void mthca_free_eq(struct mthca_dev *dev, { struct mthca_mailbox *mailbox; int err; - u8 status; int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) / PAGE_SIZE; int i; @@ -625,11 +598,9 @@ static void mthca_free_eq(struct mthca_dev *dev, if (IS_ERR(mailbox)) return; - err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn, &status); + err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn); if (err) - mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err); - if (status) - mthca_warn(dev, "HW2SW_EQ returned status 0x%02x\n", status); + mthca_warn(dev, "HW2SW_EQ returned %d\n", err); dev->eq_table.arm_mask &= ~eq->eqn_mask; @@ -648,7 +619,7 @@ static void mthca_free_eq(struct mthca_dev *dev, for (i = 0; i < npages; ++i) pci_free_consistent(dev->pdev, PAGE_SIZE, eq->page_list[i].buf, - pci_unmap_addr(&eq->page_list[i], mapping)); + dma_unmap_addr(&eq->page_list[i], mapping)); kfree(eq->page_list); mthca_free_mailbox(dev, mailbox); @@ -661,39 +632,27 @@ static void mthca_free_irqs(struct mthca_dev *dev) if (dev->eq_table.have_irq) free_irq(dev->pdev->irq, dev); for (i = 0; i < MTHCA_NUM_EQ; ++i) - if (dev->eq_table.eq[i].have_irq) + if (dev->eq_table.eq[i].have_irq) { free_irq(dev->eq_table.eq[i].msi_x_vector, dev->eq_table.eq + i); + dev->eq_table.eq[i].have_irq = 0; + } } -static int __devinit mthca_map_reg(struct mthca_dev *dev, - unsigned long offset, unsigned long size, - void __iomem **map) +static int mthca_map_reg(struct mthca_dev *dev, + unsigned long offset, unsigned long size, + void __iomem **map) { - unsigned long base = pci_resource_start(dev->pdev, 0); - - if (!request_mem_region(base + offset, size, DRV_NAME)) - return -EBUSY; + phys_addr_t base = pci_resource_start(dev->pdev, 0); *map = ioremap(base + offset, size); - if (!*map) { - release_mem_region(base + offset, size); + if (!*map) return -ENOMEM; - } return 0; } -static void mthca_unmap_reg(struct mthca_dev *dev, unsigned long offset, - unsigned long size, void __iomem *map) -{ - unsigned long base = pci_resource_start(dev->pdev, 0); - - release_mem_region(base + offset, size); - iounmap(map); -} - -static int __devinit mthca_map_eq_regs(struct mthca_dev *dev) +static int mthca_map_eq_regs(struct mthca_dev *dev) { if (mthca_is_memfree(dev)) { /* @@ -719,9 +678,7 @@ static int __devinit mthca_map_eq_regs(struct mthca_dev *dev) dev->fw.arbel.eq_arm_base) + 4, 4, &dev->eq_regs.arbel.eq_arm)) { mthca_err(dev, "Couldn't map EQ arm register, aborting.\n"); - mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE, - dev->clr_base); + iounmap(dev->clr_base); return -ENOMEM; } @@ -730,12 +687,8 @@ static int __devinit mthca_map_eq_regs(struct mthca_dev *dev) MTHCA_EQ_SET_CI_SIZE, &dev->eq_regs.arbel.eq_set_ci_base)) { mthca_err(dev, "Couldn't map EQ CI register, aborting.\n"); - mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.eq_arm_base) + 4, 4, - dev->eq_regs.arbel.eq_arm); - mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE, - dev->clr_base); + iounmap(dev->eq_regs.arbel.eq_arm); + iounmap(dev->clr_base); return -ENOMEM; } } else { @@ -751,8 +704,7 @@ static int __devinit mthca_map_eq_regs(struct mthca_dev *dev) &dev->eq_regs.tavor.ecr_base)) { mthca_err(dev, "Couldn't map ecr register, " "aborting.\n"); - mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE, - dev->clr_base); + iounmap(dev->clr_base); return -ENOMEM; } } @@ -764,29 +716,18 @@ static int __devinit mthca_map_eq_regs(struct mthca_dev *dev) static void mthca_unmap_eq_regs(struct mthca_dev *dev) { if (mthca_is_memfree(dev)) { - mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.eq_set_ci_base, - MTHCA_EQ_SET_CI_SIZE, - dev->eq_regs.arbel.eq_set_ci_base); - mthca_unmap_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.eq_arm_base) + 4, 4, - dev->eq_regs.arbel.eq_arm); - mthca_unmap_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE, - dev->clr_base); + iounmap(dev->eq_regs.arbel.eq_set_ci_base); + iounmap(dev->eq_regs.arbel.eq_arm); + iounmap(dev->clr_base); } else { - mthca_unmap_reg(dev, MTHCA_ECR_BASE, - MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE, - dev->eq_regs.tavor.ecr_base); - mthca_unmap_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE, - dev->clr_base); + iounmap(dev->eq_regs.tavor.ecr_base); + iounmap(dev->clr_base); } } -int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt) +int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt) { int ret; - u8 status; /* * We assume that mapping one page is enough for the whole EQ @@ -800,14 +741,12 @@ int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt) return -ENOMEM; dev->eq_table.icm_dma = pci_map_page(dev->pdev, dev->eq_table.icm_page, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(dev->eq_table.icm_dma)) { + if (pci_dma_mapping_error(dev->pdev, dev->eq_table.icm_dma)) { __free_page(dev->eq_table.icm_page); return -ENOMEM; } - ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt, &status); - if (!ret && status) - ret = -EINVAL; + ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt); if (ret) { pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); @@ -819,18 +758,15 @@ int __devinit mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt) void mthca_unmap_eq_icm(struct mthca_dev *dev) { - u8 status; - - mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, 1, &status); + mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, 1); pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); __free_page(dev->eq_table.icm_page); } -int __devinit mthca_init_eq_table(struct mthca_dev *dev) +int mthca_init_eq_table(struct mthca_dev *dev) { int err; - u8 status; u8 intr; int i; @@ -845,8 +781,7 @@ int __devinit mthca_init_eq_table(struct mthca_dev *dev) if (err) goto err_out_free; - if (dev->mthca_flags & MTHCA_FLAG_MSI || - dev->mthca_flags & MTHCA_FLAG_MSI_X) { + if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { dev->eq_table.clr_mask = 0; } else { dev->eq_table.clr_mask = @@ -857,8 +792,7 @@ int __devinit mthca_init_eq_table(struct mthca_dev *dev) dev->eq_table.arm_mask = 0; - intr = (dev->mthca_flags & MTHCA_FLAG_MSI) ? - 128 : dev->eq_table.inta_pin; + intr = dev->eq_table.inta_pin; err = mthca_create_eq(dev, dev->limits.num_cqs + MTHCA_NUM_SPARE_EQE, (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr, @@ -880,49 +814,50 @@ int __devinit mthca_init_eq_table(struct mthca_dev *dev) if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { static const char *eq_name[] = { - [MTHCA_EQ_COMP] = DRV_NAME " (comp)", - [MTHCA_EQ_ASYNC] = DRV_NAME " (async)", - [MTHCA_EQ_CMD] = DRV_NAME " (cmd)" + [MTHCA_EQ_COMP] = DRV_NAME "-comp", + [MTHCA_EQ_ASYNC] = DRV_NAME "-async", + [MTHCA_EQ_CMD] = DRV_NAME "-cmd" }; for (i = 0; i < MTHCA_NUM_EQ; ++i) { + snprintf(dev->eq_table.eq[i].irq_name, + IB_DEVICE_NAME_MAX, + "%s@pci:%s", eq_name[i], + pci_name(dev->pdev)); err = request_irq(dev->eq_table.eq[i].msi_x_vector, mthca_is_memfree(dev) ? mthca_arbel_msi_x_interrupt : mthca_tavor_msi_x_interrupt, - 0, eq_name[i], dev->eq_table.eq + i); + 0, dev->eq_table.eq[i].irq_name, + dev->eq_table.eq + i); if (err) goto err_out_cmd; dev->eq_table.eq[i].have_irq = 1; } } else { + snprintf(dev->eq_table.eq[0].irq_name, IB_DEVICE_NAME_MAX, + DRV_NAME "@pci:%s", pci_name(dev->pdev)); err = request_irq(dev->pdev->irq, mthca_is_memfree(dev) ? mthca_arbel_interrupt : mthca_tavor_interrupt, - IRQF_SHARED, DRV_NAME, dev); + IRQF_SHARED, dev->eq_table.eq[0].irq_name, dev); if (err) goto err_out_cmd; dev->eq_table.have_irq = 1; } err = mthca_MAP_EQ(dev, async_mask(dev), - 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status); + 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); if (err) mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err); - if (status) - mthca_warn(dev, "MAP_EQ for async EQ %d returned status 0x%02x\n", - dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, status); err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK, - 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status); + 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn); if (err) mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n", dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err); - if (status) - mthca_warn(dev, "MAP_EQ for cmd EQ %d returned status 0x%02x\n", - dev->eq_table.eq[MTHCA_EQ_CMD].eqn, status); for (i = 0; i < MTHCA_NUM_EQ; ++i) if (mthca_is_memfree(dev)) @@ -952,15 +887,14 @@ err_out_free: void mthca_cleanup_eq_table(struct mthca_dev *dev) { - u8 status; int i; mthca_free_irqs(dev); mthca_MAP_EQ(dev, async_mask(dev), - 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status); + 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK, - 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status); + 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn); for (i = 0; i < MTHCA_NUM_EQ; ++i) mthca_free_eq(dev, &dev->eq_table.eq[i]); diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c index 45e106f1480..b6f7f457fc5 100644 --- a/drivers/infiniband/hw/mthca/mthca_mad.c +++ b/drivers/infiniband/hw/mthca/mthca_mad.c @@ -30,8 +30,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_mad.c 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/string.h> @@ -106,7 +104,8 @@ static void update_sm_ah(struct mthca_dev *dev, */ static void smp_snoop(struct ib_device *ibdev, u8 port_num, - struct ib_mad *mad) + struct ib_mad *mad, + u16 prev_lid) { struct ib_event event; @@ -116,6 +115,7 @@ static void smp_snoop(struct ib_device *ibdev, if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { struct ib_port_info *pinfo = (struct ib_port_info *) ((struct ib_smp *) mad)->data; + u16 lid = be16_to_cpu(pinfo->lid); mthca_update_rate(to_mdev(ibdev), port_num); update_sm_ah(to_mdev(ibdev), port_num, @@ -125,12 +125,15 @@ static void smp_snoop(struct ib_device *ibdev, event.device = ibdev; event.element.port_num = port_num; - if(pinfo->clientrereg_resv_subnetto & 0x80) + if (pinfo->clientrereg_resv_subnetto & 0x80) { event.event = IB_EVENT_CLIENT_REREGISTER; - else - event.event = IB_EVENT_LID_CHANGE; + ib_dispatch_event(&event); + } - ib_dispatch_event(&event); + if (prev_lid != lid) { + event.event = IB_EVENT_LID_CHANGE; + ib_dispatch_event(&event); + } } if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { @@ -168,6 +171,8 @@ static void forward_trap(struct mthca_dev *dev, if (agent) { send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA, GFP_ATOMIC); + if (IS_ERR(send_buf)) + return; /* * We rely here on the fact that MLX QPs don't use the * address handle after the send is posted (this is @@ -196,8 +201,9 @@ int mthca_process_mad(struct ib_device *ibdev, struct ib_mad *out_mad) { int err; - u8 status; u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); + u16 prev_lid = 0; + struct ib_port_attr pattr; /* Forward locally generated traps to the SM */ if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && @@ -235,26 +241,26 @@ int mthca_process_mad(struct ib_device *ibdev, return IB_MAD_RESULT_SUCCESS; } else return IB_MAD_RESULT_SUCCESS; + if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || + in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && + in_mad->mad_hdr.method == IB_MGMT_METHOD_SET && + in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO && + !ib_query_port(ibdev, port_num, &pattr)) + prev_lid = pattr.lid; err = mthca_MAD_IFC(to_mdev(ibdev), mad_flags & IB_MAD_IGNORE_MKEY, mad_flags & IB_MAD_IGNORE_BKEY, - port_num, in_wc, in_grh, in_mad, out_mad, - &status); - if (err) { - mthca_err(to_mdev(ibdev), "MAD_IFC failed\n"); - return IB_MAD_RESULT_FAILURE; - } - if (status == MTHCA_CMD_STAT_BAD_PKT) + port_num, in_wc, in_grh, in_mad, out_mad); + if (err == -EBADMSG) return IB_MAD_RESULT_SUCCESS; - if (status) { - mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n", - status); + else if (err) { + mthca_err(to_mdev(ibdev), "MAD_IFC returned %d\n", err); return IB_MAD_RESULT_FAILURE; } if (!out_mad->mad_hdr.status) { - smp_snoop(ibdev, port_num, in_mad); + smp_snoop(ibdev, port_num, in_mad, prev_lid); node_desc_override(ibdev, out_mad); } @@ -317,7 +323,7 @@ err: return ret; } -void __devexit mthca_free_agents(struct mthca_dev *dev) +void mthca_free_agents(struct mthca_dev *dev) { struct ib_mad_agent *agent; int p, q; diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c index 47ea0214836..ded76c101dd 100644 --- a/drivers/infiniband/hw/mthca/mthca_main.c +++ b/drivers/infiniband/hw/mthca/mthca_main.c @@ -30,8 +30,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $ */ #include <linux/module.h> @@ -39,12 +37,14 @@ #include <linux/errno.h> #include <linux/pci.h> #include <linux/interrupt.h> +#include <linux/gfp.h> #include "mthca_dev.h" #include "mthca_config_reg.h" #include "mthca_cmd.h" #include "mthca_profile.h" #include "mthca_memfree.h" +#include "mthca_wqe.h" MODULE_AUTHOR("Roland Dreier"); MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver"); @@ -61,18 +61,13 @@ MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); #ifdef CONFIG_PCI_MSI -static int msi_x = 0; +static int msi_x = 1; module_param(msi_x, int, 0444); MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); -static int msi = 0; -module_param(msi, int, 0444); -MODULE_PARM_DESC(msi, "attempt to use MSI if nonzero"); - #else /* CONFIG_PCI_MSI */ #define msi_x (0) -#define msi (0) #endif /* CONFIG_PCI_MSI */ @@ -80,60 +75,84 @@ static int tune_pci = 0; module_param(tune_pci, int, 0444); MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero"); -struct mutex mthca_device_mutex; +DEFINE_MUTEX(mthca_device_mutex); + +#define MTHCA_DEFAULT_NUM_QP (1 << 16) +#define MTHCA_DEFAULT_RDB_PER_QP (1 << 2) +#define MTHCA_DEFAULT_NUM_CQ (1 << 16) +#define MTHCA_DEFAULT_NUM_MCG (1 << 13) +#define MTHCA_DEFAULT_NUM_MPT (1 << 17) +#define MTHCA_DEFAULT_NUM_MTT (1 << 20) +#define MTHCA_DEFAULT_NUM_UDAV (1 << 15) +#define MTHCA_DEFAULT_NUM_RESERVED_MTTS (1 << 18) +#define MTHCA_DEFAULT_NUM_UARC_SIZE (1 << 18) + +static struct mthca_profile hca_profile = { + .num_qp = MTHCA_DEFAULT_NUM_QP, + .rdb_per_qp = MTHCA_DEFAULT_RDB_PER_QP, + .num_cq = MTHCA_DEFAULT_NUM_CQ, + .num_mcg = MTHCA_DEFAULT_NUM_MCG, + .num_mpt = MTHCA_DEFAULT_NUM_MPT, + .num_mtt = MTHCA_DEFAULT_NUM_MTT, + .num_udav = MTHCA_DEFAULT_NUM_UDAV, /* Tavor only */ + .fmr_reserved_mtts = MTHCA_DEFAULT_NUM_RESERVED_MTTS, /* Tavor only */ + .uarc_size = MTHCA_DEFAULT_NUM_UARC_SIZE, /* Arbel only */ +}; + +module_param_named(num_qp, hca_profile.num_qp, int, 0444); +MODULE_PARM_DESC(num_qp, "maximum number of QPs per HCA"); + +module_param_named(rdb_per_qp, hca_profile.rdb_per_qp, int, 0444); +MODULE_PARM_DESC(rdb_per_qp, "number of RDB buffers per QP"); + +module_param_named(num_cq, hca_profile.num_cq, int, 0444); +MODULE_PARM_DESC(num_cq, "maximum number of CQs per HCA"); + +module_param_named(num_mcg, hca_profile.num_mcg, int, 0444); +MODULE_PARM_DESC(num_mcg, "maximum number of multicast groups per HCA"); + +module_param_named(num_mpt, hca_profile.num_mpt, int, 0444); +MODULE_PARM_DESC(num_mpt, + "maximum number of memory protection table entries per HCA"); -static const char mthca_version[] __devinitdata = +module_param_named(num_mtt, hca_profile.num_mtt, int, 0444); +MODULE_PARM_DESC(num_mtt, + "maximum number of memory translation table segments per HCA"); + +module_param_named(num_udav, hca_profile.num_udav, int, 0444); +MODULE_PARM_DESC(num_udav, "maximum number of UD address vectors per HCA"); + +module_param_named(fmr_reserved_mtts, hca_profile.fmr_reserved_mtts, int, 0444); +MODULE_PARM_DESC(fmr_reserved_mtts, + "number of memory translation table segments reserved for FMR"); + +static int log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); +module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); +MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-5)"); + +static char mthca_version[] = DRV_NAME ": Mellanox InfiniBand HCA driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; -static struct mthca_profile default_profile = { - .num_qp = 1 << 16, - .rdb_per_qp = 4, - .num_cq = 1 << 16, - .num_mcg = 1 << 13, - .num_mpt = 1 << 17, - .num_mtt = 1 << 20, - .num_udav = 1 << 15, /* Tavor only */ - .fmr_reserved_mtts = 1 << 18, /* Tavor only */ - .uarc_size = 1 << 18, /* Arbel only */ -}; - -static int __devinit mthca_tune_pci(struct mthca_dev *mdev) +static int mthca_tune_pci(struct mthca_dev *mdev) { - int cap; - u16 val; - if (!tune_pci) return 0; /* First try to max out Read Byte Count */ - cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX); - if (cap) { - if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) { - mthca_err(mdev, "Couldn't read PCI-X command register, " - "aborting.\n"); - return -ENODEV; - } - val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2); - if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) { - mthca_err(mdev, "Couldn't write PCI-X command register, " - "aborting.\n"); + if (pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX)) { + if (pcix_set_mmrbc(mdev->pdev, pcix_get_max_mmrbc(mdev->pdev))) { + mthca_err(mdev, "Couldn't set PCI-X max read count, " + "aborting.\n"); return -ENODEV; } } else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE)) mthca_info(mdev, "No PCI-X capability, not setting RBC.\n"); - cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP); - if (cap) { - if (pci_read_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, &val)) { - mthca_err(mdev, "Couldn't read PCI Express device control " - "register, aborting.\n"); - return -ENODEV; - } - val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12); - if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) { - mthca_err(mdev, "Couldn't write PCI Express device control " - "register, aborting.\n"); + if (pci_is_pcie(mdev->pdev)) { + if (pcie_set_readrq(mdev->pdev, 4096)) { + mthca_err(mdev, "Couldn't write PCI Express read request, " + "aborting.\n"); return -ENODEV; } } else if (mdev->mthca_flags & MTHCA_FLAG_PCIE) @@ -143,21 +162,17 @@ static int __devinit mthca_tune_pci(struct mthca_dev *mdev) return 0; } -static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim) +static int mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim) { int err; - u8 status; - err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status); + mdev->limits.mtt_seg_size = (1 << log_mtts_per_seg) * 8; + err = mthca_QUERY_DEV_LIM(mdev, dev_lim); if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); + mthca_err(mdev, "QUERY_DEV_LIM command returned %d" + ", aborting.\n", err); return err; } - if (status) { - mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } if (dev_lim->min_page_sz > PAGE_SIZE) { mthca_err(mdev, "HCA minimum page size of %d bigger than " "kernel PAGE_SIZE of %ld, aborting.\n", @@ -185,7 +200,18 @@ static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim mdev->limits.gid_table_len = dev_lim->max_gids; mdev->limits.pkey_table_len = dev_lim->max_pkeys; mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay; - mdev->limits.max_sg = dev_lim->max_sg; + /* + * Need to allow for worst case send WQE overhead and check + * whether max_desc_sz imposes a lower limit than max_sg; UD + * send has the biggest overhead. + */ + mdev->limits.max_sg = min_t(int, dev_lim->max_sg, + (dev_lim->max_desc_sz - + sizeof (struct mthca_next_seg) - + (mthca_is_memfree(mdev) ? + sizeof (struct mthca_arbel_ud_seg) : + sizeof (struct mthca_tavor_ud_seg))) / + sizeof (struct mthca_data_seg)); mdev->limits.max_wqes = dev_lim->max_qp_sz; mdev->limits.max_qp_init_rdma = dev_lim->max_requester_per_qp; mdev->limits.reserved_qps = dev_lim->reserved_qps; @@ -252,172 +278,134 @@ static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim if (dev_lim->flags & DEV_LIM_FLAG_SRQ) mdev->mthca_flags |= MTHCA_FLAG_SRQ; + if (mthca_is_memfree(mdev)) + if (dev_lim->flags & DEV_LIM_FLAG_IPOIB_CSUM) + mdev->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; + return 0; } -static int __devinit mthca_init_tavor(struct mthca_dev *mdev) +static int mthca_init_tavor(struct mthca_dev *mdev) { - u8 status; + s64 size; int err; struct mthca_dev_lim dev_lim; struct mthca_profile profile; struct mthca_init_hca_param init_hca; - err = mthca_SYS_EN(mdev, &status); + err = mthca_SYS_EN(mdev); if (err) { - mthca_err(mdev, "SYS_EN command failed, aborting.\n"); + mthca_err(mdev, "SYS_EN command returned %d, aborting.\n", err); return err; } - if (status) { - mthca_err(mdev, "SYS_EN returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - err = mthca_QUERY_FW(mdev, &status); + err = mthca_QUERY_FW(mdev); if (err) { - mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); + mthca_err(mdev, "QUERY_FW command returned %d," + " aborting.\n", err); goto err_disable; } - if (status) { - mthca_err(mdev, "QUERY_FW returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_disable; - } - err = mthca_QUERY_DDR(mdev, &status); + err = mthca_QUERY_DDR(mdev); if (err) { - mthca_err(mdev, "QUERY_DDR command failed, aborting.\n"); - goto err_disable; - } - if (status) { - mthca_err(mdev, "QUERY_DDR returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "QUERY_DDR command returned %d, aborting.\n", err); goto err_disable; } err = mthca_dev_lim(mdev, &dev_lim); if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); + mthca_err(mdev, "QUERY_DEV_LIM command returned %d, aborting.\n", err); goto err_disable; } - profile = default_profile; + profile = hca_profile; profile.num_uar = dev_lim.uar_size / PAGE_SIZE; profile.uarc_size = 0; if (mdev->mthca_flags & MTHCA_FLAG_SRQ) profile.num_srq = dev_lim.max_srqs; - err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); - if (err < 0) + size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); + if (size < 0) { + err = size; goto err_disable; + } - err = mthca_INIT_HCA(mdev, &init_hca, &status); + err = mthca_INIT_HCA(mdev, &init_hca); if (err) { - mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); - goto err_disable; - } - if (status) { - mthca_err(mdev, "INIT_HCA returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "INIT_HCA command returned %d, aborting.\n", err); goto err_disable; } return 0; err_disable: - mthca_SYS_DIS(mdev, &status); + mthca_SYS_DIS(mdev); return err; } -static int __devinit mthca_load_fw(struct mthca_dev *mdev) +static int mthca_load_fw(struct mthca_dev *mdev) { - u8 status; int err; /* FIXME: use HCA-attached memory for FW if present */ mdev->fw.arbel.fw_icm = mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages, - GFP_HIGHUSER | __GFP_NOWARN); + GFP_HIGHUSER | __GFP_NOWARN, 0); if (!mdev->fw.arbel.fw_icm) { mthca_err(mdev, "Couldn't allocate FW area, aborting.\n"); return -ENOMEM; } - err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status); + err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm); if (err) { - mthca_err(mdev, "MAP_FA command failed, aborting.\n"); + mthca_err(mdev, "MAP_FA command returned %d, aborting.\n", err); goto err_free; } - if (status) { - mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status); - err = -EINVAL; - goto err_free; - } - err = mthca_RUN_FW(mdev, &status); + err = mthca_RUN_FW(mdev); if (err) { - mthca_err(mdev, "RUN_FW command failed, aborting.\n"); - goto err_unmap_fa; - } - if (status) { - mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "RUN_FW command returned %d, aborting.\n", err); goto err_unmap_fa; } return 0; err_unmap_fa: - mthca_UNMAP_FA(mdev, &status); + mthca_UNMAP_FA(mdev); err_free: - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm); + mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); return err; } -static int __devinit mthca_init_icm(struct mthca_dev *mdev, - struct mthca_dev_lim *dev_lim, - struct mthca_init_hca_param *init_hca, - u64 icm_size) +static int mthca_init_icm(struct mthca_dev *mdev, + struct mthca_dev_lim *dev_lim, + struct mthca_init_hca_param *init_hca, + u64 icm_size) { u64 aux_pages; - u8 status; int err; - err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status); + err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages); if (err) { - mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n"); + mthca_err(mdev, "SET_ICM_SIZE command returned %d, aborting.\n", err); return err; } - if (status) { - mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n", (unsigned long long) icm_size >> 10, (unsigned long long) aux_pages << 2); mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages, - GFP_HIGHUSER | __GFP_NOWARN); + GFP_HIGHUSER | __GFP_NOWARN, 0); if (!mdev->fw.arbel.aux_icm) { mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n"); return -ENOMEM; } - err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status); + err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm); if (err) { - mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n"); - goto err_free_aux; - } - if (status) { - mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "MAP_ICM_AUX returned %d, aborting.\n", err); goto err_free_aux; } @@ -427,10 +415,15 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, goto err_unmap_aux; } + /* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */ + mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * mdev->limits.mtt_seg_size, + dma_get_cache_alignment()) / mdev->limits.mtt_seg_size; + mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base, - MTHCA_MTT_SEG_SIZE, + mdev->limits.mtt_seg_size, mdev->limits.num_mtt_segs, - mdev->limits.reserved_mtts, 1); + mdev->limits.reserved_mtts, + 1, 0); if (!mdev->mr_table.mtt_table) { mthca_err(mdev, "Failed to map MTT context memory, aborting.\n"); err = -ENOMEM; @@ -440,7 +433,8 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base, dev_lim->mpt_entry_sz, mdev->limits.num_mpts, - mdev->limits.reserved_mrws, 1); + mdev->limits.reserved_mrws, + 1, 1); if (!mdev->mr_table.mpt_table) { mthca_err(mdev, "Failed to map MPT context memory, aborting.\n"); err = -ENOMEM; @@ -450,7 +444,8 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base, dev_lim->qpc_entry_sz, mdev->limits.num_qps, - mdev->limits.reserved_qps, 0); + mdev->limits.reserved_qps, + 0, 0); if (!mdev->qp_table.qp_table) { mthca_err(mdev, "Failed to map QP context memory, aborting.\n"); err = -ENOMEM; @@ -460,7 +455,8 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base, dev_lim->eqpc_entry_sz, mdev->limits.num_qps, - mdev->limits.reserved_qps, 0); + mdev->limits.reserved_qps, + 0, 0); if (!mdev->qp_table.eqp_table) { mthca_err(mdev, "Failed to map EQP context memory, aborting.\n"); err = -ENOMEM; @@ -470,7 +466,7 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base, MTHCA_RDB_ENTRY_SIZE, mdev->limits.num_qps << - mdev->qp_table.rdb_shift, + mdev->qp_table.rdb_shift, 0, 0, 0); if (!mdev->qp_table.rdb_table) { mthca_err(mdev, "Failed to map RDB context memory, aborting\n"); @@ -481,7 +477,8 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base, dev_lim->cqc_entry_sz, mdev->limits.num_cqs, - mdev->limits.reserved_cqs, 0); + mdev->limits.reserved_cqs, + 0, 0); if (!mdev->cq_table.table) { mthca_err(mdev, "Failed to map CQ context memory, aborting.\n"); err = -ENOMEM; @@ -493,7 +490,8 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mthca_alloc_icm_table(mdev, init_hca->srqc_base, dev_lim->srq_entry_sz, mdev->limits.num_srqs, - mdev->limits.reserved_srqs, 0); + mdev->limits.reserved_srqs, + 0, 0); if (!mdev->srq_table.table) { mthca_err(mdev, "Failed to map SRQ context memory, " "aborting.\n"); @@ -513,7 +511,7 @@ static int __devinit mthca_init_icm(struct mthca_dev *mdev, mdev->limits.num_amgms, mdev->limits.num_mgms + mdev->limits.num_amgms, - 0); + 0, 0); if (!mdev->mcg_table.table) { mthca_err(mdev, "Failed to map MCG context memory, aborting.\n"); err = -ENOMEM; @@ -548,17 +546,16 @@ err_unmap_eq: mthca_unmap_eq_icm(mdev); err_unmap_aux: - mthca_UNMAP_ICM_AUX(mdev, &status); + mthca_UNMAP_ICM_AUX(mdev); err_free_aux: - mthca_free_icm(mdev, mdev->fw.arbel.aux_icm); + mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); return err; } static void mthca_free_icms(struct mthca_dev *mdev) { - u8 status; mthca_free_icm_table(mdev, mdev->mcg_table.table); if (mdev->mthca_flags & MTHCA_FLAG_SRQ) @@ -571,64 +568,53 @@ static void mthca_free_icms(struct mthca_dev *mdev) mthca_free_icm_table(mdev, mdev->mr_table.mtt_table); mthca_unmap_eq_icm(mdev); - mthca_UNMAP_ICM_AUX(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.aux_icm); + mthca_UNMAP_ICM_AUX(mdev); + mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); } -static int __devinit mthca_init_arbel(struct mthca_dev *mdev) +static int mthca_init_arbel(struct mthca_dev *mdev) { struct mthca_dev_lim dev_lim; struct mthca_profile profile; struct mthca_init_hca_param init_hca; - u64 icm_size; - u8 status; + s64 icm_size; int err; - err = mthca_QUERY_FW(mdev, &status); + err = mthca_QUERY_FW(mdev); if (err) { - mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); + mthca_err(mdev, "QUERY_FW command failed %d, aborting.\n", err); return err; } - if (status) { - mthca_err(mdev, "QUERY_FW returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - err = mthca_ENABLE_LAM(mdev, &status); - if (err) { - mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n"); - return err; - } - if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) { + err = mthca_ENABLE_LAM(mdev); + if (err == -EAGAIN) { mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n"); mdev->mthca_flags |= MTHCA_FLAG_NO_LAM; - } else if (status) { - mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; + } else if (err) { + mthca_err(mdev, "ENABLE_LAM returned %d, aborting.\n", err); + return err; } err = mthca_load_fw(mdev); if (err) { - mthca_err(mdev, "Failed to start FW, aborting.\n"); + mthca_err(mdev, "Loading FW returned %d, aborting.\n", err); goto err_disable; } err = mthca_dev_lim(mdev, &dev_lim); if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); + mthca_err(mdev, "QUERY_DEV_LIM returned %d, aborting.\n", err); goto err_stop_fw; } - profile = default_profile; + profile = hca_profile; profile.num_uar = dev_lim.uar_size / PAGE_SIZE; profile.num_udav = 0; if (mdev->mthca_flags & MTHCA_FLAG_SRQ) profile.num_srq = dev_lim.max_srqs; icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); - if ((int) icm_size < 0) { + if (icm_size < 0) { err = icm_size; goto err_stop_fw; } @@ -637,15 +623,9 @@ static int __devinit mthca_init_arbel(struct mthca_dev *mdev) if (err) goto err_stop_fw; - err = mthca_INIT_HCA(mdev, &init_hca, &status); + err = mthca_INIT_HCA(mdev, &init_hca); if (err) { - mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); - goto err_free_icm; - } - if (status) { - mthca_err(mdev, "INIT_HCA returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "INIT_HCA command returned %d, aborting.\n", err); goto err_free_icm; } @@ -655,37 +635,34 @@ err_free_icm: mthca_free_icms(mdev); err_stop_fw: - mthca_UNMAP_FA(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm); + mthca_UNMAP_FA(mdev); + mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); err_disable: if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) - mthca_DISABLE_LAM(mdev, &status); + mthca_DISABLE_LAM(mdev); return err; } static void mthca_close_hca(struct mthca_dev *mdev) { - u8 status; - - mthca_CLOSE_HCA(mdev, 0, &status); + mthca_CLOSE_HCA(mdev, 0); if (mthca_is_memfree(mdev)) { mthca_free_icms(mdev); - mthca_UNMAP_FA(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm); + mthca_UNMAP_FA(mdev); + mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) - mthca_DISABLE_LAM(mdev, &status); + mthca_DISABLE_LAM(mdev); } else - mthca_SYS_DIS(mdev, &status); + mthca_SYS_DIS(mdev); } -static int __devinit mthca_init_hca(struct mthca_dev *mdev) +static int mthca_init_hca(struct mthca_dev *mdev) { - u8 status; int err; struct mthca_adapter adapter; @@ -697,20 +674,15 @@ static int __devinit mthca_init_hca(struct mthca_dev *mdev) if (err) return err; - err = mthca_QUERY_ADAPTER(mdev, &adapter, &status); + err = mthca_QUERY_ADAPTER(mdev, &adapter); if (err) { - mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n"); - goto err_close; - } - if (status) { - mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; + mthca_err(mdev, "QUERY_ADAPTER command returned %d, aborting.\n", err); goto err_close; } mdev->eq_table.inta_pin = adapter.inta_pin; - mdev->rev_id = adapter.revision_id; + if (!mthca_is_memfree(mdev)) + mdev->rev_id = adapter.revision_id; memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id); return 0; @@ -720,10 +692,9 @@ err_close: return err; } -static int __devinit mthca_setup_hca(struct mthca_dev *dev) +static int mthca_setup_hca(struct mthca_dev *dev) { int err; - u8 status; MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock); @@ -741,7 +712,7 @@ static int __devinit mthca_setup_hca(struct mthca_dev *dev) goto err_uar_table_free; } - dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); + dev->kar = ioremap((phys_addr_t) dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); if (!dev->kar) { mthca_err(dev, "Couldn't map kernel access region, " "aborting.\n"); @@ -784,16 +755,19 @@ static int __devinit mthca_setup_hca(struct mthca_dev *dev) goto err_eq_table_free; } - err = mthca_NOP(dev, &status); - if (err || status) { - mthca_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting.\n", - dev->mthca_flags & MTHCA_FLAG_MSI_X ? - dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector : - dev->pdev->irq); - if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X)) - mthca_err(dev, "Try again with MSI/MSI-X disabled.\n"); - else + err = mthca_NOP(dev); + if (err) { + if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { + mthca_warn(dev, "NOP command failed to generate interrupt " + "(IRQ %d).\n", + dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector); + mthca_warn(dev, "Trying again with MSI-X disabled.\n"); + } else { + mthca_err(dev, "NOP command failed to generate interrupt " + "(IRQ %d), aborting.\n", + dev->pdev->irq); mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n"); + } goto err_cmd_poll; } @@ -875,60 +849,7 @@ err_uar_table_free: return err; } -static int __devinit mthca_request_regions(struct pci_dev *pdev, - int ddr_hidden) -{ - int err; - - /* - * We can't just use pci_request_regions() because the MSI-X - * table is right in the middle of the first BAR. If we did - * pci_request_region and grab all of the first BAR, then - * setting up MSI-X would fail, since the PCI core wants to do - * request_mem_region on the MSI-X vector table. - * - * So just request what we need right now, and request any - * other regions we need when setting up EQs. - */ - if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE, - MTHCA_HCR_SIZE, DRV_NAME)) - return -EBUSY; - - err = pci_request_region(pdev, 2, DRV_NAME); - if (err) - goto err_bar2_failed; - - if (!ddr_hidden) { - err = pci_request_region(pdev, 4, DRV_NAME); - if (err) - goto err_bar4_failed; - } - - return 0; - -err_bar4_failed: - pci_release_region(pdev, 2); - -err_bar2_failed: - release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE, - MTHCA_HCR_SIZE); - - return err; -} - -static void mthca_release_regions(struct pci_dev *pdev, - int ddr_hidden) -{ - if (!ddr_hidden) - pci_release_region(pdev, 4); - - pci_release_region(pdev, 2); - - release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE, - MTHCA_HCR_SIZE); -} - -static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev) +static int mthca_enable_msi_x(struct mthca_dev *mdev) { struct msix_entry entries[3]; int err; @@ -937,13 +858,9 @@ static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev) entries[1].entry = 1; entries[2].entry = 2; - err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries)); - if (err) { - if (err > 0) - mthca_info(mdev, "Only %d MSI-X vectors available, " - "not using MSI-X\n", err); + err = pci_enable_msix_exact(mdev->pdev, entries, ARRAY_SIZE(entries)); + if (err) return err; - } mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector; mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector; @@ -967,14 +884,14 @@ static struct { u64 latest_fw; u32 flags; } mthca_hca_table[] = { - [TAVOR] = { .latest_fw = MTHCA_FW_VER(3, 4, 0), + [TAVOR] = { .latest_fw = MTHCA_FW_VER(3, 5, 0), .flags = 0 }, - [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 7, 600), + [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 8, 200), .flags = MTHCA_FLAG_PCIE }, - [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 1, 400), + [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 3, 0), .flags = MTHCA_FLAG_MEMFREE | MTHCA_FLAG_PCIE }, - [SINAI] = { .latest_fw = MTHCA_FW_VER(1, 1, 0), + [SINAI] = { .latest_fw = MTHCA_FW_VER(1, 2, 0), .flags = MTHCA_FLAG_MEMFREE | MTHCA_FLAG_PCIE | MTHCA_FLAG_SINAI_OPT } @@ -1014,7 +931,7 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM)) ddr_hidden = 1; - err = mthca_request_regions(pdev, ddr_hidden); + err = pci_request_regions(pdev, DRV_NAME); if (err) { dev_err(&pdev->dev, "Cannot obtain PCI resources, " "aborting.\n"); @@ -1023,20 +940,20 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) pci_set_master(pdev); - err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); - err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); goto err_free_res; } } - err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK); + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); if (err) { dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " "consistent PCI DMA mask.\n"); - err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); + err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " "aborting.\n"); @@ -1044,6 +961,9 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) } } + /* We can handle large RDMA requests, so allow larger segments. */ + dma_set_max_seg_size(&pdev->dev, 1024 * 1024 * 1024); + mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev); if (!mdev) { dev_err(&pdev->dev, "Device struct alloc failed, " @@ -1069,12 +989,6 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) goto err_free_dev; } - if (msi_x && !mthca_enable_msi_x(mdev)) - mdev->mthca_flags |= MTHCA_FLAG_MSI_X; - if (msi && !(mdev->mthca_flags & MTHCA_FLAG_MSI_X) && - !pci_enable_msi(pdev)) - mdev->mthca_flags |= MTHCA_FLAG_MSI; - if (mthca_cmd_init(mdev)) { mthca_err(mdev, "Failed to init command interface, aborting.\n"); goto err_free_dev; @@ -1089,7 +1003,7 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) goto err_cmd; if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) { - mthca_warn(mdev, "HCA FW version %d.%d.%d is old (%d.%d.%d is current).\n", + mthca_warn(mdev, "HCA FW version %d.%d.%03d is old (%d.%d.%03d is current).\n", (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff, (int) (mdev->fw_ver & 0xffff), (int) (mthca_hca_table[hca_type].latest_fw >> 32), @@ -1098,7 +1012,18 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n"); } + if (msi_x && !mthca_enable_msi_x(mdev)) + mdev->mthca_flags |= MTHCA_FLAG_MSI_X; + err = mthca_setup_hca(mdev); + if (err == -EBUSY && (mdev->mthca_flags & MTHCA_FLAG_MSI_X)) { + if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) + pci_disable_msix(pdev); + mdev->mthca_flags &= ~MTHCA_FLAG_MSI_X; + + err = mthca_setup_hca(mdev); + } + if (err) goto err_close; @@ -1113,6 +1038,8 @@ static int __mthca_init_one(struct pci_dev *pdev, int hca_type) pci_set_drvdata(pdev, mdev); mdev->hca_type = hca_type; + mdev->active = true; + return 0; err_unregister: @@ -1134,21 +1061,19 @@ err_cleanup: mthca_cleanup_uar_table(mdev); err_close: + if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) + pci_disable_msix(pdev); + mthca_close_hca(mdev); err_cmd: mthca_cmd_cleanup(mdev); err_free_dev: - if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) - pci_disable_msix(pdev); - if (mdev->mthca_flags & MTHCA_FLAG_MSI) - pci_disable_msi(pdev); - ib_dealloc_device(&mdev->ib_dev); err_free_res: - mthca_release_regions(pdev, ddr_hidden); + pci_release_regions(pdev); err_disable_pdev: pci_disable_device(pdev); @@ -1159,7 +1084,6 @@ err_disable_pdev: static void __mthca_remove_one(struct pci_dev *pdev) { struct mthca_dev *mdev = pci_get_drvdata(pdev); - u8 status; int p; if (mdev) { @@ -1167,7 +1091,7 @@ static void __mthca_remove_one(struct pci_dev *pdev) mthca_unregister_device(mdev); for (p = 1; p <= mdev->limits.num_ports; ++p) - mthca_CLOSE_IB(mdev, p, &status); + mthca_CLOSE_IB(mdev, p); mthca_cleanup_mcg_table(mdev); mthca_cleanup_av_table(mdev); @@ -1190,12 +1114,9 @@ static void __mthca_remove_one(struct pci_dev *pdev) if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) pci_disable_msix(pdev); - if (mdev->mthca_flags & MTHCA_FLAG_MSI) - pci_disable_msi(pdev); ib_dealloc_device(&mdev->ib_dev); - mthca_release_regions(pdev, mdev->mthca_flags & - MTHCA_FLAG_DDR_HIDDEN); + pci_release_regions(pdev); pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); } @@ -1204,26 +1125,23 @@ static void __mthca_remove_one(struct pci_dev *pdev) int __mthca_restart_one(struct pci_dev *pdev) { struct mthca_dev *mdev; + int hca_type; mdev = pci_get_drvdata(pdev); if (!mdev) return -ENODEV; + hca_type = mdev->hca_type; __mthca_remove_one(pdev); - return __mthca_init_one(pdev, mdev->hca_type); + return __mthca_init_one(pdev, hca_type); } -static int __devinit mthca_init_one(struct pci_dev *pdev, - const struct pci_device_id *id) +static int mthca_init_one(struct pci_dev *pdev, const struct pci_device_id *id) { - static int mthca_version_printed = 0; int ret; mutex_lock(&mthca_device_mutex); - if (!mthca_version_printed) { - printk(KERN_INFO "%s", mthca_version); - ++mthca_version_printed; - } + printk_once(KERN_INFO "%s", mthca_version); if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) { printk(KERN_ERR PFX "%s has invalid driver data %lx\n", @@ -1239,7 +1157,7 @@ static int __devinit mthca_init_one(struct pci_dev *pdev, return ret; } -static void __devexit mthca_remove_one(struct pci_dev *pdev) +static void mthca_remove_one(struct pci_dev *pdev) { mutex_lock(&mthca_device_mutex); __mthca_remove_one(pdev); @@ -1276,14 +1194,64 @@ static struct pci_driver mthca_driver = { .name = DRV_NAME, .id_table = mthca_pci_table, .probe = mthca_init_one, - .remove = __devexit_p(mthca_remove_one) + .remove = mthca_remove_one, }; +static void __init __mthca_check_profile_val(const char *name, int *pval, + int pval_default) +{ + /* value must be positive and power of 2 */ + int old_pval = *pval; + + if (old_pval <= 0) + *pval = pval_default; + else + *pval = roundup_pow_of_two(old_pval); + + if (old_pval != *pval) { + printk(KERN_WARNING PFX "Invalid value %d for %s in module parameter.\n", + old_pval, name); + printk(KERN_WARNING PFX "Corrected %s to %d.\n", name, *pval); + } +} + +#define mthca_check_profile_val(name, default) \ + __mthca_check_profile_val(#name, &hca_profile.name, default) + +static void __init mthca_validate_profile(void) +{ + mthca_check_profile_val(num_qp, MTHCA_DEFAULT_NUM_QP); + mthca_check_profile_val(rdb_per_qp, MTHCA_DEFAULT_RDB_PER_QP); + mthca_check_profile_val(num_cq, MTHCA_DEFAULT_NUM_CQ); + mthca_check_profile_val(num_mcg, MTHCA_DEFAULT_NUM_MCG); + mthca_check_profile_val(num_mpt, MTHCA_DEFAULT_NUM_MPT); + mthca_check_profile_val(num_mtt, MTHCA_DEFAULT_NUM_MTT); + mthca_check_profile_val(num_udav, MTHCA_DEFAULT_NUM_UDAV); + mthca_check_profile_val(fmr_reserved_mtts, MTHCA_DEFAULT_NUM_RESERVED_MTTS); + + if (hca_profile.fmr_reserved_mtts >= hca_profile.num_mtt) { + printk(KERN_WARNING PFX "Invalid fmr_reserved_mtts module parameter %d.\n", + hca_profile.fmr_reserved_mtts); + printk(KERN_WARNING PFX "(Must be smaller than num_mtt %d)\n", + hca_profile.num_mtt); + hca_profile.fmr_reserved_mtts = hca_profile.num_mtt / 2; + printk(KERN_WARNING PFX "Corrected fmr_reserved_mtts to %d.\n", + hca_profile.fmr_reserved_mtts); + } + + if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 5)) { + printk(KERN_WARNING PFX "bad log_mtts_per_seg (%d). Using default - %d\n", + log_mtts_per_seg, ilog2(MTHCA_MTT_SEG_SIZE / 8)); + log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); + } +} + static int __init mthca_init(void) { int ret; - mutex_init(&mthca_device_mutex); + mthca_validate_profile(); + ret = mthca_catas_init(); if (ret) return ret; diff --git a/drivers/infiniband/hw/mthca/mthca_mcg.c b/drivers/infiniband/hw/mthca/mthca_mcg.c index 47ca8a9b724..6304ae8f4a6 100644 --- a/drivers/infiniband/hw/mthca/mthca_mcg.c +++ b/drivers/infiniband/hw/mthca/mthca_mcg.c @@ -28,13 +28,10 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_mcg.c 1349 2004-12-16 21:09:43Z roland $ */ -#include <linux/init.h> #include <linux/string.h> -#include <linux/slab.h> +#include <linux/gfp.h> #include "mthca_dev.h" #include "mthca_cmd.h" @@ -71,7 +68,6 @@ static int find_mgm(struct mthca_dev *dev, struct mthca_mgm *mgm = mgm_mailbox->buf; u8 *mgid; int err; - u8 status; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) @@ -80,38 +76,22 @@ static int find_mgm(struct mthca_dev *dev, memcpy(mgid, gid, 16); - err = mthca_MGID_HASH(dev, mailbox, hash, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "MGID_HASH returned status %02x\n", status); - err = -EINVAL; + err = mthca_MGID_HASH(dev, mailbox, hash); + if (err) { + mthca_err(dev, "MGID_HASH failed (%d)\n", err); goto out; } if (0) - mthca_dbg(dev, "Hash for %04x:%04x:%04x:%04x:" - "%04x:%04x:%04x:%04x is %04x\n", - be16_to_cpu(((__be16 *) gid)[0]), - be16_to_cpu(((__be16 *) gid)[1]), - be16_to_cpu(((__be16 *) gid)[2]), - be16_to_cpu(((__be16 *) gid)[3]), - be16_to_cpu(((__be16 *) gid)[4]), - be16_to_cpu(((__be16 *) gid)[5]), - be16_to_cpu(((__be16 *) gid)[6]), - be16_to_cpu(((__be16 *) gid)[7]), - *hash); + mthca_dbg(dev, "Hash for %pI6 is %04x\n", gid, *hash); *index = *hash; *prev = -1; do { - err = mthca_READ_MGM(dev, *index, mgm_mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_READ_MGM(dev, *index, mgm_mailbox); + if (err) { + mthca_err(dev, "READ_MGM failed (%d)\n", err); goto out; } @@ -147,7 +127,6 @@ int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) int link = 0; int i; int err; - u8 status; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) @@ -173,12 +152,9 @@ int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) goto out; } - err = mthca_READ_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_READ_MGM(dev, index, mailbox); + if (err) { + mthca_err(dev, "READ_MGM failed (%d)\n", err); goto out; } memset(mgm, 0, sizeof *mgm); @@ -202,11 +178,9 @@ int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) goto out; } - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); + err = mthca_WRITE_MGM(dev, index, mailbox); + if (err) { + mthca_err(dev, "WRITE_MGM failed %d\n", err); err = -EINVAL; goto out; } @@ -214,24 +188,17 @@ int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) if (!link) goto out; - err = mthca_READ_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_READ_MGM(dev, prev, mailbox); + if (err) { + mthca_err(dev, "READ_MGM failed %d\n", err); goto out; } mgm->next_gid_index = cpu_to_be32(index << 6); - err = mthca_WRITE_MGM(dev, prev, mailbox, &status); + err = mthca_WRITE_MGM(dev, prev, mailbox); if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - } + mthca_err(dev, "WRITE_MGM returned %d\n", err); out: if (err && link && index != -1) { @@ -253,7 +220,6 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) int prev, index; int i, loc; int err; - u8 status; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) @@ -267,16 +233,7 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) goto out; if (index == -1) { - mthca_err(dev, "MGID %04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x " - "not found\n", - be16_to_cpu(((__be16 *) gid->raw)[0]), - be16_to_cpu(((__be16 *) gid->raw)[1]), - be16_to_cpu(((__be16 *) gid->raw)[2]), - be16_to_cpu(((__be16 *) gid->raw)[3]), - be16_to_cpu(((__be16 *) gid->raw)[4]), - be16_to_cpu(((__be16 *) gid->raw)[5]), - be16_to_cpu(((__be16 *) gid->raw)[6]), - be16_to_cpu(((__be16 *) gid->raw)[7])); + mthca_err(dev, "MGID %pI6 not found\n", gid->raw); err = -EINVAL; goto out; } @@ -297,12 +254,9 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) mgm->qp[loc] = mgm->qp[i - 1]; mgm->qp[i - 1] = 0; - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_WRITE_MGM(dev, index, mailbox); + if (err) { + mthca_err(dev, "WRITE_MGM returned %d\n", err); goto out; } @@ -314,24 +268,17 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) int amgm_index_to_free = be32_to_cpu(mgm->next_gid_index) >> 6; if (amgm_index_to_free) { err = mthca_READ_MGM(dev, amgm_index_to_free, - mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", - status); - err = -EINVAL; + mailbox); + if (err) { + mthca_err(dev, "READ_MGM returned %d\n", err); goto out; } } else memset(mgm->gid, 0, 16); - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_WRITE_MGM(dev, index, mailbox); + if (err) { + mthca_err(dev, "WRITE_MGM returned %d\n", err); goto out; } if (amgm_index_to_free) { @@ -341,23 +288,17 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) } else { /* Remove entry from AMGM */ int curr_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; - err = mthca_READ_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_READ_MGM(dev, prev, mailbox); + if (err) { + mthca_err(dev, "READ_MGM returned %d\n", err); goto out; } mgm->next_gid_index = cpu_to_be32(curr_next_index << 6); - err = mthca_WRITE_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; + err = mthca_WRITE_MGM(dev, prev, mailbox); + if (err) { + mthca_err(dev, "WRITE_MGM returned %d\n", err); goto out; } BUG_ON(index < dev->limits.num_mgms); @@ -371,7 +312,7 @@ int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) return err; } -int __devinit mthca_init_mcg_table(struct mthca_dev *dev) +int mthca_init_mcg_table(struct mthca_dev *dev) { int err; int table_size = dev->limits.num_mgms + dev->limits.num_amgms; diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c index 15cc2f6eb47..7d2e42dd692 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.c +++ b/drivers/infiniband/hw/mthca/mthca_memfree.c @@ -30,11 +30,14 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id$ */ #include <linux/mm.h> +#include <linux/scatterlist.h> +#include <linux/sched.h> +#include <linux/slab.h> + +#include <asm/page.h> #include "mthca_memfree.h" #include "mthca_dev.h" @@ -58,22 +61,42 @@ struct mthca_user_db_table { } page[0]; }; -void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm) +static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) +{ + int i; + + if (chunk->nsg > 0) + pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, + PCI_DMA_BIDIRECTIONAL); + + for (i = 0; i < chunk->npages; ++i) + __free_pages(sg_page(&chunk->mem[i]), + get_order(chunk->mem[i].length)); +} + +static void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) { - struct mthca_icm_chunk *chunk, *tmp; int i; + for (i = 0; i < chunk->npages; ++i) { + dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, + lowmem_page_address(sg_page(&chunk->mem[i])), + sg_dma_address(&chunk->mem[i])); + } +} + +void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent) +{ + struct mthca_icm_chunk *chunk, *tmp; + if (!icm) return; list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { - if (chunk->nsg > 0) - pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - for (i = 0; i < chunk->npages; ++i) - __free_pages(chunk->mem[i].page, - get_order(chunk->mem[i].length)); + if (coherent) + mthca_free_icm_coherent(dev, chunk); + else + mthca_free_icm_pages(dev, chunk); kfree(chunk); } @@ -81,12 +104,46 @@ void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm) kfree(icm); } +static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) +{ + struct page *page; + + /* + * Use __GFP_ZERO because buggy firmware assumes ICM pages are + * cleared, and subtle failures are seen if they aren't. + */ + page = alloc_pages(gfp_mask | __GFP_ZERO, order); + if (!page) + return -ENOMEM; + + sg_set_page(mem, page, PAGE_SIZE << order, 0); + return 0; +} + +static int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, + int order, gfp_t gfp_mask) +{ + void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), + gfp_mask); + if (!buf) + return -ENOMEM; + + sg_set_buf(mem, buf, PAGE_SIZE << order); + BUG_ON(mem->offset); + sg_dma_len(mem) = PAGE_SIZE << order; + return 0; +} + struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, - gfp_t gfp_mask) + gfp_t gfp_mask, int coherent) { struct mthca_icm *icm; struct mthca_icm_chunk *chunk = NULL; int cur_order; + int ret; + + /* We use sg_set_buf for coherent allocs, which assumes low memory */ + BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); if (!icm) @@ -104,6 +161,7 @@ struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, if (!chunk) goto fail; + sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN); chunk->npages = 0; chunk->nsg = 0; list_add_tail(&chunk->list, &icm->chunk_list); @@ -112,21 +170,30 @@ struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, while (1 << cur_order > npages) --cur_order; - chunk->mem[chunk->npages].page = alloc_pages(gfp_mask, cur_order); - if (chunk->mem[chunk->npages].page) { - chunk->mem[chunk->npages].length = PAGE_SIZE << cur_order; - chunk->mem[chunk->npages].offset = 0; + if (coherent) + ret = mthca_alloc_icm_coherent(&dev->pdev->dev, + &chunk->mem[chunk->npages], + cur_order, gfp_mask); + else + ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages], + cur_order, gfp_mask); - if (++chunk->npages == MTHCA_ICM_CHUNK_LEN) { + if (!ret) { + ++chunk->npages; + + if (coherent) + ++chunk->nsg; + else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) { chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); if (chunk->nsg <= 0) goto fail; + } + if (chunk->npages == MTHCA_ICM_CHUNK_LEN) chunk = NULL; - } npages -= 1 << cur_order; } else { @@ -136,7 +203,7 @@ struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, } } - if (chunk) { + if (!coherent && chunk) { chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, chunk->npages, PCI_DMA_BIDIRECTIONAL); @@ -148,7 +215,7 @@ struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, return icm; fail: - mthca_free_icm(dev, icm); + mthca_free_icm(dev, icm, coherent); return NULL; } @@ -156,7 +223,6 @@ int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int ob { int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; int ret = 0; - u8 status; mutex_lock(&table->mutex); @@ -167,15 +233,15 @@ int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int ob table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN); + __GFP_NOWARN, table->coherent); if (!table->icm[i]) { ret = -ENOMEM; goto out; } - if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - &status) || status) { - mthca_free_icm(dev, table->icm[i]); + if (mthca_MAP_ICM(dev, table->icm[i], + table->virt + i * MTHCA_TABLE_CHUNK_SIZE)) { + mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; ret = -ENOMEM; goto out; @@ -191,7 +257,6 @@ out: void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) { int i; - u8 status; if (!mthca_is_memfree(dev)) return; @@ -202,18 +267,17 @@ void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int o if (--table->icm[i]->refcount == 0) { mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i]); + MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); + mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; } mutex_unlock(&table->mutex); } -void *mthca_table_find(struct mthca_icm_table *table, int obj) +void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle) { - int idx, offset, i; + int idx, offset, dma_offset, i; struct mthca_icm_chunk *chunk; struct mthca_icm *icm; struct page *page = NULL; @@ -225,15 +289,24 @@ void *mthca_table_find(struct mthca_icm_table *table, int obj) idx = (obj & (table->num_obj - 1)) * table->obj_size; icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE]; - offset = idx % MTHCA_TABLE_CHUNK_SIZE; + dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE; if (!icm) goto out; list_for_each_entry(chunk, &icm->chunk_list, list) { for (i = 0; i < chunk->npages; ++i) { - if (chunk->mem[i].length >= offset) { - page = chunk->mem[i].page; + if (dma_handle && dma_offset >= 0) { + if (sg_dma_len(&chunk->mem[i]) > dma_offset) + *dma_handle = sg_dma_address(&chunk->mem[i]) + + dma_offset; + dma_offset -= sg_dma_len(&chunk->mem[i]); + } + /* DMA mapping can merge pages but not split them, + * so if we found the page, dma_handle has already + * been assigned to. */ + if (chunk->mem[i].length > offset) { + page = sg_page(&chunk->mem[i]); goto out; } offset -= chunk->mem[i].length; @@ -283,15 +356,16 @@ void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, u64 virt, int obj_size, int nobj, int reserved, - int use_lowmem) + int use_lowmem, int use_coherent) { struct mthca_icm_table *table; + int obj_per_chunk; int num_icm; unsigned chunk_size; int i; - u8 status; - num_icm = (obj_size * nobj + MTHCA_TABLE_CHUNK_SIZE - 1) / MTHCA_TABLE_CHUNK_SIZE; + obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size; + num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); if (!table) @@ -302,6 +376,7 @@ struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, table->num_obj = nobj; table->obj_size = obj_size; table->lowmem = use_lowmem; + table->coherent = use_coherent; mutex_init(&table->mutex); for (i = 0; i < num_icm; ++i) @@ -314,12 +389,12 @@ struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT, (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN); + __GFP_NOWARN, use_coherent); if (!table->icm[i]) goto err; - if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE, - &status) || status) { - mthca_free_icm(dev, table->icm[i]); + if (mthca_MAP_ICM(dev, table->icm[i], + virt + i * MTHCA_TABLE_CHUNK_SIZE)) { + mthca_free_icm(dev, table->icm[i], table->coherent); table->icm[i] = NULL; goto err; } @@ -337,9 +412,8 @@ err: for (i = 0; i < num_icm; ++i) if (table->icm[i]) { mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i]); + MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); + mthca_free_icm(dev, table->icm[i], table->coherent); } kfree(table); @@ -350,14 +424,13 @@ err: void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) { int i; - u8 status; for (i = 0; i < table->num_icm; ++i) if (table->icm[i]) { - mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i]); + mthca_UNMAP_ICM(dev, + table->virt + i * MTHCA_TABLE_CHUNK_SIZE, + MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE); + mthca_free_icm(dev, table->icm[i], table->coherent); } kfree(table); @@ -373,8 +446,8 @@ static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int pag int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, struct mthca_user_db_table *db_tab, int index, u64 uaddr) { + struct page *pages[1]; int ret = 0; - u8 status; int i; if (!mthca_is_memfree(dev)) @@ -400,26 +473,24 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, } ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0, - &db_tab->page[i].mem.page, NULL); + pages, NULL); if (ret < 0) goto out; - db_tab->page[i].mem.length = MTHCA_ICM_PAGE_SIZE; - db_tab->page[i].mem.offset = uaddr & ~PAGE_MASK; + sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE, + uaddr & ~PAGE_MASK); ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); if (ret < 0) { - put_page(db_tab->page[i].mem.page); + put_page(pages[0]); goto out; } ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), - mthca_uarc_virt(dev, uar, i), &status); - if (!ret && status) - ret = -EINVAL; + mthca_uarc_virt(dev, uar, i)); if (ret) { pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(db_tab->page[i].mem.page); + put_page(sg_page(&db_tab->page[i].mem)); goto out; } @@ -467,6 +538,7 @@ struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) for (i = 0; i < npages; ++i) { db_tab->page[i].refcount = 0; db_tab->page[i].uvirt = 0; + sg_init_table(&db_tab->page[i].mem, 1); } return db_tab; @@ -476,16 +548,15 @@ void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, struct mthca_user_db_table *db_tab) { int i; - u8 status; if (!mthca_is_memfree(dev)) return; for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) { if (db_tab->page[i].uvirt) { - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status); + mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1); pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(db_tab->page[i].mem.page); + put_page(sg_page(&db_tab->page[i].mem)); } } @@ -500,7 +571,6 @@ int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, int i, j; struct mthca_db_page *page; int ret = 0; - u8 status; mutex_lock(&dev->db_tab->mutex); @@ -563,9 +633,7 @@ alloc: memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE); ret = mthca_MAP_ICM_page(dev, page->mapping, - mthca_uarc_virt(dev, &dev->driver_uar, i), &status); - if (!ret && status) - ret = -EINVAL; + mthca_uarc_virt(dev, &dev->driver_uar, i)); if (ret) { dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, page->db_rec, page->mapping); @@ -597,7 +665,6 @@ void mthca_free_db(struct mthca_dev *dev, int type, int db_index) { int i, j; struct mthca_db_page *page; - u8 status; i = db_index / MTHCA_DB_REC_PER_PAGE; j = db_index % MTHCA_DB_REC_PER_PAGE; @@ -613,7 +680,7 @@ void mthca_free_db(struct mthca_dev *dev, int type, int db_index) if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && i >= dev->db_tab->max_group1 - 1) { - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); + mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1); dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, page->db_rec, page->mapping); @@ -664,7 +731,6 @@ int mthca_init_db_tab(struct mthca_dev *dev) void mthca_cleanup_db_tab(struct mthca_dev *dev) { int i; - u8 status; if (!mthca_is_memfree(dev)) return; @@ -682,7 +748,7 @@ void mthca_cleanup_db_tab(struct mthca_dev *dev) if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) mthca_warn(dev, "Kernel UARC page %d not empty\n", i); - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); + mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1); dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, dev->db_tab->page[i].db_rec, diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.h b/drivers/infiniband/hw/mthca/mthca_memfree.h index 6d42947e1dc..da9b8f9b884 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.h +++ b/drivers/infiniband/hw/mthca/mthca_memfree.h @@ -30,15 +30,12 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id$ */ #ifndef MTHCA_MEMFREE_H #define MTHCA_MEMFREE_H #include <linux/list.h> -#include <linux/pci.h> #include <linux/mutex.h> #define MTHCA_ICM_CHUNK_LEN \ @@ -69,6 +66,7 @@ struct mthca_icm_table { int num_obj; int obj_size; int lowmem; + int coherent; struct mutex mutex; struct mthca_icm *icm[0]; }; @@ -82,17 +80,17 @@ struct mthca_icm_iter { struct mthca_dev; struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, - gfp_t gfp_mask); -void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm); + gfp_t gfp_mask, int coherent); +void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent); struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, u64 virt, int obj_size, int nobj, int reserved, - int use_lowmem); + int use_lowmem, int use_coherent); void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table); int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); -void *mthca_table_find(struct mthca_icm_table *table, int obj); +void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle); int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, int start, int end); void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, diff --git a/drivers/infiniband/hw/mthca/mthca_mr.c b/drivers/infiniband/hw/mthca/mthca_mr.c index a486dec1707..ed9a989e501 100644 --- a/drivers/infiniband/hw/mthca/mthca_mr.c +++ b/drivers/infiniband/hw/mthca/mthca_mr.c @@ -29,12 +29,9 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/slab.h> -#include <linux/init.h> #include <linux/errno.h> #include "mthca_dev.h" @@ -92,23 +89,26 @@ static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) spin_lock(&buddy->lock); - for (o = order; o <= buddy->max_order; ++o) { - m = 1 << (buddy->max_order - o); - seg = find_first_bit(buddy->bits[o], m); - if (seg < m) - goto found; - } + for (o = order; o <= buddy->max_order; ++o) + if (buddy->num_free[o]) { + m = 1 << (buddy->max_order - o); + seg = find_first_bit(buddy->bits[o], m); + if (seg < m) + goto found; + } spin_unlock(&buddy->lock); return -1; found: clear_bit(seg, buddy->bits[o]); + --buddy->num_free[o]; while (o > order) { --o; seg <<= 1; set_bit(seg ^ 1, buddy->bits[o]); + ++buddy->num_free[o]; } spin_unlock(&buddy->lock); @@ -126,16 +126,18 @@ static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) while (test_bit(seg ^ 1, buddy->bits[order])) { clear_bit(seg ^ 1, buddy->bits[order]); + --buddy->num_free[order]; seg >>= 1; ++order; } set_bit(seg, buddy->bits[order]); + ++buddy->num_free[order]; spin_unlock(&buddy->lock); } -static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) +static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) { int i, s; @@ -144,7 +146,9 @@ static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), GFP_KERNEL); - if (!buddy->bits) + buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, + GFP_KERNEL); + if (!buddy->bits || !buddy->num_free) goto err_out; for (i = 0; i <= buddy->max_order; ++i) { @@ -157,6 +161,7 @@ static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) } set_bit(0, buddy->bits[buddy->max_order]); + buddy->num_free[buddy->max_order] = 1; return 0; @@ -164,9 +169,10 @@ err_out_free: for (i = 0; i <= buddy->max_order; ++i) kfree(buddy->bits[i]); +err_out: kfree(buddy->bits); + kfree(buddy->num_free); -err_out: return -ENOMEM; } @@ -178,6 +184,7 @@ static void mthca_buddy_cleanup(struct mthca_buddy *buddy) kfree(buddy->bits[i]); kfree(buddy->bits); + kfree(buddy->num_free); } static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, @@ -213,7 +220,7 @@ static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, mtt->buddy = buddy; mtt->order = 0; - for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1) + for (i = dev->limits.mtt_seg_size / 8; i < size; i <<= 1) ++mtt->order; mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); @@ -244,13 +251,12 @@ void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) kfree(mtt); } -int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, - int start_index, u64 *buffer_list, int list_len) +static int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, + int start_index, u64 *buffer_list, int list_len) { struct mthca_mailbox *mailbox; __be64 *mtt_entry; int err = 0; - u8 status; int i; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); @@ -260,7 +266,7 @@ int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, while (list_len > 0) { mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + - mtt->first_seg * MTHCA_MTT_SEG_SIZE + + mtt->first_seg * dev->limits.mtt_seg_size + start_index * 8); mtt_entry[1] = 0; for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) @@ -274,17 +280,11 @@ int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, if (i & 1) mtt_entry[i + 2] = 0; - err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); + err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1); if (err) { mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); goto out; } - if (status) { - mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", - status); - err = -EINVAL; - goto out; - } list_len -= i; start_index += i; @@ -296,6 +296,92 @@ out: return err; } +int mthca_write_mtt_size(struct mthca_dev *dev) +{ + if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || + !(dev->mthca_flags & MTHCA_FLAG_FMR)) + /* + * Be friendly to WRITE_MTT command + * and leave two empty slots for the + * index and reserved fields of the + * mailbox. + */ + return PAGE_SIZE / sizeof (u64) - 2; + + /* For Arbel, all MTTs must fit in the same page. */ + return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff; +} + +static void mthca_tavor_write_mtt_seg(struct mthca_dev *dev, + struct mthca_mtt *mtt, int start_index, + u64 *buffer_list, int list_len) +{ + u64 __iomem *mtts; + int i; + + mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * dev->limits.mtt_seg_size + + start_index * sizeof (u64); + for (i = 0; i < list_len; ++i) + mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT), + mtts + i); +} + +static void mthca_arbel_write_mtt_seg(struct mthca_dev *dev, + struct mthca_mtt *mtt, int start_index, + u64 *buffer_list, int list_len) +{ + __be64 *mtts; + dma_addr_t dma_handle; + int i; + int s = start_index * sizeof (u64); + + /* For Arbel, all MTTs must fit in the same page. */ + BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE); + /* Require full segments */ + BUG_ON(s % dev->limits.mtt_seg_size); + + mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg + + s / dev->limits.mtt_seg_size, &dma_handle); + + BUG_ON(!mtts); + + dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, + list_len * sizeof (u64), DMA_TO_DEVICE); + + for (i = 0; i < list_len; ++i) + mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT); + + dma_sync_single_for_device(&dev->pdev->dev, dma_handle, + list_len * sizeof (u64), DMA_TO_DEVICE); +} + +int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, + int start_index, u64 *buffer_list, int list_len) +{ + int size = mthca_write_mtt_size(dev); + int chunk; + + if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || + !(dev->mthca_flags & MTHCA_FLAG_FMR)) + return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len); + + while (list_len > 0) { + chunk = min(size, list_len); + if (mthca_is_memfree(dev)) + mthca_arbel_write_mtt_seg(dev, mtt, start_index, + buffer_list, chunk); + else + mthca_tavor_write_mtt_seg(dev, mtt, start_index, + buffer_list, chunk); + + list_len -= chunk; + start_index += chunk; + buffer_list += chunk; + } + + return 0; +} + static inline u32 tavor_hw_index_to_key(u32 ind) { return ind; @@ -348,7 +434,6 @@ int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, u32 key; int i; int err; - u8 status; WARN_ON(buffer_size_shift >= 32); @@ -390,7 +475,7 @@ int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, if (mr->mtt) mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + - mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE); + mr->mtt->first_seg * dev->limits.mtt_seg_size); if (0) { mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); @@ -404,16 +489,10 @@ int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, } err = mthca_SW2HW_MPT(dev, mailbox, - key & (dev->limits.num_mpts - 1), - &status); + key & (dev->limits.num_mpts - 1)); if (err) { mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); goto err_out_mailbox; - } else if (status) { - mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_mailbox; } mthca_free_mailbox(dev, mailbox); @@ -474,17 +553,12 @@ static void mthca_free_region(struct mthca_dev *dev, u32 lkey) void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) { int err; - u8 status; err = mthca_HW2SW_MPT(dev, NULL, key_to_hw_index(dev, mr->ibmr.lkey) & - (dev->limits.num_mpts - 1), - &status); + (dev->limits.num_mpts - 1)); if (err) mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", - status); mthca_free_region(dev, mr->ibmr.lkey); mthca_free_mtt(dev, mr->mtt); @@ -497,7 +571,6 @@ int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, struct mthca_mailbox *mailbox; u64 mtt_seg; u32 key, idx; - u8 status; int list_len = mr->attr.max_pages; int err = -ENOMEM; int i; @@ -525,28 +598,33 @@ int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, if (err) goto err_out_mpt_free; - mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key); + mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL); BUG_ON(!mr->mem.arbel.mpt); } else mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + sizeof *(mr->mem.tavor.mpt) * idx; mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); - if (IS_ERR(mr->mtt)) + if (IS_ERR(mr->mtt)) { + err = PTR_ERR(mr->mtt); goto err_out_table; + } - mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE; + mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size; if (mthca_is_memfree(dev)) { mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, - mr->mtt->first_seg); + mr->mtt->first_seg, + &mr->mem.arbel.dma_handle); BUG_ON(!mr->mem.arbel.mtts); } else mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); goto err_out_free_mtt; + } mpt_entry = mailbox->buf; @@ -574,18 +652,11 @@ int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, } err = mthca_SW2HW_MPT(dev, mailbox, - key & (dev->limits.num_mpts - 1), - &status); + key & (dev->limits.num_mpts - 1)); if (err) { mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); goto err_out_mailbox_free; } - if (status) { - mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_mailbox_free; - } mthca_free_mailbox(dev, mailbox); return 0; @@ -600,7 +671,7 @@ err_out_table: mthca_table_put(dev, dev->mr_table.mpt_table, key); err_out_mpt_free: - mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); + mthca_free(&dev->mr_table.mpt_alloc, key); return err; } @@ -709,10 +780,16 @@ int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, wmb(); + dma_sync_single_for_cpu(&dev->pdev->dev, fmr->mem.arbel.dma_handle, + list_len * sizeof(u64), DMA_TO_DEVICE); + for (i = 0; i < list_len; ++i) fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | MTHCA_MTT_FLAG_PRESENT); + dma_sync_single_for_device(&dev->pdev->dev, fmr->mem.arbel.dma_handle, + list_len * sizeof(u64), DMA_TO_DEVICE); + fmr->mem.arbel.mpt->key = cpu_to_be32(key); fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); @@ -729,15 +806,9 @@ int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) { - u32 key; - if (!fmr->maps) return; - key = tavor_key_to_hw_index(fmr->ibmr.lkey); - key &= dev->limits.num_mpts - 1; - fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); - fmr->maps = 0; writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); @@ -745,24 +816,18 @@ void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) { - u32 key; - if (!fmr->maps) return; - key = arbel_key_to_hw_index(fmr->ibmr.lkey); - key &= dev->limits.num_mpts - 1; - fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); - fmr->maps = 0; *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; } -int __devinit mthca_init_mr_table(struct mthca_dev *dev) +int mthca_init_mr_table(struct mthca_dev *dev) { - unsigned long addr; - int err, i; + phys_addr_t addr; + int mpts, mtts, err, i; err = mthca_alloc_init(&dev->mr_table.mpt_alloc, dev->limits.num_mpts, @@ -796,13 +861,21 @@ int __devinit mthca_init_mr_table(struct mthca_dev *dev) err = -EINVAL; goto err_fmr_mpt; } + mpts = mtts = 1 << i; + } else { + mtts = dev->limits.num_mtt_segs; + mpts = dev->limits.num_mpts; + } + + if (!mthca_is_memfree(dev) && + (dev->mthca_flags & MTHCA_FLAG_FMR)) { addr = pci_resource_start(dev->pdev, 4) + ((pci_resource_len(dev->pdev, 4) - 1) & dev->mr_table.mpt_base); dev->mr_table.tavor_fmr.mpt_base = - ioremap(addr, (1 << i) * sizeof(struct mthca_mpt_entry)); + ioremap(addr, mpts * sizeof(struct mthca_mpt_entry)); if (!dev->mr_table.tavor_fmr.mpt_base) { mthca_warn(dev, "MPT ioremap for FMR failed.\n"); @@ -815,19 +888,21 @@ int __devinit mthca_init_mr_table(struct mthca_dev *dev) dev->mr_table.mtt_base); dev->mr_table.tavor_fmr.mtt_base = - ioremap(addr, (1 << i) * MTHCA_MTT_SEG_SIZE); + ioremap(addr, mtts * dev->limits.mtt_seg_size); if (!dev->mr_table.tavor_fmr.mtt_base) { mthca_warn(dev, "MTT ioremap for FMR failed.\n"); err = -ENOMEM; goto err_fmr_mtt; } + } - err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i); + if (dev->limits.fmr_reserved_mtts) { + err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1)); if (err) goto err_fmr_mtt_buddy; /* Prevent regular MRs from using FMR keys */ - err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i); + err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1)); if (err) goto err_reserve_fmr; diff --git a/drivers/infiniband/hw/mthca/mthca_pd.c b/drivers/infiniband/hw/mthca/mthca_pd.c index 59df51614c8..266f14e4740 100644 --- a/drivers/infiniband/hw/mthca/mthca_pd.c +++ b/drivers/infiniband/hw/mthca/mthca_pd.c @@ -30,11 +30,8 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_pd.c 1349 2004-12-16 21:09:43Z roland $ */ -#include <linux/init.h> #include <linux/errno.h> #include "mthca_dev.h" @@ -69,7 +66,7 @@ void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd) mthca_free(&dev->pd_table.alloc, pd->pd_num); } -int __devinit mthca_init_pd_table(struct mthca_dev *dev) +int mthca_init_pd_table(struct mthca_dev *dev) { return mthca_alloc_init(&dev->pd_table.alloc, dev->limits.num_pds, diff --git a/drivers/infiniband/hw/mthca/mthca_profile.c b/drivers/infiniband/hw/mthca/mthca_profile.c index 58d44aa3c30..8edb28a9a0e 100644 --- a/drivers/infiniband/hw/mthca/mthca_profile.c +++ b/drivers/infiniband/hw/mthca/mthca_profile.c @@ -29,8 +29,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_profile.c 1349 2004-12-16 21:09:43Z roland $ */ #include <linux/module.h> @@ -63,7 +61,7 @@ enum { MTHCA_NUM_PDS = 1 << 15 }; -u64 mthca_make_profile(struct mthca_dev *dev, +s64 mthca_make_profile(struct mthca_dev *dev, struct mthca_profile *request, struct mthca_dev_lim *dev_lim, struct mthca_init_hca_param *init_hca) @@ -77,7 +75,7 @@ u64 mthca_make_profile(struct mthca_dev *dev, }; u64 mem_base, mem_avail; - u64 total_size = 0; + s64 total_size = 0; struct mthca_resource *profile; struct mthca_resource tmp; int i, j; @@ -96,7 +94,7 @@ u64 mthca_make_profile(struct mthca_dev *dev, profile[MTHCA_RES_RDB].size = MTHCA_RDB_ENTRY_SIZE; profile[MTHCA_RES_MCG].size = MTHCA_MGM_ENTRY_SIZE; profile[MTHCA_RES_MPT].size = dev_lim->mpt_entry_sz; - profile[MTHCA_RES_MTT].size = MTHCA_MTT_SEG_SIZE; + profile[MTHCA_RES_MTT].size = dev->limits.mtt_seg_size; profile[MTHCA_RES_UAR].size = dev_lim->uar_scratch_entry_sz; profile[MTHCA_RES_UDAV].size = MTHCA_AV_SIZE; profile[MTHCA_RES_UARC].size = request->uarc_size; @@ -234,7 +232,7 @@ u64 mthca_make_profile(struct mthca_dev *dev, dev->limits.num_mtt_segs = profile[i].num; dev->mr_table.mtt_base = profile[i].start; init_hca->mtt_base = profile[i].start; - init_hca->mtt_seg_sz = ffs(MTHCA_MTT_SEG_SIZE) - 7; + init_hca->mtt_seg_sz = ffs(dev->limits.mtt_seg_size) - 7; break; case MTHCA_RES_UAR: dev->limits.num_uars = profile[i].num; @@ -277,7 +275,7 @@ u64 mthca_make_profile(struct mthca_dev *dev, * out of the MR pool. They don't use additional memory, but * we assign them as part of the HCA profile anyway. */ - if (mthca_is_memfree(dev)) + if (mthca_is_memfree(dev) || BITS_PER_LONG == 64) dev->limits.fmr_reserved_mtts = 0; else dev->limits.fmr_reserved_mtts = request->fmr_reserved_mtts; diff --git a/drivers/infiniband/hw/mthca/mthca_profile.h b/drivers/infiniband/hw/mthca/mthca_profile.h index 94641808f97..62b009cc873 100644 --- a/drivers/infiniband/hw/mthca/mthca_profile.h +++ b/drivers/infiniband/hw/mthca/mthca_profile.h @@ -29,8 +29,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_profile.h 1349 2004-12-16 21:09:43Z roland $ */ #ifndef MTHCA_PROFILE_H @@ -53,7 +51,7 @@ struct mthca_profile { int fmr_reserved_mtts; }; -u64 mthca_make_profile(struct mthca_dev *mdev, +s64 mthca_make_profile(struct mthca_dev *mdev, struct mthca_profile *request, struct mthca_dev_lim *dev_lim, struct mthca_init_hca_param *init_hca); diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 981fe2eebdf..415f8e1a54d 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -32,13 +32,17 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_provider.c 4859 2006-01-09 21:55:10Z roland $ */ #include <rdma/ib_smi.h> +#include <rdma/ib_umem.h> #include <rdma/ib_user_verbs.h> + +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/stat.h> #include <linux/mm.h> +#include <linux/export.h> #include "mthca_dev.h" #include "mthca_cmd.h" @@ -59,9 +63,7 @@ static int mthca_query_device(struct ib_device *ibdev, struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; - struct mthca_dev* mdev = to_mdev(ibdev); - - u8 status; + struct mthca_dev *mdev = to_mdev(ibdev); in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); @@ -76,14 +78,9 @@ static int mthca_query_device(struct ib_device *ibdev, in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mthca_MAD_IFC(mdev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); + 1, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } props->device_cap_flags = mdev->device_cap_flags; props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & @@ -124,7 +121,7 @@ static int mthca_query_device(struct ib_device *ibdev, props->max_map_per_fmr = 255; else props->max_map_per_fmr = - (1 << (32 - long_log2(mdev->limits.num_mpts))) - 1; + (1 << (32 - ilog2(mdev->limits.num_mpts))) - 1; err = 0; out: @@ -139,7 +136,6 @@ static int mthca_query_port(struct ib_device *ibdev, struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; - u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); @@ -153,14 +149,9 @@ static int mthca_query_port(struct ib_device *ibdev, in_mad->attr_mod = cpu_to_be32(port); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); + port, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); props->lmc = out_mad->data[34] & 0x7; @@ -179,6 +170,8 @@ static int mthca_query_port(struct ib_device *ibdev, props->max_mtu = out_mad->data[41] & 0xf; props->active_mtu = out_mad->data[36] >> 4; props->subnet_timeout = out_mad->data[51] & 0x1f; + props->max_vl_num = out_mad->data[37] >> 4; + props->init_type_reply = out_mad->data[41] >> 4; out: kfree(in_mad); @@ -210,7 +203,6 @@ static int mthca_modify_port(struct ib_device *ibdev, struct mthca_set_ib_param set_ib; struct ib_port_attr attr; int err; - u8 status; if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) return -ERESTARTSYS; @@ -225,14 +217,9 @@ static int mthca_modify_port(struct ib_device *ibdev, set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & ~props->clr_port_cap_mask; - err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status); + err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } - out: mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); return err; @@ -244,7 +231,6 @@ static int mthca_query_pkey(struct ib_device *ibdev, struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; - u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); @@ -256,14 +242,9 @@ static int mthca_query_pkey(struct ib_device *ibdev, in_mad->attr_mod = cpu_to_be32(index / 32); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); + port, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); @@ -279,7 +260,6 @@ static int mthca_query_gid(struct ib_device *ibdev, u8 port, struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; - u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); @@ -291,14 +271,9 @@ static int mthca_query_gid(struct ib_device *ibdev, u8 port, in_mad->attr_mod = cpu_to_be32(port); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); + port, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } memcpy(gid->raw, out_mad->data + 8, 8); @@ -307,14 +282,9 @@ static int mthca_query_gid(struct ib_device *ibdev, u8 port, in_mad->attr_mod = cpu_to_be32(index / 8); err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); + port, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); @@ -331,6 +301,9 @@ static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev, struct mthca_ucontext *context; int err; + if (!(to_mdev(ibdev)->active)) + return ERR_PTR(-EAGAIN); + memset(&uresp, 0, sizeof uresp); uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; @@ -364,6 +337,8 @@ static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev, return ERR_PTR(-EFAULT); } + context->reg_mr_warned = 0; + return &context->ibucontext; } @@ -465,6 +440,9 @@ static struct ib_srq *mthca_create_srq(struct ib_pd *pd, struct mthca_srq *srq; int err; + if (init_attr->srq_type != IB_SRQT_BASIC) + return ERR_PTR(-ENOSYS); + srq = kmalloc(sizeof *srq, GFP_KERNEL); if (!srq) return ERR_PTR(-ENOMEM); @@ -537,6 +515,9 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd, struct mthca_qp *qp; int err; + if (init_attr->create_flags) + return ERR_PTR(-EINVAL); + switch (init_attr->qp_type) { case IB_QPT_RC: case IB_QPT_UC: @@ -661,6 +642,7 @@ static int mthca_destroy_qp(struct ib_qp *qp) } static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries, + int comp_vector, struct ib_ucontext *context, struct ib_udata *udata) { @@ -713,6 +695,7 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, int entries, if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) { mthca_free_cq(to_mdev(ibdev), cq); + err = -EFAULT; goto err_free; } @@ -787,7 +770,6 @@ static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *uda struct mthca_cq *cq = to_mcq(ibcq); struct mthca_resize_cq ucmd; u32 lkey; - u8 status; int ret; if (entries < 1 || entries > dev->limits.max_cqes) @@ -814,9 +796,7 @@ static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *uda lkey = ucmd.lkey; } - ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, long_log2(entries), &status); - if (status) - ret = -EINVAL; + ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries)); if (ret) { if (cq->resize_buf) { @@ -905,6 +885,8 @@ static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) return ERR_PTR(err); } + mr->umem = NULL; + return &mr->ibmr; } @@ -917,17 +899,13 @@ static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd, struct mthca_mr *mr; u64 *page_list; u64 total_size; - u64 mask; + unsigned long mask; int shift; int npages; int err; int i, j, n; - /* First check that we have enough alignment */ - if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) - return ERR_PTR(-EINVAL); - - mask = 0; + mask = buffer_list[0].addr ^ *iova_start; total_size = 0; for (i = 0; i < num_phys_buf; ++i) { if (i != 0) @@ -941,17 +919,7 @@ static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd, if (mask & ~PAGE_MASK) return ERR_PTR(-EINVAL); - /* Find largest page shift we can use to cover buffers */ - for (shift = PAGE_SHIFT; shift < 31; ++shift) - if (num_phys_buf > 1) { - if ((1ULL << shift) & mask) - break; - } else { - if (1ULL << shift >= - buffer_list[0].size + - (buffer_list[0].addr & ((1ULL << shift) - 1))) - break; - } + shift = __ffs(mask | 1 << 31); buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1); buffer_list[0].addr &= ~0ull << shift; @@ -1000,34 +968,54 @@ static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd, } kfree(page_list); + mr->umem = NULL; + return &mr->ibmr; } -static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region, - int acc, struct ib_udata *udata) +static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, + u64 virt, int acc, struct ib_udata *udata) { struct mthca_dev *dev = to_mdev(pd->device); - struct ib_umem_chunk *chunk; + struct scatterlist *sg; struct mthca_mr *mr; + struct mthca_reg_mr ucmd; u64 *pages; int shift, n, len; - int i, j, k; + int i, k, entry; int err = 0; + int write_mtt_size; - shift = ffs(region->page_size) - 1; + if (udata->inlen - sizeof (struct ib_uverbs_cmd_hdr) < sizeof ucmd) { + if (!to_mucontext(pd->uobject->context)->reg_mr_warned) { + mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n", + current->comm); + mthca_warn(dev, " Update libmthca to fix this.\n"); + } + ++to_mucontext(pd->uobject->context)->reg_mr_warned; + ucmd.mr_attrs = 0; + } else if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) + return ERR_PTR(-EFAULT); mr = kmalloc(sizeof *mr, GFP_KERNEL); if (!mr) return ERR_PTR(-ENOMEM); - n = 0; - list_for_each_entry(chunk, ®ion->chunk_list, list) - n += chunk->nents; + mr->umem = ib_umem_get(pd->uobject->context, start, length, acc, + ucmd.mr_attrs & MTHCA_MR_DMASYNC); + + if (IS_ERR(mr->umem)) { + err = PTR_ERR(mr->umem); + goto err; + } + + shift = ffs(mr->umem->page_size) - 1; + n = mr->umem->nmap; mr->mtt = mthca_alloc_mtt(dev, n); if (IS_ERR(mr->mtt)) { err = PTR_ERR(mr->mtt); - goto err; + goto err_umem; } pages = (u64 *) __get_free_page(GFP_KERNEL); @@ -1038,28 +1026,26 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, struct ib_umem *region, i = n = 0; - list_for_each_entry(chunk, ®ion->chunk_list, list) - for (j = 0; j < chunk->nmap; ++j) { - len = sg_dma_len(&chunk->page_list[j]) >> shift; - for (k = 0; k < len; ++k) { - pages[i++] = sg_dma_address(&chunk->page_list[j]) + - region->page_size * k; - /* - * Be friendly to WRITE_MTT command - * and leave two empty slots for the - * index and reserved fields of the - * mailbox. - */ - if (i == PAGE_SIZE / sizeof (u64) - 2) { - err = mthca_write_mtt(dev, mr->mtt, - n, pages, i); - if (err) - goto mtt_done; - n += i; - i = 0; - } + write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); + + for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) { + len = sg_dma_len(sg) >> shift; + for (k = 0; k < len; ++k) { + pages[i++] = sg_dma_address(sg) + + mr->umem->page_size * k; + /* + * Be friendly to write_mtt and pass it chunks + * of appropriate size. + */ + if (i == write_mtt_size) { + err = mthca_write_mtt(dev, mr->mtt, n, pages, i); + if (err) + goto mtt_done; + n += i; + i = 0; } } + } if (i) err = mthca_write_mtt(dev, mr->mtt, n, pages, i); @@ -1068,8 +1054,8 @@ mtt_done: if (err) goto err_mtt; - err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, region->virt_base, - region->length, convert_access(acc), mr); + err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, virt, length, + convert_access(acc), mr); if (err) goto err_mtt; @@ -1079,6 +1065,9 @@ mtt_done: err_mtt: mthca_free_mtt(dev, mr->mtt); +err_umem: + ib_umem_release(mr->umem); + err: kfree(mr); return ERR_PTR(err); @@ -1087,8 +1076,12 @@ err: static int mthca_dereg_mr(struct ib_mr *mr) { struct mthca_mr *mmr = to_mmr(mr); + mthca_free_mr(to_mdev(mr->device), mmr); + if (mmr->umem) + ib_umem_release(mmr->umem); kfree(mmr); + return 0; } @@ -1131,7 +1124,6 @@ static int mthca_unmap_fmr(struct list_head *fmr_list) { struct ib_fmr *fmr; int err; - u8 status; struct mthca_dev *mdev = NULL; list_for_each_entry(fmr, fmr_list, list) { @@ -1152,31 +1144,33 @@ static int mthca_unmap_fmr(struct list_head *fmr_list) list_for_each_entry(fmr, fmr_list, list) mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr)); - err = mthca_SYNC_TPT(mdev, &status); - if (err) - return err; - if (status) - return -EINVAL; - return 0; + err = mthca_SYNC_TPT(mdev); + return err; } -static ssize_t show_rev(struct class_device *cdev, char *buf) +static ssize_t show_rev(struct device *device, struct device_attribute *attr, + char *buf) { - struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev); + struct mthca_dev *dev = + container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%x\n", dev->rev_id); } -static ssize_t show_fw_ver(struct class_device *cdev, char *buf) +static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, + char *buf) { - struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev); + struct mthca_dev *dev = + container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%d.%d.%d\n", (int) (dev->fw_ver >> 32), (int) (dev->fw_ver >> 16) & 0xffff, (int) dev->fw_ver & 0xffff); } -static ssize_t show_hca(struct class_device *cdev, char *buf) +static ssize_t show_hca(struct device *device, struct device_attribute *attr, + char *buf) { - struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev); + struct mthca_dev *dev = + container_of(device, struct mthca_dev, ib_dev.dev); switch (dev->pdev->device) { case PCI_DEVICE_ID_MELLANOX_TAVOR: return sprintf(buf, "MT23108\n"); @@ -1192,22 +1186,24 @@ static ssize_t show_hca(struct class_device *cdev, char *buf) } } -static ssize_t show_board(struct class_device *cdev, char *buf) +static ssize_t show_board(struct device *device, struct device_attribute *attr, + char *buf) { - struct mthca_dev *dev = container_of(cdev, struct mthca_dev, ib_dev.class_dev); + struct mthca_dev *dev = + container_of(device, struct mthca_dev, ib_dev.dev); return sprintf(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id); } -static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); -static CLASS_DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); -static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); -static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); +static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); +static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); +static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); +static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); -static struct class_device_attribute *mthca_class_attributes[] = { - &class_device_attr_hw_rev, - &class_device_attr_fw_ver, - &class_device_attr_hca_type, - &class_device_attr_board_id +static struct device_attribute *mthca_dev_attributes[] = { + &dev_attr_hw_rev, + &dev_attr_fw_ver, + &dev_attr_hca_type, + &dev_attr_board_id }; static int mthca_init_node_data(struct mthca_dev *dev) @@ -1215,7 +1211,6 @@ static int mthca_init_node_data(struct mthca_dev *dev) struct ib_smp *in_mad = NULL; struct ib_smp *out_mad = NULL; int err = -ENOMEM; - u8 status; in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); @@ -1226,29 +1221,21 @@ static int mthca_init_node_data(struct mthca_dev *dev) in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; err = mthca_MAD_IFC(dev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); + 1, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } memcpy(dev->ib_dev.node_desc, out_mad->data, 64); in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; err = mthca_MAD_IFC(dev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); + 1, NULL, NULL, in_mad, out_mad); if (err) goto out; - if (status) { - err = -EINVAL; - goto out; - } + if (mthca_is_memfree(dev)) + dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); out: @@ -1290,8 +1277,8 @@ int mthca_register_device(struct mthca_dev *dev) (1ull << IB_USER_VERBS_CMD_DETACH_MCAST); dev->ib_dev.node_type = RDMA_NODE_IB_CA; dev->ib_dev.phys_port_cnt = dev->limits.num_ports; + dev->ib_dev.num_comp_vectors = 1; dev->ib_dev.dma_device = &dev->pdev->dev; - dev->ib_dev.class_dev.dev = &dev->pdev->dev; dev->ib_dev.query_device = mthca_query_device; dev->ib_dev.query_port = mthca_query_port; dev->ib_dev.modify_device = mthca_modify_device; @@ -1363,13 +1350,13 @@ int mthca_register_device(struct mthca_dev *dev) mutex_init(&dev->cap_mask_mutex); - ret = ib_register_device(&dev->ib_dev); + ret = ib_register_device(&dev->ib_dev, NULL); if (ret) return ret; - for (i = 0; i < ARRAY_SIZE(mthca_class_attributes); ++i) { - ret = class_device_create_file(&dev->ib_dev.class_dev, - mthca_class_attributes[i]); + for (i = 0; i < ARRAY_SIZE(mthca_dev_attributes); ++i) { + ret = device_create_file(&dev->ib_dev.dev, + mthca_dev_attributes[i]); if (ret) { ib_unregister_device(&dev->ib_dev); return ret; diff --git a/drivers/infiniband/hw/mthca/mthca_provider.h b/drivers/infiniband/hw/mthca/mthca_provider.h index 9a5bece3fa5..596acc45569 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.h +++ b/drivers/infiniband/hw/mthca/mthca_provider.h @@ -30,8 +30,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_provider.h 1349 2004-12-16 21:09:43Z roland $ */ #ifndef MTHCA_PROVIDER_H @@ -48,7 +46,7 @@ struct mthca_buf_list { void *buf; - DECLARE_PCI_UNMAP_ADDR(mapping) + DEFINE_DMA_UNMAP_ADDR(mapping); }; union mthca_buf { @@ -67,12 +65,14 @@ struct mthca_ucontext { struct ib_ucontext ibucontext; struct mthca_uar uar; struct mthca_user_db_table *db_tab; + int reg_mr_warned; }; struct mthca_mtt; struct mthca_mr { struct ib_mr ibmr; + struct ib_umem *umem; struct mthca_mtt *mtt; }; @@ -89,6 +89,7 @@ struct mthca_fmr { struct { struct mthca_mpt_entry *mpt; __be64 *mtts; + dma_addr_t dma_handle; } arbel; } mem; }; @@ -112,6 +113,7 @@ struct mthca_eq { int nent; struct mthca_buf_list *page_list; struct mthca_mr mr; + char irq_name[IB_DEVICE_NAME_MAX]; }; struct mthca_av; diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c index 5e5c58b9920..e354b2f04ad 100644 --- a/drivers/infiniband/hw/mthca/mthca_qp.c +++ b/drivers/infiniband/hw/mthca/mthca_qp.c @@ -31,13 +31,13 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_qp.c 1355 2004-12-17 15:23:43Z roland $ */ -#include <linux/init.h> #include <linux/string.h> #include <linux/slab.h> +#include <linux/sched.h> + +#include <asm/io.h> #include <rdma/ib_verbs.h> #include <rdma/ib_cache.h> @@ -247,7 +247,8 @@ void mthca_qp_event(struct mthca_dev *dev, u32 qpn, spin_unlock(&dev->qp_table.lock); if (!qp) { - mthca_warn(dev, "Async event for bogus QP %08x\n", qpn); + mthca_warn(dev, "Async event %d for bogus QP %08x\n", + event_type, qpn); return; } @@ -294,7 +295,7 @@ static int to_mthca_st(int transport) } } -static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr, +static void store_attrs(struct mthca_sqp *sqp, const struct ib_qp_attr *attr, int attr_mask) { if (attr_mask & IB_QP_PKEY_INDEX) @@ -308,7 +309,6 @@ static void store_attrs(struct mthca_sqp *sqp, struct ib_qp_attr *attr, static void init_port(struct mthca_dev *dev, int port) { int err; - u8 status; struct mthca_init_ib_param param; memset(¶m, 0, sizeof param); @@ -319,14 +319,12 @@ static void init_port(struct mthca_dev *dev, int port) param.gid_cap = dev->limits.gid_table_len; param.pkey_cap = dev->limits.pkey_table_len; - err = mthca_INIT_IB(dev, ¶m, port, &status); + err = mthca_INIT_IB(dev, ¶m, port); if (err) mthca_warn(dev, "INIT_IB failed, return code %d.\n", err); - if (status) - mthca_warn(dev, "INIT_IB returned status %02x.\n", status); } -static __be32 get_hw_access_flags(struct mthca_qp *qp, struct ib_qp_attr *attr, +static __be32 get_hw_access_flags(struct mthca_qp *qp, const struct ib_qp_attr *attr, int attr_mask) { u8 dest_rd_atomic; @@ -398,7 +396,7 @@ static int to_ib_qp_access_flags(int mthca_flags) static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr, struct mthca_qp_path *path) { - memset(ib_ah_attr, 0, sizeof *path); + memset(ib_ah_attr, 0, sizeof *ib_ah_attr); ib_ah_attr->port_num = (be32_to_cpu(path->port_pkey) >> 24) & 0x3; if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->limits.num_ports) @@ -428,32 +426,37 @@ int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_m { struct mthca_dev *dev = to_mdev(ibqp->device); struct mthca_qp *qp = to_mqp(ibqp); - int err; - struct mthca_mailbox *mailbox; + int err = 0; + struct mthca_mailbox *mailbox = NULL; struct mthca_qp_param *qp_param; struct mthca_qp_context *context; int mthca_state; - u8 status; - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); + mutex_lock(&qp->mutex); - err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_warn(dev, "QUERY_QP returned status %02x\n", status); - err = -EINVAL; + if (qp->state == IB_QPS_RESET) { + qp_attr->qp_state = IB_QPS_RESET; + goto done; + } + + mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); + if (IS_ERR(mailbox)) { + err = PTR_ERR(mailbox); goto out; } + err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox); + if (err) { + mthca_warn(dev, "QUERY_QP failed (%d)\n", err); + goto out_mailbox; + } + qp_param = mailbox->buf; context = &qp_param->context; mthca_state = be32_to_cpu(context->flags) >> 28; - qp_attr->qp_state = to_ib_qp_state(mthca_state); - qp_attr->cur_qp_state = qp_attr->qp_state; + qp->state = to_ib_qp_state(mthca_state); + qp_attr->qp_state = qp->state; qp_attr->path_mtu = context->mtu_msgmax >> 5; qp_attr->path_mig_state = to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3); @@ -463,11 +466,6 @@ int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_m qp_attr->dest_qp_num = be32_to_cpu(context->remote_qpn) & 0xffffff; qp_attr->qp_access_flags = to_ib_qp_access_flags(be32_to_cpu(context->params2)); - qp_attr->cap.max_send_wr = qp->sq.max; - qp_attr->cap.max_recv_wr = qp->rq.max; - qp_attr->cap.max_send_sge = qp->sq.max_gs; - qp_attr->cap.max_recv_sge = qp->rq.max_gs; - qp_attr->cap.max_inline_data = qp->max_inline_data; if (qp->transport == RC || qp->transport == UC) { to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path); @@ -494,14 +492,27 @@ int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_m qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7; qp_attr->rnr_retry = context->pri_path.rnr_retry >> 5; qp_attr->alt_timeout = context->alt_path.ackto >> 3; - qp_init_attr->cap = qp_attr->cap; -out: +done: + qp_attr->cur_qp_state = qp_attr->qp_state; + qp_attr->cap.max_send_wr = qp->sq.max; + qp_attr->cap.max_recv_wr = qp->rq.max; + qp_attr->cap.max_send_sge = qp->sq.max_gs; + qp_attr->cap.max_recv_sge = qp->rq.max_gs; + qp_attr->cap.max_inline_data = qp->max_inline_data; + + qp_init_attr->cap = qp_attr->cap; + qp_init_attr->sq_sig_type = qp->sq_policy; + +out_mailbox: mthca_free_mailbox(dev, mailbox); + +out: + mutex_unlock(&qp->mutex); return err; } -static int mthca_path_set(struct mthca_dev *dev, struct ib_ah_attr *ah, +static int mthca_path_set(struct mthca_dev *dev, const struct ib_ah_attr *ah, struct mthca_qp_path *path, u8 port) { path->g_mylmc = ah->src_path_bits & 0x7f; @@ -529,68 +540,18 @@ static int mthca_path_set(struct mthca_dev *dev, struct ib_ah_attr *ah, return 0; } -int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, - struct ib_udata *udata) +static int __mthca_modify_qp(struct ib_qp *ibqp, + const struct ib_qp_attr *attr, int attr_mask, + enum ib_qp_state cur_state, enum ib_qp_state new_state) { struct mthca_dev *dev = to_mdev(ibqp->device); struct mthca_qp *qp = to_mqp(ibqp); - enum ib_qp_state cur_state, new_state; struct mthca_mailbox *mailbox; struct mthca_qp_param *qp_param; struct mthca_qp_context *qp_context; u32 sqd_event = 0; - u8 status; int err = -EINVAL; - mutex_lock(&qp->mutex); - - if (attr_mask & IB_QP_CUR_STATE) { - cur_state = attr->cur_qp_state; - } else { - spin_lock_irq(&qp->sq.lock); - spin_lock(&qp->rq.lock); - cur_state = qp->state; - spin_unlock(&qp->rq.lock); - spin_unlock_irq(&qp->sq.lock); - } - - new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; - - if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask)) { - mthca_dbg(dev, "Bad QP transition (transport %d) " - "%d->%d with attr 0x%08x\n", - qp->transport, cur_state, new_state, - attr_mask); - goto out; - } - - if ((attr_mask & IB_QP_PKEY_INDEX) && - attr->pkey_index >= dev->limits.pkey_table_len) { - mthca_dbg(dev, "P_Key index (%u) too large. max is %d\n", - attr->pkey_index, dev->limits.pkey_table_len-1); - goto out; - } - - if ((attr_mask & IB_QP_PORT) && - (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) { - mthca_dbg(dev, "Port number (%u) is invalid\n", attr->port_num); - goto out; - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && - attr->max_rd_atomic > dev->limits.max_qp_init_rdma) { - mthca_dbg(dev, "Max rdma_atomic as initiator %u too large (max is %d)\n", - attr->max_rd_atomic, dev->limits.max_qp_init_rdma); - goto out; - } - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && - attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) { - mthca_dbg(dev, "Max rdma_atomic as responder %u too large (max %d)\n", - attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift); - goto out; - } - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) { err = PTR_ERR(mailbox); @@ -635,11 +596,11 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, if (mthca_is_memfree(dev)) { if (qp->rq.max) - qp_context->rq_size_stride = long_log2(qp->rq.max) << 3; + qp_context->rq_size_stride = ilog2(qp->rq.max) << 3; qp_context->rq_size_stride |= qp->rq.wqe_shift - 4; if (qp->sq.max) - qp_context->sq_size_stride = long_log2(qp->sq.max) << 3; + qp_context->sq_size_stride = ilog2(qp->sq.max) << 3; qp_context->sq_size_stride |= qp->sq.wqe_shift - 4; } @@ -687,6 +648,19 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH); } + if (ibqp->qp_type == IB_QPT_RC && + cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { + u8 sched_queue = ibqp->uobject ? 0x2 : 0x1; + + if (mthca_is_memfree(dev)) + qp_context->rlkey_arbel_sched_queue |= sched_queue; + else + qp_context->tavor_sched_queue |= cpu_to_be32(sched_queue); + + qp_param->opt_param_mask |= + cpu_to_be32(MTHCA_QP_OPTPAR_SCHED_QUEUE); + } + if (attr_mask & IB_QP_TIMEOUT) { qp_context->pri_path.ackto = attr->timeout << 3; qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_ACK_TIMEOUT); @@ -801,13 +775,10 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, sqd_event = 1 << 31; err = mthca_MODIFY_QP(dev, cur_state, new_state, qp->qpn, 0, - mailbox, sqd_event, &status); - if (err) - goto out_mailbox; - if (status) { - mthca_warn(dev, "modify QP %d->%d returned status %02x.\n", - cur_state, new_state, status); - err = -EINVAL; + mailbox, sqd_event); + if (err) { + mthca_warn(dev, "modify QP %d->%d returned %d.\n", + cur_state, new_state, err); goto out_mailbox; } @@ -837,7 +808,7 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, cur_state != IB_QPS_ERR && (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) - mthca_CLOSE_IB(dev, qp->port, &status); + mthca_CLOSE_IB(dev, qp->port); } /* @@ -864,6 +835,73 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, out_mailbox: mthca_free_mailbox(dev, mailbox); +out: + return err; +} + +int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, + struct ib_udata *udata) +{ + struct mthca_dev *dev = to_mdev(ibqp->device); + struct mthca_qp *qp = to_mqp(ibqp); + enum ib_qp_state cur_state, new_state; + int err = -EINVAL; + + mutex_lock(&qp->mutex); + if (attr_mask & IB_QP_CUR_STATE) { + cur_state = attr->cur_qp_state; + } else { + spin_lock_irq(&qp->sq.lock); + spin_lock(&qp->rq.lock); + cur_state = qp->state; + spin_unlock(&qp->rq.lock); + spin_unlock_irq(&qp->sq.lock); + } + + new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; + + if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask, + IB_LINK_LAYER_UNSPECIFIED)) { + mthca_dbg(dev, "Bad QP transition (transport %d) " + "%d->%d with attr 0x%08x\n", + qp->transport, cur_state, new_state, + attr_mask); + goto out; + } + + if ((attr_mask & IB_QP_PKEY_INDEX) && + attr->pkey_index >= dev->limits.pkey_table_len) { + mthca_dbg(dev, "P_Key index (%u) too large. max is %d\n", + attr->pkey_index, dev->limits.pkey_table_len-1); + goto out; + } + + if ((attr_mask & IB_QP_PORT) && + (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) { + mthca_dbg(dev, "Port number (%u) is invalid\n", attr->port_num); + goto out; + } + + if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && + attr->max_rd_atomic > dev->limits.max_qp_init_rdma) { + mthca_dbg(dev, "Max rdma_atomic as initiator %u too large (max is %d)\n", + attr->max_rd_atomic, dev->limits.max_qp_init_rdma); + goto out; + } + + if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && + attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) { + mthca_dbg(dev, "Max rdma_atomic as responder %u too large (max %d)\n", + attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift); + goto out; + } + + if (cur_state == new_state && cur_state == IB_QPS_RESET) { + err = 0; + goto out; + } + + err = __mthca_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); out: mutex_unlock(&qp->mutex); @@ -1074,21 +1112,21 @@ static void mthca_unmap_memfree(struct mthca_dev *dev, static int mthca_alloc_memfree(struct mthca_dev *dev, struct mthca_qp *qp) { - int ret = 0; - if (mthca_is_memfree(dev)) { qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ, qp->qpn, &qp->rq.db); if (qp->rq.db_index < 0) - return ret; + return -ENOMEM; qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ, qp->qpn, &qp->sq.db); - if (qp->sq.db_index < 0) + if (qp->sq.db_index < 0) { mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index); + return -ENOMEM; + } } - return ret; + return 0; } static void mthca_free_memfree(struct mthca_dev *dev, @@ -1109,6 +1147,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev, { int ret; int i; + struct mthca_next_seg *next; qp->refcount = 1; init_waitqueue_head(&qp->wait); @@ -1151,7 +1190,6 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev, } if (mthca_is_memfree(dev)) { - struct mthca_next_seg *next; struct mthca_data_seg *scatter; int size = (sizeof (struct mthca_next_seg) + qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16; @@ -1174,6 +1212,13 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev, qp->sq.wqe_shift) + qp->send_wqe_offset); } + } else { + for (i = 0; i < qp->rq.max; ++i) { + next = get_recv_wqe(qp, i); + next->nda_op = htonl((((i + 1) % qp->rq.max) << + qp->rq.wqe_shift) | 1); + } + } qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); @@ -1196,10 +1241,10 @@ static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap, return -EINVAL; /* - * For MLX transport we need 2 extra S/G entries: + * For MLX transport we need 2 extra send gather entries: * one for the header and one for the checksum at the end */ - if (qp->transport == MLX && cap->max_recv_sge + 2 > dev->limits.max_sg) + if (qp->transport == MLX && cap->max_send_sge + 2 > dev->limits.max_sg) return -EINVAL; if (mthca_is_memfree(dev)) { @@ -1266,10 +1311,12 @@ int mthca_alloc_qp(struct mthca_dev *dev, } static void mthca_lock_cqs(struct mthca_cq *send_cq, struct mthca_cq *recv_cq) + __acquires(&send_cq->lock) __acquires(&recv_cq->lock) { - if (send_cq == recv_cq) + if (send_cq == recv_cq) { spin_lock_irq(&send_cq->lock); - else if (send_cq->cqn < recv_cq->cqn) { + __acquire(&recv_cq->lock); + } else if (send_cq->cqn < recv_cq->cqn) { spin_lock_irq(&send_cq->lock); spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); } else { @@ -1279,10 +1326,12 @@ static void mthca_lock_cqs(struct mthca_cq *send_cq, struct mthca_cq *recv_cq) } static void mthca_unlock_cqs(struct mthca_cq *send_cq, struct mthca_cq *recv_cq) + __releases(&send_cq->lock) __releases(&recv_cq->lock) { - if (send_cq == recv_cq) + if (send_cq == recv_cq) { + __release(&recv_cq->lock); spin_unlock_irq(&send_cq->lock); - else if (send_cq->cqn < recv_cq->cqn) { + } else if (send_cq->cqn < recv_cq->cqn) { spin_unlock(&recv_cq->lock); spin_unlock_irq(&send_cq->lock); } else { @@ -1372,7 +1421,6 @@ static inline int get_qp_refcount(struct mthca_dev *dev, struct mthca_qp *qp) void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp) { - u8 status; struct mthca_cq *send_cq; struct mthca_cq *recv_cq; @@ -1397,7 +1445,7 @@ void mthca_free_qp(struct mthca_dev *dev, if (qp->state != IB_QPS_RESET) mthca_MODIFY_QP(dev, qp->state, IB_QPS_RESET, qp->qpn, 0, - NULL, 0, &status); + NULL, 0); /* * If this is a userspace QP, the buffers, MR, CQs and so on @@ -1405,11 +1453,10 @@ void mthca_free_qp(struct mthca_dev *dev, * unref the mem-free tables and free the QPN in our table. */ if (!qp->ibqp.uobject) { - mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq), qp->qpn, + mthca_cq_clean(dev, recv_cq, qp->qpn, qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); - if (qp->ibqp.send_cq != qp->ibqp.recv_cq) - mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq), qp->qpn, - qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); + if (send_cq != recv_cq) + mthca_cq_clean(dev, send_cq, qp->qpn, NULL); mthca_free_memfree(dev, qp); mthca_free_wqe_buf(dev, qp); @@ -1437,8 +1484,8 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, int err; u16 pkey; - ib_ud_header_init(256, /* assume a MAD */ - mthca_ah_grh_present(to_mah(wr->wr.ud.ah)), + ib_ud_header_init(256, /* assume a MAD */ 1, 0, 0, + mthca_ah_grh_present(to_mah(wr->wr.ud.ah)), 0, &sqp->ud_header); err = mthca_read_ah(dev, to_mah(wr->wr.ud.ah), &sqp->ud_header); @@ -1460,7 +1507,7 @@ static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, case IB_WR_SEND_WITH_IMM: sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; sqp->ud_header.immediate_present = 1; - sqp->ud_header.immediate_data = wr->imm_data; + sqp->ud_header.immediate_data = wr->ex.imm_data; break; default: return -EINVAL; @@ -1513,6 +1560,45 @@ static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq, return cur + nreq >= wq->max; } +static __always_inline void set_raddr_seg(struct mthca_raddr_seg *rseg, + u64 remote_addr, u32 rkey) +{ + rseg->raddr = cpu_to_be64(remote_addr); + rseg->rkey = cpu_to_be32(rkey); + rseg->reserved = 0; +} + +static __always_inline void set_atomic_seg(struct mthca_atomic_seg *aseg, + struct ib_send_wr *wr) +{ + if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { + aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); + aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); + } else { + aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); + aseg->compare = 0; + } + +} + +static void set_tavor_ud_seg(struct mthca_tavor_ud_seg *useg, + struct ib_send_wr *wr) +{ + useg->lkey = cpu_to_be32(to_mah(wr->wr.ud.ah)->key); + useg->av_addr = cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma); + useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); + useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); + +} + +static void set_arbel_ud_seg(struct mthca_arbel_ud_seg *useg, + struct ib_send_wr *wr) +{ + memcpy(useg->av, to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE); + useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); + useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); +} + int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, struct ib_send_wr **bad_wr) { @@ -1525,8 +1611,15 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, int nreq; int i; int size; - int size0 = 0; - u32 f0; + /* + * f0 and size0 are only used if nreq != 0, and they will + * always be initialized the first time through the main loop + * before nreq is incremented. So nreq cannot become non-zero + * without initializing f0 and size0, and they are in fact + * never used uninitialized. + */ + int uninitialized_var(size0); + u32 uninitialized_var(f0); int ind; u8 op0 = 0; @@ -1561,7 +1654,7 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, cpu_to_be32(1); if (wr->opcode == IB_WR_SEND_WITH_IMM || wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) - ((struct mthca_next_seg *) wqe)->imm = wr->imm_data; + ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; wqe += sizeof (struct mthca_next_seg); size = sizeof (struct mthca_next_seg) / 16; @@ -1571,25 +1664,11 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, switch (wr->opcode) { case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.atomic.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.atomic.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - + set_raddr_seg(wqe, wr->wr.atomic.remote_addr, + wr->wr.atomic.rkey); wqe += sizeof (struct mthca_raddr_seg); - if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { - ((struct mthca_atomic_seg *) wqe)->swap_add = - cpu_to_be64(wr->wr.atomic.swap); - ((struct mthca_atomic_seg *) wqe)->compare = - cpu_to_be64(wr->wr.atomic.compare_add); - } else { - ((struct mthca_atomic_seg *) wqe)->swap_add = - cpu_to_be64(wr->wr.atomic.compare_add); - ((struct mthca_atomic_seg *) wqe)->compare = 0; - } - + set_atomic_seg(wqe, wr); wqe += sizeof (struct mthca_atomic_seg); size += (sizeof (struct mthca_raddr_seg) + sizeof (struct mthca_atomic_seg)) / 16; @@ -1598,12 +1677,9 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: case IB_WR_RDMA_READ: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.rdma.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.rdma.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - wqe += sizeof (struct mthca_raddr_seg); + set_raddr_seg(wqe, wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey); + wqe += sizeof (struct mthca_raddr_seg); size += sizeof (struct mthca_raddr_seg) / 16; break; @@ -1618,12 +1694,9 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, switch (wr->opcode) { case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.rdma.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.rdma.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - wqe += sizeof (struct mthca_raddr_seg); + set_raddr_seg(wqe, wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey); + wqe += sizeof (struct mthca_raddr_seg); size += sizeof (struct mthca_raddr_seg) / 16; break; @@ -1635,16 +1708,8 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, break; case UD: - ((struct mthca_tavor_ud_seg *) wqe)->lkey = - cpu_to_be32(to_mah(wr->wr.ud.ah)->key); - ((struct mthca_tavor_ud_seg *) wqe)->av_addr = - cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma); - ((struct mthca_tavor_ud_seg *) wqe)->dqpn = - cpu_to_be32(wr->wr.ud.remote_qpn); - ((struct mthca_tavor_ud_seg *) wqe)->qkey = - cpu_to_be32(wr->wr.ud.remote_qkey); - - wqe += sizeof (struct mthca_tavor_ud_seg); + set_tavor_ud_seg(wqe, wr); + wqe += sizeof (struct mthca_tavor_ud_seg); size += sizeof (struct mthca_tavor_ud_seg) / 16; break; @@ -1669,13 +1734,8 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); - wqe += sizeof (struct mthca_data_seg); + mthca_set_data_seg(wqe, wr->sg_list + i); + wqe += sizeof (struct mthca_data_seg); size += sizeof (struct mthca_data_seg) / 16; } @@ -1703,11 +1763,11 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, mthca_opcode[wr->opcode]); wmb(); ((struct mthca_next_seg *) prev_wqe)->ee_nds = - cpu_to_be32((size0 ? 0 : MTHCA_NEXT_DBD) | size | + cpu_to_be32((nreq ? 0 : MTHCA_NEXT_DBD) | size | ((wr->send_flags & IB_SEND_FENCE) ? MTHCA_NEXT_FENCE : 0)); - if (!size0) { + if (!nreq) { size0 = size; op0 = mthca_opcode[wr->opcode]; f0 = wr->send_flags & IB_SEND_FENCE ? @@ -1721,17 +1781,18 @@ int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, out: if (likely(nreq)) { - __be32 doorbell[2]; - - doorbell[0] = cpu_to_be32(((qp->sq.next_ind << qp->sq.wqe_shift) + - qp->send_wqe_offset) | f0 | op0); - doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0); - wmb(); - mthca_write64(doorbell, + mthca_write64(((qp->sq.next_ind << qp->sq.wqe_shift) + + qp->send_wqe_offset) | f0 | op0, + (qp->qpn << 8) | size0, dev->kar + MTHCA_SEND_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); + /* + * Make sure doorbells don't leak out of SQ spinlock + * and reach the HCA out of order: + */ + mmiowb(); } qp->sq.next_ind = ind; @@ -1746,13 +1807,19 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, { struct mthca_dev *dev = to_mdev(ibqp->device); struct mthca_qp *qp = to_mqp(ibqp); - __be32 doorbell[2]; unsigned long flags; int err = 0; int nreq; int i; int size; - int size0 = 0; + /* + * size0 is only used if nreq != 0, and it will always be + * initialized the first time through the main loop before + * nreq is incremented. So nreq cannot become non-zero + * without initializing size0, and it is in fact never used + * uninitialized. + */ + int uninitialized_var(size0); int ind; void *wqe; void *prev_wqe; @@ -1778,7 +1845,6 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, prev_wqe = qp->rq.last; qp->rq.last = wqe; - ((struct mthca_next_seg *) wqe)->nda_op = 0; ((struct mthca_next_seg *) wqe)->ee_nds = cpu_to_be32(MTHCA_NEXT_DBD); ((struct mthca_next_seg *) wqe)->flags = 0; @@ -1793,25 +1859,17 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); - wqe += sizeof (struct mthca_data_seg); + mthca_set_data_seg(wqe, wr->sg_list + i); + wqe += sizeof (struct mthca_data_seg); size += sizeof (struct mthca_data_seg) / 16; } qp->wrid[ind] = wr->wr_id; - ((struct mthca_next_seg *) prev_wqe)->nda_op = - cpu_to_be32((ind << qp->rq.wqe_shift) | 1); - wmb(); ((struct mthca_next_seg *) prev_wqe)->ee_nds = cpu_to_be32(MTHCA_NEXT_DBD | size); - if (!size0) + if (!nreq) size0 = size; ++ind; @@ -1822,35 +1880,35 @@ int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) { nreq = 0; - doorbell[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0); - doorbell[1] = cpu_to_be32(qp->qpn << 8); - wmb(); - mthca_write64(doorbell, - dev->kar + MTHCA_RECEIVE_DOORBELL, + mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, + qp->qpn << 8, dev->kar + MTHCA_RECEIVE_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); + qp->rq.next_ind = ind; qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB; - size0 = 0; } } out: if (likely(nreq)) { - doorbell[0] = cpu_to_be32((qp->rq.next_ind << qp->rq.wqe_shift) | size0); - doorbell[1] = cpu_to_be32((qp->qpn << 8) | nreq); - wmb(); - mthca_write64(doorbell, - dev->kar + MTHCA_RECEIVE_DOORBELL, + mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, + qp->qpn << 8 | nreq, dev->kar + MTHCA_RECEIVE_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } qp->rq.next_ind = ind; qp->rq.head += nreq; + /* + * Make sure doorbells don't leak out of RQ spinlock and reach + * the HCA out of order: + */ + mmiowb(); + spin_unlock_irqrestore(&qp->rq.lock, flags); return err; } @@ -1860,7 +1918,7 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, { struct mthca_dev *dev = to_mdev(ibqp->device); struct mthca_qp *qp = to_mqp(ibqp); - __be32 doorbell[2]; + u32 dbhi; void *wqe; void *prev_wqe; unsigned long flags; @@ -1868,8 +1926,15 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, int nreq; int i; int size; - int size0 = 0; - u32 f0; + /* + * f0 and size0 are only used if nreq != 0, and they will + * always be initialized the first time through the main loop + * before nreq is incremented. So nreq cannot become non-zero + * without initializing f0 and size0, and they are in fact + * never used uninitialized. + */ + int uninitialized_var(size0); + u32 uninitialized_var(f0); int ind; u8 op0 = 0; @@ -1883,13 +1948,10 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) { nreq = 0; - doorbell[0] = cpu_to_be32((MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) | - ((qp->sq.head & 0xffff) << 8) | - f0 | op0); - doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0); + dbhi = (MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) | + ((qp->sq.head & 0xffff) << 8) | f0 | op0; qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB; - size0 = 0; /* * Make sure that descriptors are written before @@ -1903,7 +1965,8 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, * write MMIO send doorbell. */ wmb(); - mthca_write64(doorbell, + + mthca_write64(dbhi, (qp->qpn << 8) | size0, dev->kar + MTHCA_SEND_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } @@ -1927,10 +1990,12 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0) | ((wr->send_flags & IB_SEND_SOLICITED) ? cpu_to_be32(MTHCA_NEXT_SOLICIT) : 0) | + ((wr->send_flags & IB_SEND_IP_CSUM) ? + cpu_to_be32(MTHCA_NEXT_IP_CSUM | MTHCA_NEXT_TCP_UDP_CSUM) : 0) | cpu_to_be32(1); if (wr->opcode == IB_WR_SEND_WITH_IMM || wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) - ((struct mthca_next_seg *) wqe)->imm = wr->imm_data; + ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; wqe += sizeof (struct mthca_next_seg); size = sizeof (struct mthca_next_seg) / 16; @@ -1940,26 +2005,12 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, switch (wr->opcode) { case IB_WR_ATOMIC_CMP_AND_SWP: case IB_WR_ATOMIC_FETCH_AND_ADD: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.atomic.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.atomic.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - + set_raddr_seg(wqe, wr->wr.atomic.remote_addr, + wr->wr.atomic.rkey); wqe += sizeof (struct mthca_raddr_seg); - if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { - ((struct mthca_atomic_seg *) wqe)->swap_add = - cpu_to_be64(wr->wr.atomic.swap); - ((struct mthca_atomic_seg *) wqe)->compare = - cpu_to_be64(wr->wr.atomic.compare_add); - } else { - ((struct mthca_atomic_seg *) wqe)->swap_add = - cpu_to_be64(wr->wr.atomic.compare_add); - ((struct mthca_atomic_seg *) wqe)->compare = 0; - } - - wqe += sizeof (struct mthca_atomic_seg); + set_atomic_seg(wqe, wr); + wqe += sizeof (struct mthca_atomic_seg); size += (sizeof (struct mthca_raddr_seg) + sizeof (struct mthca_atomic_seg)) / 16; break; @@ -1967,12 +2018,9 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, case IB_WR_RDMA_READ: case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.rdma.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.rdma.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - wqe += sizeof (struct mthca_raddr_seg); + set_raddr_seg(wqe, wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey); + wqe += sizeof (struct mthca_raddr_seg); size += sizeof (struct mthca_raddr_seg) / 16; break; @@ -1987,12 +2035,9 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, switch (wr->opcode) { case IB_WR_RDMA_WRITE: case IB_WR_RDMA_WRITE_WITH_IMM: - ((struct mthca_raddr_seg *) wqe)->raddr = - cpu_to_be64(wr->wr.rdma.remote_addr); - ((struct mthca_raddr_seg *) wqe)->rkey = - cpu_to_be32(wr->wr.rdma.rkey); - ((struct mthca_raddr_seg *) wqe)->reserved = 0; - wqe += sizeof (struct mthca_raddr_seg); + set_raddr_seg(wqe, wr->wr.rdma.remote_addr, + wr->wr.rdma.rkey); + wqe += sizeof (struct mthca_raddr_seg); size += sizeof (struct mthca_raddr_seg) / 16; break; @@ -2004,14 +2049,8 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, break; case UD: - memcpy(((struct mthca_arbel_ud_seg *) wqe)->av, - to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE); - ((struct mthca_arbel_ud_seg *) wqe)->dqpn = - cpu_to_be32(wr->wr.ud.remote_qpn); - ((struct mthca_arbel_ud_seg *) wqe)->qkey = - cpu_to_be32(wr->wr.ud.remote_qkey); - - wqe += sizeof (struct mthca_arbel_ud_seg); + set_arbel_ud_seg(wqe, wr); + wqe += sizeof (struct mthca_arbel_ud_seg); size += sizeof (struct mthca_arbel_ud_seg) / 16; break; @@ -2036,13 +2075,8 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); - wqe += sizeof (struct mthca_data_seg); + mthca_set_data_seg(wqe, wr->sg_list + i); + wqe += sizeof (struct mthca_data_seg); size += sizeof (struct mthca_data_seg) / 16; } @@ -2074,7 +2108,7 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, ((wr->send_flags & IB_SEND_FENCE) ? MTHCA_NEXT_FENCE : 0)); - if (!size0) { + if (!nreq) { size0 = size; op0 = mthca_opcode[wr->opcode]; f0 = wr->send_flags & IB_SEND_FENCE ? @@ -2088,10 +2122,7 @@ int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, out: if (likely(nreq)) { - doorbell[0] = cpu_to_be32((nreq << 24) | - ((qp->sq.head & 0xffff) << 8) | - f0 | op0); - doorbell[1] = cpu_to_be32((qp->qpn << 8) | size0); + dbhi = (nreq << 24) | ((qp->sq.head & 0xffff) << 8) | f0 | op0; qp->sq.head += nreq; @@ -2107,11 +2138,17 @@ out: * write MMIO send doorbell. */ wmb(); - mthca_write64(doorbell, - dev->kar + MTHCA_SEND_DOORBELL, + + mthca_write64(dbhi, (qp->qpn << 8) | size0, dev->kar + MTHCA_SEND_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } + /* + * Make sure doorbells don't leak out of SQ spinlock and reach + * the HCA out of order: + */ + mmiowb(); + spin_unlock_irqrestore(&qp->sq.lock, flags); return err; } @@ -2158,20 +2195,12 @@ int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); + mthca_set_data_seg(wqe, wr->sg_list + i); wqe += sizeof (struct mthca_data_seg); } - if (i < qp->rq.max_gs) { - ((struct mthca_data_seg *) wqe)->byte_count = 0; - ((struct mthca_data_seg *) wqe)->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - ((struct mthca_data_seg *) wqe)->addr = 0; - } + if (i < qp->rq.max_gs) + mthca_set_data_seg_inval(wqe); qp->wrid[ind] = wr->wr_id; @@ -2201,10 +2230,10 @@ void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, struct mthca_next_seg *next; /* - * For SRQs, all WQEs generate a CQE, so we're always at the - * end of the doorbell chain. + * For SRQs, all receive WQEs generate a CQE, so we're always + * at the end of the doorbell chain. */ - if (qp->ibqp.srq) { + if (qp->ibqp.srq && !is_send) { *new_wqe = 0; return; } @@ -2222,10 +2251,9 @@ void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, *new_wqe = 0; } -int __devinit mthca_init_qp_table(struct mthca_dev *dev) +int mthca_init_qp_table(struct mthca_dev *dev) { int err; - u8 status; int i; spin_lock_init(&dev->qp_table.lock); @@ -2252,15 +2280,10 @@ int __devinit mthca_init_qp_table(struct mthca_dev *dev) for (i = 0; i < 2; ++i) { err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_GSI : IB_QPT_SMI, - dev->qp_table.sqp_start + i * 2, - &status); - if (err) - goto err_out; - if (status) { + dev->qp_table.sqp_start + i * 2); + if (err) { mthca_warn(dev, "CONF_SPECIAL_QP returned " - "status %02x, aborting.\n", - status); - err = -EINVAL; + "%d, aborting.\n", err); goto err_out; } } @@ -2268,7 +2291,7 @@ int __devinit mthca_init_qp_table(struct mthca_dev *dev) err_out: for (i = 0; i < 2; ++i) - mthca_CONF_SPECIAL_QP(dev, i, 0, &status); + mthca_CONF_SPECIAL_QP(dev, i, 0); mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); mthca_alloc_cleanup(&dev->qp_table.alloc); @@ -2279,10 +2302,9 @@ int __devinit mthca_init_qp_table(struct mthca_dev *dev) void mthca_cleanup_qp_table(struct mthca_dev *dev) { int i; - u8 status; for (i = 0; i < 2; ++i) - mthca_CONF_SPECIAL_QP(dev, i, 0, &status); + mthca_CONF_SPECIAL_QP(dev, i, 0); mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); mthca_alloc_cleanup(&dev->qp_table.alloc); diff --git a/drivers/infiniband/hw/mthca/mthca_reset.c b/drivers/infiniband/hw/mthca/mthca_reset.c index 91934f2d9db..74c6a942604 100644 --- a/drivers/infiniband/hw/mthca/mthca_reset.c +++ b/drivers/infiniband/hw/mthca/mthca_reset.c @@ -28,11 +28,8 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_reset.c 1349 2004-12-16 21:09:43Z roland $ */ -#include <linux/init.h> #include <linux/errno.h> #include <linux/pci.h> #include <linux/delay.h> @@ -116,7 +113,7 @@ int mthca_reset(struct mthca_dev *mdev) } hca_pcix_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX); - hca_pcie_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP); + hca_pcie_cap = pci_pcie_cap(mdev->pdev); if (bridge) { bridge_header = kmalloc(256, GFP_KERNEL); @@ -244,16 +241,16 @@ good: if (hca_pcie_cap) { devctl = hca_header[(hca_pcie_cap + PCI_EXP_DEVCTL) / 4]; - if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_DEVCTL, - devctl)) { + if (pcie_capability_write_word(mdev->pdev, PCI_EXP_DEVCTL, + devctl)) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA PCI Express " "Device Control register, aborting.\n"); goto out; } linkctl = hca_header[(hca_pcie_cap + PCI_EXP_LNKCTL) / 4]; - if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_LNKCTL, - linkctl)) { + if (pcie_capability_write_word(mdev->pdev, PCI_EXP_LNKCTL, + linkctl)) { err = -ENODEV; mthca_err(mdev, "Couldn't restore HCA PCI Express " "Link control register, aborting.\n"); diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c index 0f316c87bf6..d22f970480c 100644 --- a/drivers/infiniband/hw/mthca/mthca_srq.c +++ b/drivers/infiniband/hw/mthca/mthca_srq.c @@ -28,12 +28,13 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_srq.c 3047 2005-08-10 03:59:35Z roland $ */ #include <linux/slab.h> #include <linux/string.h> +#include <linux/sched.h> + +#include <asm/io.h> #include "mthca_dev.h" #include "mthca_cmd.h" @@ -114,11 +115,16 @@ static void mthca_arbel_init_srq_context(struct mthca_dev *dev, struct mthca_srq *srq, struct mthca_arbel_srq_context *context) { - int logsize; + int logsize, max; memset(context, 0, sizeof *context); - logsize = long_log2(srq->max) + srq->wqe_shift; + /* + * Put max in a temporary variable to work around gcc bug + * triggered by ilog2() on sparc64. + */ + max = srq->max; + logsize = ilog2(max); context->state_logsize_srqn = cpu_to_be32(logsize << 24 | srq->srqn); context->lkey = cpu_to_be32(srq->mr.ibmr.lkey); context->db_index = cpu_to_be32(srq->db_index); @@ -167,9 +173,17 @@ static int mthca_alloc_srq_buf(struct mthca_dev *dev, struct mthca_pd *pd, * scatter list L_Keys to the sentry value of 0x100. */ for (i = 0; i < srq->max; ++i) { - wqe = get_wqe(srq, i); + struct mthca_next_seg *next; - *wqe_to_link(wqe) = i < srq->max - 1 ? i + 1 : -1; + next = wqe = get_wqe(srq, i); + + if (i < srq->max - 1) { + *wqe_to_link(wqe) = i + 1; + next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); + } else { + *wqe_to_link(wqe) = -1; + next->nda_op = 0; + } for (scatter = wqe + sizeof (struct mthca_next_seg); (void *) scatter < wqe + (1 << srq->wqe_shift); @@ -186,7 +200,6 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, struct ib_srq_attr *attr, struct mthca_srq *srq) { struct mthca_mailbox *mailbox; - u8 status; int ds; int err; @@ -201,6 +214,8 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, if (mthca_is_memfree(dev)) srq->max = roundup_pow_of_two(srq->max + 1); + else + srq->max = srq->max + 1; ds = max(64UL, roundup_pow_of_two(sizeof (struct mthca_next_seg) + @@ -209,7 +224,7 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, if (!mthca_is_memfree(dev) && (ds > dev->limits.max_desc_sz)) return -EINVAL; - srq->wqe_shift = long_log2(ds); + srq->wqe_shift = ilog2(ds); srq->srqn = mthca_alloc(&dev->srq_table.alloc); if (srq->srqn == -1) @@ -250,18 +265,12 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, else mthca_tavor_init_srq_context(dev, pd, srq, mailbox->buf); - err = mthca_SW2HW_SRQ(dev, mailbox, srq->srqn, &status); + err = mthca_SW2HW_SRQ(dev, mailbox, srq->srqn); if (err) { mthca_warn(dev, "SW2HW_SRQ failed (%d)\n", err); goto err_out_free_buf; } - if (status) { - mthca_warn(dev, "SW2HW_SRQ returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_free_buf; - } spin_lock_irq(&dev->srq_table.lock); if (mthca_array_set(&dev->srq_table.srq, @@ -277,17 +286,15 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, srq->first_free = 0; srq->last_free = srq->max - 1; - attr->max_wr = (mthca_is_memfree(dev)) ? srq->max - 1 : srq->max; + attr->max_wr = srq->max - 1; attr->max_sge = srq->max_gs; return 0; err_out_free_srq: - err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn, &status); + err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn); if (err) mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_SRQ returned status 0x%02x\n", status); err_out_free_buf: if (!pd->ibpd.uobject) @@ -324,7 +331,6 @@ void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq) { struct mthca_mailbox *mailbox; int err; - u8 status; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) { @@ -332,11 +338,9 @@ void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq) return; } - err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn, &status); + err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn); if (err) mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_SRQ returned status 0x%02x\n", status); spin_lock_irq(&dev->srq_table.lock); mthca_array_clear(&dev->srq_table.srq, @@ -362,8 +366,7 @@ int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, { struct mthca_dev *dev = to_mdev(ibsrq->device); struct mthca_srq *srq = to_msrq(ibsrq); - int ret; - u8 status; + int ret = 0; /* We don't support resizing SRQs (yet?) */ if (attr_mask & IB_SRQ_MAX_WR) @@ -375,16 +378,11 @@ int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, return -EINVAL; mutex_lock(&srq->mutex); - ret = mthca_ARM_SRQ(dev, srq->srqn, attr->srq_limit, &status); + ret = mthca_ARM_SRQ(dev, srq->srqn, attr->srq_limit); mutex_unlock(&srq->mutex); - - if (ret) - return ret; - if (status) - return -EINVAL; } - return 0; + return ret; } int mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) @@ -394,14 +392,13 @@ int mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) struct mthca_mailbox *mailbox; struct mthca_arbel_srq_context *arbel_ctx; struct mthca_tavor_srq_context *tavor_ctx; - u8 status; int err; mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); if (IS_ERR(mailbox)) return PTR_ERR(mailbox); - err = mthca_QUERY_SRQ(dev, srq->srqn, mailbox, &status); + err = mthca_QUERY_SRQ(dev, srq->srqn, mailbox); if (err) goto out; @@ -413,7 +410,7 @@ int mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) srq_attr->srq_limit = be16_to_cpu(tavor_ctx->limit_watermark); } - srq_attr->max_wr = (mthca_is_memfree(dev)) ? srq->max - 1 : srq->max; + srq_attr->max_wr = srq->max - 1; srq_attr->max_sge = srq->max_gs; out: @@ -460,16 +457,15 @@ out: void mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr) { int ind; + struct mthca_next_seg *last_free; ind = wqe_addr >> srq->wqe_shift; spin_lock(&srq->lock); - if (likely(srq->first_free >= 0)) - *wqe_to_link(get_wqe(srq, srq->last_free)) = ind; - else - srq->first_free = ind; - + last_free = get_wqe(srq, srq->last_free); + *wqe_to_link(last_free) = ind; + last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); *wqe_to_link(get_wqe(srq, ind)) = -1; srq->last_free = ind; @@ -481,7 +477,6 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, { struct mthca_dev *dev = to_mdev(ibsrq->device); struct mthca_srq *srq = to_msrq(ibsrq); - __be32 doorbell[2]; unsigned long flags; int err = 0; int first_ind; @@ -497,19 +492,11 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, first_ind = srq->first_free; for (nreq = 0; wr; wr = wr->next) { - ind = srq->first_free; - - if (ind < 0) { - mthca_err(dev, "SRQ %06x full\n", srq->srqn); - err = -ENOMEM; - *bad_wr = wr; - break; - } - + ind = srq->first_free; wqe = get_wqe(srq, ind); next_ind = *wqe_to_link(wqe); - if (next_ind < 0) { + if (unlikely(next_ind < 0)) { mthca_err(dev, "SRQ %06x full\n", srq->srqn); err = -ENOMEM; *bad_wr = wr; @@ -519,7 +506,6 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, prev_wqe = srq->last; srq->last = wqe; - ((struct mthca_next_seg *) wqe)->nda_op = 0; ((struct mthca_next_seg *) wqe)->ee_nds = 0; /* flags field will always remain 0 */ @@ -533,24 +519,13 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); + mthca_set_data_seg(wqe, wr->sg_list + i); wqe += sizeof (struct mthca_data_seg); } - if (i < srq->max_gs) { - ((struct mthca_data_seg *) wqe)->byte_count = 0; - ((struct mthca_data_seg *) wqe)->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - ((struct mthca_data_seg *) wqe)->addr = 0; - } + if (i < srq->max_gs) + mthca_set_data_seg_inval(wqe); - ((struct mthca_next_seg *) prev_wqe)->nda_op = - cpu_to_be32((ind << srq->wqe_shift) | 1); - wmb(); ((struct mthca_next_seg *) prev_wqe)->ee_nds = cpu_to_be32(MTHCA_NEXT_DBD); @@ -561,16 +536,13 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) { nreq = 0; - doorbell[0] = cpu_to_be32(first_ind << srq->wqe_shift); - doorbell[1] = cpu_to_be32(srq->srqn << 8); - /* * Make sure that descriptors are written * before doorbell is rung. */ wmb(); - mthca_write64(doorbell, + mthca_write64(first_ind << srq->wqe_shift, srq->srqn << 8, dev->kar + MTHCA_RECEIVE_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); @@ -579,20 +551,23 @@ int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, } if (likely(nreq)) { - doorbell[0] = cpu_to_be32(first_ind << srq->wqe_shift); - doorbell[1] = cpu_to_be32((srq->srqn << 8) | nreq); - /* * Make sure that descriptors are written before * doorbell is rung. */ wmb(); - mthca_write64(doorbell, + mthca_write64(first_ind << srq->wqe_shift, (srq->srqn << 8) | nreq, dev->kar + MTHCA_RECEIVE_DOORBELL, MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); } + /* + * Make sure doorbells don't leak out of SRQ spinlock and + * reach the HCA out of order: + */ + mmiowb(); + spin_unlock_irqrestore(&srq->lock, flags); return err; } @@ -613,27 +588,17 @@ int mthca_arbel_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, spin_lock_irqsave(&srq->lock, flags); for (nreq = 0; wr; ++nreq, wr = wr->next) { - ind = srq->first_free; - - if (ind < 0) { - mthca_err(dev, "SRQ %06x full\n", srq->srqn); - err = -ENOMEM; - *bad_wr = wr; - break; - } - + ind = srq->first_free; wqe = get_wqe(srq, ind); next_ind = *wqe_to_link(wqe); - if (next_ind < 0) { + if (unlikely(next_ind < 0)) { mthca_err(dev, "SRQ %06x full\n", srq->srqn); err = -ENOMEM; *bad_wr = wr; break; } - ((struct mthca_next_seg *) wqe)->nda_op = - cpu_to_be32((next_ind << srq->wqe_shift) | 1); ((struct mthca_next_seg *) wqe)->ee_nds = 0; /* flags field will always remain 0 */ @@ -646,20 +611,12 @@ int mthca_arbel_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, } for (i = 0; i < wr->num_sge; ++i) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32(wr->sg_list[i].length); - ((struct mthca_data_seg *) wqe)->lkey = - cpu_to_be32(wr->sg_list[i].lkey); - ((struct mthca_data_seg *) wqe)->addr = - cpu_to_be64(wr->sg_list[i].addr); + mthca_set_data_seg(wqe, wr->sg_list + i); wqe += sizeof (struct mthca_data_seg); } - if (i < srq->max_gs) { - ((struct mthca_data_seg *) wqe)->byte_count = 0; - ((struct mthca_data_seg *) wqe)->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - ((struct mthca_data_seg *) wqe)->addr = 0; - } + if (i < srq->max_gs) + mthca_set_data_seg_inval(wqe); srq->wrid[ind] = wr->wr_id; srq->first_free = next_ind; @@ -705,7 +662,7 @@ int mthca_max_srq_sge(struct mthca_dev *dev) sizeof (struct mthca_data_seg)); } -int __devinit mthca_init_srq_table(struct mthca_dev *dev) +int mthca_init_srq_table(struct mthca_dev *dev) { int err; diff --git a/drivers/infiniband/hw/mthca/mthca_uar.c b/drivers/infiniband/hw/mthca/mthca_uar.c index 8b728486410..ca5900c96fc 100644 --- a/drivers/infiniband/hw/mthca/mthca_uar.c +++ b/drivers/infiniband/hw/mthca/mthca_uar.c @@ -28,8 +28,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id$ */ #include <asm/page.h> /* PAGE_SHIFT */ diff --git a/drivers/infiniband/hw/mthca/mthca_user.h b/drivers/infiniband/hw/mthca/mthca_user.h index 02cc0a766f3..5fe56e81073 100644 --- a/drivers/infiniband/hw/mthca/mthca_user.h +++ b/drivers/infiniband/hw/mthca/mthca_user.h @@ -29,7 +29,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * */ #ifndef MTHCA_USER_H @@ -61,6 +60,16 @@ struct mthca_alloc_pd_resp { __u32 reserved; }; +struct mthca_reg_mr { +/* + * Mark the memory region with a DMA attribute that causes + * in-flight DMA to be flushed when the region is written to: + */ +#define MTHCA_MR_DMASYNC 0x1 + __u32 mr_attrs; + __u32 reserved; +}; + struct mthca_create_cq { __u32 lkey; __u32 pdn; diff --git a/drivers/infiniband/hw/mthca/mthca_wqe.h b/drivers/infiniband/hw/mthca/mthca_wqe.h index e7d2c1e8619..341a5ae881c 100644 --- a/drivers/infiniband/hw/mthca/mthca_wqe.h +++ b/drivers/infiniband/hw/mthca/mthca_wqe.h @@ -28,8 +28,6 @@ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. - * - * $Id: mthca_wqe.h 3047 2005-08-10 03:59:35Z roland $ */ #ifndef MTHCA_WQE_H @@ -38,14 +36,16 @@ #include <linux/types.h> enum { - MTHCA_NEXT_DBD = 1 << 7, - MTHCA_NEXT_FENCE = 1 << 6, - MTHCA_NEXT_CQ_UPDATE = 1 << 3, - MTHCA_NEXT_EVENT_GEN = 1 << 2, - MTHCA_NEXT_SOLICIT = 1 << 1, - - MTHCA_MLX_VL15 = 1 << 17, - MTHCA_MLX_SLR = 1 << 16 + MTHCA_NEXT_DBD = 1 << 7, + MTHCA_NEXT_FENCE = 1 << 6, + MTHCA_NEXT_CQ_UPDATE = 1 << 3, + MTHCA_NEXT_EVENT_GEN = 1 << 2, + MTHCA_NEXT_SOLICIT = 1 << 1, + MTHCA_NEXT_IP_CSUM = 1 << 4, + MTHCA_NEXT_TCP_UDP_CSUM = 1 << 5, + + MTHCA_MLX_VL15 = 1 << 17, + MTHCA_MLX_SLR = 1 << 16 }; enum { @@ -113,4 +113,19 @@ struct mthca_mlx_seg { __be16 vcrc; }; +static __always_inline void mthca_set_data_seg(struct mthca_data_seg *dseg, + struct ib_sge *sg) +{ + dseg->byte_count = cpu_to_be32(sg->length); + dseg->lkey = cpu_to_be32(sg->lkey); + dseg->addr = cpu_to_be64(sg->addr); +} + +static __always_inline void mthca_set_data_seg_inval(struct mthca_data_seg *dseg) +{ + dseg->byte_count = 0; + dseg->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); + dseg->addr = 0; +} + #endif /* MTHCA_WQE_H */ |