diff options
Diffstat (limited to 'drivers/scsi/ipr.c')
| -rw-r--r-- | drivers/scsi/ipr.c | 2338 |
1 files changed, 1712 insertions, 626 deletions
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c index fa60d7df44b..924b0ba74df 100644 --- a/drivers/scsi/ipr.c +++ b/drivers/scsi/ipr.c @@ -60,6 +60,7 @@ #include <linux/errno.h> #include <linux/kernel.h> #include <linux/slab.h> +#include <linux/vmalloc.h> #include <linux/ioport.h> #include <linux/delay.h> #include <linux/pci.h> @@ -97,13 +98,17 @@ static unsigned int ipr_transop_timeout = 0; static unsigned int ipr_debug = 0; static unsigned int ipr_max_devs = IPR_DEFAULT_SIS64_DEVS; static unsigned int ipr_dual_ioa_raid = 1; +static unsigned int ipr_number_of_msix = 2; static DEFINE_SPINLOCK(ipr_driver_lock); /* This table describes the differences between DMA controller chips */ static const struct ipr_chip_cfg_t ipr_chip_cfg[] = { { /* Gemstone, Citrine, Obsidian, and Obsidian-E */ .mailbox = 0x0042C, + .max_cmds = 100, .cache_line_size = 0x20, + .clear_isr = 1, + .iopoll_weight = 0, { .set_interrupt_mask_reg = 0x0022C, .clr_interrupt_mask_reg = 0x00230, @@ -125,7 +130,10 @@ static const struct ipr_chip_cfg_t ipr_chip_cfg[] = { }, { /* Snipe and Scamp */ .mailbox = 0x0052C, + .max_cmds = 100, .cache_line_size = 0x20, + .clear_isr = 1, + .iopoll_weight = 0, { .set_interrupt_mask_reg = 0x00288, .clr_interrupt_mask_reg = 0x0028C, @@ -146,8 +154,11 @@ static const struct ipr_chip_cfg_t ipr_chip_cfg[] = { } }, { /* CRoC */ - .mailbox = 0x00040, + .mailbox = 0x00044, + .max_cmds = 1000, .cache_line_size = 0x20, + .clear_isr = 0, + .iopoll_weight = 64, { .set_interrupt_mask_reg = 0x00010, .clr_interrupt_mask_reg = 0x00018, @@ -182,10 +193,10 @@ static const struct ipr_chip_t ipr_chip[] = { { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_SNIPE, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, { PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_SCAMP, IPR_USE_LSI, IPR_SIS32, IPR_PCI_CFG, &ipr_chip_cfg[1] }, { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] } + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, IPR_USE_MSI, IPR_SIS64, IPR_MMIO, &ipr_chip_cfg[2] } }; -static int ipr_max_bus_speeds [] = { +static int ipr_max_bus_speeds[] = { IPR_80MBs_SCSI_RATE, IPR_U160_SCSI_RATE, IPR_U320_SCSI_RATE }; @@ -208,6 +219,8 @@ MODULE_PARM_DESC(dual_ioa_raid, "Enable dual adapter RAID support. Set to 1 to e module_param_named(max_devs, ipr_max_devs, int, 0); MODULE_PARM_DESC(max_devs, "Specify the maximum number of physical devices. " "[Default=" __stringify(IPR_DEFAULT_SIS64_DEVS) "]"); +module_param_named(number_of_msix, ipr_number_of_msix, int, 0); +MODULE_PARM_DESC(number_of_msix, "Specify the number of MSIX interrupts to use on capable adapters (1 - 16). (default:2)"); MODULE_LICENSE("GPL"); MODULE_VERSION(IPR_DRIVER_VERSION); @@ -268,12 +281,22 @@ struct ipr_error_table_t ipr_error_table[] = { "FFF6: Failure prediction threshold exceeded"}, {0x015D9200, 0, IPR_DEFAULT_LOG_LEVEL, "8009: Impending cache battery pack failure"}, + {0x02040100, 0, 0, + "Logical Unit in process of becoming ready"}, + {0x02040200, 0, 0, + "Initializing command required"}, {0x02040400, 0, 0, "34FF: Disk device format in progress"}, + {0x02040C00, 0, 0, + "Logical unit not accessible, target port in unavailable state"}, {0x02048000, 0, IPR_DEFAULT_LOG_LEVEL, "9070: IOA requested reset"}, {0x023F0000, 0, 0, "Synchronization required"}, + {0x02408500, 0, 0, + "IOA microcode download required"}, + {0x02408600, 0, 0, + "Device bus connection is prohibited by host"}, {0x024E0000, 0, 0, "No ready, IOA shutdown"}, {0x025A0000, 0, 0, @@ -372,6 +395,8 @@ struct ipr_error_table_t ipr_error_table[] = { "4030: Incorrect multipath connection"}, {0x04679000, 0, IPR_DEFAULT_LOG_LEVEL, "4110: Unsupported enclosure function"}, + {0x04679800, 0, IPR_DEFAULT_LOG_LEVEL, + "4120: SAS cable VPD cannot be read"}, {0x046E0000, 0, IPR_DEFAULT_LOG_LEVEL, "FFF4: Command to logical unit failed"}, {0x05240000, 1, 0, @@ -394,10 +419,18 @@ struct ipr_error_table_t ipr_error_table[] = { "Illegal request, command sequence error"}, {0x052C8000, 1, 0, "Illegal request, dual adapter support not enabled"}, + {0x052C8100, 1, 0, + "Illegal request, another cable connector was physically disabled"}, + {0x054E8000, 1, 0, + "Illegal request, inconsistent group id/group count"}, {0x06040500, 0, IPR_DEFAULT_LOG_LEVEL, "9031: Array protection temporarily suspended, protection resuming"}, {0x06040600, 0, IPR_DEFAULT_LOG_LEVEL, "9040: Array protection temporarily suspended, protection resuming"}, + {0x060B0100, 0, IPR_DEFAULT_LOG_LEVEL, + "4080: IOA exceeded maximum operating temperature"}, + {0x060B8000, 0, IPR_DEFAULT_LOG_LEVEL, + "4085: Service required"}, {0x06288000, 0, IPR_DEFAULT_LOG_LEVEL, "3140: Device bus not ready to ready transition"}, {0x06290000, 0, IPR_DEFAULT_LOG_LEVEL, @@ -410,6 +443,8 @@ struct ipr_error_table_t ipr_error_table[] = { "FFFB: SCSI bus was reset by another initiator"}, {0x063F0300, 0, IPR_DEFAULT_LOG_LEVEL, "3029: A device replacement has occurred"}, + {0x063F8300, 0, IPR_DEFAULT_LOG_LEVEL, + "4102: Device bus fabric performance degradation"}, {0x064C8000, 0, IPR_DEFAULT_LOG_LEVEL, "9051: IOA cache data exists for a missing or failed device"}, {0x064C8100, 0, IPR_DEFAULT_LOG_LEVEL, @@ -432,6 +467,14 @@ struct ipr_error_table_t ipr_error_table[] = { "9076: Configuration error, missing remote IOA"}, {0x06679100, 0, IPR_DEFAULT_LOG_LEVEL, "4050: Enclosure does not support a required multipath function"}, + {0x06679800, 0, IPR_DEFAULT_LOG_LEVEL, + "4121: Configuration error, required cable is missing"}, + {0x06679900, 0, IPR_DEFAULT_LOG_LEVEL, + "4122: Cable is not plugged into the correct location on remote IOA"}, + {0x06679A00, 0, IPR_DEFAULT_LOG_LEVEL, + "4123: Configuration error, invalid cable vital product data"}, + {0x06679B00, 0, IPR_DEFAULT_LOG_LEVEL, + "4124: Configuration error, both cable ends are plugged into the same IOA"}, {0x06690000, 0, IPR_DEFAULT_LOG_LEVEL, "4070: Logically bad block written on device"}, {0x06690200, 0, IPR_DEFAULT_LOG_LEVEL, @@ -494,10 +537,18 @@ struct ipr_error_table_t ipr_error_table[] = { "9062: One or more disks are missing from an array"}, {0x07279900, 0, IPR_DEFAULT_LOG_LEVEL, "9063: Maximum number of functional arrays has been exceeded"}, + {0x07279A00, 0, 0, + "Data protect, other volume set problem"}, {0x0B260000, 0, 0, "Aborted command, invalid descriptor"}, + {0x0B3F9000, 0, 0, + "Target operating conditions have changed, dual adapter takeover"}, + {0x0B530200, 0, 0, + "Aborted command, medium removal prevented"}, {0x0B5A0000, 0, 0, - "Command terminated by host"} + "Command terminated by host"}, + {0x0B5B8000, 0, 0, + "Aborted command, command terminated by host"} }; static const struct ipr_ses_table_entry ipr_ses_table[] = { @@ -542,7 +593,8 @@ static void ipr_trc_hook(struct ipr_cmnd *ipr_cmd, struct ipr_trace_entry *trace_entry; struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; - trace_entry = &ioa_cfg->trace[ioa_cfg->trace_index++]; + trace_entry = &ioa_cfg->trace[atomic_add_return + (1, &ioa_cfg->trace_index)%IPR_NUM_TRACE_ENTRIES]; trace_entry->time = jiffies; trace_entry->op_code = ipr_cmd->ioarcb.cmd_pkt.cdb[0]; trace_entry->type = type; @@ -553,12 +605,30 @@ static void ipr_trc_hook(struct ipr_cmnd *ipr_cmd, trace_entry->cmd_index = ipr_cmd->cmd_index & 0xff; trace_entry->res_handle = ipr_cmd->ioarcb.res_handle; trace_entry->u.add_data = add_data; + wmb(); } #else -#define ipr_trc_hook(ipr_cmd, type, add_data) do { } while(0) +#define ipr_trc_hook(ipr_cmd, type, add_data) do { } while (0) #endif /** + * ipr_lock_and_done - Acquire lock and complete command + * @ipr_cmd: ipr command struct + * + * Return value: + * none + **/ +static void ipr_lock_and_done(struct ipr_cmnd *ipr_cmd) +{ + unsigned long lock_flags; + struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); + ipr_cmd->done(ipr_cmd); + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); +} + +/** * ipr_reinit_ipr_cmnd - Re-initialize an IPR Cmnd block for reuse * @ipr_cmd: ipr command struct * @@ -571,8 +641,11 @@ static void ipr_reinit_ipr_cmnd(struct ipr_cmnd *ipr_cmd) struct ipr_ioasa *ioasa = &ipr_cmd->s.ioasa; struct ipr_ioasa64 *ioasa64 = &ipr_cmd->s.ioasa64; dma_addr_t dma_addr = ipr_cmd->dma_addr; + int hrrq_id; + hrrq_id = ioarcb->cmd_pkt.hrrq_id; memset(&ioarcb->cmd_pkt, 0, sizeof(struct ipr_cmd_pkt)); + ioarcb->cmd_pkt.hrrq_id = hrrq_id; ioarcb->data_transfer_length = 0; ioarcb->read_data_transfer_length = 0; ioarcb->ioadl_len = 0; @@ -604,30 +677,51 @@ static void ipr_reinit_ipr_cmnd(struct ipr_cmnd *ipr_cmd) * Return value: * none **/ -static void ipr_init_ipr_cmnd(struct ipr_cmnd *ipr_cmd) +static void ipr_init_ipr_cmnd(struct ipr_cmnd *ipr_cmd, + void (*fast_done) (struct ipr_cmnd *)) { ipr_reinit_ipr_cmnd(ipr_cmd); ipr_cmd->u.scratch = 0; ipr_cmd->sibling = NULL; + ipr_cmd->fast_done = fast_done; init_timer(&ipr_cmd->timer); } /** - * ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block + * __ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block * @ioa_cfg: ioa config struct * * Return value: * pointer to ipr command struct **/ static -struct ipr_cmnd *ipr_get_free_ipr_cmnd(struct ipr_ioa_cfg *ioa_cfg) +struct ipr_cmnd *__ipr_get_free_ipr_cmnd(struct ipr_hrr_queue *hrrq) { - struct ipr_cmnd *ipr_cmd; + struct ipr_cmnd *ipr_cmd = NULL; + + if (likely(!list_empty(&hrrq->hrrq_free_q))) { + ipr_cmd = list_entry(hrrq->hrrq_free_q.next, + struct ipr_cmnd, queue); + list_del(&ipr_cmd->queue); + } + - ipr_cmd = list_entry(ioa_cfg->free_q.next, struct ipr_cmnd, queue); - list_del(&ipr_cmd->queue); - ipr_init_ipr_cmnd(ipr_cmd); + return ipr_cmd; +} +/** + * ipr_get_free_ipr_cmnd - Get a free IPR Cmnd block and initialize it + * @ioa_cfg: ioa config struct + * + * Return value: + * pointer to ipr command struct + **/ +static +struct ipr_cmnd *ipr_get_free_ipr_cmnd(struct ipr_ioa_cfg *ioa_cfg) +{ + struct ipr_cmnd *ipr_cmd = + __ipr_get_free_ipr_cmnd(&ioa_cfg->hrrq[IPR_INIT_HRRQ]); + ipr_init_ipr_cmnd(ipr_cmd, ipr_lock_and_done); return ipr_cmd; } @@ -646,9 +740,15 @@ static void ipr_mask_and_clear_interrupts(struct ipr_ioa_cfg *ioa_cfg, u32 clr_ints) { volatile u32 int_reg; + int i; /* Stop new interrupts */ - ioa_cfg->allow_interrupts = 0; + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].allow_interrupts = 0; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); /* Set interrupt mask to stop all new interrupts */ if (ioa_cfg->sis64) @@ -721,13 +821,12 @@ static int ipr_set_pcix_cmd_reg(struct ipr_ioa_cfg *ioa_cfg) **/ static void ipr_sata_eh_done(struct ipr_cmnd *ipr_cmd) { - struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; struct ata_queued_cmd *qc = ipr_cmd->qc; struct ipr_sata_port *sata_port = qc->ap->private_data; qc->err_mask |= AC_ERR_OTHER; sata_port->ioasa.status |= ATA_BUSY; - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); ata_qc_complete(qc); } @@ -743,14 +842,13 @@ static void ipr_sata_eh_done(struct ipr_cmnd *ipr_cmd) **/ static void ipr_scsi_eh_done(struct ipr_cmnd *ipr_cmd) { - struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; struct scsi_cmnd *scsi_cmd = ipr_cmd->scsi_cmd; scsi_cmd->result |= (DID_ERROR << 16); scsi_dma_unmap(ipr_cmd->scsi_cmd); scsi_cmd->scsi_done(scsi_cmd); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); } /** @@ -765,24 +863,32 @@ static void ipr_scsi_eh_done(struct ipr_cmnd *ipr_cmd) static void ipr_fail_all_ops(struct ipr_ioa_cfg *ioa_cfg) { struct ipr_cmnd *ipr_cmd, *temp; + struct ipr_hrr_queue *hrrq; ENTER; - list_for_each_entry_safe(ipr_cmd, temp, &ioa_cfg->pending_q, queue) { - list_del(&ipr_cmd->queue); + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + list_for_each_entry_safe(ipr_cmd, + temp, &hrrq->hrrq_pending_q, queue) { + list_del(&ipr_cmd->queue); - ipr_cmd->s.ioasa.hdr.ioasc = cpu_to_be32(IPR_IOASC_IOA_WAS_RESET); - ipr_cmd->s.ioasa.hdr.ilid = cpu_to_be32(IPR_DRIVER_ILID); + ipr_cmd->s.ioasa.hdr.ioasc = + cpu_to_be32(IPR_IOASC_IOA_WAS_RESET); + ipr_cmd->s.ioasa.hdr.ilid = + cpu_to_be32(IPR_DRIVER_ILID); - if (ipr_cmd->scsi_cmd) - ipr_cmd->done = ipr_scsi_eh_done; - else if (ipr_cmd->qc) - ipr_cmd->done = ipr_sata_eh_done; + if (ipr_cmd->scsi_cmd) + ipr_cmd->done = ipr_scsi_eh_done; + else if (ipr_cmd->qc) + ipr_cmd->done = ipr_sata_eh_done; - ipr_trc_hook(ipr_cmd, IPR_TRACE_FINISH, IPR_IOASC_IOA_WAS_RESET); - del_timer(&ipr_cmd->timer); - ipr_cmd->done(ipr_cmd); + ipr_trc_hook(ipr_cmd, IPR_TRACE_FINISH, + IPR_IOASC_IOA_WAS_RESET); + del_timer(&ipr_cmd->timer); + ipr_cmd->done(ipr_cmd); + } + spin_unlock(&hrrq->_lock); } - LEAVE; } @@ -832,9 +938,7 @@ static void ipr_do_req(struct ipr_cmnd *ipr_cmd, void (*done) (struct ipr_cmnd *), void (*timeout_func) (struct ipr_cmnd *), u32 timeout) { - struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; - - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); ipr_cmd->done = done; @@ -846,8 +950,6 @@ static void ipr_do_req(struct ipr_cmnd *ipr_cmd, ipr_trc_hook(ipr_cmd, IPR_TRACE_START, 0); - mb(); - ipr_send_command(ipr_cmd); } @@ -937,6 +1039,14 @@ static void ipr_send_blocking_cmd(struct ipr_cmnd *ipr_cmd, spin_lock_irq(ioa_cfg->host->host_lock); } +static int ipr_get_hrrq_index(struct ipr_ioa_cfg *ioa_cfg) +{ + if (ioa_cfg->hrrq_num == 1) + return 0; + else + return (atomic_add_return(1, &ioa_cfg->hrrq_index) % (ioa_cfg->hrrq_num - 1)) + 1; +} + /** * ipr_send_hcam - Send an HCAM to the adapter. * @ioa_cfg: ioa config struct @@ -956,9 +1066,9 @@ static void ipr_send_hcam(struct ipr_ioa_cfg *ioa_cfg, u8 type, struct ipr_cmnd *ipr_cmd; struct ipr_ioarcb *ioarcb; - if (ioa_cfg->allow_cmds) { + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds) { ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); list_add_tail(&hostrcb->queue, &ioa_cfg->hostrcb_pending_q); ipr_cmd->u.hostrcb = hostrcb; @@ -981,8 +1091,6 @@ static void ipr_send_hcam(struct ipr_ioa_cfg *ioa_cfg, u8 type, ipr_trc_hook(ipr_cmd, IPR_TRACE_START, IPR_IOA_RES_ADDR); - mb(); - ipr_send_command(ipr_cmd); } else { list_add_tail(&hostrcb->queue, &ioa_cfg->hostrcb_free_q); @@ -999,7 +1107,7 @@ static void ipr_send_hcam(struct ipr_ioa_cfg *ioa_cfg, u8 type, **/ static void ipr_update_ata_class(struct ipr_resource_entry *res, unsigned int proto) { - switch(proto) { + switch (proto) { case IPR_PROTO_SATA: case IPR_PROTO_SAS_STP: res->ata_class = ATA_DEV_ATA; @@ -1035,6 +1143,7 @@ static void ipr_init_res_entry(struct ipr_resource_entry *res, res->add_to_ml = 0; res->del_from_ml = 0; res->resetting_device = 0; + res->reset_occurred = 0; res->sdev = NULL; res->sata_port = NULL; @@ -1048,6 +1157,8 @@ static void ipr_init_res_entry(struct ipr_resource_entry *res, sizeof(res->res_path)); res->bus = 0; + memcpy(&res->dev_lun.scsi_lun, &cfgtew->u.cfgte64->lun, + sizeof(res->dev_lun.scsi_lun)); res->lun = scsilun_to_int(&res->dev_lun); if (res->type == IPR_RES_TYPE_GENERIC_SCSI) { @@ -1063,9 +1174,6 @@ static void ipr_init_res_entry(struct ipr_resource_entry *res, ioa_cfg->max_devs_supported); set_bit(res->target, ioa_cfg->target_ids); } - - memcpy(&res->dev_lun.scsi_lun, &cfgtew->u.cfgte64->lun, - sizeof(res->dev_lun.scsi_lun)); } else if (res->type == IPR_RES_TYPE_IOAFP) { res->bus = IPR_IOAFP_VIRTUAL_BUS; res->target = 0; @@ -1116,7 +1224,7 @@ static int ipr_is_same_device(struct ipr_resource_entry *res, if (res->ioa_cfg->sis64) { if (!memcmp(&res->dev_id, &cfgtew->u.cfgte64->dev_id, sizeof(cfgtew->u.cfgte64->dev_id)) && - !memcmp(&res->lun, &cfgtew->u.cfgte64->lun, + !memcmp(&res->dev_lun.scsi_lun, &cfgtew->u.cfgte64->lun, sizeof(cfgtew->u.cfgte64->lun))) { return 1; } @@ -1131,14 +1239,15 @@ static int ipr_is_same_device(struct ipr_resource_entry *res, } /** - * ipr_format_res_path - Format the resource path for printing. + * __ipr_format_res_path - Format the resource path for printing. * @res_path: resource path * @buf: buffer + * @len: length of buffer provided * * Return value: * pointer to buffer **/ -static char *ipr_format_res_path(u8 *res_path, char *buffer, int len) +static char *__ipr_format_res_path(u8 *res_path, char *buffer, int len) { int i; char *p = buffer; @@ -1152,6 +1261,27 @@ static char *ipr_format_res_path(u8 *res_path, char *buffer, int len) } /** + * ipr_format_res_path - Format the resource path for printing. + * @ioa_cfg: ioa config struct + * @res_path: resource path + * @buf: buffer + * @len: length of buffer provided + * + * Return value: + * pointer to buffer + **/ +static char *ipr_format_res_path(struct ipr_ioa_cfg *ioa_cfg, + u8 *res_path, char *buffer, int len) +{ + char *p = buffer; + + *p = '\0'; + p += snprintf(p, buffer + len - p, "%d/", ioa_cfg->host->host_no); + __ipr_format_res_path(res_path, p, len - (buffer - p)); + return buffer; +} + +/** * ipr_update_res_entry - Update the resource entry. * @res: resource entry struct * @cfgtew: config table entry wrapper struct @@ -1191,8 +1321,8 @@ static void ipr_update_res_entry(struct ipr_resource_entry *res, if (res->sdev && new_path) sdev_printk(KERN_INFO, res->sdev, "Resource path: %s\n", - ipr_format_res_path(res->res_path, buffer, - sizeof(buffer))); + ipr_format_res_path(res->ioa_cfg, + res->res_path, buffer, sizeof(buffer))); } else { res->flags = cfgtew->u.cfgte->flags; if (res->flags & IPR_IS_IOA_RESOURCE) @@ -1302,7 +1432,7 @@ static void ipr_handle_config_change(struct ipr_ioa_cfg *ioa_cfg, ipr_clear_res_target(res); list_move_tail(&res->queue, &ioa_cfg->free_res_q); } - } else if (!res->sdev) { + } else if (!res->sdev || res->del_from_ml) { res->add_to_ml = 1; if (ioa_cfg->allow_ml_add_del) schedule_work(&ioa_cfg->work_q); @@ -1328,7 +1458,7 @@ static void ipr_process_ccn(struct ipr_cmnd *ipr_cmd) u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); list_del(&hostrcb->queue); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); if (ioasc) { if (ioasc != IPR_IOASC_IOA_WAS_RESET) @@ -1578,8 +1708,8 @@ static void ipr_log_sis64_config_error(struct ipr_ioa_cfg *ioa_cfg, ipr_err_separator; ipr_err("Device %d : %s", i + 1, - ipr_format_res_path(dev_entry->res_path, buffer, - sizeof(buffer))); + __ipr_format_res_path(dev_entry->res_path, + buffer, sizeof(buffer))); ipr_log_ext_vpd(&dev_entry->vpd); ipr_err("-----New Device Information-----\n"); @@ -1925,14 +2055,16 @@ static void ipr_log64_fabric_path(struct ipr_hostrcb *hostrcb, ipr_hcam_err(hostrcb, "%s %s: Resource Path=%s\n", path_active_desc[i].desc, path_state_desc[j].desc, - ipr_format_res_path(fabric->res_path, buffer, - sizeof(buffer))); + ipr_format_res_path(hostrcb->ioa_cfg, + fabric->res_path, + buffer, sizeof(buffer))); return; } } ipr_err("Path state=%02X Resource Path=%s\n", path_state, - ipr_format_res_path(fabric->res_path, buffer, sizeof(buffer))); + ipr_format_res_path(hostrcb->ioa_cfg, fabric->res_path, + buffer, sizeof(buffer))); } static const struct { @@ -2073,18 +2205,20 @@ static void ipr_log64_path_elem(struct ipr_hostrcb *hostrcb, ipr_hcam_err(hostrcb, "%s %s: Resource Path=%s, Link rate=%s, WWN=%08X%08X\n", path_status_desc[j].desc, path_type_desc[i].desc, - ipr_format_res_path(cfg->res_path, buffer, - sizeof(buffer)), - link_rate[cfg->link_rate & IPR_PHY_LINK_RATE_MASK], - be32_to_cpu(cfg->wwid[0]), be32_to_cpu(cfg->wwid[1])); + ipr_format_res_path(hostrcb->ioa_cfg, + cfg->res_path, buffer, sizeof(buffer)), + link_rate[cfg->link_rate & IPR_PHY_LINK_RATE_MASK], + be32_to_cpu(cfg->wwid[0]), + be32_to_cpu(cfg->wwid[1])); return; } } ipr_hcam_err(hostrcb, "Path element=%02X: Resource Path=%s, Link rate=%s " "WWN=%08X%08X\n", cfg->type_status, - ipr_format_res_path(cfg->res_path, buffer, sizeof(buffer)), - link_rate[cfg->link_rate & IPR_PHY_LINK_RATE_MASK], - be32_to_cpu(cfg->wwid[0]), be32_to_cpu(cfg->wwid[1])); + ipr_format_res_path(hostrcb->ioa_cfg, + cfg->res_path, buffer, sizeof(buffer)), + link_rate[cfg->link_rate & IPR_PHY_LINK_RATE_MASK], + be32_to_cpu(cfg->wwid[0]), be32_to_cpu(cfg->wwid[1])); } /** @@ -2147,7 +2281,8 @@ static void ipr_log_sis64_array_error(struct ipr_ioa_cfg *ioa_cfg, ipr_err("RAID %s Array Configuration: %s\n", error->protection_level, - ipr_format_res_path(error->last_res_path, buffer, sizeof(buffer))); + ipr_format_res_path(ioa_cfg, error->last_res_path, + buffer, sizeof(buffer))); ipr_err_separator; @@ -2168,11 +2303,12 @@ static void ipr_log_sis64_array_error(struct ipr_ioa_cfg *ioa_cfg, ipr_err("Array Member %d:\n", i); ipr_log_ext_vpd(&array_entry->vpd); ipr_err("Current Location: %s\n", - ipr_format_res_path(array_entry->res_path, buffer, - sizeof(buffer))); + ipr_format_res_path(ioa_cfg, array_entry->res_path, + buffer, sizeof(buffer))); ipr_err("Expected Location: %s\n", - ipr_format_res_path(array_entry->expected_res_path, - buffer, sizeof(buffer))); + ipr_format_res_path(ioa_cfg, + array_entry->expected_res_path, + buffer, sizeof(buffer))); ipr_err_separator; } @@ -2232,6 +2368,42 @@ static void ipr_log_generic_error(struct ipr_ioa_cfg *ioa_cfg, } /** + * ipr_log_sis64_device_error - Log a cache error. + * @ioa_cfg: ioa config struct + * @hostrcb: hostrcb struct + * + * Return value: + * none + **/ +static void ipr_log_sis64_device_error(struct ipr_ioa_cfg *ioa_cfg, + struct ipr_hostrcb *hostrcb) +{ + struct ipr_hostrcb_type_21_error *error; + char buffer[IPR_MAX_RES_PATH_LENGTH]; + + error = &hostrcb->hcam.u.error64.u.type_21_error; + + ipr_err("-----Failing Device Information-----\n"); + ipr_err("World Wide Unique ID: %08X%08X%08X%08X\n", + be32_to_cpu(error->wwn[0]), be32_to_cpu(error->wwn[1]), + be32_to_cpu(error->wwn[2]), be32_to_cpu(error->wwn[3])); + ipr_err("Device Resource Path: %s\n", + __ipr_format_res_path(error->res_path, + buffer, sizeof(buffer))); + error->primary_problem_desc[sizeof(error->primary_problem_desc) - 1] = '\0'; + error->second_problem_desc[sizeof(error->second_problem_desc) - 1] = '\0'; + ipr_err("Primary Problem Description: %s\n", error->primary_problem_desc); + ipr_err("Secondary Problem Description: %s\n", error->second_problem_desc); + ipr_err("SCSI Sense Data:\n"); + ipr_log_hex_data(ioa_cfg, error->sense_data, sizeof(error->sense_data)); + ipr_err("SCSI Command Descriptor Block: \n"); + ipr_log_hex_data(ioa_cfg, error->cdb, sizeof(error->cdb)); + + ipr_err("Additional IOA Data:\n"); + ipr_log_hex_data(ioa_cfg, error->ioa_data, be32_to_cpu(error->length_of_error)); +} + +/** * ipr_get_error - Find the specfied IOASC in the ipr_error_table. * @ioasc: IOASC * @@ -2332,6 +2504,9 @@ static void ipr_handle_log_data(struct ipr_ioa_cfg *ioa_cfg, case IPR_HOST_RCB_OVERLAY_ID_20: ipr_log_fabric_error(ioa_cfg, hostrcb); break; + case IPR_HOST_RCB_OVERLAY_ID_21: + ipr_log_sis64_device_error(ioa_cfg, hostrcb); + break; case IPR_HOST_RCB_OVERLAY_ID_23: ipr_log_sis64_config_error(ioa_cfg, hostrcb); break; @@ -2374,7 +2549,7 @@ static void ipr_process_error(struct ipr_cmnd *ipr_cmd) fd_ioasc = be32_to_cpu(hostrcb->hcam.u.error.fd_ioasc); list_del(&hostrcb->queue); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); if (!ioasc) { ipr_handle_log_data(ioa_cfg, hostrcb); @@ -2456,36 +2631,6 @@ static void ipr_oper_timeout(struct ipr_cmnd *ipr_cmd) } /** - * ipr_reset_reload - Reset/Reload the IOA - * @ioa_cfg: ioa config struct - * @shutdown_type: shutdown type - * - * This function resets the adapter and re-initializes it. - * This function assumes that all new host commands have been stopped. - * Return value: - * SUCCESS / FAILED - **/ -static int ipr_reset_reload(struct ipr_ioa_cfg *ioa_cfg, - enum ipr_shutdown_type shutdown_type) -{ - if (!ioa_cfg->in_reset_reload) - ipr_initiate_ioa_reset(ioa_cfg, shutdown_type); - - spin_unlock_irq(ioa_cfg->host->host_lock); - wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); - spin_lock_irq(ioa_cfg->host->host_lock); - - /* If we got hit with a host reset while we were already resetting - the adapter for some reason, and the reset failed. */ - if (ioa_cfg->ioa_is_dead) { - ipr_trace; - return FAILED; - } - - return SUCCESS; -} - -/** * ipr_find_ses_entry - Find matching SES in SES table * @res: resource entry struct of SES * @@ -2718,13 +2863,18 @@ static int ipr_sdt_copy(struct ipr_ioa_cfg *ioa_cfg, unsigned long pci_address, u32 length) { int bytes_copied = 0; - int cur_len, rc, rem_len, rem_page_len; + int cur_len, rc, rem_len, rem_page_len, max_dump_size; __be32 *page; unsigned long lock_flags = 0; struct ipr_ioa_dump *ioa_dump = &ioa_cfg->dump->ioa_dump; + if (ioa_cfg->sis64) + max_dump_size = IPR_FMT3_MAX_IOA_DUMP_SIZE; + else + max_dump_size = IPR_FMT2_MAX_IOA_DUMP_SIZE; + while (bytes_copied < length && - (ioa_dump->hdr.len + bytes_copied) < IPR_MAX_IOA_DUMP_SIZE) { + (ioa_dump->hdr.len + bytes_copied) < max_dump_size) { if (ioa_dump->page_offset >= PAGE_SIZE || ioa_dump->page_offset == 0) { page = (__be32 *)__get_free_page(GFP_ATOMIC); @@ -2886,8 +3036,8 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) unsigned long lock_flags = 0; struct ipr_driver_dump *driver_dump = &dump->driver_dump; struct ipr_ioa_dump *ioa_dump = &dump->ioa_dump; - u32 num_entries, start_off, end_off; - u32 bytes_to_copy, bytes_copied, rc; + u32 num_entries, max_num_entries, start_off, end_off; + u32 max_dump_size, bytes_to_copy, bytes_copied, rc; struct ipr_sdt *sdt; int valid = 1; int i; @@ -2896,11 +3046,17 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->sdt_state != GET_DUMP) { + if (ioa_cfg->sdt_state != READ_DUMP) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); return; } + if (ioa_cfg->sis64) { + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + ssleep(IPR_DUMP_DELAY_SECONDS); + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); + } + start_addr = readl(ioa_cfg->ioa_mailbox); if (!ioa_cfg->sis64 && !ipr_sdt_is_fmt2(start_addr)) { @@ -2942,8 +3098,18 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) on entries in this table */ sdt = &ioa_dump->sdt; + if (ioa_cfg->sis64) { + max_num_entries = IPR_FMT3_NUM_SDT_ENTRIES; + max_dump_size = IPR_FMT3_MAX_IOA_DUMP_SIZE; + } else { + max_num_entries = IPR_FMT2_NUM_SDT_ENTRIES; + max_dump_size = IPR_FMT2_MAX_IOA_DUMP_SIZE; + } + + bytes_to_copy = offsetof(struct ipr_sdt, entry) + + (max_num_entries * sizeof(struct ipr_sdt_entry)); rc = ipr_get_ldump_data_section(ioa_cfg, start_addr, (__be32 *)sdt, - sizeof(struct ipr_sdt) / sizeof(__be32)); + bytes_to_copy / sizeof(__be32)); /* Smart Dump table is ready to use and the first entry is valid */ if (rc || ((be32_to_cpu(sdt->hdr.state) != IPR_FMT3_SDT_READY_TO_USE) && @@ -2959,13 +3125,20 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) num_entries = be32_to_cpu(sdt->hdr.num_entries_used); - if (num_entries > IPR_NUM_SDT_ENTRIES) - num_entries = IPR_NUM_SDT_ENTRIES; + if (num_entries > max_num_entries) + num_entries = max_num_entries; + + /* Update dump length to the actual data to be copied */ + dump->driver_dump.hdr.len += sizeof(struct ipr_sdt_header); + if (ioa_cfg->sis64) + dump->driver_dump.hdr.len += num_entries * sizeof(struct ipr_sdt_entry); + else + dump->driver_dump.hdr.len += max_num_entries * sizeof(struct ipr_sdt_entry); spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); for (i = 0; i < num_entries; i++) { - if (ioa_dump->hdr.len > IPR_MAX_IOA_DUMP_SIZE) { + if (ioa_dump->hdr.len > max_dump_size) { driver_dump->hdr.status = IPR_DUMP_STATUS_QUAL_SUCCESS; break; } @@ -2984,7 +3157,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) valid = 0; } if (valid) { - if (bytes_to_copy > IPR_MAX_IOA_DUMP_SIZE) { + if (bytes_to_copy > max_dump_size) { sdt->entry[i].flags &= ~IPR_SDT_VALID_ENTRY; continue; } @@ -3013,7 +3186,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) } #else -#define ipr_get_ioa_dump(ioa_cfg, dump) do { } while(0) +#define ipr_get_ioa_dump(ioa_cfg, dump) do { } while (0) #endif /** @@ -3025,7 +3198,7 @@ static void ipr_get_ioa_dump(struct ipr_ioa_cfg *ioa_cfg, struct ipr_dump *dump) **/ static void ipr_release_dump(struct kref *kref) { - struct ipr_dump *dump = container_of(kref,struct ipr_dump,kref); + struct ipr_dump *dump = container_of(kref, struct ipr_dump, kref); struct ipr_ioa_cfg *ioa_cfg = dump->ioa_cfg; unsigned long lock_flags = 0; int i; @@ -3039,6 +3212,7 @@ static void ipr_release_dump(struct kref *kref) for (i = 0; i < dump->ioa_dump.next_page_index; i++) free_page((unsigned long) dump->ioa_dump.ioa_data[i]); + vfree(dump->ioa_dump.ioa_data); kfree(dump); LEAVE; } @@ -3068,7 +3242,7 @@ static void ipr_worker_thread(struct work_struct *work) ENTER; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->sdt_state == GET_DUMP) { + if (ioa_cfg->sdt_state == READ_DUMP) { dump = ioa_cfg->dump; if (!dump) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); @@ -3080,7 +3254,7 @@ static void ipr_worker_thread(struct work_struct *work) kref_put(&dump->kref, ipr_release_dump); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->sdt_state == DUMP_OBTAINED) + if (ioa_cfg->sdt_state == DUMP_OBTAINED && !ioa_cfg->dump_timeout) ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); return; @@ -3089,7 +3263,8 @@ static void ipr_worker_thread(struct work_struct *work) restart: do { did_work = 0; - if (!ioa_cfg->allow_cmds || !ioa_cfg->allow_ml_add_del) { + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds || + !ioa_cfg->allow_ml_add_del) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); return; } @@ -3099,7 +3274,10 @@ restart: did_work = 1; sdev = res->sdev; if (!scsi_device_get(sdev)) { - list_move_tail(&res->queue, &ioa_cfg->free_res_q); + if (!res->add_to_ml) + list_move_tail(&res->queue, &ioa_cfg->free_res_q); + else + res->del_from_ml = 0; spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); scsi_remove_device(sdev); scsi_device_put(sdev); @@ -3108,7 +3286,7 @@ restart: break; } } - } while(did_work); + } while (did_work); list_for_each_entry(res, &ioa_cfg->used_res_q, queue) { if (res->add_to_ml) { @@ -3234,7 +3412,7 @@ static ssize_t ipr_show_log_level(struct device *dev, * number of bytes printed to buffer **/ static ssize_t ipr_store_log_level(struct device *dev, - struct device_attribute *attr, + struct device_attribute *attr, const char *buf, size_t count) { struct Scsi_Host *shost = class_to_shost(dev); @@ -3281,7 +3459,7 @@ static ssize_t ipr_store_diagnostics(struct device *dev, return -EACCES; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); @@ -3334,7 +3512,7 @@ static ssize_t ipr_show_adapter_state(struct device *dev, int len; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->ioa_is_dead) + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) len = snprintf(buf, PAGE_SIZE, "offline\n"); else len = snprintf(buf, PAGE_SIZE, "online\n"); @@ -3360,14 +3538,20 @@ static ssize_t ipr_store_adapter_state(struct device *dev, struct Scsi_Host *shost = class_to_shost(dev); struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata; unsigned long lock_flags; - int result = count; + int result = count, i; if (!capable(CAP_SYS_ADMIN)) return -EACCES; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - if (ioa_cfg->ioa_is_dead && !strncmp(buf, "online", 6)) { - ioa_cfg->ioa_is_dead = 0; + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead && + !strncmp(buf, "online", 6)) { + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].ioa_is_dead = 0; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); ioa_cfg->reset_retries = 0; ioa_cfg->in_ioa_bringdown = 0; ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); @@ -3427,6 +3611,93 @@ static struct device_attribute ipr_ioa_reset_attr = { .store = ipr_store_reset_adapter }; +static int ipr_iopoll(struct blk_iopoll *iop, int budget); + /** + * ipr_show_iopoll_weight - Show ipr polling mode + * @dev: class device struct + * @buf: buffer + * + * Return value: + * number of bytes printed to buffer + **/ +static ssize_t ipr_show_iopoll_weight(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata; + unsigned long lock_flags = 0; + int len; + + spin_lock_irqsave(shost->host_lock, lock_flags); + len = snprintf(buf, PAGE_SIZE, "%d\n", ioa_cfg->iopoll_weight); + spin_unlock_irqrestore(shost->host_lock, lock_flags); + + return len; +} + +/** + * ipr_store_iopoll_weight - Change the adapter's polling mode + * @dev: class device struct + * @buf: buffer + * + * Return value: + * number of bytes printed to buffer + **/ +static ssize_t ipr_store_iopoll_weight(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata; + unsigned long user_iopoll_weight; + unsigned long lock_flags = 0; + int i; + + if (!ioa_cfg->sis64) { + dev_info(&ioa_cfg->pdev->dev, "blk-iopoll not supported on this adapter\n"); + return -EINVAL; + } + if (kstrtoul(buf, 10, &user_iopoll_weight)) + return -EINVAL; + + if (user_iopoll_weight > 256) { + dev_info(&ioa_cfg->pdev->dev, "Invalid blk-iopoll weight. It must be less than 256\n"); + return -EINVAL; + } + + if (user_iopoll_weight == ioa_cfg->iopoll_weight) { + dev_info(&ioa_cfg->pdev->dev, "Current blk-iopoll weight has the same weight\n"); + return strlen(buf); + } + + if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { + for (i = 1; i < ioa_cfg->hrrq_num; i++) + blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); + } + + spin_lock_irqsave(shost->host_lock, lock_flags); + ioa_cfg->iopoll_weight = user_iopoll_weight; + if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { + for (i = 1; i < ioa_cfg->hrrq_num; i++) { + blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, + ioa_cfg->iopoll_weight, ipr_iopoll); + blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll); + } + } + spin_unlock_irqrestore(shost->host_lock, lock_flags); + + return strlen(buf); +} + +static struct device_attribute ipr_iopoll_weight_attr = { + .attr = { + .name = "iopoll_weight", + .mode = S_IRUGO | S_IWUSR, + }, + .show = ipr_show_iopoll_weight, + .store = ipr_store_iopoll_weight +}; + /** * ipr_alloc_ucode_buffer - Allocates a microcode download buffer * @buf_len: buffer length @@ -3648,7 +3919,7 @@ static int ipr_update_ioa_ucode(struct ipr_ioa_cfg *ioa_cfg, unsigned long lock_flags; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); @@ -3712,21 +3983,13 @@ static ssize_t ipr_store_update_fw(struct device *dev, len = snprintf(fname, 99, "%s", buf); fname[len-1] = '\0'; - if(request_firmware(&fw_entry, fname, &ioa_cfg->pdev->dev)) { + if (request_firmware(&fw_entry, fname, &ioa_cfg->pdev->dev)) { dev_err(&ioa_cfg->pdev->dev, "Firmware file %s not found\n", fname); return -EIO; } image_hdr = (struct ipr_ucode_image_header *)fw_entry->data; - if (be32_to_cpu(image_hdr->header_length) > fw_entry->size || - (ioa_cfg->vpd_cbs->page3_data.card_type && - ioa_cfg->vpd_cbs->page3_data.card_type != image_hdr->card_type)) { - dev_err(&ioa_cfg->pdev->dev, "Invalid microcode buffer\n"); - release_firmware(fw_entry); - return -EINVAL; - } - src = (u8 *)image_hdr + be32_to_cpu(image_hdr->header_length); dnld_size = fw_entry->size - be32_to_cpu(image_hdr->header_length); sglist = ipr_alloc_ucode_buffer(dnld_size); @@ -3745,6 +4008,8 @@ static ssize_t ipr_store_update_fw(struct device *dev, goto out; } + ipr_info("Updating microcode, please be patient. This may take up to 30 minutes.\n"); + result = ipr_update_ioa_ucode(ioa_cfg, sglist); if (!result) @@ -3801,6 +4066,7 @@ static struct device_attribute *ipr_ioa_attrs[] = { &ipr_ioa_reset_attr, &ipr_update_fw_attr, &ipr_ioa_fw_type_attr, + &ipr_iopoll_weight_attr, NULL, }; @@ -3827,7 +4093,7 @@ static ssize_t ipr_read_dump(struct file *filp, struct kobject *kobj, struct ipr_dump *dump; unsigned long lock_flags = 0; char *src; - int len; + int len, sdt_end; size_t rc = count; if (!capable(CAP_SYS_ADMIN)) @@ -3867,9 +4133,17 @@ static ssize_t ipr_read_dump(struct file *filp, struct kobject *kobj, off -= sizeof(dump->driver_dump); - if (count && off < offsetof(struct ipr_ioa_dump, ioa_data)) { - if (off + count > offsetof(struct ipr_ioa_dump, ioa_data)) - len = offsetof(struct ipr_ioa_dump, ioa_data) - off; + if (ioa_cfg->sis64) + sdt_end = offsetof(struct ipr_ioa_dump, sdt.entry) + + (be32_to_cpu(dump->ioa_dump.sdt.hdr.num_entries_used) * + sizeof(struct ipr_sdt_entry)); + else + sdt_end = offsetof(struct ipr_ioa_dump, sdt.entry) + + (IPR_FMT2_NUM_SDT_ENTRIES * sizeof(struct ipr_sdt_entry)); + + if (count && off < sdt_end) { + if (off + count > sdt_end) + len = sdt_end - off; else len = count; src = (u8 *)&dump->ioa_dump + off; @@ -3879,7 +4153,7 @@ static ssize_t ipr_read_dump(struct file *filp, struct kobject *kobj, count -= len; } - off -= offsetof(struct ipr_ioa_dump, ioa_data); + off -= sdt_end; while (count) { if ((off & PAGE_MASK) != ((off + count) & PAGE_MASK)) @@ -3908,6 +4182,7 @@ static ssize_t ipr_read_dump(struct file *filp, struct kobject *kobj, static int ipr_alloc_dump(struct ipr_ioa_cfg *ioa_cfg) { struct ipr_dump *dump; + __be32 **ioa_data; unsigned long lock_flags = 0; dump = kzalloc(sizeof(struct ipr_dump), GFP_KERNEL); @@ -3917,6 +4192,19 @@ static int ipr_alloc_dump(struct ipr_ioa_cfg *ioa_cfg) return -ENOMEM; } + if (ioa_cfg->sis64) + ioa_data = vmalloc(IPR_FMT3_MAX_NUM_DUMP_PAGES * sizeof(__be32 *)); + else + ioa_data = vmalloc(IPR_FMT2_MAX_NUM_DUMP_PAGES * sizeof(__be32 *)); + + if (!ioa_data) { + ipr_err("Dump memory allocation failed\n"); + kfree(dump); + return -ENOMEM; + } + + dump->ioa_dump.ioa_data = ioa_data; + kref_init(&dump->kref); dump->ioa_cfg = ioa_cfg; @@ -3924,13 +4212,14 @@ static int ipr_alloc_dump(struct ipr_ioa_cfg *ioa_cfg) if (INACTIVE != ioa_cfg->sdt_state) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + vfree(dump->ioa_dump.ioa_data); kfree(dump); return 0; } ioa_cfg->dump = dump; ioa_cfg->sdt_state = WAIT_FOR_DUMP; - if (ioa_cfg->ioa_is_dead && !ioa_cfg->dump_taken) { + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead && !ioa_cfg->dump_taken) { ioa_cfg->dump_taken = 1; schedule_work(&ioa_cfg->work_q); } @@ -4143,8 +4432,8 @@ static ssize_t ipr_show_resource_path(struct device *dev, struct device_attribut res = (struct ipr_resource_entry *)sdev->hostdata; if (res && ioa_cfg->sis64) len = snprintf(buf, PAGE_SIZE, "%s\n", - ipr_format_res_path(res->res_path, buffer, - sizeof(buffer))); + __ipr_format_res_path(res->res_path, buffer, + sizeof(buffer))); else if (res) len = snprintf(buf, PAGE_SIZE, "%d:%d:%d:%d\n", ioa_cfg->host->host_no, res->bus, res->target, res->lun); @@ -4290,8 +4579,7 @@ static struct ipr_resource_entry *ipr_find_starget(struct scsi_target *starget) list_for_each_entry(res, &ioa_cfg->used_res_q, queue) { if ((res->bus == starget->channel) && - (res->target == starget->id) && - (res->lun == 0)) { + (res->target == starget->id)) { return res; } } @@ -4365,12 +4653,14 @@ static void ipr_target_destroy(struct scsi_target *starget) struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *) shost->hostdata; if (ioa_cfg->sis64) { - if (starget->channel == IPR_ARRAY_VIRTUAL_BUS) - clear_bit(starget->id, ioa_cfg->array_ids); - else if (starget->channel == IPR_VSET_VIRTUAL_BUS) - clear_bit(starget->id, ioa_cfg->vset_ids); - else if (starget->channel == 0) - clear_bit(starget->id, ioa_cfg->target_ids); + if (!ipr_find_starget(starget)) { + if (starget->channel == IPR_ARRAY_VIRTUAL_BUS) + clear_bit(starget->id, ioa_cfg->array_ids); + else if (starget->channel == IPR_VSET_VIRTUAL_BUS) + clear_bit(starget->id, ioa_cfg->vset_ids); + else if (starget->channel == 0) + clear_bit(starget->id, ioa_cfg->target_ids); + } } if (sata_port) { @@ -4471,8 +4761,8 @@ static int ipr_slave_configure(struct scsi_device *sdev) scsi_adjust_queue_depth(sdev, 0, sdev->host->cmd_per_lun); if (ioa_cfg->sis64) sdev_printk(KERN_INFO, sdev, "Resource path: %s\n", - ipr_format_res_path(res->res_path, buffer, - sizeof(buffer))); + ipr_format_res_path(ioa_cfg, + res->res_path, buffer, sizeof(buffer))); return 0; } spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); @@ -4497,8 +4787,12 @@ static int ipr_ata_slave_alloc(struct scsi_device *sdev) ENTER; if (sdev->sdev_target) sata_port = sdev->sdev_target->hostdata; - if (sata_port) + if (sata_port) { rc = ata_sas_port_init(sata_port->ap); + if (rc == 0) + rc = ata_sas_sync_probe(sata_port->ap); + } + if (rc) ipr_slave_destroy(sdev); @@ -4549,41 +4843,38 @@ static int ipr_slave_alloc(struct scsi_device *sdev) return rc; } -/** - * ipr_eh_host_reset - Reset the host adapter - * @scsi_cmd: scsi command struct - * - * Return value: - * SUCCESS / FAILED - **/ -static int __ipr_eh_host_reset(struct scsi_cmnd * scsi_cmd) +static int ipr_eh_host_reset(struct scsi_cmnd *cmd) { struct ipr_ioa_cfg *ioa_cfg; - int rc; + unsigned long lock_flags = 0; + int rc = SUCCESS; ENTER; - ioa_cfg = (struct ipr_ioa_cfg *) scsi_cmd->device->host->hostdata; - - dev_err(&ioa_cfg->pdev->dev, - "Adapter being reset as a result of error recovery.\n"); - - if (WAIT_FOR_DUMP == ioa_cfg->sdt_state) - ioa_cfg->sdt_state = GET_DUMP; + ioa_cfg = (struct ipr_ioa_cfg *) cmd->device->host->hostdata; + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - rc = ipr_reset_reload(ioa_cfg, IPR_SHUTDOWN_ABBREV); + if (!ioa_cfg->in_reset_reload && !ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) { + ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_ABBREV); + dev_err(&ioa_cfg->pdev->dev, + "Adapter being reset as a result of error recovery.\n"); - LEAVE; - return rc; -} + if (WAIT_FOR_DUMP == ioa_cfg->sdt_state) + ioa_cfg->sdt_state = GET_DUMP; + } -static int ipr_eh_host_reset(struct scsi_cmnd * cmd) -{ - int rc; + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - spin_lock_irq(cmd->device->host->host_lock); - rc = __ipr_eh_host_reset(cmd); - spin_unlock_irq(cmd->device->host->host_lock); + /* If we got hit with a host reset while we were already resetting + the adapter for some reason, and the reset failed. */ + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) { + ipr_trace; + rc = FAILED; + } + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + LEAVE; return rc; } @@ -4632,7 +4923,7 @@ static int ipr_device_reset(struct ipr_ioa_cfg *ioa_cfg, ipr_send_blocking_cmd(ipr_cmd, ipr_timeout, IPR_DEVICE_RESET_TIMEOUT); ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); if (ipr_is_gata(res) && res->sata_port && ioasc != IPR_IOASC_IOA_WAS_RESET) { if (ipr_cmd->ioa_cfg->sis64) memcpy(&res->sata_port->ioasa, &ipr_cmd->s.ioasa64.u.gata, @@ -4643,7 +4934,7 @@ static int ipr_device_reset(struct ipr_ioa_cfg *ioa_cfg, } LEAVE; - return (IPR_IOASC_SENSE_KEY(ioasc) ? -EIO : 0); + return IPR_IOASC_SENSE_KEY(ioasc) ? -EIO : 0; } /** @@ -4667,7 +4958,7 @@ static int ipr_sata_reset(struct ata_link *link, unsigned int *classes, ENTER; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); @@ -4695,13 +4986,14 @@ static int ipr_sata_reset(struct ata_link *link, unsigned int *classes, * Return value: * SUCCESS / FAILED **/ -static int __ipr_eh_dev_reset(struct scsi_cmnd * scsi_cmd) +static int __ipr_eh_dev_reset(struct scsi_cmnd *scsi_cmd) { struct ipr_cmnd *ipr_cmd; struct ipr_ioa_cfg *ioa_cfg; struct ipr_resource_entry *res; struct ata_port *ap; int rc = 0; + struct ipr_hrr_queue *hrrq; ENTER; ioa_cfg = (struct ipr_ioa_cfg *) scsi_cmd->device->host->hostdata; @@ -4717,22 +5009,26 @@ static int __ipr_eh_dev_reset(struct scsi_cmnd * scsi_cmd) */ if (ioa_cfg->in_reset_reload) return FAILED; - if (ioa_cfg->ioa_is_dead) + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) return FAILED; - list_for_each_entry(ipr_cmd, &ioa_cfg->pending_q, queue) { - if (ipr_cmd->ioarcb.res_handle == res->res_handle) { - if (ipr_cmd->scsi_cmd) - ipr_cmd->done = ipr_scsi_eh_done; - if (ipr_cmd->qc) - ipr_cmd->done = ipr_sata_eh_done; - if (ipr_cmd->qc && !(ipr_cmd->qc->flags & ATA_QCFLAG_FAILED)) { - ipr_cmd->qc->err_mask |= AC_ERR_TIMEOUT; - ipr_cmd->qc->flags |= ATA_QCFLAG_FAILED; + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + list_for_each_entry(ipr_cmd, &hrrq->hrrq_pending_q, queue) { + if (ipr_cmd->ioarcb.res_handle == res->res_handle) { + if (ipr_cmd->scsi_cmd) + ipr_cmd->done = ipr_scsi_eh_done; + if (ipr_cmd->qc) + ipr_cmd->done = ipr_sata_eh_done; + if (ipr_cmd->qc && + !(ipr_cmd->qc->flags & ATA_QCFLAG_FAILED)) { + ipr_cmd->qc->err_mask |= AC_ERR_TIMEOUT; + ipr_cmd->qc->flags |= ATA_QCFLAG_FAILED; + } } } + spin_unlock(&hrrq->_lock); } - res->resetting_device = 1; scmd_printk(KERN_ERR, scsi_cmd, "Resetting device\n"); @@ -4742,21 +5038,28 @@ static int __ipr_eh_dev_reset(struct scsi_cmnd * scsi_cmd) ata_std_error_handler(ap); spin_lock_irq(scsi_cmd->device->host->host_lock); - list_for_each_entry(ipr_cmd, &ioa_cfg->pending_q, queue) { - if (ipr_cmd->ioarcb.res_handle == res->res_handle) { - rc = -EIO; - break; + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + list_for_each_entry(ipr_cmd, + &hrrq->hrrq_pending_q, queue) { + if (ipr_cmd->ioarcb.res_handle == + res->res_handle) { + rc = -EIO; + break; + } } + spin_unlock(&hrrq->_lock); } } else rc = ipr_device_reset(ioa_cfg, res); res->resetting_device = 0; + res->reset_occurred = 1; LEAVE; - return (rc ? FAILED : SUCCESS); + return rc ? FAILED : SUCCESS; } -static int ipr_eh_dev_reset(struct scsi_cmnd * cmd) +static int ipr_eh_dev_reset(struct scsi_cmnd *cmd) { int rc; @@ -4799,7 +5102,7 @@ static void ipr_bus_reset_done(struct ipr_cmnd *ipr_cmd) else ipr_cmd->sibling->done(ipr_cmd->sibling); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); LEAVE; } @@ -4852,14 +5155,15 @@ static void ipr_abort_timeout(struct ipr_cmnd *ipr_cmd) * Return value: * SUCCESS / FAILED **/ -static int ipr_cancel_op(struct scsi_cmnd * scsi_cmd) +static int ipr_cancel_op(struct scsi_cmnd *scsi_cmd) { struct ipr_cmnd *ipr_cmd; struct ipr_ioa_cfg *ioa_cfg; struct ipr_resource_entry *res; struct ipr_cmd_pkt *cmd_pkt; - u32 ioasc; + u32 ioasc, int_reg; int op_found = 0; + struct ipr_hrr_queue *hrrq; ENTER; ioa_cfg = (struct ipr_ioa_cfg *)scsi_cmd->device->host->hostdata; @@ -4869,17 +5173,32 @@ static int ipr_cancel_op(struct scsi_cmnd * scsi_cmd) * This will force the mid-layer to call ipr_eh_host_reset, * which will then go to sleep and wait for the reset to complete */ - if (ioa_cfg->in_reset_reload || ioa_cfg->ioa_is_dead) + if (ioa_cfg->in_reset_reload || + ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) return FAILED; - if (!res || !ipr_is_gscsi(res)) + if (!res) return FAILED; - list_for_each_entry(ipr_cmd, &ioa_cfg->pending_q, queue) { - if (ipr_cmd->scsi_cmd == scsi_cmd) { - ipr_cmd->done = ipr_scsi_eh_done; - op_found = 1; - break; + /* + * If we are aborting a timed out op, chances are that the timeout was caused + * by a still not detected EEH error. In such cases, reading a register will + * trigger the EEH recovery infrastructure. + */ + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg); + + if (!ipr_is_gscsi(res)) + return FAILED; + + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + list_for_each_entry(ipr_cmd, &hrrq->hrrq_pending_q, queue) { + if (ipr_cmd->scsi_cmd == scsi_cmd) { + ipr_cmd->done = ipr_scsi_eh_done; + op_found = 1; + break; + } } + spin_unlock(&hrrq->_lock); } if (!op_found) @@ -4906,12 +5225,12 @@ static int ipr_cancel_op(struct scsi_cmnd * scsi_cmd) ipr_trace; } - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); if (!ipr_is_naca_model(res)) res->needs_sync_complete = 1; LEAVE; - return (IPR_IOASC_SENSE_KEY(ioasc) ? FAILED : SUCCESS); + return IPR_IOASC_SENSE_KEY(ioasc) ? FAILED : SUCCESS; } /** @@ -4921,7 +5240,7 @@ static int ipr_cancel_op(struct scsi_cmnd * scsi_cmd) * Return value: * SUCCESS / FAILED **/ -static int ipr_eh_abort(struct scsi_cmnd * scsi_cmd) +static int ipr_eh_abort(struct scsi_cmnd *scsi_cmd) { unsigned long flags; int rc; @@ -4945,9 +5264,35 @@ static int ipr_eh_abort(struct scsi_cmnd * scsi_cmd) * IRQ_NONE / IRQ_HANDLED **/ static irqreturn_t ipr_handle_other_interrupt(struct ipr_ioa_cfg *ioa_cfg, - volatile u32 int_reg) + u32 int_reg) { irqreturn_t rc = IRQ_HANDLED; + u32 int_mask_reg; + + int_mask_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg32); + int_reg &= ~int_mask_reg; + + /* If an interrupt on the adapter did not occur, ignore it. + * Or in the case of SIS 64, check for a stage change interrupt. + */ + if ((int_reg & IPR_PCII_OPER_INTERRUPTS) == 0) { + if (ioa_cfg->sis64) { + int_mask_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg); + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg) & ~int_mask_reg; + if (int_reg & IPR_PCII_IPL_STAGE_CHANGE) { + + /* clear stage change */ + writel(IPR_PCII_IPL_STAGE_CHANGE, ioa_cfg->regs.clr_interrupt_reg); + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg) & ~int_mask_reg; + list_del(&ioa_cfg->reset_cmd->queue); + del_timer(&ioa_cfg->reset_cmd->timer); + ipr_reset_ioa_job(ioa_cfg->reset_cmd); + return IRQ_HANDLED; + } + } + + return IRQ_NONE; + } if (int_reg & IPR_PCII_IOA_TRANS_TO_OPER) { /* Mask the interrupt */ @@ -4960,9 +5305,21 @@ static irqreturn_t ipr_handle_other_interrupt(struct ipr_ioa_cfg *ioa_cfg, list_del(&ioa_cfg->reset_cmd->queue); del_timer(&ioa_cfg->reset_cmd->timer); ipr_reset_ioa_job(ioa_cfg->reset_cmd); + } else if ((int_reg & IPR_PCII_HRRQ_UPDATED) == int_reg) { + if (ioa_cfg->clear_isr) { + if (ipr_debug && printk_ratelimit()) + dev_err(&ioa_cfg->pdev->dev, + "Spurious interrupt detected. 0x%08X\n", int_reg); + writel(IPR_PCII_HRRQ_UPDATED, ioa_cfg->regs.clr_interrupt_reg32); + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg32); + return IRQ_NONE; + } } else { if (int_reg & IPR_PCII_IOA_UNIT_CHECKED) ioa_cfg->ioa_unit_checked = 1; + else if (int_reg & IPR_PCII_NO_HOST_RRQ) + dev_err(&ioa_cfg->pdev->dev, + "No Host RRQ. 0x%08X\n", int_reg); else dev_err(&ioa_cfg->pdev->dev, "Permanent IOA failure. 0x%08X\n", int_reg); @@ -4985,10 +5342,10 @@ static irqreturn_t ipr_handle_other_interrupt(struct ipr_ioa_cfg *ioa_cfg, * Return value: * none **/ -static void ipr_isr_eh(struct ipr_ioa_cfg *ioa_cfg, char *msg) +static void ipr_isr_eh(struct ipr_ioa_cfg *ioa_cfg, char *msg, u16 number) { ioa_cfg->errors_logged++; - dev_err(&ioa_cfg->pdev->dev, "%s\n", msg); + dev_err(&ioa_cfg->pdev->dev, "%s %d\n", msg, number); if (WAIT_FOR_DUMP == ioa_cfg->sdt_state) ioa_cfg->sdt_state = GET_DUMP; @@ -4996,109 +5353,135 @@ static void ipr_isr_eh(struct ipr_ioa_cfg *ioa_cfg, char *msg) ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); } -/** - * ipr_isr - Interrupt service routine - * @irq: irq number - * @devp: pointer to ioa config struct - * - * Return value: - * IRQ_NONE / IRQ_HANDLED - **/ -static irqreturn_t ipr_isr(int irq, void *devp) +static int ipr_process_hrrq(struct ipr_hrr_queue *hrr_queue, int budget, + struct list_head *doneq) { - struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)devp; - unsigned long lock_flags = 0; - volatile u32 int_reg, int_mask_reg; u32 ioasc; u16 cmd_index; - int num_hrrq = 0; struct ipr_cmnd *ipr_cmd; - irqreturn_t rc = IRQ_NONE; - - spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); + struct ipr_ioa_cfg *ioa_cfg = hrr_queue->ioa_cfg; + int num_hrrq = 0; /* If interrupts are disabled, ignore the interrupt */ - if (!ioa_cfg->allow_interrupts) { - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - return IRQ_NONE; - } + if (!hrr_queue->allow_interrupts) + return 0; - int_mask_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg32); - int_reg = readl(ioa_cfg->regs.sense_interrupt_reg32) & ~int_mask_reg; + while ((be32_to_cpu(*hrr_queue->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == + hrr_queue->toggle_bit) { - /* If an interrupt on the adapter did not occur, ignore it. - * Or in the case of SIS 64, check for a stage change interrupt. - */ - if (unlikely((int_reg & IPR_PCII_OPER_INTERRUPTS) == 0)) { - if (ioa_cfg->sis64) { - int_mask_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg); - int_reg = readl(ioa_cfg->regs.sense_interrupt_reg) & ~int_mask_reg; - if (int_reg & IPR_PCII_IPL_STAGE_CHANGE) { + cmd_index = (be32_to_cpu(*hrr_queue->hrrq_curr) & + IPR_HRRQ_REQ_RESP_HANDLE_MASK) >> + IPR_HRRQ_REQ_RESP_HANDLE_SHIFT; - /* clear stage change */ - writel(IPR_PCII_IPL_STAGE_CHANGE, ioa_cfg->regs.clr_interrupt_reg); - int_reg = readl(ioa_cfg->regs.sense_interrupt_reg) & ~int_mask_reg; - list_del(&ioa_cfg->reset_cmd->queue); - del_timer(&ioa_cfg->reset_cmd->timer); - ipr_reset_ioa_job(ioa_cfg->reset_cmd); - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - return IRQ_HANDLED; - } + if (unlikely(cmd_index > hrr_queue->max_cmd_id || + cmd_index < hrr_queue->min_cmd_id)) { + ipr_isr_eh(ioa_cfg, + "Invalid response handle from IOA: ", + cmd_index); + break; } - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - return IRQ_NONE; + ipr_cmd = ioa_cfg->ipr_cmnd_list[cmd_index]; + ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); + + ipr_trc_hook(ipr_cmd, IPR_TRACE_FINISH, ioasc); + + list_move_tail(&ipr_cmd->queue, doneq); + + if (hrr_queue->hrrq_curr < hrr_queue->hrrq_end) { + hrr_queue->hrrq_curr++; + } else { + hrr_queue->hrrq_curr = hrr_queue->hrrq_start; + hrr_queue->toggle_bit ^= 1u; + } + num_hrrq++; + if (budget > 0 && num_hrrq >= budget) + break; } - while (1) { - ipr_cmd = NULL; + return num_hrrq; +} - while ((be32_to_cpu(*ioa_cfg->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == - ioa_cfg->toggle_bit) { +static int ipr_iopoll(struct blk_iopoll *iop, int budget) +{ + struct ipr_ioa_cfg *ioa_cfg; + struct ipr_hrr_queue *hrrq; + struct ipr_cmnd *ipr_cmd, *temp; + unsigned long hrrq_flags; + int completed_ops; + LIST_HEAD(doneq); - cmd_index = (be32_to_cpu(*ioa_cfg->hrrq_curr) & - IPR_HRRQ_REQ_RESP_HANDLE_MASK) >> IPR_HRRQ_REQ_RESP_HANDLE_SHIFT; + hrrq = container_of(iop, struct ipr_hrr_queue, iopoll); + ioa_cfg = hrrq->ioa_cfg; - if (unlikely(cmd_index >= IPR_NUM_CMD_BLKS)) { - ipr_isr_eh(ioa_cfg, "Invalid response handle from IOA"); - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - return IRQ_HANDLED; - } + spin_lock_irqsave(hrrq->lock, hrrq_flags); + completed_ops = ipr_process_hrrq(hrrq, budget, &doneq); - ipr_cmd = ioa_cfg->ipr_cmnd_list[cmd_index]; + if (completed_ops < budget) + blk_iopoll_complete(iop); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); - ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); + list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) { + list_del(&ipr_cmd->queue); + del_timer(&ipr_cmd->timer); + ipr_cmd->fast_done(ipr_cmd); + } - ipr_trc_hook(ipr_cmd, IPR_TRACE_FINISH, ioasc); + return completed_ops; +} - list_del(&ipr_cmd->queue); - del_timer(&ipr_cmd->timer); - ipr_cmd->done(ipr_cmd); +/** + * ipr_isr - Interrupt service routine + * @irq: irq number + * @devp: pointer to ioa config struct + * + * Return value: + * IRQ_NONE / IRQ_HANDLED + **/ +static irqreturn_t ipr_isr(int irq, void *devp) +{ + struct ipr_hrr_queue *hrrq = (struct ipr_hrr_queue *)devp; + struct ipr_ioa_cfg *ioa_cfg = hrrq->ioa_cfg; + unsigned long hrrq_flags = 0; + u32 int_reg = 0; + int num_hrrq = 0; + int irq_none = 0; + struct ipr_cmnd *ipr_cmd, *temp; + irqreturn_t rc = IRQ_NONE; + LIST_HEAD(doneq); - rc = IRQ_HANDLED; + spin_lock_irqsave(hrrq->lock, hrrq_flags); + /* If interrupts are disabled, ignore the interrupt */ + if (!hrrq->allow_interrupts) { + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + return IRQ_NONE; + } - if (ioa_cfg->hrrq_curr < ioa_cfg->hrrq_end) { - ioa_cfg->hrrq_curr++; - } else { - ioa_cfg->hrrq_curr = ioa_cfg->hrrq_start; - ioa_cfg->toggle_bit ^= 1u; - } - } + while (1) { + if (ipr_process_hrrq(hrrq, -1, &doneq)) { + rc = IRQ_HANDLED; + + if (!ioa_cfg->clear_isr) + break; - if (ipr_cmd != NULL) { /* Clear the PCI interrupt */ + num_hrrq = 0; do { - writel(IPR_PCII_HRRQ_UPDATED, ioa_cfg->regs.clr_interrupt_reg32); - int_reg = readl(ioa_cfg->regs.sense_interrupt_reg32) & ~int_mask_reg; + writel(IPR_PCII_HRRQ_UPDATED, + ioa_cfg->regs.clr_interrupt_reg32); + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg32); } while (int_reg & IPR_PCII_HRRQ_UPDATED && - num_hrrq++ < IPR_MAX_HRRQ_RETRIES); - - if (int_reg & IPR_PCII_HRRQ_UPDATED) { - ipr_isr_eh(ioa_cfg, "Error clearing HRRQ"); - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - return IRQ_HANDLED; - } - + num_hrrq++ < IPR_MAX_HRRQ_RETRIES); + + } else if (rc == IRQ_NONE && irq_none == 0) { + int_reg = readl(ioa_cfg->regs.sense_interrupt_reg32); + irq_none++; + } else if (num_hrrq == IPR_MAX_HRRQ_RETRIES && + int_reg & IPR_PCII_HRRQ_UPDATED) { + ipr_isr_eh(ioa_cfg, + "Error clearing HRRQ: ", num_hrrq); + rc = IRQ_HANDLED; + break; } else break; } @@ -5106,7 +5489,63 @@ static irqreturn_t ipr_isr(int irq, void *devp) if (unlikely(rc == IRQ_NONE)) rc = ipr_handle_other_interrupt(ioa_cfg, int_reg); - spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) { + list_del(&ipr_cmd->queue); + del_timer(&ipr_cmd->timer); + ipr_cmd->fast_done(ipr_cmd); + } + return rc; +} + +/** + * ipr_isr_mhrrq - Interrupt service routine + * @irq: irq number + * @devp: pointer to ioa config struct + * + * Return value: + * IRQ_NONE / IRQ_HANDLED + **/ +static irqreturn_t ipr_isr_mhrrq(int irq, void *devp) +{ + struct ipr_hrr_queue *hrrq = (struct ipr_hrr_queue *)devp; + struct ipr_ioa_cfg *ioa_cfg = hrrq->ioa_cfg; + unsigned long hrrq_flags = 0; + struct ipr_cmnd *ipr_cmd, *temp; + irqreturn_t rc = IRQ_NONE; + LIST_HEAD(doneq); + + spin_lock_irqsave(hrrq->lock, hrrq_flags); + + /* If interrupts are disabled, ignore the interrupt */ + if (!hrrq->allow_interrupts) { + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + return IRQ_NONE; + } + + if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { + if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == + hrrq->toggle_bit) { + if (!blk_iopoll_sched_prep(&hrrq->iopoll)) + blk_iopoll_sched(&hrrq->iopoll); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + return IRQ_HANDLED; + } + } else { + if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == + hrrq->toggle_bit) + + if (ipr_process_hrrq(hrrq, -1, &doneq)) + rc = IRQ_HANDLED; + } + + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + + list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) { + list_del(&ipr_cmd->queue); + del_timer(&ipr_cmd->timer); + ipr_cmd->fast_done(ipr_cmd); + } return rc; } @@ -5135,7 +5574,8 @@ static int ipr_build_ioadl64(struct ipr_ioa_cfg *ioa_cfg, nseg = scsi_dma_map(scsi_cmd); if (nseg < 0) { - dev_err(&ioa_cfg->pdev->dev, "pci_map_sg failed!\n"); + if (printk_ratelimit()) + dev_err(&ioa_cfg->pdev->dev, "pci_map_sg failed!\n"); return -1; } @@ -5265,7 +5705,6 @@ static void ipr_erp_done(struct ipr_cmnd *ipr_cmd) { struct scsi_cmnd *scsi_cmd = ipr_cmd->scsi_cmd; struct ipr_resource_entry *res = scsi_cmd->device->hostdata; - struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); if (IPR_IOASC_SENSE_KEY(ioasc) > 0) { @@ -5283,7 +5722,7 @@ static void ipr_erp_done(struct ipr_cmnd *ipr_cmd) res->in_erp = 0; } scsi_dma_unmap(ipr_cmd->scsi_cmd); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); scsi_cmd->scsi_done(scsi_cmd); } @@ -5667,7 +6106,7 @@ static void ipr_erp_start(struct ipr_ioa_cfg *ioa_cfg, } scsi_dma_unmap(ipr_cmd->scsi_cmd); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); scsi_cmd->scsi_done(scsi_cmd); } @@ -5686,21 +6125,28 @@ static void ipr_scsi_done(struct ipr_cmnd *ipr_cmd) struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; struct scsi_cmnd *scsi_cmd = ipr_cmd->scsi_cmd; u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); + unsigned long hrrq_flags; scsi_set_resid(scsi_cmd, be32_to_cpu(ipr_cmd->s.ioasa.hdr.residual_data_len)); if (likely(IPR_IOASC_SENSE_KEY(ioasc) == 0)) { - scsi_dma_unmap(ipr_cmd->scsi_cmd); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + scsi_dma_unmap(scsi_cmd); + + spin_lock_irqsave(ipr_cmd->hrrq->lock, hrrq_flags); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); scsi_cmd->scsi_done(scsi_cmd); - } else + spin_unlock_irqrestore(ipr_cmd->hrrq->lock, hrrq_flags); + } else { + spin_lock_irqsave(ipr_cmd->hrrq->lock, hrrq_flags); ipr_erp_start(ioa_cfg, ipr_cmd); + spin_unlock_irqrestore(ipr_cmd->hrrq->lock, hrrq_flags); + } } /** * ipr_queuecommand - Queue a mid-layer request + * @shost: scsi host struct * @scsi_cmd: scsi command struct - * @done: done function * * This function queues a request generated by the mid-layer. * @@ -5709,86 +6155,123 @@ static void ipr_scsi_done(struct ipr_cmnd *ipr_cmd) * SCSI_MLQUEUE_DEVICE_BUSY if device is busy * SCSI_MLQUEUE_HOST_BUSY if host is busy **/ -static int ipr_queuecommand(struct scsi_cmnd *scsi_cmd, - void (*done) (struct scsi_cmnd *)) +static int ipr_queuecommand(struct Scsi_Host *shost, + struct scsi_cmnd *scsi_cmd) { struct ipr_ioa_cfg *ioa_cfg; struct ipr_resource_entry *res; struct ipr_ioarcb *ioarcb; struct ipr_cmnd *ipr_cmd; - int rc = 0; + unsigned long hrrq_flags, lock_flags; + int rc; + struct ipr_hrr_queue *hrrq; + int hrrq_id; + + ioa_cfg = (struct ipr_ioa_cfg *)shost->hostdata; - scsi_cmd->scsi_done = done; - ioa_cfg = (struct ipr_ioa_cfg *)scsi_cmd->device->host->hostdata; - res = scsi_cmd->device->hostdata; scsi_cmd->result = (DID_OK << 16); + res = scsi_cmd->device->hostdata; + + if (ipr_is_gata(res) && res->sata_port) { + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); + rc = ata_sas_queuecmd(scsi_cmd, res->sata_port->ap); + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + return rc; + } + + hrrq_id = ipr_get_hrrq_index(ioa_cfg); + hrrq = &ioa_cfg->hrrq[hrrq_id]; + spin_lock_irqsave(hrrq->lock, hrrq_flags); /* * We are currently blocking all devices due to a host reset * We have told the host to stop giving us new requests, but * ERP ops don't count. FIXME */ - if (unlikely(!ioa_cfg->allow_cmds && !ioa_cfg->ioa_is_dead)) + if (unlikely(!hrrq->allow_cmds && !hrrq->ioa_is_dead && !hrrq->removing_ioa)) { + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); return SCSI_MLQUEUE_HOST_BUSY; + } /* * FIXME - Create scsi_set_host_offline interface * and the ioa_is_dead check can be removed */ - if (unlikely(ioa_cfg->ioa_is_dead || !res)) { - memset(scsi_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); - scsi_cmd->result = (DID_NO_CONNECT << 16); - scsi_cmd->scsi_done(scsi_cmd); - return 0; + if (unlikely(hrrq->ioa_is_dead || hrrq->removing_ioa || !res)) { + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + goto err_nodev; } - if (ipr_is_gata(res) && res->sata_port) - return ata_sas_queuecmd(scsi_cmd, done, res->sata_port->ap); + ipr_cmd = __ipr_get_free_ipr_cmnd(hrrq); + if (ipr_cmd == NULL) { + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + return SCSI_MLQUEUE_HOST_BUSY; + } + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); - ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg); + ipr_init_ipr_cmnd(ipr_cmd, ipr_scsi_done); ioarcb = &ipr_cmd->ioarcb; - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); memcpy(ioarcb->cmd_pkt.cdb, scsi_cmd->cmnd, scsi_cmd->cmd_len); ipr_cmd->scsi_cmd = scsi_cmd; - ioarcb->res_handle = res->res_handle; - ipr_cmd->done = ipr_scsi_done; - ipr_trc_hook(ipr_cmd, IPR_TRACE_START, IPR_GET_RES_PHYS_LOC(res)); + ipr_cmd->done = ipr_scsi_eh_done; if (ipr_is_gscsi(res) || ipr_is_vset_device(res)) { if (scsi_cmd->underflow == 0) ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_NO_ULEN_CHK; - if (res->needs_sync_complete) { - ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_SYNC_COMPLETE; - res->needs_sync_complete = 0; - } - ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_NO_LINK_DESC; - ioarcb->cmd_pkt.flags_lo |= IPR_FLAGS_LO_DELAY_AFTER_RST; + if (ipr_is_gscsi(res) && res->reset_occurred) { + res->reset_occurred = 0; + ioarcb->cmd_pkt.flags_lo |= IPR_FLAGS_LO_DELAY_AFTER_RST; + } ioarcb->cmd_pkt.flags_lo |= IPR_FLAGS_LO_ALIGNED_BFR; ioarcb->cmd_pkt.flags_lo |= ipr_get_task_attributes(scsi_cmd); } if (scsi_cmd->cmnd[0] >= 0xC0 && - (!ipr_is_gscsi(res) || scsi_cmd->cmnd[0] == IPR_QUERY_RSRC_STATE)) + (!ipr_is_gscsi(res) || scsi_cmd->cmnd[0] == IPR_QUERY_RSRC_STATE)) { ioarcb->cmd_pkt.request_type = IPR_RQTYPE_IOACMD; + } - if (likely(rc == 0)) { - if (ioa_cfg->sis64) - rc = ipr_build_ioadl64(ioa_cfg, ipr_cmd); - else - rc = ipr_build_ioadl(ioa_cfg, ipr_cmd); + if (ioa_cfg->sis64) + rc = ipr_build_ioadl64(ioa_cfg, ipr_cmd); + else + rc = ipr_build_ioadl(ioa_cfg, ipr_cmd); + + spin_lock_irqsave(hrrq->lock, hrrq_flags); + if (unlikely(rc || (!hrrq->allow_cmds && !hrrq->ioa_is_dead))) { + list_add_tail(&ipr_cmd->queue, &hrrq->hrrq_free_q); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + if (!rc) + scsi_dma_unmap(scsi_cmd); + return SCSI_MLQUEUE_HOST_BUSY; } - if (likely(rc == 0)) { - mb(); - ipr_send_command(ipr_cmd); - } else { - list_move_tail(&ipr_cmd->queue, &ioa_cfg->free_q); - return SCSI_MLQUEUE_HOST_BUSY; + if (unlikely(hrrq->ioa_is_dead)) { + list_add_tail(&ipr_cmd->queue, &hrrq->hrrq_free_q); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + scsi_dma_unmap(scsi_cmd); + goto err_nodev; + } + + ioarcb->res_handle = res->res_handle; + if (res->needs_sync_complete) { + ioarcb->cmd_pkt.flags_hi |= IPR_FLAGS_HI_SYNC_COMPLETE; + res->needs_sync_complete = 0; } + list_add_tail(&ipr_cmd->queue, &hrrq->hrrq_pending_q); + ipr_trc_hook(ipr_cmd, IPR_TRACE_START, IPR_GET_RES_PHYS_LOC(res)); + ipr_send_command(ipr_cmd); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); + return 0; +err_nodev: + spin_lock_irqsave(hrrq->lock, hrrq_flags); + memset(scsi_cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE); + scsi_cmd->result = (DID_NO_CONNECT << 16); + scsi_cmd->scsi_done(scsi_cmd); + spin_unlock_irqrestore(hrrq->lock, hrrq_flags); return 0; } @@ -5822,7 +6305,7 @@ static int ipr_ioctl(struct scsi_device *sdev, int cmd, void __user *arg) * Return value: * pointer to buffer with description string **/ -static const char * ipr_ioa_info(struct Scsi_Host *host) +static const char *ipr_ioa_info(struct Scsi_Host *host) { static char buffer[512]; struct ipr_ioa_cfg *ioa_cfg; @@ -5862,7 +6345,8 @@ static struct scsi_host_template driver_template = { .use_clustering = ENABLE_CLUSTERING, .shost_attrs = ipr_ioa_attrs, .sdev_attrs = ipr_dev_attrs, - .proc_name = IPR_NAME + .proc_name = IPR_NAME, + .no_write_same = 1, }; /** @@ -5880,13 +6364,13 @@ static void ipr_ata_phy_reset(struct ata_port *ap) ENTER; spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, flags); } - if (!ioa_cfg->allow_cmds) + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds) goto out_unlock; rc = ipr_device_reset(ioa_cfg, res); @@ -5917,20 +6401,25 @@ static void ipr_ata_post_internal(struct ata_queued_cmd *qc) struct ipr_sata_port *sata_port = qc->ap->private_data; struct ipr_ioa_cfg *ioa_cfg = sata_port->ioa_cfg; struct ipr_cmnd *ipr_cmd; + struct ipr_hrr_queue *hrrq; unsigned long flags; spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, flags); } - list_for_each_entry(ipr_cmd, &ioa_cfg->pending_q, queue) { - if (ipr_cmd->qc == qc) { - ipr_device_reset(ioa_cfg, sata_port->res); - break; + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + list_for_each_entry(ipr_cmd, &hrrq->hrrq_pending_q, queue) { + if (ipr_cmd->qc == qc) { + ipr_device_reset(ioa_cfg, sata_port->res); + break; + } } + spin_unlock(&hrrq->_lock); } spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); } @@ -5979,6 +6468,7 @@ static void ipr_sata_done(struct ipr_cmnd *ipr_cmd) struct ipr_resource_entry *res = sata_port->res; u32 ioasc = be32_to_cpu(ipr_cmd->s.ioasa.hdr.ioasc); + spin_lock(&ipr_cmd->hrrq->_lock); if (ipr_cmd->ioa_cfg->sis64) memcpy(&sata_port->ioasa, &ipr_cmd->s.ioasa64.u.gata, sizeof(struct ipr_ioasa_gata)); @@ -5994,7 +6484,8 @@ static void ipr_sata_done(struct ipr_cmnd *ipr_cmd) qc->err_mask |= __ac_err_mask(sata_port->ioasa.status); else qc->err_mask |= ac_err_mask(sata_port->ioasa.status); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); + spin_unlock(&ipr_cmd->hrrq->_lock); ata_qc_complete(qc); } @@ -6009,7 +6500,7 @@ static void ipr_build_ata_ioadl64(struct ipr_cmnd *ipr_cmd, { u32 ioadl_flags = 0; struct ipr_ioarcb *ioarcb = &ipr_cmd->ioarcb; - struct ipr_ioadl64_desc *ioadl64 = ipr_cmd->i.ioadl64; + struct ipr_ioadl64_desc *ioadl64 = ipr_cmd->i.ata_ioadl.ioadl64; struct ipr_ioadl64_desc *last_ioadl64 = NULL; int len = qc->nbytes; struct scatterlist *sg; @@ -6029,7 +6520,7 @@ static void ipr_build_ata_ioadl64(struct ipr_cmnd *ipr_cmd, ioarcb->ioadl_len = cpu_to_be32(sizeof(struct ipr_ioadl64_desc) * ipr_cmd->dma_use_sg); ioarcb->u.sis64_addr_data.data_ioadl_addr = - cpu_to_be64(dma_addr + offsetof(struct ipr_cmnd, i.ata_ioadl)); + cpu_to_be64(dma_addr + offsetof(struct ipr_cmnd, i.ata_ioadl.ioadl64)); for_each_sg(qc->sg, sg, qc->n_elem, si) { ioadl64->flags = cpu_to_be32(ioadl_flags); @@ -6090,6 +6581,48 @@ static void ipr_build_ata_ioadl(struct ipr_cmnd *ipr_cmd, } /** + * ipr_qc_defer - Get a free ipr_cmd + * @qc: queued command + * + * Return value: + * 0 if success + **/ +static int ipr_qc_defer(struct ata_queued_cmd *qc) +{ + struct ata_port *ap = qc->ap; + struct ipr_sata_port *sata_port = ap->private_data; + struct ipr_ioa_cfg *ioa_cfg = sata_port->ioa_cfg; + struct ipr_cmnd *ipr_cmd; + struct ipr_hrr_queue *hrrq; + int hrrq_id; + + hrrq_id = ipr_get_hrrq_index(ioa_cfg); + hrrq = &ioa_cfg->hrrq[hrrq_id]; + + qc->lldd_task = NULL; + spin_lock(&hrrq->_lock); + if (unlikely(hrrq->ioa_is_dead)) { + spin_unlock(&hrrq->_lock); + return 0; + } + + if (unlikely(!hrrq->allow_cmds)) { + spin_unlock(&hrrq->_lock); + return ATA_DEFER_LINK; + } + + ipr_cmd = __ipr_get_free_ipr_cmnd(hrrq); + if (ipr_cmd == NULL) { + spin_unlock(&hrrq->_lock); + return ATA_DEFER_LINK; + } + + qc->lldd_task = ipr_cmd; + spin_unlock(&hrrq->_lock); + return 0; +} + +/** * ipr_qc_issue - Issue a SATA qc to a device * @qc: queued command * @@ -6106,10 +6639,23 @@ static unsigned int ipr_qc_issue(struct ata_queued_cmd *qc) struct ipr_ioarcb *ioarcb; struct ipr_ioarcb_ata_regs *regs; - if (unlikely(!ioa_cfg->allow_cmds || ioa_cfg->ioa_is_dead)) + if (qc->lldd_task == NULL) + ipr_qc_defer(qc); + + ipr_cmd = qc->lldd_task; + if (ipr_cmd == NULL) return AC_ERR_SYSTEM; - ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg); + qc->lldd_task = NULL; + spin_lock(&ipr_cmd->hrrq->_lock); + if (unlikely(!ipr_cmd->hrrq->allow_cmds || + ipr_cmd->hrrq->ioa_is_dead)) { + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); + spin_unlock(&ipr_cmd->hrrq->_lock); + return AC_ERR_SYSTEM; + } + + ipr_init_ipr_cmnd(ipr_cmd, ipr_lock_and_done); ioarcb = &ipr_cmd->ioarcb; if (ioa_cfg->sis64) { @@ -6121,7 +6667,7 @@ static unsigned int ipr_qc_issue(struct ata_queued_cmd *qc) memset(regs, 0, sizeof(*regs)); ioarcb->add_cmd_parms_len = cpu_to_be16(sizeof(*regs)); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); ipr_cmd->qc = qc; ipr_cmd->done = ipr_sata_done; ipr_cmd->ioarcb.res_handle = res->res_handle; @@ -6161,12 +6707,12 @@ static unsigned int ipr_qc_issue(struct ata_queued_cmd *qc) default: WARN_ON(1); + spin_unlock(&ipr_cmd->hrrq->_lock); return AC_ERR_INVALID; } - mb(); - ipr_send_command(ipr_cmd); + spin_unlock(&ipr_cmd->hrrq->_lock); return 0; } @@ -6195,7 +6741,6 @@ static bool ipr_qc_fill_rtf(struct ata_queued_cmd *qc) tf->hob_lbal = g->hob_lbal; tf->hob_lbam = g->hob_lbam; tf->hob_lbah = g->hob_lbah; - tf->ctl = g->alt_status; return true; } @@ -6205,6 +6750,7 @@ static struct ata_port_operations ipr_sata_ops = { .hardreset = ipr_sata_reset, .post_internal_cmd = ipr_ata_post_internal, .qc_prep = ata_noop_qc_prep, + .qc_defer = ipr_qc_defer, .qc_issue = ipr_qc_issue, .qc_fill_rtf = ipr_qc_fill_rtf, .port_start = ata_sas_port_start, @@ -6212,24 +6758,23 @@ static struct ata_port_operations ipr_sata_ops = { }; static struct ata_port_info sata_port_info = { - .flags = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | ATA_FLAG_SATA_RESET | - ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA, - .pio_mask = 0x10, /* pio4 */ - .mwdma_mask = 0x07, - .udma_mask = 0x7f, /* udma0-6 */ + .flags = ATA_FLAG_SATA | ATA_FLAG_PIO_DMA, + .pio_mask = ATA_PIO4_ONLY, + .mwdma_mask = ATA_MWDMA2, + .udma_mask = ATA_UDMA6, .port_ops = &ipr_sata_ops }; #ifdef CONFIG_PPC_PSERIES static const u16 ipr_blocked_processors[] = { - PV_NORTHSTAR, - PV_PULSAR, - PV_POWER4, - PV_ICESTAR, - PV_SSTAR, - PV_POWER4p, - PV_630, - PV_630p + PVR_NORTHSTAR, + PVR_PULSAR, + PVR_POWER4, + PVR_ICESTAR, + PVR_SSTAR, + PVR_POWER4p, + PVR_630, + PVR_630p }; /** @@ -6248,8 +6793,8 @@ static int ipr_invalid_adapter(struct ipr_ioa_cfg *ioa_cfg) int i; if ((ioa_cfg->type == 0x5702) && (ioa_cfg->pdev->revision < 4)) { - for (i = 0; i < ARRAY_SIZE(ipr_blocked_processors); i++){ - if (__is_processor(ipr_blocked_processors[i])) + for (i = 0; i < ARRAY_SIZE(ipr_blocked_processors); i++) { + if (pvr_version_is(ipr_blocked_processors[i])) return 1; } } @@ -6272,16 +6817,27 @@ static int ipr_invalid_adapter(struct ipr_ioa_cfg *ioa_cfg) static int ipr_ioa_bringdown_done(struct ipr_cmnd *ipr_cmd) { struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + int i; ENTER; + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) { + ipr_trace; + spin_unlock_irq(ioa_cfg->host->host_lock); + scsi_unblock_requests(ioa_cfg->host); + spin_lock_irq(ioa_cfg->host->host_lock); + } + ioa_cfg->in_reset_reload = 0; ioa_cfg->reset_retries = 0; - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); - wake_up_all(&ioa_cfg->reset_wait_q); + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].ioa_is_dead = 1; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); - spin_unlock_irq(ioa_cfg->host->host_lock); - scsi_unblock_requests(ioa_cfg->host); - spin_lock_irq(ioa_cfg->host->host_lock); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); + wake_up_all(&ioa_cfg->reset_wait_q); LEAVE; return IPR_RC_JOB_RETURN; @@ -6303,11 +6859,16 @@ static int ipr_ioa_reset_done(struct ipr_cmnd *ipr_cmd) struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; struct ipr_resource_entry *res; struct ipr_hostrcb *hostrcb, *temp; - int i = 0; + int i = 0, j; ENTER; ioa_cfg->in_reset_reload = 0; - ioa_cfg->allow_cmds = 1; + for (j = 0; j < ioa_cfg->hrrq_num; j++) { + spin_lock(&ioa_cfg->hrrq[j]._lock); + ioa_cfg->hrrq[j].allow_cmds = 1; + spin_unlock(&ioa_cfg->hrrq[j]._lock); + } + wmb(); ioa_cfg->reset_cmd = NULL; ioa_cfg->doorbell |= IPR_RUNTIME_RESET; @@ -6331,14 +6892,14 @@ static int ipr_ioa_reset_done(struct ipr_cmnd *ipr_cmd) dev_info(&ioa_cfg->pdev->dev, "IOA initialized.\n"); ioa_cfg->reset_retries = 0; - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); wake_up_all(&ioa_cfg->reset_wait_q); spin_unlock(ioa_cfg->host->host_lock); scsi_unblock_requests(ioa_cfg->host); spin_lock(ioa_cfg->host->host_lock); - if (!ioa_cfg->allow_cmds) + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds) scsi_block_requests(ioa_cfg->host); LEAVE; @@ -6409,9 +6970,11 @@ static int ipr_set_supported_devs(struct ipr_cmnd *ipr_cmd) if (!ioa_cfg->sis64) ipr_cmd->job_step = ipr_set_supported_devs; + LEAVE; return IPR_RC_JOB_RETURN; } + LEAVE; return IPR_RC_JOB_CONTINUE; } @@ -6526,7 +7089,7 @@ static void ipr_scsi_bus_speed_limit(struct ipr_ioa_cfg *ioa_cfg) * none **/ static void ipr_modify_ioafp_mode_page_28(struct ipr_ioa_cfg *ioa_cfg, - struct ipr_mode_pages *mode_pages) + struct ipr_mode_pages *mode_pages) { int i, entry_length; struct ipr_dev_bus_entry *bus; @@ -6669,7 +7232,7 @@ static int ipr_reset_cmd_failed(struct ipr_cmnd *ipr_cmd) ipr_cmd->ioarcb.cmd_pkt.cdb[0], ioasc); ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); return IPR_RC_JOB_RETURN; } @@ -7127,46 +7690,71 @@ static int ipr_ioafp_identify_hrrq(struct ipr_cmnd *ipr_cmd) { struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; struct ipr_ioarcb *ioarcb = &ipr_cmd->ioarcb; + struct ipr_hrr_queue *hrrq; ENTER; + ipr_cmd->job_step = ipr_ioafp_std_inquiry; dev_info(&ioa_cfg->pdev->dev, "Starting IOA initialization sequence.\n"); - ioarcb->cmd_pkt.cdb[0] = IPR_ID_HOST_RR_Q; - ioarcb->res_handle = cpu_to_be32(IPR_IOA_RES_HANDLE); + if (ioa_cfg->identify_hrrq_index < ioa_cfg->hrrq_num) { + hrrq = &ioa_cfg->hrrq[ioa_cfg->identify_hrrq_index]; - ioarcb->cmd_pkt.request_type = IPR_RQTYPE_IOACMD; - if (ioa_cfg->sis64) - ioarcb->cmd_pkt.cdb[1] = 0x1; - ioarcb->cmd_pkt.cdb[2] = - ((u64) ioa_cfg->host_rrq_dma >> 24) & 0xff; - ioarcb->cmd_pkt.cdb[3] = - ((u64) ioa_cfg->host_rrq_dma >> 16) & 0xff; - ioarcb->cmd_pkt.cdb[4] = - ((u64) ioa_cfg->host_rrq_dma >> 8) & 0xff; - ioarcb->cmd_pkt.cdb[5] = - ((u64) ioa_cfg->host_rrq_dma) & 0xff; - ioarcb->cmd_pkt.cdb[7] = - ((sizeof(u32) * IPR_NUM_CMD_BLKS) >> 8) & 0xff; - ioarcb->cmd_pkt.cdb[8] = - (sizeof(u32) * IPR_NUM_CMD_BLKS) & 0xff; + ioarcb->cmd_pkt.cdb[0] = IPR_ID_HOST_RR_Q; + ioarcb->res_handle = cpu_to_be32(IPR_IOA_RES_HANDLE); - if (ioa_cfg->sis64) { - ioarcb->cmd_pkt.cdb[10] = - ((u64) ioa_cfg->host_rrq_dma >> 56) & 0xff; - ioarcb->cmd_pkt.cdb[11] = - ((u64) ioa_cfg->host_rrq_dma >> 48) & 0xff; - ioarcb->cmd_pkt.cdb[12] = - ((u64) ioa_cfg->host_rrq_dma >> 40) & 0xff; - ioarcb->cmd_pkt.cdb[13] = - ((u64) ioa_cfg->host_rrq_dma >> 32) & 0xff; - } + ioarcb->cmd_pkt.request_type = IPR_RQTYPE_IOACMD; + if (ioa_cfg->sis64) + ioarcb->cmd_pkt.cdb[1] = 0x1; - ipr_cmd->job_step = ipr_ioafp_std_inquiry; + if (ioa_cfg->nvectors == 1) + ioarcb->cmd_pkt.cdb[1] &= ~IPR_ID_HRRQ_SELE_ENABLE; + else + ioarcb->cmd_pkt.cdb[1] |= IPR_ID_HRRQ_SELE_ENABLE; + + ioarcb->cmd_pkt.cdb[2] = + ((u64) hrrq->host_rrq_dma >> 24) & 0xff; + ioarcb->cmd_pkt.cdb[3] = + ((u64) hrrq->host_rrq_dma >> 16) & 0xff; + ioarcb->cmd_pkt.cdb[4] = + ((u64) hrrq->host_rrq_dma >> 8) & 0xff; + ioarcb->cmd_pkt.cdb[5] = + ((u64) hrrq->host_rrq_dma) & 0xff; + ioarcb->cmd_pkt.cdb[7] = + ((sizeof(u32) * hrrq->size) >> 8) & 0xff; + ioarcb->cmd_pkt.cdb[8] = + (sizeof(u32) * hrrq->size) & 0xff; + + if (ioarcb->cmd_pkt.cdb[1] & IPR_ID_HRRQ_SELE_ENABLE) + ioarcb->cmd_pkt.cdb[9] = + ioa_cfg->identify_hrrq_index; - ipr_do_req(ipr_cmd, ipr_reset_ioa_job, ipr_timeout, IPR_INTERNAL_TIMEOUT); + if (ioa_cfg->sis64) { + ioarcb->cmd_pkt.cdb[10] = + ((u64) hrrq->host_rrq_dma >> 56) & 0xff; + ioarcb->cmd_pkt.cdb[11] = + ((u64) hrrq->host_rrq_dma >> 48) & 0xff; + ioarcb->cmd_pkt.cdb[12] = + ((u64) hrrq->host_rrq_dma >> 40) & 0xff; + ioarcb->cmd_pkt.cdb[13] = + ((u64) hrrq->host_rrq_dma >> 32) & 0xff; + } + + if (ioarcb->cmd_pkt.cdb[1] & IPR_ID_HRRQ_SELE_ENABLE) + ioarcb->cmd_pkt.cdb[14] = + ioa_cfg->identify_hrrq_index; + + ipr_do_req(ipr_cmd, ipr_reset_ioa_job, ipr_timeout, + IPR_INTERNAL_TIMEOUT); + + if (++ioa_cfg->identify_hrrq_index < ioa_cfg->hrrq_num) + ipr_cmd->job_step = ipr_ioafp_identify_hrrq; + + LEAVE; + return IPR_RC_JOB_RETURN; + } LEAVE; - return IPR_RC_JOB_RETURN; + return IPR_RC_JOB_CONTINUE; } /** @@ -7214,7 +7802,9 @@ static void ipr_reset_timer_done(struct ipr_cmnd *ipr_cmd) static void ipr_reset_start_timer(struct ipr_cmnd *ipr_cmd, unsigned long timeout) { - list_add_tail(&ipr_cmd->queue, &ipr_cmd->ioa_cfg->pending_q); + + ENTER; + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); ipr_cmd->done = ipr_reset_ioa_job; ipr_cmd->timer.data = (unsigned long) ipr_cmd; @@ -7232,13 +7822,26 @@ static void ipr_reset_start_timer(struct ipr_cmnd *ipr_cmd, **/ static void ipr_init_ioa_mem(struct ipr_ioa_cfg *ioa_cfg) { - memset(ioa_cfg->host_rrq, 0, sizeof(u32) * IPR_NUM_CMD_BLKS); + struct ipr_hrr_queue *hrrq; + + for_each_hrrq(hrrq, ioa_cfg) { + spin_lock(&hrrq->_lock); + memset(hrrq->host_rrq, 0, sizeof(u32) * hrrq->size); + + /* Initialize Host RRQ pointers */ + hrrq->hrrq_start = hrrq->host_rrq; + hrrq->hrrq_end = &hrrq->host_rrq[hrrq->size - 1]; + hrrq->hrrq_curr = hrrq->hrrq_start; + hrrq->toggle_bit = 1; + spin_unlock(&hrrq->_lock); + } + wmb(); - /* Initialize Host RRQ pointers */ - ioa_cfg->hrrq_start = ioa_cfg->host_rrq; - ioa_cfg->hrrq_end = &ioa_cfg->host_rrq[IPR_NUM_CMD_BLKS - 1]; - ioa_cfg->hrrq_curr = ioa_cfg->hrrq_start; - ioa_cfg->toggle_bit = 1; + ioa_cfg->identify_hrrq_index = 0; + if (ioa_cfg->hrrq_num == 1) + atomic_set(&ioa_cfg->hrrq_index, 0); + else + atomic_set(&ioa_cfg->hrrq_index, 1); /* Zero out config table */ memset(ioa_cfg->u.cfg_table, 0, ioa_cfg->cfg_table_size); @@ -7295,7 +7898,8 @@ static int ipr_reset_next_stage(struct ipr_cmnd *ipr_cmd) ipr_cmd->timer.function = (void (*)(unsigned long))ipr_oper_timeout; ipr_cmd->done = ipr_reset_ioa_job; add_timer(&ipr_cmd->timer); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); + + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); return IPR_RC_JOB_RETURN; } @@ -7315,12 +7919,18 @@ static int ipr_reset_enable_ioa(struct ipr_cmnd *ipr_cmd) struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; volatile u32 int_reg; volatile u64 maskval; + int i; ENTER; ipr_cmd->job_step = ipr_ioafp_identify_hrrq; ipr_init_ioa_mem(ioa_cfg); - ioa_cfg->allow_interrupts = 1; + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].allow_interrupts = 1; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); if (ioa_cfg->sis64) { /* Set the adapter to the correct endian mode. */ writel(IPR_ENDIAN_SWAP_KEY, ioa_cfg->regs.endian_swap_reg); @@ -7360,7 +7970,7 @@ static int ipr_reset_enable_ioa(struct ipr_cmnd *ipr_cmd) ipr_cmd->timer.function = (void (*)(unsigned long))ipr_oper_timeout; ipr_cmd->done = ipr_reset_ioa_job; add_timer(&ipr_cmd->timer); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->pending_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); LEAVE; return IPR_RC_JOB_RETURN; @@ -7381,8 +7991,11 @@ static int ipr_reset_wait_for_dump(struct ipr_cmnd *ipr_cmd) struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; if (ioa_cfg->sdt_state == GET_DUMP) + ioa_cfg->sdt_state = WAIT_FOR_DUMP; + else if (ioa_cfg->sdt_state == READ_DUMP) ioa_cfg->sdt_state = ABORT_DUMP; + ioa_cfg->dump_timeout = 1; ipr_cmd->job_step = ipr_reset_alert; return IPR_RC_JOB_CONTINUE; @@ -7471,6 +8084,29 @@ static void ipr_get_unit_check_buffer(struct ipr_ioa_cfg *ioa_cfg) } /** + * ipr_reset_get_unit_check_job - Call to get the unit check buffer. + * @ipr_cmd: ipr command struct + * + * Description: This function will call to get the unit check buffer. + * + * Return value: + * IPR_RC_JOB_RETURN + **/ +static int ipr_reset_get_unit_check_job(struct ipr_cmnd *ipr_cmd) +{ + struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + + ENTER; + ioa_cfg->ioa_unit_checked = 0; + ipr_get_unit_check_buffer(ioa_cfg); + ipr_cmd->job_step = ipr_reset_alert; + ipr_reset_start_timer(ipr_cmd, 0); + + LEAVE; + return IPR_RC_JOB_RETURN; +} + +/** * ipr_reset_restore_cfg_space - Restore PCI config space. * @ipr_cmd: ipr command struct * @@ -7484,17 +8120,11 @@ static void ipr_get_unit_check_buffer(struct ipr_ioa_cfg *ioa_cfg) static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) { struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; - volatile u32 int_reg; - int rc; + u32 int_reg; ENTER; ioa_cfg->pdev->state_saved = true; - rc = pci_restore_state(ioa_cfg->pdev); - - if (rc != PCIBIOS_SUCCESSFUL) { - ipr_cmd->s.ioasa.hdr.ioasc = cpu_to_be32(IPR_IOASC_PCI_ACCESS_ERROR); - return IPR_RC_JOB_CONTINUE; - } + pci_restore_state(ioa_cfg->pdev); if (ipr_set_pcix_cmd_reg(ioa_cfg)) { ipr_cmd->s.ioasa.hdr.ioasc = cpu_to_be32(IPR_IOASC_PCI_ACCESS_ERROR); @@ -7510,11 +8140,17 @@ static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) } if (ioa_cfg->ioa_unit_checked) { - ioa_cfg->ioa_unit_checked = 0; - ipr_get_unit_check_buffer(ioa_cfg); - ipr_cmd->job_step = ipr_reset_alert; - ipr_reset_start_timer(ipr_cmd, 0); - return IPR_RC_JOB_RETURN; + if (ioa_cfg->sis64) { + ipr_cmd->job_step = ipr_reset_get_unit_check_job; + ipr_reset_start_timer(ipr_cmd, IPR_DUMP_DELAY_TIMEOUT); + return IPR_RC_JOB_RETURN; + } else { + ioa_cfg->ioa_unit_checked = 0; + ipr_get_unit_check_buffer(ioa_cfg); + ipr_cmd->job_step = ipr_reset_alert; + ipr_reset_start_timer(ipr_cmd, 0); + return IPR_RC_JOB_RETURN; + } } if (ioa_cfg->in_ioa_bringdown) { @@ -7523,7 +8159,12 @@ static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) ipr_cmd->job_step = ipr_reset_enable_ioa; if (GET_DUMP == ioa_cfg->sdt_state) { - ipr_reset_start_timer(ipr_cmd, IPR_DUMP_TIMEOUT); + ioa_cfg->sdt_state = READ_DUMP; + ioa_cfg->dump_timeout = 0; + if (ioa_cfg->sis64) + ipr_reset_start_timer(ipr_cmd, IPR_SIS64_DUMP_TIMEOUT); + else + ipr_reset_start_timer(ipr_cmd, IPR_SIS32_DUMP_TIMEOUT); ipr_cmd->job_step = ipr_reset_wait_for_dump; schedule_work(&ioa_cfg->work_q); return IPR_RC_JOB_RETURN; @@ -7545,8 +8186,12 @@ static int ipr_reset_restore_cfg_space(struct ipr_cmnd *ipr_cmd) **/ static int ipr_reset_bist_done(struct ipr_cmnd *ipr_cmd) { + struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + ENTER; - pci_unblock_user_cfg_access(ipr_cmd->ioa_cfg->pdev); + if (ioa_cfg->cfg_locked) + pci_cfg_access_unlock(ioa_cfg->pdev); + ioa_cfg->cfg_locked = 0; ipr_cmd->job_step = ipr_reset_restore_cfg_space; LEAVE; return IPR_RC_JOB_CONTINUE; @@ -7567,8 +8212,6 @@ static int ipr_reset_start_bist(struct ipr_cmnd *ipr_cmd) int rc = PCIBIOS_SUCCESSFUL; ENTER; - pci_block_user_cfg_access(ioa_cfg->pdev); - if (ioa_cfg->ipr_chip->bist_method == IPR_MMIO) writel(IPR_UPROCI_SIS64_START_BIST, ioa_cfg->regs.set_uproc_interrupt_reg32); @@ -7580,7 +8223,9 @@ static int ipr_reset_start_bist(struct ipr_cmnd *ipr_cmd) ipr_reset_start_timer(ipr_cmd, IPR_WAIT_FOR_BIST_TIMEOUT); rc = IPR_RC_JOB_RETURN; } else { - pci_unblock_user_cfg_access(ipr_cmd->ioa_cfg->pdev); + if (ioa_cfg->cfg_locked) + pci_cfg_access_unlock(ipr_cmd->ioa_cfg->pdev); + ioa_cfg->cfg_locked = 0; ipr_cmd->s.ioasa.hdr.ioasc = cpu_to_be32(IPR_IOASC_PCI_ACCESS_ERROR); rc = IPR_RC_JOB_CONTINUE; } @@ -7623,7 +8268,6 @@ static int ipr_reset_slot_reset(struct ipr_cmnd *ipr_cmd) struct pci_dev *pdev = ioa_cfg->pdev; ENTER; - pci_block_user_cfg_access(pdev); pci_set_pcie_reset_state(pdev, pcie_warm_reset); ipr_cmd->job_step = ipr_reset_slot_reset_done; ipr_reset_start_timer(ipr_cmd, IPR_PCI_RESET_TIMEOUT); @@ -7632,6 +8276,56 @@ static int ipr_reset_slot_reset(struct ipr_cmnd *ipr_cmd) } /** + * ipr_reset_block_config_access_wait - Wait for permission to block config access + * @ipr_cmd: ipr command struct + * + * Description: This attempts to block config access to the IOA. + * + * Return value: + * IPR_RC_JOB_CONTINUE / IPR_RC_JOB_RETURN + **/ +static int ipr_reset_block_config_access_wait(struct ipr_cmnd *ipr_cmd) +{ + struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + int rc = IPR_RC_JOB_CONTINUE; + + if (pci_cfg_access_trylock(ioa_cfg->pdev)) { + ioa_cfg->cfg_locked = 1; + ipr_cmd->job_step = ioa_cfg->reset; + } else { + if (ipr_cmd->u.time_left) { + rc = IPR_RC_JOB_RETURN; + ipr_cmd->u.time_left -= IPR_CHECK_FOR_RESET_TIMEOUT; + ipr_reset_start_timer(ipr_cmd, + IPR_CHECK_FOR_RESET_TIMEOUT); + } else { + ipr_cmd->job_step = ioa_cfg->reset; + dev_err(&ioa_cfg->pdev->dev, + "Timed out waiting to lock config access. Resetting anyway.\n"); + } + } + + return rc; +} + +/** + * ipr_reset_block_config_access - Block config access to the IOA + * @ipr_cmd: ipr command struct + * + * Description: This attempts to block config access to the IOA + * + * Return value: + * IPR_RC_JOB_CONTINUE + **/ +static int ipr_reset_block_config_access(struct ipr_cmnd *ipr_cmd) +{ + ipr_cmd->ioa_cfg->cfg_locked = 0; + ipr_cmd->job_step = ipr_reset_block_config_access_wait; + ipr_cmd->u.time_left = IPR_WAIT_FOR_RESET_TIMEOUT; + return IPR_RC_JOB_CONTINUE; +} + +/** * ipr_reset_allowed - Query whether or not IOA can be reset * @ioa_cfg: ioa config struct * @@ -7670,7 +8364,7 @@ static int ipr_reset_wait_to_start_bist(struct ipr_cmnd *ipr_cmd) ipr_cmd->u.time_left -= IPR_CHECK_FOR_RESET_TIMEOUT; ipr_reset_start_timer(ipr_cmd, IPR_CHECK_FOR_RESET_TIMEOUT); } else { - ipr_cmd->job_step = ioa_cfg->reset; + ipr_cmd->job_step = ipr_reset_block_config_access; rc = IPR_RC_JOB_CONTINUE; } @@ -7703,7 +8397,7 @@ static int ipr_reset_alert(struct ipr_cmnd *ipr_cmd) writel(IPR_UPROCI_RESET_ALERT, ioa_cfg->regs.set_uproc_interrupt_reg32); ipr_cmd->job_step = ipr_reset_wait_to_start_bist; } else { - ipr_cmd->job_step = ioa_cfg->reset; + ipr_cmd->job_step = ipr_reset_block_config_access; } ipr_cmd->u.time_left = IPR_WAIT_FOR_RESET_TIMEOUT; @@ -7795,7 +8489,8 @@ static int ipr_reset_shutdown_ioa(struct ipr_cmnd *ipr_cmd) int rc = IPR_RC_JOB_CONTINUE; ENTER; - if (shutdown_type != IPR_SHUTDOWN_NONE && !ioa_cfg->ioa_is_dead) { + if (shutdown_type != IPR_SHUTDOWN_NONE && + !ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) { ipr_cmd->ioarcb.res_handle = cpu_to_be32(IPR_IOA_RES_HANDLE); ipr_cmd->ioarcb.cmd_pkt.request_type = IPR_RQTYPE_IOACMD; ipr_cmd->ioarcb.cmd_pkt.cdb[0] = IPR_IOA_SHUTDOWN; @@ -7843,7 +8538,8 @@ static void ipr_reset_ioa_job(struct ipr_cmnd *ipr_cmd) * We are doing nested adapter resets and this is * not the current reset job. */ - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, + &ipr_cmd->hrrq->hrrq_free_q); return; } @@ -7856,7 +8552,7 @@ static void ipr_reset_ioa_job(struct ipr_cmnd *ipr_cmd) ipr_reinit_ipr_cmnd(ipr_cmd); ipr_cmd->job_step_failed = ipr_reset_cmd_failed; rc = ipr_cmd->job_step(ipr_cmd); - } while(rc == IPR_RC_JOB_CONTINUE); + } while (rc == IPR_RC_JOB_CONTINUE); } /** @@ -7878,10 +8574,17 @@ static void _ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg, enum ipr_shutdown_type shutdown_type) { struct ipr_cmnd *ipr_cmd; + int i; ioa_cfg->in_reset_reload = 1; - ioa_cfg->allow_cmds = 0; - scsi_block_requests(ioa_cfg->host); + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].allow_cmds = 0; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) + scsi_block_requests(ioa_cfg->host); ipr_cmd = ipr_get_free_ipr_cmnd(ioa_cfg); ioa_cfg->reset_cmd = ipr_cmd; @@ -7906,18 +8609,29 @@ static void _ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg, static void ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg, enum ipr_shutdown_type shutdown_type) { - if (ioa_cfg->ioa_is_dead) + int i; + + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) return; - if (ioa_cfg->in_reset_reload && ioa_cfg->sdt_state == GET_DUMP) - ioa_cfg->sdt_state = ABORT_DUMP; + if (ioa_cfg->in_reset_reload) { + if (ioa_cfg->sdt_state == GET_DUMP) + ioa_cfg->sdt_state = WAIT_FOR_DUMP; + else if (ioa_cfg->sdt_state == READ_DUMP) + ioa_cfg->sdt_state = ABORT_DUMP; + } if (ioa_cfg->reset_retries++ >= IPR_NUM_RESET_RELOAD_RETRIES) { dev_err(&ioa_cfg->pdev->dev, "IOA taken offline - error recovery failed\n"); ioa_cfg->reset_retries = 0; - ioa_cfg->ioa_is_dead = 1; + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].ioa_is_dead = 1; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); if (ioa_cfg->in_ioa_bringdown) { ioa_cfg->reset_cmd = NULL; @@ -7925,9 +8639,11 @@ static void ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg, ipr_fail_all_ops(ioa_cfg); wake_up_all(&ioa_cfg->reset_wait_q); - spin_unlock_irq(ioa_cfg->host->host_lock); - scsi_unblock_requests(ioa_cfg->host); - spin_lock_irq(ioa_cfg->host->host_lock); + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].removing_ioa) { + spin_unlock_irq(ioa_cfg->host->host_lock); + scsi_unblock_requests(ioa_cfg->host); + spin_lock_irq(ioa_cfg->host->host_lock); + } return; } else { ioa_cfg->in_ioa_bringdown = 1; @@ -7949,14 +8665,41 @@ static void ipr_initiate_ioa_reset(struct ipr_ioa_cfg *ioa_cfg, */ static int ipr_reset_freeze(struct ipr_cmnd *ipr_cmd) { + struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; + int i; + /* Disallow new interrupts, avoid loop */ - ipr_cmd->ioa_cfg->allow_interrupts = 0; - list_add_tail(&ipr_cmd->queue, &ipr_cmd->ioa_cfg->pending_q); + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].allow_interrupts = 0; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_pending_q); ipr_cmd->done = ipr_reset_ioa_job; return IPR_RC_JOB_RETURN; } /** + * ipr_pci_mmio_enabled - Called when MMIO has been re-enabled + * @pdev: PCI device struct + * + * Description: This routine is called to tell us that the MMIO + * access to the IOA has been restored + */ +static pci_ers_result_t ipr_pci_mmio_enabled(struct pci_dev *pdev) +{ + unsigned long flags = 0; + struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); + + spin_lock_irqsave(ioa_cfg->host->host_lock, flags); + if (!ioa_cfg->probe_done) + pci_save_state(pdev); + spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); + return PCI_ERS_RESULT_NEED_RESET; +} + +/** * ipr_pci_frozen - Called when slot has experienced a PCI bus error. * @pdev: PCI device struct * @@ -7970,7 +8713,8 @@ static void ipr_pci_frozen(struct pci_dev *pdev) struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - _ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_freeze, IPR_SHUTDOWN_NONE); + if (ioa_cfg->probe_done) + _ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_freeze, IPR_SHUTDOWN_NONE); spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); } @@ -7988,11 +8732,14 @@ static pci_ers_result_t ipr_pci_slot_reset(struct pci_dev *pdev) struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - if (ioa_cfg->needs_warm_reset) - ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); - else - _ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_restore_cfg_space, - IPR_SHUTDOWN_NONE); + if (ioa_cfg->probe_done) { + if (ioa_cfg->needs_warm_reset) + ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); + else + _ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_restore_cfg_space, + IPR_SHUTDOWN_NONE); + } else + wake_up_all(&ioa_cfg->eeh_wait_q); spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); return PCI_ERS_RESULT_RECOVERED; } @@ -8008,14 +8755,23 @@ static void ipr_pci_perm_failure(struct pci_dev *pdev) { unsigned long flags = 0; struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); + int i; spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - if (ioa_cfg->sdt_state == WAIT_FOR_DUMP) - ioa_cfg->sdt_state = ABORT_DUMP; - ioa_cfg->reset_retries = IPR_NUM_RESET_RELOAD_RETRIES; - ioa_cfg->in_ioa_bringdown = 1; - ioa_cfg->allow_cmds = 0; - ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); + if (ioa_cfg->probe_done) { + if (ioa_cfg->sdt_state == WAIT_FOR_DUMP) + ioa_cfg->sdt_state = ABORT_DUMP; + ioa_cfg->reset_retries = IPR_NUM_RESET_RELOAD_RETRIES - 1; + ioa_cfg->in_ioa_bringdown = 1; + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].allow_cmds = 0; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); + ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); + } else + wake_up_all(&ioa_cfg->eeh_wait_q); spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); } @@ -8035,7 +8791,7 @@ static pci_ers_result_t ipr_pci_error_detected(struct pci_dev *pdev, switch (state) { case pci_channel_io_frozen: ipr_pci_frozen(pdev); - return PCI_ERS_RESULT_NEED_RESET; + return PCI_ERS_RESULT_CAN_RECOVER; case pci_channel_io_perm_failure: ipr_pci_perm_failure(pdev); return PCI_ERS_RESULT_DISCONNECT; @@ -8057,7 +8813,7 @@ static pci_ers_result_t ipr_pci_error_detected(struct pci_dev *pdev, * Return value: * 0 on success / -EIO on failure **/ -static int __devinit ipr_probe_ioa_part2(struct ipr_ioa_cfg *ioa_cfg) +static int ipr_probe_ioa_part2(struct ipr_ioa_cfg *ioa_cfg) { int rc = 0; unsigned long host_lock_flags = 0; @@ -8065,18 +8821,18 @@ static int __devinit ipr_probe_ioa_part2(struct ipr_ioa_cfg *ioa_cfg) ENTER; spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags); dev_dbg(&ioa_cfg->pdev->dev, "ioa_cfg adx: 0x%p\n", ioa_cfg); + ioa_cfg->probe_done = 1; if (ioa_cfg->needs_hard_reset) { ioa_cfg->needs_hard_reset = 0; ipr_initiate_ioa_reset(ioa_cfg, IPR_SHUTDOWN_NONE); } else _ipr_initiate_ioa_reset(ioa_cfg, ipr_reset_enable_ioa, IPR_SHUTDOWN_NONE); - spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags); - if (ioa_cfg->ioa_is_dead) { + if (ioa_cfg->hrrq[IPR_INIT_HRRQ].ioa_is_dead) { rc = -EIO; } else if (ipr_invalid_adapter(ioa_cfg)) { if (!ipr_testmode) @@ -8113,8 +8869,12 @@ static void ipr_free_cmd_blks(struct ipr_ioa_cfg *ioa_cfg) } if (ioa_cfg->ipr_cmd_pool) - pci_pool_destroy (ioa_cfg->ipr_cmd_pool); + pci_pool_destroy(ioa_cfg->ipr_cmd_pool); + kfree(ioa_cfg->ipr_cmnd_list); + kfree(ioa_cfg->ipr_cmnd_list_dma); + ioa_cfg->ipr_cmnd_list = NULL; + ioa_cfg->ipr_cmnd_list_dma = NULL; ioa_cfg->ipr_cmd_pool = NULL; } @@ -8133,8 +8893,13 @@ static void ipr_free_mem(struct ipr_ioa_cfg *ioa_cfg) pci_free_consistent(ioa_cfg->pdev, sizeof(struct ipr_misc_cbs), ioa_cfg->vpd_cbs, ioa_cfg->vpd_cbs_dma); ipr_free_cmd_blks(ioa_cfg); - pci_free_consistent(ioa_cfg->pdev, sizeof(u32) * IPR_NUM_CMD_BLKS, - ioa_cfg->host_rrq, ioa_cfg->host_rrq_dma); + + for (i = 0; i < ioa_cfg->hrrq_num; i++) + pci_free_consistent(ioa_cfg->pdev, + sizeof(u32) * ioa_cfg->hrrq[i].size, + ioa_cfg->hrrq[i].host_rrq, + ioa_cfg->hrrq[i].host_rrq_dma); + pci_free_consistent(ioa_cfg->pdev, ioa_cfg->cfg_table_size, ioa_cfg->u.cfg_table, ioa_cfg->cfg_table_dma); @@ -8165,8 +8930,23 @@ static void ipr_free_all_resources(struct ipr_ioa_cfg *ioa_cfg) struct pci_dev *pdev = ioa_cfg->pdev; ENTER; - free_irq(pdev->irq, ioa_cfg); - pci_disable_msi(pdev); + if (ioa_cfg->intr_flag == IPR_USE_MSI || + ioa_cfg->intr_flag == IPR_USE_MSIX) { + int i; + for (i = 0; i < ioa_cfg->nvectors; i++) + free_irq(ioa_cfg->vectors_info[i].vec, + &ioa_cfg->hrrq[i]); + } else + free_irq(pdev->irq, &ioa_cfg->hrrq[0]); + + if (ioa_cfg->intr_flag == IPR_USE_MSI) { + pci_disable_msi(pdev); + ioa_cfg->intr_flag &= ~IPR_USE_MSI; + } else if (ioa_cfg->intr_flag == IPR_USE_MSIX) { + pci_disable_msix(pdev); + ioa_cfg->intr_flag &= ~IPR_USE_MSIX; + } + iounmap(ioa_cfg->hdw_dma_regs); pci_release_regions(pdev); ipr_free_mem(ioa_cfg); @@ -8182,21 +8962,64 @@ static void ipr_free_all_resources(struct ipr_ioa_cfg *ioa_cfg) * Return value: * 0 on success / -ENOMEM on allocation failure **/ -static int __devinit ipr_alloc_cmd_blks(struct ipr_ioa_cfg *ioa_cfg) +static int ipr_alloc_cmd_blks(struct ipr_ioa_cfg *ioa_cfg) { struct ipr_cmnd *ipr_cmd; struct ipr_ioarcb *ioarcb; dma_addr_t dma_addr; - int i; + int i, entries_each_hrrq, hrrq_id = 0; - ioa_cfg->ipr_cmd_pool = pci_pool_create (IPR_NAME, ioa_cfg->pdev, - sizeof(struct ipr_cmnd), 16, 0); + ioa_cfg->ipr_cmd_pool = pci_pool_create(IPR_NAME, ioa_cfg->pdev, + sizeof(struct ipr_cmnd), 512, 0); if (!ioa_cfg->ipr_cmd_pool) return -ENOMEM; + ioa_cfg->ipr_cmnd_list = kcalloc(IPR_NUM_CMD_BLKS, sizeof(struct ipr_cmnd *), GFP_KERNEL); + ioa_cfg->ipr_cmnd_list_dma = kcalloc(IPR_NUM_CMD_BLKS, sizeof(dma_addr_t), GFP_KERNEL); + + if (!ioa_cfg->ipr_cmnd_list || !ioa_cfg->ipr_cmnd_list_dma) { + ipr_free_cmd_blks(ioa_cfg); + return -ENOMEM; + } + + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + if (ioa_cfg->hrrq_num > 1) { + if (i == 0) { + entries_each_hrrq = IPR_NUM_INTERNAL_CMD_BLKS; + ioa_cfg->hrrq[i].min_cmd_id = 0; + ioa_cfg->hrrq[i].max_cmd_id = + (entries_each_hrrq - 1); + } else { + entries_each_hrrq = + IPR_NUM_BASE_CMD_BLKS/ + (ioa_cfg->hrrq_num - 1); + ioa_cfg->hrrq[i].min_cmd_id = + IPR_NUM_INTERNAL_CMD_BLKS + + (i - 1) * entries_each_hrrq; + ioa_cfg->hrrq[i].max_cmd_id = + (IPR_NUM_INTERNAL_CMD_BLKS + + i * entries_each_hrrq - 1); + } + } else { + entries_each_hrrq = IPR_NUM_CMD_BLKS; + ioa_cfg->hrrq[i].min_cmd_id = 0; + ioa_cfg->hrrq[i].max_cmd_id = (entries_each_hrrq - 1); + } + ioa_cfg->hrrq[i].size = entries_each_hrrq; + } + + BUG_ON(ioa_cfg->hrrq_num == 0); + + i = IPR_NUM_CMD_BLKS - + ioa_cfg->hrrq[ioa_cfg->hrrq_num - 1].max_cmd_id - 1; + if (i > 0) { + ioa_cfg->hrrq[ioa_cfg->hrrq_num - 1].size += i; + ioa_cfg->hrrq[ioa_cfg->hrrq_num - 1].max_cmd_id += i; + } + for (i = 0; i < IPR_NUM_CMD_BLKS; i++) { - ipr_cmd = pci_pool_alloc (ioa_cfg->ipr_cmd_pool, GFP_KERNEL, &dma_addr); + ipr_cmd = pci_pool_alloc(ioa_cfg->ipr_cmd_pool, GFP_KERNEL, &dma_addr); if (!ipr_cmd) { ipr_free_cmd_blks(ioa_cfg); @@ -8233,7 +9056,11 @@ static int __devinit ipr_alloc_cmd_blks(struct ipr_ioa_cfg *ioa_cfg) ipr_cmd->sense_buffer_dma = dma_addr + offsetof(struct ipr_cmnd, sense_buffer); - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + ipr_cmd->ioarcb.cmd_pkt.hrrq_id = hrrq_id; + ipr_cmd->hrrq = &ioa_cfg->hrrq[hrrq_id]; + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); + if (i >= ioa_cfg->hrrq[hrrq_id].max_cmd_id) + hrrq_id++; } return 0; @@ -8246,7 +9073,7 @@ static int __devinit ipr_alloc_cmd_blks(struct ipr_ioa_cfg *ioa_cfg) * Return value: * 0 on success / non-zero for error **/ -static int __devinit ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg) +static int ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg) { struct pci_dev *pdev = ioa_cfg->pdev; int i, rc = -ENOMEM; @@ -8258,15 +9085,6 @@ static int __devinit ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg) if (!ioa_cfg->res_entries) goto out; - if (ioa_cfg->sis64) { - ioa_cfg->target_ids = kzalloc(sizeof(unsigned long) * - BITS_TO_LONGS(ioa_cfg->max_devs_supported), GFP_KERNEL); - ioa_cfg->array_ids = kzalloc(sizeof(unsigned long) * - BITS_TO_LONGS(ioa_cfg->max_devs_supported), GFP_KERNEL); - ioa_cfg->vset_ids = kzalloc(sizeof(unsigned long) * - BITS_TO_LONGS(ioa_cfg->max_devs_supported), GFP_KERNEL); - } - for (i = 0; i < ioa_cfg->max_devs_supported; i++) { list_add_tail(&ioa_cfg->res_entries[i].queue, &ioa_cfg->free_res_q); ioa_cfg->res_entries[i].ioa_cfg = ioa_cfg; @@ -8282,12 +9100,21 @@ static int __devinit ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg) if (ipr_alloc_cmd_blks(ioa_cfg)) goto out_free_vpd_cbs; - ioa_cfg->host_rrq = pci_alloc_consistent(ioa_cfg->pdev, - sizeof(u32) * IPR_NUM_CMD_BLKS, - &ioa_cfg->host_rrq_dma); - - if (!ioa_cfg->host_rrq) - goto out_ipr_free_cmd_blocks; + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + ioa_cfg->hrrq[i].host_rrq = pci_alloc_consistent(ioa_cfg->pdev, + sizeof(u32) * ioa_cfg->hrrq[i].size, + &ioa_cfg->hrrq[i].host_rrq_dma); + + if (!ioa_cfg->hrrq[i].host_rrq) { + while (--i > 0) + pci_free_consistent(pdev, + sizeof(u32) * ioa_cfg->hrrq[i].size, + ioa_cfg->hrrq[i].host_rrq, + ioa_cfg->hrrq[i].host_rrq_dma); + goto out_ipr_free_cmd_blocks; + } + ioa_cfg->hrrq[i].ioa_cfg = ioa_cfg; + } ioa_cfg->u.cfg_table = pci_alloc_consistent(ioa_cfg->pdev, ioa_cfg->cfg_table_size, @@ -8331,8 +9158,12 @@ out_free_hostrcb_dma: ioa_cfg->u.cfg_table, ioa_cfg->cfg_table_dma); out_free_host_rrq: - pci_free_consistent(pdev, sizeof(u32) * IPR_NUM_CMD_BLKS, - ioa_cfg->host_rrq, ioa_cfg->host_rrq_dma); + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + pci_free_consistent(pdev, + sizeof(u32) * ioa_cfg->hrrq[i].size, + ioa_cfg->hrrq[i].host_rrq, + ioa_cfg->hrrq[i].host_rrq_dma); + } out_ipr_free_cmd_blocks: ipr_free_cmd_blks(ioa_cfg); out_free_vpd_cbs: @@ -8350,7 +9181,7 @@ out_free_res_entries: * Return value: * none **/ -static void __devinit ipr_initialize_bus_attr(struct ipr_ioa_cfg *ioa_cfg) +static void ipr_initialize_bus_attr(struct ipr_ioa_cfg *ioa_cfg) { int i; @@ -8366,6 +9197,48 @@ static void __devinit ipr_initialize_bus_attr(struct ipr_ioa_cfg *ioa_cfg) } /** + * ipr_init_regs - Initialize IOA registers + * @ioa_cfg: ioa config struct + * + * Return value: + * none + **/ +static void ipr_init_regs(struct ipr_ioa_cfg *ioa_cfg) +{ + const struct ipr_interrupt_offsets *p; + struct ipr_interrupts *t; + void __iomem *base; + + p = &ioa_cfg->chip_cfg->regs; + t = &ioa_cfg->regs; + base = ioa_cfg->hdw_dma_regs; + + t->set_interrupt_mask_reg = base + p->set_interrupt_mask_reg; + t->clr_interrupt_mask_reg = base + p->clr_interrupt_mask_reg; + t->clr_interrupt_mask_reg32 = base + p->clr_interrupt_mask_reg32; + t->sense_interrupt_mask_reg = base + p->sense_interrupt_mask_reg; + t->sense_interrupt_mask_reg32 = base + p->sense_interrupt_mask_reg32; + t->clr_interrupt_reg = base + p->clr_interrupt_reg; + t->clr_interrupt_reg32 = base + p->clr_interrupt_reg32; + t->sense_interrupt_reg = base + p->sense_interrupt_reg; + t->sense_interrupt_reg32 = base + p->sense_interrupt_reg32; + t->ioarrin_reg = base + p->ioarrin_reg; + t->sense_uproc_interrupt_reg = base + p->sense_uproc_interrupt_reg; + t->sense_uproc_interrupt_reg32 = base + p->sense_uproc_interrupt_reg32; + t->set_uproc_interrupt_reg = base + p->set_uproc_interrupt_reg; + t->set_uproc_interrupt_reg32 = base + p->set_uproc_interrupt_reg32; + t->clr_uproc_interrupt_reg = base + p->clr_uproc_interrupt_reg; + t->clr_uproc_interrupt_reg32 = base + p->clr_uproc_interrupt_reg32; + + if (ioa_cfg->sis64) { + t->init_feedback_reg = base + p->init_feedback_reg; + t->dump_addr_reg = base + p->dump_addr_reg; + t->dump_data_reg = base + p->dump_data_reg; + t->endian_swap_reg = base + p->endian_swap_reg; + } +} + +/** * ipr_init_ioa_cfg - Initialize IOA config struct * @ioa_cfg: ioa config struct * @host: scsi host struct @@ -8374,12 +9247,10 @@ static void __devinit ipr_initialize_bus_attr(struct ipr_ioa_cfg *ioa_cfg) * Return value: * none **/ -static void __devinit ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, - struct Scsi_Host *host, struct pci_dev *pdev) +static void ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, + struct Scsi_Host *host, struct pci_dev *pdev) { - const struct ipr_interrupt_offsets *p; - struct ipr_interrupts *t; - void __iomem *base; + int i; ioa_cfg->host = host; ioa_cfg->pdev = pdev; @@ -8387,15 +9258,11 @@ static void __devinit ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, ioa_cfg->doorbell = IPR_DOORBELL; sprintf(ioa_cfg->eye_catcher, IPR_EYECATCHER); sprintf(ioa_cfg->trace_start, IPR_TRACE_START_LABEL); - sprintf(ioa_cfg->ipr_free_label, IPR_FREEQ_LABEL); - sprintf(ioa_cfg->ipr_pending_label, IPR_PENDQ_LABEL); sprintf(ioa_cfg->cfg_table_start, IPR_CFG_TBL_START); sprintf(ioa_cfg->resource_table_label, IPR_RES_TABLE_LABEL); sprintf(ioa_cfg->ipr_hcam_label, IPR_HCAM_LABEL); sprintf(ioa_cfg->ipr_cmd_label, IPR_CMD_LABEL); - INIT_LIST_HEAD(&ioa_cfg->free_q); - INIT_LIST_HEAD(&ioa_cfg->pending_q); INIT_LIST_HEAD(&ioa_cfg->hostrcb_free_q); INIT_LIST_HEAD(&ioa_cfg->hostrcb_pending_q); INIT_LIST_HEAD(&ioa_cfg->free_res_q); @@ -8403,6 +9270,7 @@ static void __devinit ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, INIT_WORK(&ioa_cfg->work_q, ipr_worker_thread); init_waitqueue_head(&ioa_cfg->reset_wait_q); init_waitqueue_head(&ioa_cfg->msi_wait_q); + init_waitqueue_head(&ioa_cfg->eeh_wait_q); ioa_cfg->sdt_state = INACTIVE; ipr_initialize_bus_attr(ioa_cfg); @@ -8413,43 +9281,33 @@ static void __devinit ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, host->max_lun = IPR_MAX_SIS64_LUNS_PER_TARGET; if (ipr_max_devs > IPR_MAX_SIS64_DEVS) ioa_cfg->max_devs_supported = IPR_MAX_SIS64_DEVS; + ioa_cfg->cfg_table_size = (sizeof(struct ipr_config_table_hdr64) + + ((sizeof(struct ipr_config_table_entry64) + * ioa_cfg->max_devs_supported))); } else { host->max_id = IPR_MAX_NUM_TARGETS_PER_BUS; host->max_lun = IPR_MAX_NUM_LUNS_PER_TARGET; if (ipr_max_devs > IPR_MAX_PHYSICAL_DEVS) ioa_cfg->max_devs_supported = IPR_MAX_PHYSICAL_DEVS; + ioa_cfg->cfg_table_size = (sizeof(struct ipr_config_table_hdr) + + ((sizeof(struct ipr_config_table_entry) + * ioa_cfg->max_devs_supported))); } + host->max_channel = IPR_MAX_BUS_TO_SCAN; host->unique_id = host->host_no; host->max_cmd_len = IPR_MAX_CDB_LEN; + host->can_queue = ioa_cfg->max_cmds; pci_set_drvdata(pdev, ioa_cfg); - p = &ioa_cfg->chip_cfg->regs; - t = &ioa_cfg->regs; - base = ioa_cfg->hdw_dma_regs; - - t->set_interrupt_mask_reg = base + p->set_interrupt_mask_reg; - t->clr_interrupt_mask_reg = base + p->clr_interrupt_mask_reg; - t->clr_interrupt_mask_reg32 = base + p->clr_interrupt_mask_reg32; - t->sense_interrupt_mask_reg = base + p->sense_interrupt_mask_reg; - t->sense_interrupt_mask_reg32 = base + p->sense_interrupt_mask_reg32; - t->clr_interrupt_reg = base + p->clr_interrupt_reg; - t->clr_interrupt_reg32 = base + p->clr_interrupt_reg32; - t->sense_interrupt_reg = base + p->sense_interrupt_reg; - t->sense_interrupt_reg32 = base + p->sense_interrupt_reg32; - t->ioarrin_reg = base + p->ioarrin_reg; - t->sense_uproc_interrupt_reg = base + p->sense_uproc_interrupt_reg; - t->sense_uproc_interrupt_reg32 = base + p->sense_uproc_interrupt_reg32; - t->set_uproc_interrupt_reg = base + p->set_uproc_interrupt_reg; - t->set_uproc_interrupt_reg32 = base + p->set_uproc_interrupt_reg32; - t->clr_uproc_interrupt_reg = base + p->clr_uproc_interrupt_reg; - t->clr_uproc_interrupt_reg32 = base + p->clr_uproc_interrupt_reg32; - - if (ioa_cfg->sis64) { - t->init_feedback_reg = base + p->init_feedback_reg; - t->dump_addr_reg = base + p->dump_addr_reg; - t->dump_data_reg = base + p->dump_data_reg; - t->endian_swap_reg = base + p->endian_swap_reg; + for (i = 0; i < ARRAY_SIZE(ioa_cfg->hrrq); i++) { + INIT_LIST_HEAD(&ioa_cfg->hrrq[i].hrrq_free_q); + INIT_LIST_HEAD(&ioa_cfg->hrrq[i].hrrq_pending_q); + spin_lock_init(&ioa_cfg->hrrq[i]._lock); + if (i == 0) + ioa_cfg->hrrq[i].lock = ioa_cfg->host->host_lock; + else + ioa_cfg->hrrq[i].lock = &ioa_cfg->hrrq[i]._lock; } } @@ -8460,7 +9318,7 @@ static void __devinit ipr_init_ioa_cfg(struct ipr_ioa_cfg *ioa_cfg, * Return value: * ptr to chip information on success / NULL on failure **/ -static const struct ipr_chip_t * __devinit +static const struct ipr_chip_t * ipr_get_chip_info(const struct pci_device_id *dev_id) { int i; @@ -8473,6 +9331,97 @@ ipr_get_chip_info(const struct pci_device_id *dev_id) } /** + * ipr_wait_for_pci_err_recovery - Wait for any PCI error recovery to complete + * during probe time + * @ioa_cfg: ioa config struct + * + * Return value: + * None + **/ +static void ipr_wait_for_pci_err_recovery(struct ipr_ioa_cfg *ioa_cfg) +{ + struct pci_dev *pdev = ioa_cfg->pdev; + + if (pci_channel_offline(pdev)) { + wait_event_timeout(ioa_cfg->eeh_wait_q, + !pci_channel_offline(pdev), + IPR_PCI_ERROR_RECOVERY_TIMEOUT); + pci_restore_state(pdev); + } +} + +static int ipr_enable_msix(struct ipr_ioa_cfg *ioa_cfg) +{ + struct msix_entry entries[IPR_MAX_MSIX_VECTORS]; + int i, vectors; + + for (i = 0; i < ARRAY_SIZE(entries); ++i) + entries[i].entry = i; + + vectors = pci_enable_msix_range(ioa_cfg->pdev, + entries, 1, ipr_number_of_msix); + if (vectors < 0) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + return vectors; + } + + for (i = 0; i < vectors; i++) + ioa_cfg->vectors_info[i].vec = entries[i].vector; + ioa_cfg->nvectors = vectors; + + return 0; +} + +static int ipr_enable_msi(struct ipr_ioa_cfg *ioa_cfg) +{ + int i, vectors; + + vectors = pci_enable_msi_range(ioa_cfg->pdev, 1, ipr_number_of_msix); + if (vectors < 0) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + return vectors; + } + + for (i = 0; i < vectors; i++) + ioa_cfg->vectors_info[i].vec = ioa_cfg->pdev->irq + i; + ioa_cfg->nvectors = vectors; + + return 0; +} + +static void name_msi_vectors(struct ipr_ioa_cfg *ioa_cfg) +{ + int vec_idx, n = sizeof(ioa_cfg->vectors_info[0].desc) - 1; + + for (vec_idx = 0; vec_idx < ioa_cfg->nvectors; vec_idx++) { + snprintf(ioa_cfg->vectors_info[vec_idx].desc, n, + "host%d-%d", ioa_cfg->host->host_no, vec_idx); + ioa_cfg->vectors_info[vec_idx]. + desc[strlen(ioa_cfg->vectors_info[vec_idx].desc)] = 0; + } +} + +static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg) +{ + int i, rc; + + for (i = 1; i < ioa_cfg->nvectors; i++) { + rc = request_irq(ioa_cfg->vectors_info[i].vec, + ipr_isr_mhrrq, + 0, + ioa_cfg->vectors_info[i].desc, + &ioa_cfg->hrrq[i]); + if (rc) { + while (--i >= 0) + free_irq(ioa_cfg->vectors_info[i].vec, + &ioa_cfg->hrrq[i]); + return rc; + } + } + return 0; +} + +/** * ipr_test_intr - Handle the interrupt generated in ipr_test_msi(). * @pdev: PCI device struct * @@ -8482,12 +9431,13 @@ ipr_get_chip_info(const struct pci_device_id *dev_id) * Return value: * 0 on success / non-zero on failure **/ -static irqreturn_t __devinit ipr_test_intr(int irq, void *devp) +static irqreturn_t ipr_test_intr(int irq, void *devp) { struct ipr_ioa_cfg *ioa_cfg = (struct ipr_ioa_cfg *)devp; unsigned long lock_flags = 0; irqreturn_t rc = IRQ_HANDLED; + dev_info(&ioa_cfg->pdev->dev, "Received IRQ : %d\n", irq); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); ioa_cfg->msi_received = 1; @@ -8501,7 +9451,7 @@ static irqreturn_t __devinit ipr_test_intr(int irq, void *devp) * ipr_test_msi - Test for Message Signaled Interrupt (MSI) support. * @pdev: PCI device struct * - * Description: The return value from pci_enable_msi() can not always be + * Description: The return value from pci_enable_msi_range() can not always be * trusted. This routine sets up and initiates a test interrupt to determine * if the interrupt is received via the ipr_test_intr() service routine. * If the tests fails, the driver will fall back to LSI. @@ -8509,8 +9459,7 @@ static irqreturn_t __devinit ipr_test_intr(int irq, void *devp) * Return value: * 0 on success / non-zero on failure **/ -static int __devinit ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, - struct pci_dev *pdev) +static int ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, struct pci_dev *pdev) { int rc; volatile u32 int_reg; @@ -8526,7 +9475,10 @@ static int __devinit ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, int_reg = readl(ioa_cfg->regs.sense_interrupt_mask_reg); spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - rc = request_irq(pdev->irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg); + if (ioa_cfg->intr_flag == IPR_USE_MSIX) + rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_test_intr, 0, IPR_NAME, ioa_cfg); + else + rc = request_irq(pdev->irq, ipr_test_intr, 0, IPR_NAME, ioa_cfg); if (rc) { dev_err(&pdev->dev, "Can not assign irq %d\n", pdev->irq); return rc; @@ -8536,9 +9488,9 @@ static int __devinit ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, writel(IPR_PCII_IO_DEBUG_ACKNOWLEDGE, ioa_cfg->regs.sense_interrupt_reg32); int_reg = readl(ioa_cfg->regs.sense_interrupt_reg); wait_event_timeout(ioa_cfg->msi_wait_q, ioa_cfg->msi_received, HZ); + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER); - spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); if (!ioa_cfg->msi_received) { /* MSI test failed */ dev_info(&pdev->dev, "MSI test failed. Falling back to LSI.\n"); @@ -8548,23 +9500,25 @@ static int __devinit ipr_test_msi(struct ipr_ioa_cfg *ioa_cfg, spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); - free_irq(pdev->irq, ioa_cfg); + if (ioa_cfg->intr_flag == IPR_USE_MSIX) + free_irq(ioa_cfg->vectors_info[0].vec, ioa_cfg); + else + free_irq(pdev->irq, ioa_cfg); LEAVE; return rc; } -/** - * ipr_probe_ioa - Allocates memory and does first stage of initialization + /* ipr_probe_ioa - Allocates memory and does first stage of initialization * @pdev: PCI device struct * @dev_id: PCI device id struct * * Return value: * 0 on success / non-zero on failure **/ -static int __devinit ipr_probe_ioa(struct pci_dev *pdev, - const struct pci_device_id *dev_id) +static int ipr_probe_ioa(struct pci_dev *pdev, + const struct pci_device_id *dev_id) { struct ipr_ioa_cfg *ioa_cfg; struct Scsi_Host *host; @@ -8572,28 +9526,22 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, void __iomem *ipr_regs; int rc = PCIBIOS_SUCCESSFUL; volatile u32 mask, uproc, interrupts; + unsigned long lock_flags, driver_lock_flags; ENTER; - if ((rc = pci_enable_device(pdev))) { - dev_err(&pdev->dev, "Cannot enable adapter\n"); - goto out; - } - dev_info(&pdev->dev, "Found IOA with IRQ: %d\n", pdev->irq); - host = scsi_host_alloc(&driver_template, sizeof(*ioa_cfg)); if (!host) { dev_err(&pdev->dev, "call to scsi_host_alloc failed!\n"); rc = -ENOMEM; - goto out_disable; + goto out; } ioa_cfg = (struct ipr_ioa_cfg *)host->hostdata; memset(ioa_cfg, 0, sizeof(struct ipr_ioa_cfg)); - ata_host_init(&ioa_cfg->ata_host, &pdev->dev, - sata_port_info.flags, &ipr_sata_ops); + ata_host_init(&ioa_cfg->ata_host, &pdev->dev, &ipr_sata_ops); ioa_cfg->ipr_chip = ipr_get_chip_info(dev_id); @@ -8606,6 +9554,8 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, /* set SIS 32 or SIS 64 */ ioa_cfg->sis64 = ioa_cfg->ipr_chip->sis_type == IPR_SIS64 ? 1 : 0; ioa_cfg->chip_cfg = ioa_cfg->ipr_chip->cfg; + ioa_cfg->clear_isr = ioa_cfg->chip_cfg->clear_isr; + ioa_cfg->max_cmds = ioa_cfg->chip_cfg->max_cmds; if (ipr_transop_timeout) ioa_cfg->transop_timeout = ipr_transop_timeout; @@ -8616,6 +9566,8 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, ioa_cfg->revid = pdev->revision; + ipr_init_ioa_cfg(ioa_cfg, host, pdev); + ipr_regs_pci = pci_resource_start(pdev, 0); rc = pci_request_regions(pdev, IPR_NAME); @@ -8625,22 +9577,35 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, goto out_scsi_host_put; } + rc = pci_enable_device(pdev); + + if (rc || pci_channel_offline(pdev)) { + if (pci_channel_offline(pdev)) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + rc = pci_enable_device(pdev); + } + + if (rc) { + dev_err(&pdev->dev, "Cannot enable adapter\n"); + ipr_wait_for_pci_err_recovery(ioa_cfg); + goto out_release_regions; + } + } + ipr_regs = pci_ioremap_bar(pdev, 0); if (!ipr_regs) { dev_err(&pdev->dev, "Couldn't map memory range of registers\n"); rc = -ENOMEM; - goto out_release_regions; + goto out_disable; } ioa_cfg->hdw_dma_regs = ipr_regs; ioa_cfg->hdw_dma_regs_pci = ipr_regs_pci; ioa_cfg->ioa_mailbox = ioa_cfg->chip_cfg->mailbox + ipr_regs; - ipr_init_ioa_cfg(ioa_cfg, host, pdev); - - pci_set_master(pdev); + ipr_init_regs(ioa_cfg); if (ioa_cfg->sis64) { rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); @@ -8648,7 +9613,6 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, dev_dbg(&pdev->dev, "Failed to set 64 bit PCI DMA mask\n"); rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); } - } else rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); @@ -8662,51 +9626,98 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, if (rc != PCIBIOS_SUCCESSFUL) { dev_err(&pdev->dev, "Write of cache line size failed\n"); + ipr_wait_for_pci_err_recovery(ioa_cfg); rc = -EIO; goto cleanup_nomem; } - /* Enable MSI style interrupts if they are supported. */ - if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI && !pci_enable_msi(pdev)) { - rc = ipr_test_msi(ioa_cfg, pdev); - if (rc == -EOPNOTSUPP) - pci_disable_msi(pdev); - else if (rc) - goto out_msi_disable; - else - dev_info(&pdev->dev, "MSI enabled with IRQ: %d\n", pdev->irq); - } else if (ipr_debug) + /* Issue MMIO read to ensure card is not in EEH */ + interrupts = readl(ioa_cfg->regs.sense_interrupt_reg); + ipr_wait_for_pci_err_recovery(ioa_cfg); + + if (ipr_number_of_msix > IPR_MAX_MSIX_VECTORS) { + dev_err(&pdev->dev, "The max number of MSIX is %d\n", + IPR_MAX_MSIX_VECTORS); + ipr_number_of_msix = IPR_MAX_MSIX_VECTORS; + } + + if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI && + ipr_enable_msix(ioa_cfg) == 0) + ioa_cfg->intr_flag = IPR_USE_MSIX; + else if (ioa_cfg->ipr_chip->intr_type == IPR_USE_MSI && + ipr_enable_msi(ioa_cfg) == 0) + ioa_cfg->intr_flag = IPR_USE_MSI; + else { + ioa_cfg->intr_flag = IPR_USE_LSI; + ioa_cfg->nvectors = 1; dev_info(&pdev->dev, "Cannot enable MSI.\n"); + } - /* Save away PCI config space for use following IOA reset */ - rc = pci_save_state(pdev); + pci_set_master(pdev); - if (rc != PCIBIOS_SUCCESSFUL) { - dev_err(&pdev->dev, "Failed to save PCI config space\n"); - rc = -EIO; - goto cleanup_nomem; + if (pci_channel_offline(pdev)) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + pci_set_master(pdev); + if (pci_channel_offline(pdev)) { + rc = -EIO; + goto out_msi_disable; + } + } + + if (ioa_cfg->intr_flag == IPR_USE_MSI || + ioa_cfg->intr_flag == IPR_USE_MSIX) { + rc = ipr_test_msi(ioa_cfg, pdev); + if (rc == -EOPNOTSUPP) { + ipr_wait_for_pci_err_recovery(ioa_cfg); + if (ioa_cfg->intr_flag == IPR_USE_MSI) { + ioa_cfg->intr_flag &= ~IPR_USE_MSI; + pci_disable_msi(pdev); + } else if (ioa_cfg->intr_flag == IPR_USE_MSIX) { + ioa_cfg->intr_flag &= ~IPR_USE_MSIX; + pci_disable_msix(pdev); + } + + ioa_cfg->intr_flag = IPR_USE_LSI; + ioa_cfg->nvectors = 1; + } + else if (rc) + goto out_msi_disable; + else { + if (ioa_cfg->intr_flag == IPR_USE_MSI) + dev_info(&pdev->dev, + "Request for %d MSIs succeeded with starting IRQ: %d\n", + ioa_cfg->nvectors, pdev->irq); + else if (ioa_cfg->intr_flag == IPR_USE_MSIX) + dev_info(&pdev->dev, + "Request for %d MSIXs succeeded.", + ioa_cfg->nvectors); + } } + ioa_cfg->hrrq_num = min3(ioa_cfg->nvectors, + (unsigned int)num_online_cpus(), + (unsigned int)IPR_MAX_HRRQ_NUM); + if ((rc = ipr_save_pcix_cmd_reg(ioa_cfg))) - goto cleanup_nomem; + goto out_msi_disable; if ((rc = ipr_set_pcix_cmd_reg(ioa_cfg))) - goto cleanup_nomem; - - if (ioa_cfg->sis64) - ioa_cfg->cfg_table_size = (sizeof(struct ipr_config_table_hdr64) - + ((sizeof(struct ipr_config_table_entry64) - * ioa_cfg->max_devs_supported))); - else - ioa_cfg->cfg_table_size = (sizeof(struct ipr_config_table_hdr) - + ((sizeof(struct ipr_config_table_entry) - * ioa_cfg->max_devs_supported))); + goto out_msi_disable; rc = ipr_alloc_mem(ioa_cfg); if (rc < 0) { dev_err(&pdev->dev, "Couldn't allocate enough memory for device driver!\n"); - goto cleanup_nomem; + goto out_msi_disable; + } + + /* Save away PCI config space for use following IOA reset */ + rc = pci_save_state(pdev); + + if (rc != PCIBIOS_SUCCESSFUL) { + dev_err(&pdev->dev, "Failed to save PCI config space\n"); + rc = -EIO; + goto cleanup_nolog; } /* @@ -8718,16 +9729,29 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, uproc = readl(ioa_cfg->regs.sense_uproc_interrupt_reg32); if ((mask & IPR_PCII_HRRQ_UPDATED) == 0 || (uproc & IPR_UPROCI_RESET_ALERT)) ioa_cfg->needs_hard_reset = 1; - if (interrupts & IPR_PCII_ERROR_INTERRUPTS) + if ((interrupts & IPR_PCII_ERROR_INTERRUPTS) || reset_devices) ioa_cfg->needs_hard_reset = 1; if (interrupts & IPR_PCII_IOA_UNIT_CHECKED) ioa_cfg->ioa_unit_checked = 1; + spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); ipr_mask_and_clear_interrupts(ioa_cfg, ~IPR_PCII_IOA_TRANS_TO_OPER); - rc = request_irq(pdev->irq, ipr_isr, - ioa_cfg->msi_received ? 0 : IRQF_SHARED, - IPR_NAME, ioa_cfg); + spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); + if (ioa_cfg->intr_flag == IPR_USE_MSI + || ioa_cfg->intr_flag == IPR_USE_MSIX) { + name_msi_vectors(ioa_cfg); + rc = request_irq(ioa_cfg->vectors_info[0].vec, ipr_isr, + 0, + ioa_cfg->vectors_info[0].desc, + &ioa_cfg->hrrq[0]); + if (!rc) + rc = ipr_request_other_msi_irqs(ioa_cfg); + } else { + rc = request_irq(pdev->irq, ipr_isr, + IRQF_SHARED, + IPR_NAME, &ioa_cfg->hrrq[0]); + } if (rc) { dev_err(&pdev->dev, "Couldn't register IRQ %d! rc=%d\n", pdev->irq, rc); @@ -8741,9 +9765,9 @@ static int __devinit ipr_probe_ioa(struct pci_dev *pdev, } else ioa_cfg->reset = ipr_reset_start_bist; - spin_lock(&ipr_driver_lock); + spin_lock_irqsave(&ipr_driver_lock, driver_lock_flags); list_add_tail(&ioa_cfg->queue, &ipr_ioa_head); - spin_unlock(&ipr_driver_lock); + spin_unlock_irqrestore(&ipr_driver_lock, driver_lock_flags); LEAVE; out: @@ -8751,16 +9775,20 @@ out: cleanup_nolog: ipr_free_mem(ioa_cfg); +out_msi_disable: + ipr_wait_for_pci_err_recovery(ioa_cfg); + if (ioa_cfg->intr_flag == IPR_USE_MSI) + pci_disable_msi(pdev); + else if (ioa_cfg->intr_flag == IPR_USE_MSIX) + pci_disable_msix(pdev); cleanup_nomem: iounmap(ipr_regs); -out_msi_disable: - pci_disable_msi(pdev); +out_disable: + pci_disable_device(pdev); out_release_regions: pci_release_regions(pdev); out_scsi_host_put: scsi_host_put(host); -out_disable: - pci_disable_device(pdev); goto out; } @@ -8779,7 +9807,7 @@ static void ipr_scan_vsets(struct ipr_ioa_cfg *ioa_cfg) int target, lun; for (target = 0; target < IPR_MAX_NUM_TARGETS_PER_BUS; target++) - for (lun = 0; lun < IPR_MAX_NUM_VSET_LUNS_PER_TARGET; lun++ ) + for (lun = 0; lun < IPR_MAX_NUM_VSET_LUNS_PER_TARGET; lun++) scsi_add_device(ioa_cfg->host, IPR_VSET_BUS, target, lun); } @@ -8822,25 +9850,34 @@ static void __ipr_remove(struct pci_dev *pdev) { unsigned long host_lock_flags = 0; struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); + int i; + unsigned long driver_lock_flags; ENTER; spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags); - while(ioa_cfg->in_reset_reload) { + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags); } + for (i = 0; i < ioa_cfg->hrrq_num; i++) { + spin_lock(&ioa_cfg->hrrq[i]._lock); + ioa_cfg->hrrq[i].removing_ioa = 1; + spin_unlock(&ioa_cfg->hrrq[i]._lock); + } + wmb(); ipr_initiate_ioa_bringdown(ioa_cfg, IPR_SHUTDOWN_NORMAL); spin_unlock_irqrestore(ioa_cfg->host->host_lock, host_lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); - flush_scheduled_work(); + flush_work(&ioa_cfg->work_q); + INIT_LIST_HEAD(&ioa_cfg->used_res_q); spin_lock_irqsave(ioa_cfg->host->host_lock, host_lock_flags); - spin_lock(&ipr_driver_lock); + spin_lock_irqsave(&ipr_driver_lock, driver_lock_flags); list_del(&ioa_cfg->queue); - spin_unlock(&ipr_driver_lock); + spin_unlock_irqrestore(&ipr_driver_lock, driver_lock_flags); if (ioa_cfg->sdt_state == ABORT_DUMP) ioa_cfg->sdt_state = WAIT_FOR_DUMP; @@ -8860,7 +9897,7 @@ static void __ipr_remove(struct pci_dev *pdev) * Return value: * none **/ -static void __devexit ipr_remove(struct pci_dev *pdev) +static void ipr_remove(struct pci_dev *pdev) { struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); @@ -8883,11 +9920,10 @@ static void __devexit ipr_remove(struct pci_dev *pdev) * Return value: * 0 on success / non-zero on failure **/ -static int __devinit ipr_probe(struct pci_dev *pdev, - const struct pci_device_id *dev_id) +static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id) { struct ipr_ioa_cfg *ioa_cfg; - int rc; + int rc, i; rc = ipr_probe_ioa(pdev, dev_id); @@ -8934,6 +9970,16 @@ static int __devinit ipr_probe(struct pci_dev *pdev, scsi_add_device(ioa_cfg->host, IPR_IOA_BUS, IPR_IOA_TARGET, IPR_IOA_LUN); ioa_cfg->allow_ml_add_del = 1; ioa_cfg->host->max_channel = IPR_VSET_BUS; + ioa_cfg->iopoll_weight = ioa_cfg->chip_cfg->iopoll_weight; + + if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { + for (i = 1; i < ioa_cfg->hrrq_num; i++) { + blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, + ioa_cfg->iopoll_weight, ipr_iopoll); + blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll); + } + } + schedule_work(&ioa_cfg->work_q); return 0; } @@ -8952,9 +9998,16 @@ static void ipr_shutdown(struct pci_dev *pdev) { struct ipr_ioa_cfg *ioa_cfg = pci_get_drvdata(pdev); unsigned long lock_flags = 0; + int i; spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); - while(ioa_cfg->in_reset_reload) { + if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) { + ioa_cfg->iopoll_weight = 0; + for (i = 1; i < ioa_cfg->hrrq_num; i++) + blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); + } + + while (ioa_cfg->in_reset_reload) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, lock_flags); wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); @@ -8965,7 +10018,7 @@ static void ipr_shutdown(struct pci_dev *pdev) wait_event(ioa_cfg->reset_wait_q, !ioa_cfg->in_reset_reload); } -static struct pci_device_id ipr_pci_table[] __devinitdata = { +static struct pci_device_id ipr_pci_table[] = { { PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_5702, 0, 0, 0 }, { PCI_VENDOR_ID_MYLEX, PCI_DEVICE_ID_IBM_GEMSTONE, @@ -9026,23 +10079,58 @@ static struct pci_device_id ipr_pci_table[] __devinitdata = { { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57B2, 0, 0, 0 }, { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57C0, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57C3, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_FPGA_E2, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57C4, 0, 0, 0 }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57B4, 0, 0, 0 }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57B1, 0, 0, 0 }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57C6, 0, 0, 0 }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, - PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_575D, 0, 0, 0 }, - { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROC_ASIC_E2, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57C8, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57CE, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57D5, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57D6, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57D7, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57D8, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57D9, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57DA, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57EB, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57EC, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57ED, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57EE, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57EF, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_57F0, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_2CCA, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_2CD2, 0, 0, 0 }, + { PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_IBM_CROCODILE, + PCI_VENDOR_ID_IBM, IPR_SUBS_DEV_ID_2CCD, 0, 0, 0 }, { } }; MODULE_DEVICE_TABLE(pci, ipr_pci_table); -static struct pci_error_handlers ipr_err_handler = { +static const struct pci_error_handlers ipr_err_handler = { .error_detected = ipr_pci_error_detected, + .mmio_enabled = ipr_pci_mmio_enabled, .slot_reset = ipr_pci_slot_reset, }; @@ -9050,7 +10138,7 @@ static struct pci_driver ipr_driver = { .name = IPR_NAME, .id_table = ipr_pci_table, .probe = ipr_probe, - .remove = __devexit_p(ipr_remove), + .remove = ipr_remove, .shutdown = ipr_shutdown, .err_handler = &ipr_err_handler, }; @@ -9063,9 +10151,7 @@ static struct pci_driver ipr_driver = { **/ static void ipr_halt_done(struct ipr_cmnd *ipr_cmd) { - struct ipr_ioa_cfg *ioa_cfg = ipr_cmd->ioa_cfg; - - list_add_tail(&ipr_cmd->queue, &ioa_cfg->free_q); + list_add_tail(&ipr_cmd->queue, &ipr_cmd->hrrq->hrrq_free_q); } /** @@ -9078,16 +10164,16 @@ static int ipr_halt(struct notifier_block *nb, ulong event, void *buf) { struct ipr_cmnd *ipr_cmd; struct ipr_ioa_cfg *ioa_cfg; - unsigned long flags = 0; + unsigned long flags = 0, driver_lock_flags; if (event != SYS_RESTART && event != SYS_HALT && event != SYS_POWER_OFF) return NOTIFY_DONE; - spin_lock(&ipr_driver_lock); + spin_lock_irqsave(&ipr_driver_lock, driver_lock_flags); list_for_each_entry(ioa_cfg, &ipr_ioa_head, queue) { spin_lock_irqsave(ioa_cfg->host->host_lock, flags); - if (!ioa_cfg->allow_cmds) { + if (!ioa_cfg->hrrq[IPR_INIT_HRRQ].allow_cmds) { spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); continue; } @@ -9101,7 +10187,7 @@ static int ipr_halt(struct notifier_block *nb, ulong event, void *buf) ipr_do_req(ipr_cmd, ipr_halt_done, ipr_timeout, IPR_DEVICE_RESET_TIMEOUT); spin_unlock_irqrestore(ioa_cfg->host->host_lock, flags); } - spin_unlock(&ipr_driver_lock); + spin_unlock_irqrestore(&ipr_driver_lock, driver_lock_flags); return NOTIFY_OK; } |