diff options
Diffstat (limited to 'drivers/target/target_core_transport.c')
| -rw-r--r-- | drivers/target/target_core_transport.c | 3392 |
1 files changed, 966 insertions, 2426 deletions
diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 634d0f31a28..7fa62fc93e0 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -3,10 +3,7 @@ * * This file contains the Generic Target Engine Core. * - * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc. - * Copyright (c) 2005, 2006, 2007 SBE, Inc. - * Copyright (c) 2007-2010 Rising Tide Systems - * Copyright (c) 2008-2010 Linux-iSCSI.org + * (c) Copyright 2002-2013 Datera, Inc. * * Nicholas A. Bellinger <nab@kernel.org> * @@ -31,7 +28,6 @@ #include <linux/string.h> #include <linux/timer.h> #include <linux/slab.h> -#include <linux/blkdev.h> #include <linux/spinlock.h> #include <linux/kthread.h> #include <linux/in.h> @@ -55,7 +51,8 @@ #include "target_core_pr.h" #include "target_core_ua.h" -static int sub_api_initialized; +#define CREATE_TRACE_POINTS +#include <trace/events/target.h> static struct workqueue_struct *target_completion_wq; static struct kmem_cache *se_sess_cache; @@ -65,17 +62,13 @@ struct kmem_cache *t10_alua_lu_gp_cache; struct kmem_cache *t10_alua_lu_gp_mem_cache; struct kmem_cache *t10_alua_tg_pt_gp_cache; struct kmem_cache *t10_alua_tg_pt_gp_mem_cache; +struct kmem_cache *t10_alua_lba_map_cache; +struct kmem_cache *t10_alua_lba_map_mem_cache; -static int transport_generic_write_pending(struct se_cmd *); -static int transport_processing_thread(void *param); -static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *); static void transport_complete_task_attr(struct se_cmd *cmd); static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev); -static int transport_generic_get_mem(struct se_cmd *cmd); -static void transport_put_cmd(struct se_cmd *cmd); -static void transport_remove_cmd_from_queue(struct se_cmd *cmd); -static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq); +static int transport_put_cmd(struct se_cmd *cmd); static void target_complete_ok_work(struct work_struct *work); int init_se_kmem_caches(void) @@ -137,14 +130,36 @@ int init_se_kmem_caches(void) "mem_t failed\n"); goto out_free_tg_pt_gp_cache; } + t10_alua_lba_map_cache = kmem_cache_create( + "t10_alua_lba_map_cache", + sizeof(struct t10_alua_lba_map), + __alignof__(struct t10_alua_lba_map), 0, NULL); + if (!t10_alua_lba_map_cache) { + pr_err("kmem_cache_create() for t10_alua_lba_map_" + "cache failed\n"); + goto out_free_tg_pt_gp_mem_cache; + } + t10_alua_lba_map_mem_cache = kmem_cache_create( + "t10_alua_lba_map_mem_cache", + sizeof(struct t10_alua_lba_map_member), + __alignof__(struct t10_alua_lba_map_member), 0, NULL); + if (!t10_alua_lba_map_mem_cache) { + pr_err("kmem_cache_create() for t10_alua_lba_map_mem_" + "cache failed\n"); + goto out_free_lba_map_cache; + } target_completion_wq = alloc_workqueue("target_completion", WQ_MEM_RECLAIM, 0); if (!target_completion_wq) - goto out_free_tg_pt_gp_mem_cache; + goto out_free_lba_map_mem_cache; return 0; +out_free_lba_map_mem_cache: + kmem_cache_destroy(t10_alua_lba_map_mem_cache); +out_free_lba_map_cache: + kmem_cache_destroy(t10_alua_lba_map_cache); out_free_tg_pt_gp_mem_cache: kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache); out_free_tg_pt_gp_cache: @@ -173,6 +188,8 @@ void release_se_kmem_caches(void) kmem_cache_destroy(t10_alua_lu_gp_mem_cache); kmem_cache_destroy(t10_alua_tg_pt_gp_cache); kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache); + kmem_cache_destroy(t10_alua_lba_map_cache); + kmem_cache_destroy(t10_alua_lba_map_mem_cache); } /* This code ensures unique mib indexes are handed out. */ @@ -195,17 +212,10 @@ u32 scsi_get_new_index(scsi_index_t type) return new_index; } -static void transport_init_queue_obj(struct se_queue_obj *qobj) -{ - atomic_set(&qobj->queue_cnt, 0); - INIT_LIST_HEAD(&qobj->qobj_list); - init_waitqueue_head(&qobj->thread_wq); - spin_lock_init(&qobj->cmd_queue_lock); -} - void transport_subsystem_check_init(void) { int ret; + static int sub_api_initialized; if (sub_api_initialized) return; @@ -222,15 +232,10 @@ void transport_subsystem_check_init(void) if (ret != 0) pr_err("Unable to load target_core_pscsi\n"); - ret = request_module("target_core_stgt"); - if (ret != 0) - pr_err("Unable to load target_core_stgt\n"); - sub_api_initialized = 1; - return; } -struct se_session *transport_init_session(void) +struct se_session *transport_init_session(enum target_prot_op sup_prot_ops) { struct se_session *se_sess; @@ -246,11 +251,64 @@ struct se_session *transport_init_session(void) INIT_LIST_HEAD(&se_sess->sess_wait_list); spin_lock_init(&se_sess->sess_cmd_lock); kref_init(&se_sess->sess_kref); + se_sess->sup_prot_ops = sup_prot_ops; return se_sess; } EXPORT_SYMBOL(transport_init_session); +int transport_alloc_session_tags(struct se_session *se_sess, + unsigned int tag_num, unsigned int tag_size) +{ + int rc; + + se_sess->sess_cmd_map = kzalloc(tag_num * tag_size, + GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); + if (!se_sess->sess_cmd_map) { + se_sess->sess_cmd_map = vzalloc(tag_num * tag_size); + if (!se_sess->sess_cmd_map) { + pr_err("Unable to allocate se_sess->sess_cmd_map\n"); + return -ENOMEM; + } + } + + rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num); + if (rc < 0) { + pr_err("Unable to init se_sess->sess_tag_pool," + " tag_num: %u\n", tag_num); + if (is_vmalloc_addr(se_sess->sess_cmd_map)) + vfree(se_sess->sess_cmd_map); + else + kfree(se_sess->sess_cmd_map); + se_sess->sess_cmd_map = NULL; + return -ENOMEM; + } + + return 0; +} +EXPORT_SYMBOL(transport_alloc_session_tags); + +struct se_session *transport_init_session_tags(unsigned int tag_num, + unsigned int tag_size, + enum target_prot_op sup_prot_ops) +{ + struct se_session *se_sess; + int rc; + + se_sess = transport_init_session(sup_prot_ops); + if (IS_ERR(se_sess)) + return se_sess; + + rc = transport_alloc_session_tags(se_sess, tag_num, tag_size); + if (rc < 0) { + transport_free_session(se_sess); + return ERR_PTR(-ENOMEM); + } + + return se_sess; +} +EXPORT_SYMBOL(transport_init_session_tags); + /* * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called. */ @@ -315,7 +373,7 @@ void transport_register_session( } EXPORT_SYMBOL(transport_register_session); -void target_release_session(struct kref *kref) +static void target_release_session(struct kref *kref) { struct se_session *se_sess = container_of(kref, struct se_session, sess_kref); @@ -386,6 +444,13 @@ EXPORT_SYMBOL(transport_deregister_session_configfs); void transport_free_session(struct se_session *se_sess) { + if (se_sess->sess_cmd_map) { + percpu_ida_destroy(&se_sess->sess_tag_pool); + if (is_vmalloc_addr(se_sess->sess_cmd_map)) + vfree(se_sess->sess_cmd_map); + else + kfree(se_sess->sess_cmd_map); + } kmem_cache_free(se_sess_cache, se_sess); } EXPORT_SYMBOL(transport_free_session); @@ -435,11 +500,11 @@ void transport_deregister_session(struct se_session *se_sess) pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n", se_tpg->se_tpg_tfo->get_fabric_name()); /* - * If last kref is dropping now for an explict NodeACL, awake sleeping + * If last kref is dropping now for an explicit NodeACL, awake sleeping * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group * removal context. */ - if (se_nacl && comp_nacl == true) + if (se_nacl && comp_nacl) target_put_nacl(se_nacl); transport_free_session(se_sess); @@ -468,38 +533,24 @@ static void target_remove_from_state_list(struct se_cmd *cmd) spin_unlock_irqrestore(&dev->execute_task_lock, flags); } -/* transport_cmd_check_stop(): - * - * 'transport_off = 1' determines if CMD_T_ACTIVE should be cleared. - * 'transport_off = 2' determines if task_dev_state should be removed. - * - * A non-zero u8 t_state sets cmd->t_state. - * Returns 1 when command is stopped, else 0. - */ -static int transport_cmd_check_stop( - struct se_cmd *cmd, - int transport_off, - u8 t_state) +static int transport_cmd_check_stop(struct se_cmd *cmd, bool remove_from_lists, + bool write_pending) { unsigned long flags; spin_lock_irqsave(&cmd->t_state_lock, flags); - /* - * Determine if IOCTL context caller in requesting the stopping of this - * command for LUN shutdown purposes. - */ - if (cmd->transport_state & CMD_T_LUN_STOP) { - pr_debug("%s:%d CMD_T_LUN_STOP for ITT: 0x%08x\n", - __func__, __LINE__, cmd->se_tfo->get_task_tag(cmd)); + if (write_pending) + cmd->t_state = TRANSPORT_WRITE_PENDING; - cmd->transport_state &= ~CMD_T_ACTIVE; - if (transport_off == 2) - target_remove_from_state_list(cmd); - spin_unlock_irqrestore(&cmd->t_state_lock, flags); + if (remove_from_lists) { + target_remove_from_state_list(cmd); - complete(&cmd->transport_lun_stop_comp); - return 1; + /* + * Clear struct se_cmd->se_lun before the handoff to FE. + */ + cmd->se_lun = NULL; } + /* * Determine if frontend context caller is requesting the stopping of * this command for frontend exceptions. @@ -509,168 +560,95 @@ static int transport_cmd_check_stop( __func__, __LINE__, cmd->se_tfo->get_task_tag(cmd)); - if (transport_off == 2) - target_remove_from_state_list(cmd); - - /* - * Clear struct se_cmd->se_lun before the transport_off == 2 handoff - * to FE. - */ - if (transport_off == 2) - cmd->se_lun = NULL; spin_unlock_irqrestore(&cmd->t_state_lock, flags); - complete(&cmd->t_transport_stop_comp); + complete_all(&cmd->t_transport_stop_comp); return 1; } - if (transport_off) { - cmd->transport_state &= ~CMD_T_ACTIVE; - if (transport_off == 2) { - target_remove_from_state_list(cmd); - /* - * Clear struct se_cmd->se_lun before the transport_off == 2 - * handoff to fabric module. - */ - cmd->se_lun = NULL; - /* - * Some fabric modules like tcm_loop can release - * their internally allocated I/O reference now and - * struct se_cmd now. - * - * Fabric modules are expected to return '1' here if the - * se_cmd being passed is released at this point, - * or zero if not being released. - */ - if (cmd->se_tfo->check_stop_free != NULL) { - spin_unlock_irqrestore( - &cmd->t_state_lock, flags); - - return cmd->se_tfo->check_stop_free(cmd); - } + + cmd->transport_state &= ~CMD_T_ACTIVE; + if (remove_from_lists) { + /* + * Some fabric modules like tcm_loop can release + * their internally allocated I/O reference now and + * struct se_cmd now. + * + * Fabric modules are expected to return '1' here if the + * se_cmd being passed is released at this point, + * or zero if not being released. + */ + if (cmd->se_tfo->check_stop_free != NULL) { + spin_unlock_irqrestore(&cmd->t_state_lock, flags); + return cmd->se_tfo->check_stop_free(cmd); } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); + } - return 0; - } else if (t_state) - cmd->t_state = t_state; spin_unlock_irqrestore(&cmd->t_state_lock, flags); - return 0; } static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd) { - return transport_cmd_check_stop(cmd, 2, 0); + return transport_cmd_check_stop(cmd, true, false); } static void transport_lun_remove_cmd(struct se_cmd *cmd) { struct se_lun *lun = cmd->se_lun; - unsigned long flags; if (!lun) return; - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->transport_state & CMD_T_DEV_ACTIVE) { - cmd->transport_state &= ~CMD_T_DEV_ACTIVE; - target_remove_from_state_list(cmd); - } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - spin_lock_irqsave(&lun->lun_cmd_lock, flags); - if (!list_empty(&cmd->se_lun_node)) - list_del_init(&cmd->se_lun_node); - spin_unlock_irqrestore(&lun->lun_cmd_lock, flags); + if (cmpxchg(&cmd->lun_ref_active, true, false)) + percpu_ref_put(&lun->lun_ref); } void transport_cmd_finish_abort(struct se_cmd *cmd, int remove) { - if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) + if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) transport_lun_remove_cmd(cmd); + /* + * Allow the fabric driver to unmap any resources before + * releasing the descriptor via TFO->release_cmd() + */ + if (remove) + cmd->se_tfo->aborted_task(cmd); if (transport_cmd_check_stop_to_fabric(cmd)) return; - if (remove) { - transport_remove_cmd_from_queue(cmd); + if (remove) transport_put_cmd(cmd); - } } -static void transport_add_cmd_to_queue(struct se_cmd *cmd, int t_state, - bool at_head) +static void target_complete_failure_work(struct work_struct *work) { - struct se_device *dev = cmd->se_dev; - struct se_queue_obj *qobj = &dev->dev_queue_obj; - unsigned long flags; - - if (t_state) { - spin_lock_irqsave(&cmd->t_state_lock, flags); - cmd->t_state = t_state; - cmd->transport_state |= CMD_T_ACTIVE; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - } - - spin_lock_irqsave(&qobj->cmd_queue_lock, flags); - - /* If the cmd is already on the list, remove it before we add it */ - if (!list_empty(&cmd->se_queue_node)) - list_del(&cmd->se_queue_node); - else - atomic_inc(&qobj->queue_cnt); - - if (at_head) - list_add(&cmd->se_queue_node, &qobj->qobj_list); - else - list_add_tail(&cmd->se_queue_node, &qobj->qobj_list); - cmd->transport_state |= CMD_T_QUEUED; - spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags); + struct se_cmd *cmd = container_of(work, struct se_cmd, work); - wake_up_interruptible(&qobj->thread_wq); + transport_generic_request_failure(cmd, + TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE); } -static struct se_cmd * -transport_get_cmd_from_queue(struct se_queue_obj *qobj) +/* + * Used when asking transport to copy Sense Data from the underlying + * Linux/SCSI struct scsi_cmnd + */ +static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd) { - struct se_cmd *cmd; - unsigned long flags; - - spin_lock_irqsave(&qobj->cmd_queue_lock, flags); - if (list_empty(&qobj->qobj_list)) { - spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags); - return NULL; - } - cmd = list_first_entry(&qobj->qobj_list, struct se_cmd, se_queue_node); - - cmd->transport_state &= ~CMD_T_QUEUED; - list_del_init(&cmd->se_queue_node); - atomic_dec(&qobj->queue_cnt); - spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags); + struct se_device *dev = cmd->se_dev; - return cmd; -} + WARN_ON(!cmd->se_lun); -static void transport_remove_cmd_from_queue(struct se_cmd *cmd) -{ - struct se_queue_obj *qobj = &cmd->se_dev->dev_queue_obj; - unsigned long flags; + if (!dev) + return NULL; - spin_lock_irqsave(&qobj->cmd_queue_lock, flags); - if (!(cmd->transport_state & CMD_T_QUEUED)) { - spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags); - return; - } - cmd->transport_state &= ~CMD_T_QUEUED; - atomic_dec(&qobj->queue_cnt); - list_del_init(&cmd->se_queue_node); - spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags); -} + if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) + return NULL; -static void target_complete_failure_work(struct work_struct *work) -{ - struct se_cmd *cmd = container_of(work, struct se_cmd, work); + cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; - transport_generic_request_failure(cmd); + pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n", + dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); + return cmd->sense_buffer; } void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) @@ -686,11 +664,11 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) cmd->transport_state &= ~CMD_T_BUSY; if (dev && dev->transport->transport_complete) { - if (dev->transport->transport_complete(cmd, - cmd->t_data_sg) != 0) { - cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE; + dev->transport->transport_complete(cmd, + cmd->t_data_sg, + transport_get_sense_buffer(cmd)); + if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) success = 1; - } } /* @@ -702,20 +680,16 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) return; } - if (!success) - cmd->transport_state |= CMD_T_FAILED; - /* - * Check for case where an explict ABORT_TASK has been received + * Check for case where an explicit ABORT_TASK has been received * and transport_wait_for_tasks() will be waiting for completion.. */ if (cmd->transport_state & CMD_T_ABORTED && cmd->transport_state & CMD_T_STOP) { spin_unlock_irqrestore(&cmd->t_state_lock, flags); - complete(&cmd->t_transport_stop_comp); + complete_all(&cmd->t_transport_stop_comp); return; - } else if (cmd->transport_state & CMD_T_FAILED) { - cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } else if (!success) { INIT_WORK(&cmd->work, target_complete_failure_work); } else { INIT_WORK(&cmd->work, target_complete_ok_work); @@ -729,83 +703,43 @@ void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status) } EXPORT_SYMBOL(target_complete_cmd); -static void target_add_to_state_list(struct se_cmd *cmd) +void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length) { - struct se_device *dev = cmd->se_dev; - unsigned long flags; + if (scsi_status == SAM_STAT_GOOD && length < cmd->data_length) { + if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) { + cmd->residual_count += cmd->data_length - length; + } else { + cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; + cmd->residual_count = cmd->data_length - length; + } - spin_lock_irqsave(&dev->execute_task_lock, flags); - if (!cmd->state_active) { - list_add_tail(&cmd->state_list, &dev->state_list); - cmd->state_active = true; + cmd->data_length = length; } - spin_unlock_irqrestore(&dev->execute_task_lock, flags); -} - -static void __target_add_to_execute_list(struct se_cmd *cmd) -{ - struct se_device *dev = cmd->se_dev; - bool head_of_queue = false; - - if (!list_empty(&cmd->execute_list)) - return; - - if (dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED && - cmd->sam_task_attr == MSG_HEAD_TAG) - head_of_queue = true; - - if (head_of_queue) - list_add(&cmd->execute_list, &dev->execute_list); - else - list_add_tail(&cmd->execute_list, &dev->execute_list); - - atomic_inc(&dev->execute_tasks); - - if (cmd->state_active) - return; - - if (head_of_queue) - list_add(&cmd->state_list, &dev->state_list); - else - list_add_tail(&cmd->state_list, &dev->state_list); - - cmd->state_active = true; -} - -static void target_add_to_execute_list(struct se_cmd *cmd) -{ - unsigned long flags; - struct se_device *dev = cmd->se_dev; - - spin_lock_irqsave(&dev->execute_task_lock, flags); - __target_add_to_execute_list(cmd); - spin_unlock_irqrestore(&dev->execute_task_lock, flags); -} -void __target_remove_from_execute_list(struct se_cmd *cmd) -{ - list_del_init(&cmd->execute_list); - atomic_dec(&cmd->se_dev->execute_tasks); + target_complete_cmd(cmd, scsi_status); } +EXPORT_SYMBOL(target_complete_cmd_with_length); -static void target_remove_from_execute_list(struct se_cmd *cmd) +static void target_add_to_state_list(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; unsigned long flags; - if (WARN_ON(list_empty(&cmd->execute_list))) - return; - spin_lock_irqsave(&dev->execute_task_lock, flags); - __target_remove_from_execute_list(cmd); + if (!cmd->state_active) { + list_add_tail(&cmd->state_list, &dev->state_list); + cmd->state_active = true; + } spin_unlock_irqrestore(&dev->execute_task_lock, flags); } /* * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status */ +static void transport_write_pending_qf(struct se_cmd *cmd); +static void transport_complete_qf(struct se_cmd *cmd); -static void target_qf_do_work(struct work_struct *work) +void target_qf_do_work(struct work_struct *work) { struct se_device *dev = container_of(work, struct se_device, qf_work_queue); @@ -819,7 +753,7 @@ static void target_qf_do_work(struct work_struct *work) list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) { list_del(&cmd->se_qf_node); atomic_dec(&dev->dev_qf_count); - smp_mb__after_atomic_dec(); + smp_mb__after_atomic(); pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue" " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd, @@ -827,7 +761,10 @@ static void target_qf_do_work(struct work_struct *work) (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING" : "UNKNOWN"); - transport_add_cmd_to_queue(cmd, cmd->t_state, true); + if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) + transport_write_pending_qf(cmd); + else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) + transport_complete_qf(cmd); } } @@ -855,30 +792,15 @@ void transport_dump_dev_state( int *bl) { *bl += sprintf(b + *bl, "Status: "); - switch (dev->dev_status) { - case TRANSPORT_DEVICE_ACTIVATED: + if (dev->export_count) *bl += sprintf(b + *bl, "ACTIVATED"); - break; - case TRANSPORT_DEVICE_DEACTIVATED: + else *bl += sprintf(b + *bl, "DEACTIVATED"); - break; - case TRANSPORT_DEVICE_SHUTDOWN: - *bl += sprintf(b + *bl, "SHUTDOWN"); - break; - case TRANSPORT_DEVICE_OFFLINE_ACTIVATED: - case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED: - *bl += sprintf(b + *bl, "OFFLINE"); - break; - default: - *bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status); - break; - } - *bl += sprintf(b + *bl, " Execute/Max Queue Depth: %d/%d", - atomic_read(&dev->execute_tasks), dev->queue_depth); + *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth); *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n", - dev->se_sub_dev->se_dev_attrib.block_size, - dev->se_sub_dev->se_dev_attrib.hw_max_sectors); + dev->dev_attrib.block_size, + dev->dev_attrib.hw_max_sectors); *bl += sprintf(b + *bl, " "); } @@ -1072,15 +994,18 @@ int transport_dump_vpd_ident( switch (vpd->device_identifier_code_set) { case 0x01: /* Binary */ - sprintf(buf, "T10 VPD Binary Device Identifier: %s\n", + snprintf(buf, sizeof(buf), + "T10 VPD Binary Device Identifier: %s\n", &vpd->device_identifier[0]); break; case 0x02: /* ASCII */ - sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n", + snprintf(buf, sizeof(buf), + "T10 VPD ASCII Device Identifier: %s\n", &vpd->device_identifier[0]); break; case 0x03: /* UTF-8 */ - sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n", + snprintf(buf, sizeof(buf), + "T10 VPD UTF-8 Device Identifier: %s\n", &vpd->device_identifier[0]); break; default: @@ -1102,7 +1027,7 @@ int transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) { static const char hex_str[] = "0123456789abcdef"; - int j = 0, i = 4; /* offset to start of the identifer */ + int j = 0, i = 4; /* offset to start of the identifier */ /* * The VPD Code Set (encoding) @@ -1135,216 +1060,55 @@ transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83) } EXPORT_SYMBOL(transport_set_vpd_ident); -static void core_setup_task_attr_emulation(struct se_device *dev) -{ - /* - * If this device is from Target_Core_Mod/pSCSI, disable the - * SAM Task Attribute emulation. - * - * This is currently not available in upsream Linux/SCSI Target - * mode code, and is assumed to be disabled while using TCM/pSCSI. - */ - if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) { - dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH; - return; - } - - dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED; - pr_debug("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x" - " device\n", dev->transport->name, - dev->transport->get_device_rev(dev)); -} - -static void scsi_dump_inquiry(struct se_device *dev) +sense_reason_t +target_cmd_size_check(struct se_cmd *cmd, unsigned int size) { - struct t10_wwn *wwn = &dev->se_sub_dev->t10_wwn; - char buf[17]; - int i, device_type; - /* - * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer - */ - for (i = 0; i < 8; i++) - if (wwn->vendor[i] >= 0x20) - buf[i] = wwn->vendor[i]; - else - buf[i] = ' '; - buf[i] = '\0'; - pr_debug(" Vendor: %s\n", buf); - - for (i = 0; i < 16; i++) - if (wwn->model[i] >= 0x20) - buf[i] = wwn->model[i]; - else - buf[i] = ' '; - buf[i] = '\0'; - pr_debug(" Model: %s\n", buf); - - for (i = 0; i < 4; i++) - if (wwn->revision[i] >= 0x20) - buf[i] = wwn->revision[i]; - else - buf[i] = ' '; - buf[i] = '\0'; - pr_debug(" Revision: %s\n", buf); - - device_type = dev->transport->get_device_type(dev); - pr_debug(" Type: %s ", scsi_device_type(device_type)); - pr_debug(" ANSI SCSI revision: %02x\n", - dev->transport->get_device_rev(dev)); -} - -struct se_device *transport_add_device_to_core_hba( - struct se_hba *hba, - struct se_subsystem_api *transport, - struct se_subsystem_dev *se_dev, - u32 device_flags, - void *transport_dev, - struct se_dev_limits *dev_limits, - const char *inquiry_prod, - const char *inquiry_rev) -{ - int force_pt; - struct se_device *dev; - - dev = kzalloc(sizeof(struct se_device), GFP_KERNEL); - if (!dev) { - pr_err("Unable to allocate memory for se_dev_t\n"); - return NULL; - } + struct se_device *dev = cmd->se_dev; - transport_init_queue_obj(&dev->dev_queue_obj); - dev->dev_flags = device_flags; - dev->dev_status |= TRANSPORT_DEVICE_DEACTIVATED; - dev->dev_ptr = transport_dev; - dev->se_hba = hba; - dev->se_sub_dev = se_dev; - dev->transport = transport; - INIT_LIST_HEAD(&dev->dev_list); - INIT_LIST_HEAD(&dev->dev_sep_list); - INIT_LIST_HEAD(&dev->dev_tmr_list); - INIT_LIST_HEAD(&dev->execute_list); - INIT_LIST_HEAD(&dev->delayed_cmd_list); - INIT_LIST_HEAD(&dev->state_list); - INIT_LIST_HEAD(&dev->qf_cmd_list); - spin_lock_init(&dev->execute_task_lock); - spin_lock_init(&dev->delayed_cmd_lock); - spin_lock_init(&dev->dev_reservation_lock); - spin_lock_init(&dev->dev_status_lock); - spin_lock_init(&dev->se_port_lock); - spin_lock_init(&dev->se_tmr_lock); - spin_lock_init(&dev->qf_cmd_lock); - atomic_set(&dev->dev_ordered_id, 0); - - se_dev_set_default_attribs(dev, dev_limits); - - dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX); - dev->creation_time = get_jiffies_64(); - spin_lock_init(&dev->stats_lock); - - spin_lock(&hba->device_lock); - list_add_tail(&dev->dev_list, &hba->hba_dev_list); - hba->dev_count++; - spin_unlock(&hba->device_lock); - /* - * Setup the SAM Task Attribute emulation for struct se_device - */ - core_setup_task_attr_emulation(dev); - /* - * Force PR and ALUA passthrough emulation with internal object use. - */ - force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE); - /* - * Setup the Reservations infrastructure for struct se_device - */ - core_setup_reservations(dev, force_pt); - /* - * Setup the Asymmetric Logical Unit Assignment for struct se_device - */ - if (core_setup_alua(dev, force_pt) < 0) - goto out; + if (cmd->unknown_data_length) { + cmd->data_length = size; + } else if (size != cmd->data_length) { + pr_warn("TARGET_CORE[%s]: Expected Transfer Length:" + " %u does not match SCSI CDB Length: %u for SAM Opcode:" + " 0x%02x\n", cmd->se_tfo->get_fabric_name(), + cmd->data_length, size, cmd->t_task_cdb[0]); - /* - * Startup the struct se_device processing thread - */ - dev->process_thread = kthread_run(transport_processing_thread, dev, - "LIO_%s", dev->transport->name); - if (IS_ERR(dev->process_thread)) { - pr_err("Unable to create kthread: LIO_%s\n", - dev->transport->name); - goto out; - } - /* - * Setup work_queue for QUEUE_FULL - */ - INIT_WORK(&dev->qf_work_queue, target_qf_do_work); - /* - * Preload the initial INQUIRY const values if we are doing - * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI - * passthrough because this is being provided by the backend LLD. - * This is required so that transport_get_inquiry() copies these - * originals once back into DEV_T10_WWN(dev) for the virtual device - * setup. - */ - if (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) { - if (!inquiry_prod || !inquiry_rev) { - pr_err("All non TCM/pSCSI plugins require" - " INQUIRY consts\n"); - goto out; + if (cmd->data_direction == DMA_TO_DEVICE) { + pr_err("Rejecting underflow/overflow" + " WRITE data\n"); + return TCM_INVALID_CDB_FIELD; + } + /* + * Reject READ_* or WRITE_* with overflow/underflow for + * type SCF_SCSI_DATA_CDB. + */ + if (dev->dev_attrib.block_size != 512) { + pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" + " CDB on non 512-byte sector setup subsystem" + " plugin: %s\n", dev->transport->name); + /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ + return TCM_INVALID_CDB_FIELD; + } + /* + * For the overflow case keep the existing fabric provided + * ->data_length. Otherwise for the underflow case, reset + * ->data_length to the smaller SCSI expected data transfer + * length. + */ + if (size > cmd->data_length) { + cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; + cmd->residual_count = (size - cmd->data_length); + } else { + cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; + cmd->residual_count = (cmd->data_length - size); + cmd->data_length = size; } - - strncpy(&dev->se_sub_dev->t10_wwn.vendor[0], "LIO-ORG", 8); - strncpy(&dev->se_sub_dev->t10_wwn.model[0], inquiry_prod, 16); - strncpy(&dev->se_sub_dev->t10_wwn.revision[0], inquiry_rev, 4); } - scsi_dump_inquiry(dev); - - return dev; -out: - kthread_stop(dev->process_thread); - - spin_lock(&hba->device_lock); - list_del(&dev->dev_list); - hba->dev_count--; - spin_unlock(&hba->device_lock); - - se_release_vpd_for_dev(dev); - - kfree(dev); - return NULL; -} -EXPORT_SYMBOL(transport_add_device_to_core_hba); + return 0; -/* transport_generic_prepare_cdb(): - * - * Since the Initiator sees iSCSI devices as LUNs, the SCSI CDB will - * contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2. - * The point of this is since we are mapping iSCSI LUNs to - * SCSI Target IDs having a non-zero LUN in the CDB will throw the - * devices and HBAs for a loop. - */ -static inline void transport_generic_prepare_cdb( - unsigned char *cdb) -{ - switch (cdb[0]) { - case READ_10: /* SBC - RDProtect */ - case READ_12: /* SBC - RDProtect */ - case READ_16: /* SBC - RDProtect */ - case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */ - case VERIFY: /* SBC - VRProtect */ - case VERIFY_16: /* SBC - VRProtect */ - case WRITE_VERIFY: /* SBC - VRProtect */ - case WRITE_VERIFY_12: /* SBC - VRProtect */ - case MAINTENANCE_IN: /* SPC - Parameter Data Format for SA RTPG */ - break; - default: - cdb[1] &= 0x1f; /* clear logical unit number */ - break; - } } -static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *); - /* * Used by fabric modules containing a local struct se_cmd within their * fabric dependent per I/O descriptor. @@ -1358,19 +1122,15 @@ void transport_init_se_cmd( int task_attr, unsigned char *sense_buffer) { - INIT_LIST_HEAD(&cmd->se_lun_node); INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); - INIT_LIST_HEAD(&cmd->se_queue_node); INIT_LIST_HEAD(&cmd->se_cmd_list); - INIT_LIST_HEAD(&cmd->execute_list); INIT_LIST_HEAD(&cmd->state_list); - init_completion(&cmd->transport_lun_fe_stop_comp); - init_completion(&cmd->transport_lun_stop_comp); init_completion(&cmd->t_transport_stop_comp); init_completion(&cmd->cmd_wait_comp); init_completion(&cmd->task_stop_comp); spin_lock_init(&cmd->t_state_lock); + kref_init(&cmd->cmd_kref); cmd->transport_state = CMD_T_DEV_ACTIVE; cmd->se_tfo = tfo; @@ -1384,43 +1144,41 @@ void transport_init_se_cmd( } EXPORT_SYMBOL(transport_init_se_cmd); -static int transport_check_alloc_task_attr(struct se_cmd *cmd) +static sense_reason_t +transport_check_alloc_task_attr(struct se_cmd *cmd) { + struct se_device *dev = cmd->se_dev; + /* * Check if SAM Task Attribute emulation is enabled for this * struct se_device storage object */ - if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) + if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) return 0; if (cmd->sam_task_attr == MSG_ACA_TAG) { pr_debug("SAM Task Attribute ACA" " emulation is not supported\n"); - return -EINVAL; + return TCM_INVALID_CDB_FIELD; } /* * Used to determine when ORDERED commands should go from * Dormant to Active status. */ - cmd->se_ordered_id = atomic_inc_return(&cmd->se_dev->dev_ordered_id); - smp_mb__after_atomic_inc(); + cmd->se_ordered_id = atomic_inc_return(&dev->dev_ordered_id); + smp_mb__after_atomic(); pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n", cmd->se_ordered_id, cmd->sam_task_attr, - cmd->se_dev->transport->name); + dev->transport->name); return 0; } -/* target_setup_cmd_from_cdb(): - * - * Called from fabric RX Thread. - */ -int target_setup_cmd_from_cdb( - struct se_cmd *cmd, - unsigned char *cdb) +sense_reason_t +target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) { - int ret; + struct se_device *dev = cmd->se_dev; + sense_reason_t ret; - transport_generic_prepare_cdb(cdb); /* * Ensure that the received CDB is less than the max (252 + 8) bytes * for VARIABLE_LENGTH_CMD @@ -1429,9 +1187,7 @@ int target_setup_cmd_from_cdb( pr_err("Received SCSI CDB with command_size: %d that" " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD; - return -EINVAL; + return TCM_INVALID_CDB_FIELD; } /* * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, @@ -1446,10 +1202,7 @@ int target_setup_cmd_from_cdb( " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", scsi_command_size(cdb), (unsigned long)sizeof(cmd->__t_task_cdb)); - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = - TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - return -ENOMEM; + return TCM_OUT_OF_RESOURCES; } } else cmd->t_task_cdb = &cmd->__t_task_cdb[0]; @@ -1457,23 +1210,36 @@ int target_setup_cmd_from_cdb( * Copy the original CDB into cmd-> */ memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); + + trace_target_sequencer_start(cmd); + /* - * Setup the received CDB based on SCSI defined opcodes and - * perform unit attention, persistent reservations and ALUA - * checks for virtual device backends. The cmd->t_task_cdb - * pointer is expected to be setup before we reach this point. + * Check for an existing UNIT ATTENTION condition */ - ret = transport_generic_cmd_sequencer(cmd, cdb); - if (ret < 0) + ret = target_scsi3_ua_check(cmd); + if (ret) + return ret; + + ret = target_alua_state_check(cmd); + if (ret) + return ret; + + ret = target_check_reservation(cmd); + if (ret) { + cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; return ret; - /* - * Check for SAM Task Attribute Emulation - */ - if (transport_check_alloc_task_attr(cmd) < 0) { - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD; - return -EINVAL; } + + ret = dev->transport->parse_cdb(cmd); + if (ret) + return ret; + + ret = transport_check_alloc_task_attr(cmd); + if (ret) + return ret; + + cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; + spin_lock(&cmd->se_lun->lun_sep_lock); if (cmd->se_lun->lun_sep) cmd->se_lun->lun_sep->sep_stats.cmd_pdus++; @@ -1489,7 +1255,7 @@ EXPORT_SYMBOL(target_setup_cmd_from_cdb); int transport_handle_cdb_direct( struct se_cmd *cmd) { - int ret; + sense_reason_t ret; if (!cmd->se_lun) { dump_stack(); @@ -1503,10 +1269,9 @@ int transport_handle_cdb_direct( return -EINVAL; } /* - * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE following - * transport_generic_handle_cdb*() -> transport_add_cmd_to_queue() - * in existing usage to ensure that outstanding descriptors are handled - * correctly during shutdown via transport_wait_for_tasks() + * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that + * outstanding descriptors are handled correctly during shutdown via + * transport_wait_for_tasks() * * Also, we don't take cmd->t_state_lock here as we only expect * this to be called for initial descriptor submission. @@ -1520,15 +1285,44 @@ int transport_handle_cdb_direct( * and call transport_generic_request_failure() if necessary.. */ ret = transport_generic_new_cmd(cmd); - if (ret < 0) - transport_generic_request_failure(cmd); - + if (ret) + transport_generic_request_failure(cmd, ret); return 0; } EXPORT_SYMBOL(transport_handle_cdb_direct); -/** - * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd +sense_reason_t +transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl, + u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count) +{ + if (!sgl || !sgl_count) + return 0; + + /* + * Reject SCSI data overflow with map_mem_to_cmd() as incoming + * scatterlists already have been set to follow what the fabric + * passes for the original expected data transfer length. + */ + if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { + pr_warn("Rejecting SCSI DATA overflow for fabric using" + " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); + return TCM_INVALID_CDB_FIELD; + } + + cmd->t_data_sg = sgl; + cmd->t_data_nents = sgl_count; + + if (sgl_bidi && sgl_bidi_count) { + cmd->t_bidi_data_sg = sgl_bidi; + cmd->t_bidi_data_nents = sgl_bidi_count; + } + cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; + return 0; +} + +/* + * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized + * se_cmd + use pre-allocated SGL memory. * * @se_cmd: command descriptor to submit * @se_sess: associated se_sess for endpoint @@ -1539,16 +1333,30 @@ EXPORT_SYMBOL(transport_handle_cdb_direct); * @task_addr: SAM task attribute * @data_dir: DMA data direction * @flags: flags for command submission from target_sc_flags_tables + * @sgl: struct scatterlist memory for unidirectional mapping + * @sgl_count: scatterlist count for unidirectional mapping + * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping + * @sgl_bidi_count: scatterlist count for bidirectional READ mapping + * @sgl_prot: struct scatterlist memory protection information + * @sgl_prot_count: scatterlist count for protection information + * + * Returns non zero to signal active I/O shutdown failure. All other + * setup exceptions will be returned as a SCSI CHECK_CONDITION response, + * but still return zero here. * * This may only be called from process context, and also currently * assumes internal allocation of fabric payload buffer by target-core. - **/ -void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, + */ +int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess, unsigned char *cdb, unsigned char *sense, u32 unpacked_lun, - u32 data_length, int task_attr, int data_dir, int flags) + u32 data_length, int task_attr, int data_dir, int flags, + struct scatterlist *sgl, u32 sgl_count, + struct scatterlist *sgl_bidi, u32 sgl_bidi_count, + struct scatterlist *sgl_prot, u32 sgl_prot_count) { struct se_portal_group *se_tpg; - int rc; + sense_reason_t rc; + int ret; se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); @@ -1569,7 +1377,9 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, * for fabrics using TARGET_SCF_ACK_KREF that expect a second * kref_put() to happen during fabric packet acknowledgement. */ - target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF)); + ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF)); + if (ret) + return ret; /* * Signal bidirectional data payloads to target-core */ @@ -1578,20 +1388,63 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, /* * Locate se_lun pointer and attach it to struct se_cmd */ - if (transport_lookup_cmd_lun(se_cmd, unpacked_lun) < 0) { - transport_send_check_condition_and_sense(se_cmd, - se_cmd->scsi_sense_reason, 0); + rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); + if (rc) { + transport_send_check_condition_and_sense(se_cmd, rc, 0); target_put_sess_cmd(se_sess, se_cmd); - return; + return 0; } - /* - * Sanitize CDBs via transport_generic_cmd_sequencer() and - * allocate the necessary tasks to complete the received CDB+data - */ + rc = target_setup_cmd_from_cdb(se_cmd, cdb); if (rc != 0) { - transport_generic_request_failure(se_cmd); - return; + transport_generic_request_failure(se_cmd, rc); + return 0; + } + + /* + * Save pointers for SGLs containing protection information, + * if present. + */ + if (sgl_prot_count) { + se_cmd->t_prot_sg = sgl_prot; + se_cmd->t_prot_nents = sgl_prot_count; + } + + /* + * When a non zero sgl_count has been passed perform SGL passthrough + * mapping for pre-allocated fabric memory instead of having target + * core perform an internal SGL allocation.. + */ + if (sgl_count != 0) { + BUG_ON(!sgl); + + /* + * A work-around for tcm_loop as some userspace code via + * scsi-generic do not memset their associated read buffers, + * so go ahead and do that here for type non-data CDBs. Also + * note that this is currently guaranteed to be a single SGL + * for this case by target core in target_setup_cmd_from_cdb() + * -> transport_generic_cmd_sequencer(). + */ + if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) && + se_cmd->data_direction == DMA_FROM_DEVICE) { + unsigned char *buf = NULL; + + if (sgl) + buf = kmap(sg_page(sgl)) + sgl->offset; + + if (buf) { + memset(buf, 0, sgl->length); + kunmap(sg_page(sgl)); + } + } + + rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count, + sgl_bidi, sgl_bidi_count); + if (rc != 0) { + transport_generic_request_failure(se_cmd, rc); + return 0; + } } /* @@ -1600,14 +1453,40 @@ void target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, */ core_alua_check_nonop_delay(se_cmd); - /* - * Dispatch se_cmd descriptor to se_lun->lun_se_dev backend - * for immediate execution of READs, otherwise wait for - * transport_generic_handle_data() to be called for WRITEs - * when fabric has filled the incoming buffer. - */ transport_handle_cdb_direct(se_cmd); - return; + return 0; +} +EXPORT_SYMBOL(target_submit_cmd_map_sgls); + +/* + * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd + * + * @se_cmd: command descriptor to submit + * @se_sess: associated se_sess for endpoint + * @cdb: pointer to SCSI CDB + * @sense: pointer to SCSI sense buffer + * @unpacked_lun: unpacked LUN to reference for struct se_lun + * @data_length: fabric expected data transfer length + * @task_addr: SAM task attribute + * @data_dir: DMA data direction + * @flags: flags for command submission from target_sc_flags_tables + * + * Returns non zero to signal active I/O shutdown failure. All other + * setup exceptions will be returned as a SCSI CHECK_CONDITION response, + * but still return zero here. + * + * This may only be called from process context, and also currently + * assumes internal allocation of fabric payload buffer by target-core. + * + * It also assumes interal target core SGL memory allocation. + */ +int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, + unsigned char *cdb, unsigned char *sense, u32 unpacked_lun, + u32 data_length, int task_attr, int data_dir, int flags) +{ + return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense, + unpacked_lun, data_length, task_attr, data_dir, + flags, NULL, 0, NULL, 0, NULL, 0); } EXPORT_SYMBOL(target_submit_cmd); @@ -1617,7 +1496,8 @@ static void target_complete_tmr_failure(struct work_struct *work) se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST; se_cmd->se_tfo->queue_tm_rsp(se_cmd); - transport_generic_free_cmd(se_cmd, 0); + + transport_cmd_check_stop_to_fabric(se_cmd); } /** @@ -1662,7 +1542,11 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, se_cmd->se_tmr_req->ref_task_tag = tag; /* See target_submit_cmd for commentary */ - target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF)); + ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF)); + if (ret) { + core_tmr_release_req(se_cmd->se_tmr_req); + return ret; + } ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); if (ret) { @@ -1680,67 +1564,6 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, EXPORT_SYMBOL(target_submit_tmr); /* - * Used by fabric module frontends defining a TFO->new_cmd_map() caller - * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to - * complete setup in TCM process context w/ TFO->new_cmd_map(). - */ -int transport_generic_handle_cdb_map( - struct se_cmd *cmd) -{ - if (!cmd->se_lun) { - dump_stack(); - pr_err("cmd->se_lun is NULL\n"); - return -EINVAL; - } - - transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP, false); - return 0; -} -EXPORT_SYMBOL(transport_generic_handle_cdb_map); - -/* transport_generic_handle_data(): - * - * - */ -int transport_generic_handle_data( - struct se_cmd *cmd) -{ - /* - * For the software fabric case, then we assume the nexus is being - * failed/shutdown when signals are pending from the kthread context - * caller, so we return a failure. For the HW target mode case running - * in interrupt code, the signal_pending() check is skipped. - */ - if (!in_interrupt() && signal_pending(current)) - return -EPERM; - /* - * If the received CDB has aleady been ABORTED by the generic - * target engine, we now call transport_check_aborted_status() - * to queue any delated TASK_ABORTED status for the received CDB to the - * fabric module as we are expecting no further incoming DATA OUT - * sequences at this point. - */ - if (transport_check_aborted_status(cmd, 1) != 0) - return 0; - - transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE, false); - return 0; -} -EXPORT_SYMBOL(transport_generic_handle_data); - -/* transport_generic_handle_tmr(): - * - * - */ -int transport_generic_handle_tmr( - struct se_cmd *cmd) -{ - transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR, false); - return 0; -} -EXPORT_SYMBOL(transport_generic_handle_tmr); - -/* * If the cmd is active, request it to be stopped and sleep until it * has completed. */ @@ -1768,16 +1591,17 @@ bool target_stop_cmd(struct se_cmd *cmd, unsigned long *flags) /* * Handle SAM-esque emulation for generic transport request failures. */ -void transport_generic_request_failure(struct se_cmd *cmd) +void transport_generic_request_failure(struct se_cmd *cmd, + sense_reason_t sense_reason) { int ret = 0; pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x" " CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd), cmd->t_task_cdb[0]); - pr_debug("-----[ i_state: %d t_state: %d scsi_sense_reason: %d\n", + pr_debug("-----[ i_state: %d t_state: %d sense_reason: %d\n", cmd->se_tfo->get_cmd_state(cmd), - cmd->t_state, cmd->scsi_sense_reason); + cmd->t_state, sense_reason); pr_debug("-----[ CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n", (cmd->transport_state & CMD_T_ACTIVE) != 0, (cmd->transport_state & CMD_T_STOP) != 0, @@ -1786,20 +1610,34 @@ void transport_generic_request_failure(struct se_cmd *cmd) /* * For SAM Task Attribute emulation for failed struct se_cmd */ - if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED) - transport_complete_task_attr(cmd); + transport_complete_task_attr(cmd); + /* + * Handle special case for COMPARE_AND_WRITE failure, where the + * callback is expected to drop the per device ->caw_mutex. + */ + if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) && + cmd->transport_complete_callback) + cmd->transport_complete_callback(cmd); - switch (cmd->scsi_sense_reason) { + switch (sense_reason) { case TCM_NON_EXISTENT_LUN: case TCM_UNSUPPORTED_SCSI_OPCODE: case TCM_INVALID_CDB_FIELD: case TCM_INVALID_PARAMETER_LIST: + case TCM_PARAMETER_LIST_LENGTH_ERROR: case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: case TCM_UNKNOWN_MODE_PAGE: case TCM_WRITE_PROTECTED: + case TCM_ADDRESS_OUT_OF_RANGE: case TCM_CHECK_CONDITION_ABORT_CMD: case TCM_CHECK_CONDITION_UNIT_ATTENTION: case TCM_CHECK_CONDITION_NOT_READY: + case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: + case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: + case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: + break; + case TCM_OUT_OF_RESOURCES: + sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; break; case TCM_RESERVATION_CONFLICT: /* @@ -1817,30 +1655,24 @@ void transport_generic_request_failure(struct se_cmd *cmd) * See spc4r17, section 7.4.6 Control Mode Page, Table 349 */ if (cmd->se_sess && - cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2) + cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) core_scsi3_ua_allocate(cmd->se_sess->se_node_acl, cmd->orig_fe_lun, 0x2C, ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS); - ret = cmd->se_tfo->queue_status(cmd); + trace_target_cmd_complete(cmd); + ret = cmd->se_tfo-> queue_status(cmd); if (ret == -EAGAIN || ret == -ENOMEM) goto queue_full; goto check_stop; default: pr_err("Unknown transport error for CDB 0x%02x: %d\n", - cmd->t_task_cdb[0], cmd->scsi_sense_reason); - cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; + cmd->t_task_cdb[0], sense_reason); + sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; break; } - /* - * If a fabric does not define a cmd->se_tfo->new_cmd_map caller, - * make the call to transport_send_check_condition_and_sense() - * directly. Otherwise expect the fabric to make the call to - * transport_send_check_condition_and_sense() after handling - * possible unsoliticied write data payloads. - */ - ret = transport_send_check_condition_and_sense(cmd, - cmd->scsi_sense_reason, 0); + + ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0); if (ret == -EAGAIN || ret == -ENOMEM) goto queue_full; @@ -1856,1193 +1688,149 @@ queue_full: } EXPORT_SYMBOL(transport_generic_request_failure); -static inline u32 transport_lba_21(unsigned char *cdb) -{ - return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3]; -} - -static inline u32 transport_lba_32(unsigned char *cdb) -{ - return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5]; -} - -static inline unsigned long long transport_lba_64(unsigned char *cdb) -{ - unsigned int __v1, __v2; - - __v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5]; - __v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9]; - - return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32; -} - -/* - * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs - */ -static inline unsigned long long transport_lba_64_ext(unsigned char *cdb) +void __target_execute_cmd(struct se_cmd *cmd) { - unsigned int __v1, __v2; + sense_reason_t ret; - __v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15]; - __v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19]; + if (cmd->execute_cmd) { + ret = cmd->execute_cmd(cmd); + if (ret) { + spin_lock_irq(&cmd->t_state_lock); + cmd->transport_state &= ~(CMD_T_BUSY|CMD_T_SENT); + spin_unlock_irq(&cmd->t_state_lock); - return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32; + transport_generic_request_failure(cmd, ret); + } + } } -static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd) +static bool target_handle_task_attr(struct se_cmd *cmd) { - unsigned long flags; + struct se_device *dev = cmd->se_dev; - spin_lock_irqsave(&se_cmd->t_state_lock, flags); - se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE; - spin_unlock_irqrestore(&se_cmd->t_state_lock, flags); -} + if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) + return false; -/* - * Called from Fabric Module context from transport_execute_tasks() - * - * The return of this function determins if the tasks from struct se_cmd - * get added to the execution queue in transport_execute_tasks(), - * or are added to the delayed or ordered lists here. - */ -static inline int transport_execute_task_attr(struct se_cmd *cmd) -{ - if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) - return 1; /* * Check for the existence of HEAD_OF_QUEUE, and if true return 1 * to allow the passed struct se_cmd list of tasks to the front of the list. */ - if (cmd->sam_task_attr == MSG_HEAD_TAG) { - pr_debug("Added HEAD_OF_QUEUE for CDB:" - " 0x%02x, se_ordered_id: %u\n", - cmd->t_task_cdb[0], - cmd->se_ordered_id); - return 1; - } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) { - atomic_inc(&cmd->se_dev->dev_ordered_sync); - smp_mb__after_atomic_inc(); + switch (cmd->sam_task_attr) { + case MSG_HEAD_TAG: + pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x, " + "se_ordered_id: %u\n", + cmd->t_task_cdb[0], cmd->se_ordered_id); + return false; + case MSG_ORDERED_TAG: + atomic_inc(&dev->dev_ordered_sync); + smp_mb__after_atomic(); + + pr_debug("Added ORDERED for CDB: 0x%02x to ordered list, " + " se_ordered_id: %u\n", + cmd->t_task_cdb[0], cmd->se_ordered_id); - pr_debug("Added ORDERED for CDB: 0x%02x to ordered" - " list, se_ordered_id: %u\n", - cmd->t_task_cdb[0], - cmd->se_ordered_id); /* - * Add ORDERED command to tail of execution queue if - * no other older commands exist that need to be - * completed first. + * Execute an ORDERED command if no other older commands + * exist that need to be completed first. */ - if (!atomic_read(&cmd->se_dev->simple_cmds)) - return 1; - } else { + if (!atomic_read(&dev->simple_cmds)) + return false; + break; + default: /* * For SIMPLE and UNTAGGED Task Attribute commands */ - atomic_inc(&cmd->se_dev->simple_cmds); - smp_mb__after_atomic_inc(); - } - /* - * Otherwise if one or more outstanding ORDERED task attribute exist, - * add the dormant task(s) built for the passed struct se_cmd to the - * execution queue and become in Active state for this struct se_device. - */ - if (atomic_read(&cmd->se_dev->dev_ordered_sync) != 0) { - /* - * Otherwise, add cmd w/ tasks to delayed cmd queue that - * will be drained upon completion of HEAD_OF_QUEUE task. - */ - spin_lock(&cmd->se_dev->delayed_cmd_lock); - cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR; - list_add_tail(&cmd->se_delayed_node, - &cmd->se_dev->delayed_cmd_list); - spin_unlock(&cmd->se_dev->delayed_cmd_lock); - - pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to" - " delayed CMD list, se_ordered_id: %u\n", - cmd->t_task_cdb[0], cmd->sam_task_attr, - cmd->se_ordered_id); - /* - * Return zero to let transport_execute_tasks() know - * not to add the delayed tasks to the execution list. - */ - return 0; - } - /* - * Otherwise, no ORDERED task attributes exist.. - */ - return 1; -} - -/* - * Called from fabric module context in transport_generic_new_cmd() and - * transport_generic_process_write() - */ -static void transport_execute_tasks(struct se_cmd *cmd) -{ - int add_tasks; - struct se_device *se_dev = cmd->se_dev; - /* - * Call transport_cmd_check_stop() to see if a fabric exception - * has occurred that prevents execution. - */ - if (!transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING)) { - /* - * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE - * attribute for the tasks of the received struct se_cmd CDB - */ - add_tasks = transport_execute_task_attr(cmd); - if (add_tasks) { - __transport_execute_tasks(se_dev, cmd); - return; - } - } - __transport_execute_tasks(se_dev, NULL); -} - -static int __transport_execute_tasks(struct se_device *dev, struct se_cmd *new_cmd) -{ - int error; - struct se_cmd *cmd = NULL; - unsigned long flags; - -check_depth: - spin_lock_irq(&dev->execute_task_lock); - if (new_cmd != NULL) - __target_add_to_execute_list(new_cmd); - - if (list_empty(&dev->execute_list)) { - spin_unlock_irq(&dev->execute_task_lock); - return 0; - } - cmd = list_first_entry(&dev->execute_list, struct se_cmd, execute_list); - __target_remove_from_execute_list(cmd); - spin_unlock_irq(&dev->execute_task_lock); - - spin_lock_irqsave(&cmd->t_state_lock, flags); - cmd->transport_state |= CMD_T_BUSY; - cmd->transport_state |= CMD_T_SENT; - - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - if (cmd->execute_cmd) - error = cmd->execute_cmd(cmd); - else { - error = dev->transport->execute_cmd(cmd, cmd->t_data_sg, - cmd->t_data_nents, cmd->data_direction); - } - - if (error != 0) { - spin_lock_irqsave(&cmd->t_state_lock, flags); - cmd->transport_state &= ~CMD_T_BUSY; - cmd->transport_state &= ~CMD_T_SENT; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - transport_generic_request_failure(cmd); + atomic_inc(&dev->simple_cmds); + smp_mb__after_atomic(); + break; } - new_cmd = NULL; - goto check_depth; - - return 0; -} - -static inline u32 transport_get_sectors_6( - unsigned char *cdb, - struct se_cmd *cmd, - int *ret) -{ - struct se_device *dev = cmd->se_dev; - - /* - * Assume TYPE_DISK for non struct se_device objects. - * Use 8-bit sector value. - */ - if (!dev) - goto type_disk; + if (atomic_read(&dev->dev_ordered_sync) == 0) + return false; - /* - * Use 24-bit allocation length for TYPE_TAPE. - */ - if (dev->transport->get_device_type(dev) == TYPE_TAPE) - return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4]; + spin_lock(&dev->delayed_cmd_lock); + list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list); + spin_unlock(&dev->delayed_cmd_lock); - /* - * Everything else assume TYPE_DISK Sector CDB location. - * Use 8-bit sector value. SBC-3 says: - * - * A TRANSFER LENGTH field set to zero specifies that 256 - * logical blocks shall be written. Any other value - * specifies the number of logical blocks that shall be - * written. - */ -type_disk: - return cdb[4] ? : 256; + pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to" + " delayed CMD list, se_ordered_id: %u\n", + cmd->t_task_cdb[0], cmd->sam_task_attr, + cmd->se_ordered_id); + return true; } -static inline u32 transport_get_sectors_10( - unsigned char *cdb, - struct se_cmd *cmd, - int *ret) +void target_execute_cmd(struct se_cmd *cmd) { - struct se_device *dev = cmd->se_dev; - - /* - * Assume TYPE_DISK for non struct se_device objects. - * Use 16-bit sector value. - */ - if (!dev) - goto type_disk; - /* - * XXX_10 is not defined in SSC, throw an exception + * If the received CDB has aleady been aborted stop processing it here. */ - if (dev->transport->get_device_type(dev) == TYPE_TAPE) { - *ret = -EINVAL; - return 0; - } - - /* - * Everything else assume TYPE_DISK Sector CDB location. - * Use 16-bit sector value. - */ -type_disk: - return (u32)(cdb[7] << 8) + cdb[8]; -} - -static inline u32 transport_get_sectors_12( - unsigned char *cdb, - struct se_cmd *cmd, - int *ret) -{ - struct se_device *dev = cmd->se_dev; + if (transport_check_aborted_status(cmd, 1)) + return; /* - * Assume TYPE_DISK for non struct se_device objects. - * Use 32-bit sector value. + * Determine if frontend context caller is requesting the stopping of + * this command for frontend exceptions. */ - if (!dev) - goto type_disk; + spin_lock_irq(&cmd->t_state_lock); + if (cmd->transport_state & CMD_T_STOP) { + pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n", + __func__, __LINE__, + cmd->se_tfo->get_task_tag(cmd)); - /* - * XXX_12 is not defined in SSC, throw an exception - */ - if (dev->transport->get_device_type(dev) == TYPE_TAPE) { - *ret = -EINVAL; - return 0; + spin_unlock_irq(&cmd->t_state_lock); + complete_all(&cmd->t_transport_stop_comp); + return; } + cmd->t_state = TRANSPORT_PROCESSING; + cmd->transport_state |= CMD_T_ACTIVE|CMD_T_BUSY|CMD_T_SENT; + spin_unlock_irq(&cmd->t_state_lock); /* - * Everything else assume TYPE_DISK Sector CDB location. - * Use 32-bit sector value. - */ -type_disk: - return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9]; -} - -static inline u32 transport_get_sectors_16( - unsigned char *cdb, - struct se_cmd *cmd, - int *ret) -{ - struct se_device *dev = cmd->se_dev; - - /* - * Assume TYPE_DISK for non struct se_device objects. - * Use 32-bit sector value. - */ - if (!dev) - goto type_disk; - - /* - * Use 24-bit allocation length for TYPE_TAPE. - */ - if (dev->transport->get_device_type(dev) == TYPE_TAPE) - return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14]; - -type_disk: - return (u32)(cdb[10] << 24) + (cdb[11] << 16) + - (cdb[12] << 8) + cdb[13]; -} - -/* - * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants - */ -static inline u32 transport_get_sectors_32( - unsigned char *cdb, - struct se_cmd *cmd, - int *ret) -{ - /* - * Assume TYPE_DISK for non struct se_device objects. - * Use 32-bit sector value. + * Perform WRITE_INSERT of PI using software emulation when backend + * device has PI enabled, if the transport has not already generated + * PI using hardware WRITE_INSERT offload. */ - return (u32)(cdb[28] << 24) + (cdb[29] << 16) + - (cdb[30] << 8) + cdb[31]; - -} - -static inline u32 transport_get_size( - u32 sectors, - unsigned char *cdb, - struct se_cmd *cmd) -{ - struct se_device *dev = cmd->se_dev; - - if (dev->transport->get_device_type(dev) == TYPE_TAPE) { - if (cdb[1] & 1) { /* sectors */ - return dev->se_sub_dev->se_dev_attrib.block_size * sectors; - } else /* bytes */ - return sectors; + if (cmd->prot_op == TARGET_PROT_DOUT_INSERT) { + if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT)) + sbc_dif_generate(cmd); } - pr_debug("Returning block_size: %u, sectors: %u == %u for" - " %s object\n", dev->se_sub_dev->se_dev_attrib.block_size, - sectors, dev->se_sub_dev->se_dev_attrib.block_size * sectors, - dev->transport->name); - - return dev->se_sub_dev->se_dev_attrib.block_size * sectors; -} - -static void transport_xor_callback(struct se_cmd *cmd) -{ - unsigned char *buf, *addr; - struct scatterlist *sg; - unsigned int offset; - int i; - int count; - /* - * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command - * - * 1) read the specified logical block(s); - * 2) transfer logical blocks from the data-out buffer; - * 3) XOR the logical blocks transferred from the data-out buffer with - * the logical blocks read, storing the resulting XOR data in a buffer; - * 4) if the DISABLE WRITE bit is set to zero, then write the logical - * blocks transferred from the data-out buffer; and - * 5) transfer the resulting XOR data to the data-in buffer. - */ - buf = kmalloc(cmd->data_length, GFP_KERNEL); - if (!buf) { - pr_err("Unable to allocate xor_callback buf\n"); + if (target_handle_task_attr(cmd)) { + spin_lock_irq(&cmd->t_state_lock); + cmd->transport_state &= ~CMD_T_BUSY|CMD_T_SENT; + spin_unlock_irq(&cmd->t_state_lock); return; } - /* - * Copy the scatterlist WRITE buffer located at cmd->t_data_sg - * into the locally allocated *buf - */ - sg_copy_to_buffer(cmd->t_data_sg, - cmd->t_data_nents, - buf, - cmd->data_length); - - /* - * Now perform the XOR against the BIDI read memory located at - * cmd->t_mem_bidi_list - */ - - offset = 0; - for_each_sg(cmd->t_bidi_data_sg, sg, cmd->t_bidi_data_nents, count) { - addr = kmap_atomic(sg_page(sg)); - if (!addr) - goto out; - - for (i = 0; i < sg->length; i++) - *(addr + sg->offset + i) ^= *(buf + offset + i); - - offset += sg->length; - kunmap_atomic(addr); - } -out: - kfree(buf); + __target_execute_cmd(cmd); } +EXPORT_SYMBOL(target_execute_cmd); /* - * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd + * Process all commands up to the last received ORDERED task attribute which + * requires another blocking boundary */ -static int transport_get_sense_data(struct se_cmd *cmd) +static void target_restart_delayed_cmds(struct se_device *dev) { - unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL; - struct se_device *dev = cmd->se_dev; - unsigned long flags; - u32 offset = 0; + for (;;) { + struct se_cmd *cmd; - WARN_ON(!cmd->se_lun); - - if (!dev) - return 0; - - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - return 0; - } - - if (!(cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)) - goto out; - - if (!dev->transport->get_sense_buffer) { - pr_err("dev->transport->get_sense_buffer is NULL\n"); - goto out; - } - - sense_buffer = dev->transport->get_sense_buffer(cmd); - if (!sense_buffer) { - pr_err("ITT 0x%08x cmd %p: Unable to locate" - " sense buffer for task with sense\n", - cmd->se_tfo->get_task_tag(cmd), cmd); - goto out; - } - - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - offset = cmd->se_tfo->set_fabric_sense_len(cmd, TRANSPORT_SENSE_BUFFER); - - memcpy(&buffer[offset], sense_buffer, TRANSPORT_SENSE_BUFFER); - - /* Automatically padded */ - cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset; - - pr_debug("HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x and sense\n", - dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status); - return 0; - -out: - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - return -1; -} - -static inline long long transport_dev_end_lba(struct se_device *dev) -{ - return dev->transport->get_blocks(dev) + 1; -} - -static int transport_cmd_get_valid_sectors(struct se_cmd *cmd) -{ - struct se_device *dev = cmd->se_dev; - u32 sectors; - - if (dev->transport->get_device_type(dev) != TYPE_DISK) - return 0; - - sectors = (cmd->data_length / dev->se_sub_dev->se_dev_attrib.block_size); - - if ((cmd->t_task_lba + sectors) > transport_dev_end_lba(dev)) { - pr_err("LBA: %llu Sectors: %u exceeds" - " transport_dev_end_lba(): %llu\n", - cmd->t_task_lba, sectors, - transport_dev_end_lba(dev)); - return -EINVAL; - } - - return 0; -} - -static int target_check_write_same_discard(unsigned char *flags, struct se_device *dev) -{ - /* - * Determine if the received WRITE_SAME is used to for direct - * passthrough into Linux/SCSI with struct request via TCM/pSCSI - * or we are signaling the use of internal WRITE_SAME + UNMAP=1 - * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK code. - */ - int passthrough = (dev->transport->transport_type == - TRANSPORT_PLUGIN_PHBA_PDEV); - - if (!passthrough) { - if ((flags[0] & 0x04) || (flags[0] & 0x02)) { - pr_err("WRITE_SAME PBDATA and LBDATA" - " bits not supported for Block Discard" - " Emulation\n"); - return -ENOSYS; - } - /* - * Currently for the emulated case we only accept - * tpws with the UNMAP=1 bit set. - */ - if (!(flags[0] & 0x08)) { - pr_err("WRITE_SAME w/o UNMAP bit not" - " supported for Block Discard Emulation\n"); - return -ENOSYS; - } - } - - return 0; -} - -/* transport_generic_cmd_sequencer(): - * - * Generic Command Sequencer that should work for most DAS transport - * drivers. - * - * Called from target_setup_cmd_from_cdb() in the $FABRIC_MOD - * RX Thread. - * - * FIXME: Need to support other SCSI OPCODES where as well. - */ -static int transport_generic_cmd_sequencer( - struct se_cmd *cmd, - unsigned char *cdb) -{ - struct se_device *dev = cmd->se_dev; - struct se_subsystem_dev *su_dev = dev->se_sub_dev; - int ret = 0, sector_ret = 0, passthrough; - u32 sectors = 0, size = 0, pr_reg_type = 0; - u16 service_action; - u8 alua_ascq = 0; - /* - * Check for an existing UNIT ATTENTION condition - */ - if (core_scsi3_ua_check(cmd, cdb) < 0) { - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION; - return -EINVAL; - } - /* - * Check status of Asymmetric Logical Unit Assignment port - */ - ret = su_dev->t10_alua.alua_state_check(cmd, cdb, &alua_ascq); - if (ret != 0) { - /* - * Set SCSI additional sense code (ASC) to 'LUN Not Accessible'; - * The ALUA additional sense code qualifier (ASCQ) is determined - * by the ALUA primary or secondary access state.. - */ - if (ret > 0) { - pr_debug("[%s]: ALUA TG Port not available," - " SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n", - cmd->se_tfo->get_fabric_name(), alua_ascq); - - transport_set_sense_codes(cmd, 0x04, alua_ascq); - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY; - return -EINVAL; - } - goto out_invalid_cdb_field; - } - /* - * Check status for SPC-3 Persistent Reservations - */ - if (su_dev->t10_pr.pr_ops.t10_reservation_check(cmd, &pr_reg_type) != 0) { - if (su_dev->t10_pr.pr_ops.t10_seq_non_holder( - cmd, cdb, pr_reg_type) != 0) { - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT; - cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT; - cmd->scsi_sense_reason = TCM_RESERVATION_CONFLICT; - return -EBUSY; - } - /* - * This means the CDB is allowed for the SCSI Initiator port - * when said port is *NOT* holding the legacy SPC-2 or - * SPC-3 Persistent Reservation. - */ - } - - /* - * If we operate in passthrough mode we skip most CDB emulation and - * instead hand the commands down to the physical SCSI device. - */ - passthrough = - (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV); - - switch (cdb[0]) { - case READ_6: - sectors = transport_get_sectors_6(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_21(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case READ_10: - sectors = transport_get_sectors_10(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_32(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case READ_12: - sectors = transport_get_sectors_12(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_32(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case READ_16: - sectors = transport_get_sectors_16(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_64(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case WRITE_6: - sectors = transport_get_sectors_6(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_21(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case WRITE_10: - case WRITE_VERIFY: - sectors = transport_get_sectors_10(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_32(cdb); - if (cdb[1] & 0x8) - cmd->se_cmd_flags |= SCF_FUA; - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case WRITE_12: - sectors = transport_get_sectors_12(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_32(cdb); - if (cdb[1] & 0x8) - cmd->se_cmd_flags |= SCF_FUA; - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case WRITE_16: - sectors = transport_get_sectors_16(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_64(cdb); - if (cdb[1] & 0x8) - cmd->se_cmd_flags |= SCF_FUA; - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - break; - case XDWRITEREAD_10: - if ((cmd->data_direction != DMA_TO_DEVICE) || - !(cmd->se_cmd_flags & SCF_BIDI)) - goto out_invalid_cdb_field; - sectors = transport_get_sectors_10(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - cmd->t_task_lba = transport_lba_32(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - - /* - * Do now allow BIDI commands for passthrough mode. - */ - if (passthrough) - goto out_unsupported_cdb; - - /* - * Setup BIDI XOR callback to be run after I/O completion. - */ - cmd->transport_complete_callback = &transport_xor_callback; - if (cdb[1] & 0x8) - cmd->se_cmd_flags |= SCF_FUA; - break; - case VARIABLE_LENGTH_CMD: - service_action = get_unaligned_be16(&cdb[8]); - switch (service_action) { - case XDWRITEREAD_32: - sectors = transport_get_sectors_32(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - size = transport_get_size(sectors, cdb, cmd); - /* - * Use WRITE_32 and READ_32 opcodes for the emulated - * XDWRITE_READ_32 logic. - */ - cmd->t_task_lba = transport_lba_64_ext(cdb); - cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB; - - /* - * Do now allow BIDI commands for passthrough mode. - */ - if (passthrough) - goto out_unsupported_cdb; - - /* - * Setup BIDI XOR callback to be run during after I/O - * completion. - */ - cmd->transport_complete_callback = &transport_xor_callback; - if (cdb[1] & 0x8) - cmd->se_cmd_flags |= SCF_FUA; - break; - case WRITE_SAME_32: - sectors = transport_get_sectors_32(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - - if (sectors) - size = transport_get_size(1, cdb, cmd); - else { - pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not" - " supported\n"); - goto out_invalid_cdb_field; - } - - cmd->t_task_lba = get_unaligned_be64(&cdb[12]); - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - - if (target_check_write_same_discard(&cdb[10], dev) < 0) - goto out_unsupported_cdb; - if (!passthrough) - cmd->execute_cmd = target_emulate_write_same; - break; - default: - pr_err("VARIABLE_LENGTH_CMD service action" - " 0x%04x not supported\n", service_action); - goto out_unsupported_cdb; - } - break; - case MAINTENANCE_IN: - if (dev->transport->get_device_type(dev) != TYPE_ROM) { - /* MAINTENANCE_IN from SCC-2 */ - /* - * Check for emulated MI_REPORT_TARGET_PGS. - */ - if ((cdb[1] & 0x1f) == MI_REPORT_TARGET_PGS && - su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) { - cmd->execute_cmd = - target_emulate_report_target_port_groups; - } - size = (cdb[6] << 24) | (cdb[7] << 16) | - (cdb[8] << 8) | cdb[9]; - } else { - /* GPCMD_SEND_KEY from multi media commands */ - size = (cdb[8] << 8) + cdb[9]; - } - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case MODE_SELECT: - size = cdb[4]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case MODE_SELECT_10: - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case MODE_SENSE: - size = cdb[4]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_modesense; - break; - case MODE_SENSE_10: - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_modesense; - break; - case GPCMD_READ_BUFFER_CAPACITY: - case GPCMD_SEND_OPC: - case LOG_SELECT: - case LOG_SENSE: - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case READ_BLOCK_LIMITS: - size = READ_BLOCK_LEN; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case GPCMD_GET_CONFIGURATION: - case GPCMD_READ_FORMAT_CAPACITIES: - case GPCMD_READ_DISC_INFO: - case GPCMD_READ_TRACK_RZONE_INFO: - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case PERSISTENT_RESERVE_IN: - if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS) - cmd->execute_cmd = target_scsi3_emulate_pr_in; - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case PERSISTENT_RESERVE_OUT: - if (su_dev->t10_pr.res_type == SPC3_PERSISTENT_RESERVATIONS) - cmd->execute_cmd = target_scsi3_emulate_pr_out; - size = (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case GPCMD_MECHANISM_STATUS: - case GPCMD_READ_DVD_STRUCTURE: - size = (cdb[8] << 8) + cdb[9]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case READ_POSITION: - size = READ_POSITION_LEN; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case MAINTENANCE_OUT: - if (dev->transport->get_device_type(dev) != TYPE_ROM) { - /* MAINTENANCE_OUT from SCC-2 - * - * Check for emulated MO_SET_TARGET_PGS. - */ - if (cdb[1] == MO_SET_TARGET_PGS && - su_dev->t10_alua.alua_type == SPC3_ALUA_EMULATED) { - cmd->execute_cmd = - target_emulate_set_target_port_groups; - } - - size = (cdb[6] << 24) | (cdb[7] << 16) | - (cdb[8] << 8) | cdb[9]; - } else { - /* GPCMD_REPORT_KEY from multi media commands */ - size = (cdb[8] << 8) + cdb[9]; - } - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case INQUIRY: - size = (cdb[3] << 8) + cdb[4]; - /* - * Do implict HEAD_OF_QUEUE processing for INQUIRY. - * See spc4r17 section 5.3 - */ - if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED) - cmd->sam_task_attr = MSG_HEAD_TAG; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_inquiry; - break; - case READ_BUFFER: - size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case READ_CAPACITY: - size = READ_CAP_LEN; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_readcapacity; - break; - case READ_MEDIA_SERIAL_NUMBER: - case SECURITY_PROTOCOL_IN: - case SECURITY_PROTOCOL_OUT: - size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case SERVICE_ACTION_IN: - switch (cmd->t_task_cdb[1] & 0x1f) { - case SAI_READ_CAPACITY_16: - if (!passthrough) - cmd->execute_cmd = - target_emulate_readcapacity_16; - break; - default: - if (passthrough) - break; - - pr_err("Unsupported SA: 0x%02x\n", - cmd->t_task_cdb[1] & 0x1f); - goto out_invalid_cdb_field; - } - /*FALLTHROUGH*/ - case ACCESS_CONTROL_IN: - case ACCESS_CONTROL_OUT: - case EXTENDED_COPY: - case READ_ATTRIBUTE: - case RECEIVE_COPY_RESULTS: - case WRITE_ATTRIBUTE: - size = (cdb[10] << 24) | (cdb[11] << 16) | - (cdb[12] << 8) | cdb[13]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case RECEIVE_DIAGNOSTIC: - case SEND_DIAGNOSTIC: - size = (cdb[3] << 8) | cdb[4]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; -/* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */ -#if 0 - case GPCMD_READ_CD: - sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8]; - size = (2336 * sectors); - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; -#endif - case READ_TOC: - size = cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case REQUEST_SENSE: - size = cdb[4]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_request_sense; - break; - case READ_ELEMENT_STATUS: - size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case WRITE_BUFFER: - size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case RESERVE: - case RESERVE_10: - /* - * The SPC-2 RESERVE does not contain a size in the SCSI CDB. - * Assume the passthrough or $FABRIC_MOD will tell us about it. - */ - if (cdb[0] == RESERVE_10) - size = (cdb[7] << 8) | cdb[8]; - else - size = cmd->data_length; - - /* - * Setup the legacy emulated handler for SPC-2 and - * >= SPC-3 compatible reservation handling (CRH=1) - * Otherwise, we assume the underlying SCSI logic is - * is running in SPC_PASSTHROUGH, and wants reservations - * emulation disabled. - */ - if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH) - cmd->execute_cmd = target_scsi2_reservation_reserve; - cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB; - break; - case RELEASE: - case RELEASE_10: - /* - * The SPC-2 RELEASE does not contain a size in the SCSI CDB. - * Assume the passthrough or $FABRIC_MOD will tell us about it. - */ - if (cdb[0] == RELEASE_10) - size = (cdb[7] << 8) | cdb[8]; - else - size = cmd->data_length; - - if (su_dev->t10_pr.res_type != SPC_PASSTHROUGH) - cmd->execute_cmd = target_scsi2_reservation_release; - cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB; - break; - case SYNCHRONIZE_CACHE: - case SYNCHRONIZE_CACHE_16: - /* - * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE - */ - if (cdb[0] == SYNCHRONIZE_CACHE) { - sectors = transport_get_sectors_10(cdb, cmd, §or_ret); - cmd->t_task_lba = transport_lba_32(cdb); - } else { - sectors = transport_get_sectors_16(cdb, cmd, §or_ret); - cmd->t_task_lba = transport_lba_64(cdb); - } - if (sector_ret) - goto out_unsupported_cdb; - - size = transport_get_size(sectors, cdb, cmd); - cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB; - - if (passthrough) + spin_lock(&dev->delayed_cmd_lock); + if (list_empty(&dev->delayed_cmd_list)) { + spin_unlock(&dev->delayed_cmd_lock); break; - - /* - * Check to ensure that LBA + Range does not exceed past end of - * device for IBLOCK and FILEIO ->do_sync_cache() backend calls - */ - if ((cmd->t_task_lba != 0) || (sectors != 0)) { - if (transport_cmd_get_valid_sectors(cmd) < 0) - goto out_invalid_cdb_field; - } - cmd->execute_cmd = target_emulate_synchronize_cache; - break; - case UNMAP: - size = get_unaligned_be16(&cdb[7]); - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_unmap; - break; - case WRITE_SAME_16: - sectors = transport_get_sectors_16(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; - - if (sectors) - size = transport_get_size(1, cdb, cmd); - else { - pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n"); - goto out_invalid_cdb_field; } - cmd->t_task_lba = get_unaligned_be64(&cdb[2]); - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - - if (target_check_write_same_discard(&cdb[1], dev) < 0) - goto out_unsupported_cdb; - if (!passthrough) - cmd->execute_cmd = target_emulate_write_same; - break; - case WRITE_SAME: - sectors = transport_get_sectors_10(cdb, cmd, §or_ret); - if (sector_ret) - goto out_unsupported_cdb; + cmd = list_entry(dev->delayed_cmd_list.next, + struct se_cmd, se_delayed_node); + list_del(&cmd->se_delayed_node); + spin_unlock(&dev->delayed_cmd_lock); - if (sectors) - size = transport_get_size(1, cdb, cmd); - else { - pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n"); - goto out_invalid_cdb_field; - } + __target_execute_cmd(cmd); - cmd->t_task_lba = get_unaligned_be32(&cdb[2]); - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - /* - * Follow sbcr26 with WRITE_SAME (10) and check for the existence - * of byte 1 bit 3 UNMAP instead of original reserved field - */ - if (target_check_write_same_discard(&cdb[1], dev) < 0) - goto out_unsupported_cdb; - if (!passthrough) - cmd->execute_cmd = target_emulate_write_same; - break; - case ALLOW_MEDIUM_REMOVAL: - case ERASE: - case REZERO_UNIT: - case SEEK_10: - case SPACE: - case START_STOP: - case TEST_UNIT_READY: - case VERIFY: - case WRITE_FILEMARKS: - cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB; - if (!passthrough) - cmd->execute_cmd = target_emulate_noop; - break; - case GPCMD_CLOSE_TRACK: - case INITIALIZE_ELEMENT_STATUS: - case GPCMD_LOAD_UNLOAD: - case GPCMD_SET_SPEED: - case MOVE_MEDIUM: - cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB; - break; - case REPORT_LUNS: - cmd->execute_cmd = target_report_luns; - size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9]; - /* - * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS - * See spc4r17 section 5.3 - */ - if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED) - cmd->sam_task_attr = MSG_HEAD_TAG; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case GET_EVENT_STATUS_NOTIFICATION: - size = (cdb[7] << 8) | cdb[8]; - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - case ATA_16: - /* Only support ATA passthrough to pSCSI backends.. */ - if (!passthrough) - goto out_unsupported_cdb; - - /* T_LENGTH */ - switch (cdb[2] & 0x3) { - case 0x0: - sectors = 0; - break; - case 0x1: - sectors = (((cdb[1] & 0x1) ? cdb[3] : 0) << 8) | cdb[4]; - break; - case 0x2: - sectors = (((cdb[1] & 0x1) ? cdb[5] : 0) << 8) | cdb[6]; + if (cmd->sam_task_attr == MSG_ORDERED_TAG) break; - case 0x3: - pr_err("T_LENGTH=0x3 not supported for ATA_16\n"); - goto out_invalid_cdb_field; - } - - /* BYTE_BLOCK */ - if (cdb[2] & 0x4) { - /* BLOCK T_TYPE: 512 or sector */ - size = sectors * ((cdb[2] & 0x10) ? - dev->se_sub_dev->se_dev_attrib.block_size : 512); - } else { - /* BYTE */ - size = sectors; - } - cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB; - break; - default: - pr_warn("TARGET_CORE[%s]: Unsupported SCSI Opcode" - " 0x%02x, sending CHECK_CONDITION.\n", - cmd->se_tfo->get_fabric_name(), cdb[0]); - goto out_unsupported_cdb; } - - if (cmd->unknown_data_length) - cmd->data_length = size; - - if (size != cmd->data_length) { - pr_warn("TARGET_CORE[%s]: Expected Transfer Length:" - " %u does not match SCSI CDB Length: %u for SAM Opcode:" - " 0x%02x\n", cmd->se_tfo->get_fabric_name(), - cmd->data_length, size, cdb[0]); - - cmd->cmd_spdtl = size; - - if (cmd->data_direction == DMA_TO_DEVICE) { - pr_err("Rejecting underflow/overflow" - " WRITE data\n"); - goto out_invalid_cdb_field; - } - /* - * Reject READ_* or WRITE_* with overflow/underflow for - * type SCF_SCSI_DATA_SG_IO_CDB. - */ - if (!ret && (dev->se_sub_dev->se_dev_attrib.block_size != 512)) { - pr_err("Failing OVERFLOW/UNDERFLOW for LBA op" - " CDB on non 512-byte sector setup subsystem" - " plugin: %s\n", dev->transport->name); - /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */ - goto out_invalid_cdb_field; - } - - if (size > cmd->data_length) { - cmd->se_cmd_flags |= SCF_OVERFLOW_BIT; - cmd->residual_count = (size - cmd->data_length); - } else { - cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT; - cmd->residual_count = (cmd->data_length - size); - } - cmd->data_length = size; - } - - if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) { - if (sectors > su_dev->se_dev_attrib.fabric_max_sectors) { - printk_ratelimited(KERN_ERR "SCSI OP %02xh with too" - " big sectors %u exceeds fabric_max_sectors:" - " %u\n", cdb[0], sectors, - su_dev->se_dev_attrib.fabric_max_sectors); - goto out_invalid_cdb_field; - } - if (sectors > su_dev->se_dev_attrib.hw_max_sectors) { - printk_ratelimited(KERN_ERR "SCSI OP %02xh with too" - " big sectors %u exceeds backend hw_max_sectors:" - " %u\n", cdb[0], sectors, - su_dev->se_dev_attrib.hw_max_sectors); - goto out_invalid_cdb_field; - } - } - - /* reject any command that we don't have a handler for */ - if (!(passthrough || cmd->execute_cmd || - (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB))) - goto out_unsupported_cdb; - - transport_set_supported_SAM_opcode(cmd); - return ret; - -out_unsupported_cdb: - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE; - return -EINVAL; -out_invalid_cdb_field: - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD; - return -EINVAL; } /* @@ -3052,12 +1840,13 @@ out_invalid_cdb_field: static void transport_complete_task_attr(struct se_cmd *cmd) { struct se_device *dev = cmd->se_dev; - struct se_cmd *cmd_p, *cmd_tmp; - int new_active_tasks = 0; + + if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) + return; if (cmd->sam_task_attr == MSG_SIMPLE_TAG) { atomic_dec(&dev->simple_cmds); - smp_mb__after_atomic_dec(); + smp_mb__after_atomic(); dev->dev_cur_ordered_id++; pr_debug("Incremented dev->dev_cur_ordered_id: %u for" " SIMPLE: %u\n", dev->dev_cur_ordered_id, @@ -3069,54 +1858,24 @@ static void transport_complete_task_attr(struct se_cmd *cmd) cmd->se_ordered_id); } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) { atomic_dec(&dev->dev_ordered_sync); - smp_mb__after_atomic_dec(); + smp_mb__after_atomic(); dev->dev_cur_ordered_id++; pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:" " %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id); } - /* - * Process all commands up to the last received - * ORDERED task attribute which requires another blocking - * boundary - */ - spin_lock(&dev->delayed_cmd_lock); - list_for_each_entry_safe(cmd_p, cmd_tmp, - &dev->delayed_cmd_list, se_delayed_node) { - - list_del(&cmd_p->se_delayed_node); - spin_unlock(&dev->delayed_cmd_lock); - pr_debug("Calling add_tasks() for" - " cmd_p: 0x%02x Task Attr: 0x%02x" - " Dormant -> Active, se_ordered_id: %u\n", - cmd_p->t_task_cdb[0], - cmd_p->sam_task_attr, cmd_p->se_ordered_id); - - target_add_to_execute_list(cmd_p); - new_active_tasks++; - - spin_lock(&dev->delayed_cmd_lock); - if (cmd_p->sam_task_attr == MSG_ORDERED_TAG) - break; - } - spin_unlock(&dev->delayed_cmd_lock); - /* - * If new tasks have become active, wake up the transport thread - * to do the processing of the Active tasks. - */ - if (new_active_tasks != 0) - wake_up_interruptible(&dev->dev_queue_obj.thread_wq); + target_restart_delayed_cmds(dev); } static void transport_complete_qf(struct se_cmd *cmd) { int ret = 0; - if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED) - transport_complete_task_attr(cmd); + transport_complete_task_attr(cmd); if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_status(cmd); if (ret) goto out; @@ -3124,16 +1883,18 @@ static void transport_complete_qf(struct se_cmd *cmd) switch (cmd->data_direction) { case DMA_FROM_DEVICE: + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_data_in(cmd); break; case DMA_TO_DEVICE: - if (cmd->t_bidi_data_sg) { + if (cmd->se_cmd_flags & SCF_BIDI) { ret = cmd->se_tfo->queue_data_in(cmd); if (ret < 0) break; } /* Fall through for DMA_TO_DEVICE */ case DMA_NONE: + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_status(cmd); break; default: @@ -3156,24 +1917,39 @@ static void transport_handle_queue_full( spin_lock_irq(&dev->qf_cmd_lock); list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list); atomic_inc(&dev->dev_qf_count); - smp_mb__after_atomic_inc(); + smp_mb__after_atomic(); spin_unlock_irq(&cmd->se_dev->qf_cmd_lock); schedule_work(&cmd->se_dev->qf_work_queue); } +static bool target_check_read_strip(struct se_cmd *cmd) +{ + sense_reason_t rc; + + if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) { + rc = sbc_dif_read_strip(cmd); + if (rc) { + cmd->pi_err = rc; + return true; + } + } + + return false; +} + static void target_complete_ok_work(struct work_struct *work) { struct se_cmd *cmd = container_of(work, struct se_cmd, work); - int reason = 0, ret; + int ret; /* * Check if we need to move delayed/dormant tasks from cmds on the * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task * Attribute. */ - if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED) - transport_complete_task_attr(cmd); + transport_complete_task_attr(cmd); + /* * Check to schedule QUEUE_FULL work, or execute an existing * cmd->transport_qf_callback() @@ -3182,16 +1958,33 @@ static void target_complete_ok_work(struct work_struct *work) schedule_work(&cmd->se_dev->qf_work_queue); /* - * Check if we need to retrieve a sense buffer from + * Check if we need to send a sense buffer from * the struct se_cmd in question. */ if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) { - if (transport_get_sense_data(cmd) < 0) - reason = TCM_NON_EXISTENT_LUN; + WARN_ON(!cmd->scsi_status); + ret = transport_send_check_condition_and_sense( + cmd, 0, 1); + if (ret == -EAGAIN || ret == -ENOMEM) + goto queue_full; - if (cmd->scsi_status) { - ret = transport_send_check_condition_and_sense( - cmd, reason, 1); + transport_lun_remove_cmd(cmd); + transport_cmd_check_stop_to_fabric(cmd); + return; + } + /* + * Check for a callback, used by amongst other things + * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation. + */ + if (cmd->transport_complete_callback) { + sense_reason_t rc; + + rc = cmd->transport_complete_callback(cmd); + if (!rc && !(cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE_POST)) { + return; + } else if (rc) { + ret = transport_send_check_condition_and_sense(cmd, + rc, 0); if (ret == -EAGAIN || ret == -ENOMEM) goto queue_full; @@ -3200,12 +1993,6 @@ static void target_complete_ok_work(struct work_struct *work) return; } } - /* - * Check for a callback, used by amongst other things - * XDWRITE_READ_10 emulation. - */ - if (cmd->transport_complete_callback) - cmd->transport_complete_callback(cmd); switch (cmd->data_direction) { case DMA_FROM_DEVICE: @@ -3215,7 +2002,24 @@ static void target_complete_ok_work(struct work_struct *work) cmd->data_length; } spin_unlock(&cmd->se_lun->lun_sep_lock); + /* + * Perform READ_STRIP of PI using software emulation when + * backend had PI enabled, if the transport will not be + * performing hardware READ_STRIP offload. + */ + if (cmd->prot_op == TARGET_PROT_DIN_STRIP && + target_check_read_strip(cmd)) { + ret = transport_send_check_condition_and_sense(cmd, + cmd->pi_err, 0); + if (ret == -EAGAIN || ret == -ENOMEM) + goto queue_full; + + transport_lun_remove_cmd(cmd); + transport_cmd_check_stop_to_fabric(cmd); + return; + } + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_data_in(cmd); if (ret == -EAGAIN || ret == -ENOMEM) goto queue_full; @@ -3230,7 +2034,7 @@ static void target_complete_ok_work(struct work_struct *work) /* * Check if we need to send READ payload for BIDI-COMMAND */ - if (cmd->t_bidi_data_sg) { + if (cmd->se_cmd_flags & SCF_BIDI) { spin_lock(&cmd->se_lun->lun_sep_lock); if (cmd->se_lun->lun_sep) { cmd->se_lun->lun_sep->sep_stats.tx_data_octets += @@ -3244,6 +2048,7 @@ static void target_complete_ok_work(struct work_struct *work) } /* Fall through for DMA_TO_DEVICE */ case DMA_NONE: + trace_target_cmd_complete(cmd); ret = cmd->se_tfo->queue_status(cmd); if (ret == -EAGAIN || ret == -ENOMEM) goto queue_full; @@ -3274,10 +2079,29 @@ static inline void transport_free_sgl(struct scatterlist *sgl, int nents) kfree(sgl); } +static inline void transport_reset_sgl_orig(struct se_cmd *cmd) +{ + /* + * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE + * emulation, and free + reset pointers if necessary.. + */ + if (!cmd->t_data_sg_orig) + return; + + kfree(cmd->t_data_sg); + cmd->t_data_sg = cmd->t_data_sg_orig; + cmd->t_data_sg_orig = NULL; + cmd->t_data_nents = cmd->t_data_nents_orig; + cmd->t_data_nents_orig = 0; +} + static inline void transport_free_pages(struct se_cmd *cmd) { - if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) + if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { + transport_reset_sgl_orig(cmd); return; + } + transport_reset_sgl_orig(cmd); transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents); cmd->t_data_sg = NULL; @@ -3286,6 +2110,10 @@ static inline void transport_free_pages(struct se_cmd *cmd) transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents); cmd->t_bidi_data_sg = NULL; cmd->t_bidi_data_nents = 0; + + transport_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents); + cmd->t_prot_sg = NULL; + cmd->t_prot_nents = 0; } /** @@ -3295,7 +2123,7 @@ static inline void transport_free_pages(struct se_cmd *cmd) * This routine unconditionally frees a command, and reference counting * or list removal must be done in the caller. */ -static void transport_release_cmd(struct se_cmd *cmd) +static int transport_release_cmd(struct se_cmd *cmd) { BUG_ON(!cmd->se_tfo); @@ -3307,11 +2135,7 @@ static void transport_release_cmd(struct se_cmd *cmd) * If this cmd has been setup with target_get_sess_cmd(), drop * the kref and call ->release_cmd() in kref callback. */ - if (cmd->check_release != 0) { - target_put_sess_cmd(cmd->se_sess, cmd); - return; - } - cmd->se_tfo->release_cmd(cmd); + return target_put_sess_cmd(cmd->se_sess, cmd); } /** @@ -3320,87 +2144,18 @@ static void transport_release_cmd(struct se_cmd *cmd) * * This routine releases our reference to the command and frees it if possible. */ -static void transport_put_cmd(struct se_cmd *cmd) +static int transport_put_cmd(struct se_cmd *cmd) { - unsigned long flags; - - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (atomic_read(&cmd->t_fe_count)) { - if (!atomic_dec_and_test(&cmd->t_fe_count)) - goto out_busy; - } - - if (cmd->transport_state & CMD_T_DEV_ACTIVE) { - cmd->transport_state &= ~CMD_T_DEV_ACTIVE; - target_remove_from_state_list(cmd); - } - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - transport_free_pages(cmd); - transport_release_cmd(cmd); - return; -out_busy: - spin_unlock_irqrestore(&cmd->t_state_lock, flags); + return transport_release_cmd(cmd); } -/* - * transport_generic_map_mem_to_cmd - Use fabric-alloced pages instead of - * allocating in the core. - * @cmd: Associated se_cmd descriptor - * @mem: SGL style memory for TCM WRITE / READ - * @sg_mem_num: Number of SGL elements - * @mem_bidi_in: SGL style memory for TCM BIDI READ - * @sg_mem_bidi_num: Number of BIDI READ SGL elements - * - * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage - * of parameters. - */ -int transport_generic_map_mem_to_cmd( - struct se_cmd *cmd, - struct scatterlist *sgl, - u32 sgl_count, - struct scatterlist *sgl_bidi, - u32 sgl_bidi_count) -{ - if (!sgl || !sgl_count) - return 0; - - if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) || - (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) { - /* - * Reject SCSI data overflow with map_mem_to_cmd() as incoming - * scatterlists already have been set to follow what the fabric - * passes for the original expected data transfer length. - */ - if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) { - pr_warn("Rejecting SCSI DATA overflow for fabric using" - " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n"); - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD; - return -EINVAL; - } - - cmd->t_data_sg = sgl; - cmd->t_data_nents = sgl_count; - - if (sgl_bidi && sgl_bidi_count) { - cmd->t_bidi_data_sg = sgl_bidi; - cmd->t_bidi_data_nents = sgl_bidi_count; - } - cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; - } - - return 0; -} -EXPORT_SYMBOL(transport_generic_map_mem_to_cmd); - void *transport_kmap_data_sg(struct se_cmd *cmd) { struct scatterlist *sg = cmd->t_data_sg; struct page **pages; int i; - BUG_ON(!sg); /* * We need to take into account a possible offset here for fabrics like * tcm_loop who may be using a contig buffer from the SCSI midlayer for @@ -3408,7 +2163,9 @@ void *transport_kmap_data_sg(struct se_cmd *cmd) */ if (!cmd->t_data_nents) return NULL; - else if (cmd->t_data_nents == 1) + + BUG_ON(!sg); + if (cmd->t_data_nents == 1) return kmap(sg_page(sg)) + sg->offset; /* >1 page. use vmap */ @@ -3444,24 +2201,22 @@ void transport_kunmap_data_sg(struct se_cmd *cmd) } EXPORT_SYMBOL(transport_kunmap_data_sg); -static int -transport_generic_get_mem(struct se_cmd *cmd) +int +target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length, + bool zero_page) { - u32 length = cmd->data_length; - unsigned int nents; + struct scatterlist *sg; struct page *page; - gfp_t zero_flag; + gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0; + unsigned int nent; int i = 0; - nents = DIV_ROUND_UP(length, PAGE_SIZE); - cmd->t_data_sg = kmalloc(sizeof(struct scatterlist) * nents, GFP_KERNEL); - if (!cmd->t_data_sg) + nent = DIV_ROUND_UP(length, PAGE_SIZE); + sg = kmalloc(sizeof(struct scatterlist) * nent, GFP_KERNEL); + if (!sg) return -ENOMEM; - cmd->t_data_nents = nents; - sg_init_table(cmd->t_data_sg, nents); - - zero_flag = cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB ? 0 : __GFP_ZERO; + sg_init_table(sg, nent); while (length) { u32 page_len = min_t(u32, length, PAGE_SIZE); @@ -3469,19 +2224,20 @@ transport_generic_get_mem(struct se_cmd *cmd) if (!page) goto out; - sg_set_page(&cmd->t_data_sg[i], page, page_len, 0); + sg_set_page(&sg[i], page, page_len, 0); length -= page_len; i++; } + *sgl = sg; + *nents = nent; return 0; out: - while (i >= 0) { - __free_page(sg_page(&cmd->t_data_sg[i])); + while (i > 0) { i--; + __free_page(sg_page(&sg[i])); } - kfree(cmd->t_data_sg); - cmd->t_data_sg = NULL; + kfree(sg); return -ENOMEM; } @@ -3490,9 +2246,9 @@ out: * might not have the payload yet, so notify the fabric via a call to * ->write_pending instead. Otherwise place it on the execution queue. */ -int transport_generic_new_cmd(struct se_cmd *cmd) +sense_reason_t +transport_generic_new_cmd(struct se_cmd *cmd) { - struct se_device *dev = cmd->se_dev; int ret = 0; /* @@ -3502,77 +2258,67 @@ int transport_generic_new_cmd(struct se_cmd *cmd) */ if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) && cmd->data_length) { - ret = transport_generic_get_mem(cmd); - if (ret < 0) - goto out_fail; - } + bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB); - /* Workaround for handling zero-length control CDBs */ - if ((cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) && - !cmd->data_length) { - spin_lock_irq(&cmd->t_state_lock); - cmd->t_state = TRANSPORT_COMPLETE; - cmd->transport_state |= CMD_T_ACTIVE; - spin_unlock_irq(&cmd->t_state_lock); + if ((cmd->se_cmd_flags & SCF_BIDI) || + (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) { + u32 bidi_length; - if (cmd->t_task_cdb[0] == REQUEST_SENSE) { - u8 ua_asc = 0, ua_ascq = 0; + if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) + bidi_length = cmd->t_task_nolb * + cmd->se_dev->dev_attrib.block_size; + else + bidi_length = cmd->data_length; - core_scsi3_ua_clear_for_request_sense(cmd, - &ua_asc, &ua_ascq); + ret = target_alloc_sgl(&cmd->t_bidi_data_sg, + &cmd->t_bidi_data_nents, + bidi_length, zero_flag); + if (ret < 0) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; } - INIT_WORK(&cmd->work, target_complete_ok_work); - queue_work(target_completion_wq, &cmd->work); + if (cmd->prot_op != TARGET_PROT_NORMAL) { + ret = target_alloc_sgl(&cmd->t_prot_sg, + &cmd->t_prot_nents, + cmd->prot_length, true); + if (ret < 0) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } + + ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents, + cmd->data_length, zero_flag); + if (ret < 0) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } + /* + * If this command is not a write we can execute it right here, + * for write buffers we need to notify the fabric driver first + * and let it call back once the write buffers are ready. + */ + target_add_to_state_list(cmd); + if (cmd->data_direction != DMA_TO_DEVICE) { + target_execute_cmd(cmd); return 0; } + transport_cmd_check_stop(cmd, false, true); - if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) { - struct se_dev_attrib *attr = &dev->se_sub_dev->se_dev_attrib; - - if (transport_cmd_get_valid_sectors(cmd) < 0) - return -EINVAL; + ret = cmd->se_tfo->write_pending(cmd); + if (ret == -EAGAIN || ret == -ENOMEM) + goto queue_full; - BUG_ON(cmd->data_length % attr->block_size); - BUG_ON(DIV_ROUND_UP(cmd->data_length, attr->block_size) > - attr->hw_max_sectors); - } + /* fabric drivers should only return -EAGAIN or -ENOMEM as error */ + WARN_ON(ret); - atomic_inc(&cmd->t_fe_count); + return (!ret) ? 0 : TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - /* - * For WRITEs, let the fabric know its buffer is ready. - * - * The command will be added to the execution queue after its write - * data has arrived. - */ - if (cmd->data_direction == DMA_TO_DEVICE) { - target_add_to_state_list(cmd); - return transport_generic_write_pending(cmd); - } - /* - * Everything else but a WRITE, add the command to the execution queue. - */ - transport_execute_tasks(cmd); +queue_full: + pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); + cmd->t_state = TRANSPORT_COMPLETE_QF_WP; + transport_handle_queue_full(cmd, cmd->se_dev); return 0; - -out_fail: - cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION; - cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - return -EINVAL; } EXPORT_SYMBOL(transport_generic_new_cmd); -/* transport_generic_process_write(): - * - * - */ -void transport_generic_process_write(struct se_cmd *cmd) -{ - transport_execute_tasks(cmd); -} -EXPORT_SYMBOL(transport_generic_process_write); - static void transport_write_pending_qf(struct se_cmd *cmd) { int ret; @@ -3585,61 +2331,36 @@ static void transport_write_pending_qf(struct se_cmd *cmd) } } -static int transport_generic_write_pending(struct se_cmd *cmd) +int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) { unsigned long flags; - int ret; - - spin_lock_irqsave(&cmd->t_state_lock, flags); - cmd->t_state = TRANSPORT_WRITE_PENDING; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - /* - * Clear the se_cmd for WRITE_PENDING status in order to set - * CMD_T_ACTIVE so that transport_generic_handle_data can be called - * from HW target mode interrupt code. This is safe to be called - * with transport_off=1 before the cmd->se_tfo->write_pending - * because the se_cmd->se_lun pointer is not being cleared. - */ - transport_cmd_check_stop(cmd, 1, 0); - - /* - * Call the fabric write_pending function here to let the - * frontend know that WRITE buffers are ready. - */ - ret = cmd->se_tfo->write_pending(cmd); - if (ret == -EAGAIN || ret == -ENOMEM) - goto queue_full; - else if (ret < 0) - return ret; - - return 1; - -queue_full: - pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd); - cmd->t_state = TRANSPORT_COMPLETE_QF_WP; - transport_handle_queue_full(cmd, cmd->se_dev); - return 0; -} + int ret = 0; -void transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks) -{ if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) { if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) transport_wait_for_tasks(cmd); - transport_release_cmd(cmd); + ret = transport_release_cmd(cmd); } else { if (wait_for_tasks) transport_wait_for_tasks(cmd); - - core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd); + /* + * Handle WRITE failure case where transport_generic_new_cmd() + * has already added se_cmd to state_list, but fabric has + * failed command before I/O submission. + */ + if (cmd->state_active) { + spin_lock_irqsave(&cmd->t_state_lock, flags); + target_remove_from_state_list(cmd); + spin_unlock_irqrestore(&cmd->t_state_lock, flags); + } if (cmd->se_lun) transport_lun_remove_cmd(cmd); - transport_put_cmd(cmd); + ret = transport_put_cmd(cmd); } + return ret; } EXPORT_SYMBOL(transport_generic_free_cmd); @@ -3648,26 +2369,31 @@ EXPORT_SYMBOL(transport_generic_free_cmd); * @se_cmd: command descriptor to add * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() */ -void target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd, - bool ack_kref) +int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd, + bool ack_kref) { unsigned long flags; + int ret = 0; - kref_init(&se_cmd->cmd_kref); /* * Add a second kref if the fabric caller is expecting to handle * fabric acknowledgement that requires two target_put_sess_cmd() * invocations before se_cmd descriptor release. */ - if (ack_kref == true) { + if (ack_kref) { kref_get(&se_cmd->cmd_kref); se_cmd->se_cmd_flags |= SCF_ACK_KREF; } spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); + if (se_sess->sess_tearing_down) { + ret = -ESHUTDOWN; + goto out; + } list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list); - se_cmd->check_release = 1; +out: spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + return ret; } EXPORT_SYMBOL(target_get_sess_cmd); @@ -3675,21 +2401,19 @@ static void target_release_cmd_kref(struct kref *kref) { struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref); struct se_session *se_sess = se_cmd->se_sess; - unsigned long flags; - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); if (list_empty(&se_cmd->se_cmd_list)) { - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + spin_unlock(&se_sess->sess_cmd_lock); se_cmd->se_tfo->release_cmd(se_cmd); return; } if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) { - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + spin_unlock(&se_sess->sess_cmd_lock); complete(&se_cmd->cmd_wait_comp); return; } list_del(&se_cmd->se_cmd_list); - spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + spin_unlock(&se_sess->sess_cmd_lock); se_cmd->se_tfo->release_cmd(se_cmd); } @@ -3700,24 +2424,31 @@ static void target_release_cmd_kref(struct kref *kref) */ int target_put_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd) { - return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref); + if (!se_sess) { + se_cmd->se_tfo->release_cmd(se_cmd); + return 1; + } + return kref_put_spinlock_irqsave(&se_cmd->cmd_kref, target_release_cmd_kref, + &se_sess->sess_cmd_lock); } EXPORT_SYMBOL(target_put_sess_cmd); -/* target_splice_sess_cmd_list - Split active cmds into sess_wait_list - * @se_sess: session to split +/* target_sess_cmd_list_set_waiting - Flag all commands in + * sess_cmd_list to complete cmd_wait_comp. Set + * sess_tearing_down so no more commands are queued. + * @se_sess: session to flag */ -void target_splice_sess_cmd_list(struct se_session *se_sess) +void target_sess_cmd_list_set_waiting(struct se_session *se_sess) { struct se_cmd *se_cmd; unsigned long flags; - WARN_ON(!list_empty(&se_sess->sess_wait_list)); - INIT_LIST_HEAD(&se_sess->sess_wait_list); - spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); + if (se_sess->sess_tearing_down) { + spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); + return; + } se_sess->sess_tearing_down = 1; - list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list); list_for_each_entry(se_cmd, &se_sess->sess_wait_list, se_cmd_list) @@ -3725,18 +2456,15 @@ void target_splice_sess_cmd_list(struct se_session *se_sess) spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); } -EXPORT_SYMBOL(target_splice_sess_cmd_list); +EXPORT_SYMBOL(target_sess_cmd_list_set_waiting); /* target_wait_for_sess_cmds - Wait for outstanding descriptors * @se_sess: session to wait for active I/O - * @wait_for_tasks: Make extra transport_wait_for_tasks call */ -void target_wait_for_sess_cmds( - struct se_session *se_sess, - int wait_for_tasks) +void target_wait_for_sess_cmds(struct se_session *se_sess) { struct se_cmd *se_cmd, *tmp_cmd; - bool rc = false; + unsigned long flags; list_for_each_entry_safe(se_cmd, tmp_cmd, &se_sess->sess_wait_list, se_cmd_list) { @@ -3746,200 +2474,38 @@ void target_wait_for_sess_cmds( " %d\n", se_cmd, se_cmd->t_state, se_cmd->se_tfo->get_cmd_state(se_cmd)); - if (wait_for_tasks) { - pr_debug("Calling transport_wait_for_tasks se_cmd: %p t_state: %d," - " fabric state: %d\n", se_cmd, se_cmd->t_state, - se_cmd->se_tfo->get_cmd_state(se_cmd)); - - rc = transport_wait_for_tasks(se_cmd); - - pr_debug("After transport_wait_for_tasks se_cmd: %p t_state: %d," - " fabric state: %d\n", se_cmd, se_cmd->t_state, - se_cmd->se_tfo->get_cmd_state(se_cmd)); - } - - if (!rc) { - wait_for_completion(&se_cmd->cmd_wait_comp); - pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" - " fabric state: %d\n", se_cmd, se_cmd->t_state, - se_cmd->se_tfo->get_cmd_state(se_cmd)); - } + wait_for_completion(&se_cmd->cmd_wait_comp); + pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d" + " fabric state: %d\n", se_cmd, se_cmd->t_state, + se_cmd->se_tfo->get_cmd_state(se_cmd)); se_cmd->se_tfo->release_cmd(se_cmd); } -} -EXPORT_SYMBOL(target_wait_for_sess_cmds); -/* transport_lun_wait_for_tasks(): - * - * Called from ConfigFS context to stop the passed struct se_cmd to allow - * an struct se_lun to be successfully shutdown. - */ -static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun) -{ - unsigned long flags; - int ret = 0; - - /* - * If the frontend has already requested this struct se_cmd to - * be stopped, we can safely ignore this struct se_cmd. - */ - spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->transport_state & CMD_T_STOP) { - cmd->transport_state &= ~CMD_T_LUN_STOP; - - pr_debug("ConfigFS ITT[0x%08x] - CMD_T_STOP, skipping\n", - cmd->se_tfo->get_task_tag(cmd)); - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - transport_cmd_check_stop(cmd, 1, 0); - return -EPERM; - } - cmd->transport_state |= CMD_T_LUN_FE_STOP; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - - wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq); - - // XXX: audit task_flags checks. - spin_lock_irqsave(&cmd->t_state_lock, flags); - if ((cmd->transport_state & CMD_T_BUSY) && - (cmd->transport_state & CMD_T_SENT)) { - if (!target_stop_cmd(cmd, &flags)) - ret++; - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - } else { - spin_unlock_irqrestore(&cmd->t_state_lock, - flags); - target_remove_from_execute_list(cmd); - } - - pr_debug("ConfigFS: cmd: %p stop tasks ret:" - " %d\n", cmd, ret); - if (!ret) { - pr_debug("ConfigFS: ITT[0x%08x] - stopping cmd....\n", - cmd->se_tfo->get_task_tag(cmd)); - wait_for_completion(&cmd->transport_lun_stop_comp); - pr_debug("ConfigFS: ITT[0x%08x] - stopped cmd....\n", - cmd->se_tfo->get_task_tag(cmd)); - } - transport_remove_cmd_from_queue(cmd); - - return 0; -} - -static void __transport_clear_lun_from_sessions(struct se_lun *lun) -{ - struct se_cmd *cmd = NULL; - unsigned long lun_flags, cmd_flags; - /* - * Do exception processing and return CHECK_CONDITION status to the - * Initiator Port. - */ - spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags); - while (!list_empty(&lun->lun_cmd_list)) { - cmd = list_first_entry(&lun->lun_cmd_list, - struct se_cmd, se_lun_node); - list_del_init(&cmd->se_lun_node); - - /* - * This will notify iscsi_target_transport.c: - * transport_cmd_check_stop() that a LUN shutdown is in - * progress for the iscsi_cmd_t. - */ - spin_lock(&cmd->t_state_lock); - pr_debug("SE_LUN[%d] - Setting cmd->transport" - "_lun_stop for ITT: 0x%08x\n", - cmd->se_lun->unpacked_lun, - cmd->se_tfo->get_task_tag(cmd)); - cmd->transport_state |= CMD_T_LUN_STOP; - spin_unlock(&cmd->t_state_lock); - - spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags); - - if (!cmd->se_lun) { - pr_err("ITT: 0x%08x, [i,t]_state: %u/%u\n", - cmd->se_tfo->get_task_tag(cmd), - cmd->se_tfo->get_cmd_state(cmd), cmd->t_state); - BUG(); - } - /* - * If the Storage engine still owns the iscsi_cmd_t, determine - * and/or stop its context. - */ - pr_debug("SE_LUN[%d] - ITT: 0x%08x before transport" - "_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun, - cmd->se_tfo->get_task_tag(cmd)); - - if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) { - spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags); - continue; - } - - pr_debug("SE_LUN[%d] - ITT: 0x%08x after transport_lun" - "_wait_for_tasks(): SUCCESS\n", - cmd->se_lun->unpacked_lun, - cmd->se_tfo->get_task_tag(cmd)); - - spin_lock_irqsave(&cmd->t_state_lock, cmd_flags); - if (!(cmd->transport_state & CMD_T_DEV_ACTIVE)) { - spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags); - goto check_cond; - } - cmd->transport_state &= ~CMD_T_DEV_ACTIVE; - target_remove_from_state_list(cmd); - spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags); - - /* - * The Storage engine stopped this struct se_cmd before it was - * send to the fabric frontend for delivery back to the - * Initiator Node. Return this SCSI CDB back with an - * CHECK_CONDITION status. - */ -check_cond: - transport_send_check_condition_and_sense(cmd, - TCM_NON_EXISTENT_LUN, 0); - /* - * If the fabric frontend is waiting for this iscsi_cmd_t to - * be released, notify the waiting thread now that LU has - * finished accessing it. - */ - spin_lock_irqsave(&cmd->t_state_lock, cmd_flags); - if (cmd->transport_state & CMD_T_LUN_FE_STOP) { - pr_debug("SE_LUN[%d] - Detected FE stop for" - " struct se_cmd: %p ITT: 0x%08x\n", - lun->unpacked_lun, - cmd, cmd->se_tfo->get_task_tag(cmd)); - - spin_unlock_irqrestore(&cmd->t_state_lock, - cmd_flags); - transport_cmd_check_stop(cmd, 1, 0); - complete(&cmd->transport_lun_fe_stop_comp); - spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags); - continue; - } - pr_debug("SE_LUN[%d] - ITT: 0x%08x finished processing\n", - lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd)); + spin_lock_irqsave(&se_sess->sess_cmd_lock, flags); + WARN_ON(!list_empty(&se_sess->sess_cmd_list)); + spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); - spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags); - spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags); - } - spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags); } +EXPORT_SYMBOL(target_wait_for_sess_cmds); -static int transport_clear_lun_thread(void *p) +static int transport_clear_lun_ref_thread(void *p) { struct se_lun *lun = p; - __transport_clear_lun_from_sessions(lun); + percpu_ref_kill(&lun->lun_ref); + + wait_for_completion(&lun->lun_ref_comp); complete(&lun->lun_shutdown_comp); return 0; } -int transport_clear_lun_from_sessions(struct se_lun *lun) +int transport_clear_lun_ref(struct se_lun *lun) { struct task_struct *kt; - kt = kthread_run(transport_clear_lun_thread, lun, + kt = kthread_run(transport_clear_lun_ref_thread, lun, "tcm_cl_%u", lun->unpacked_lun); if (IS_ERR(kt)) { pr_err("Unable to start clear_lun thread\n"); @@ -3967,52 +2533,12 @@ bool transport_wait_for_tasks(struct se_cmd *cmd) spin_unlock_irqrestore(&cmd->t_state_lock, flags); return false; } - /* - * Only perform a possible wait_for_tasks if SCF_SUPPORTED_SAM_OPCODE - * has been set in transport_set_supported_SAM_opcode(). - */ + if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) { spin_unlock_irqrestore(&cmd->t_state_lock, flags); return false; } - /* - * If we are already stopped due to an external event (ie: LUN shutdown) - * sleep until the connection can have the passed struct se_cmd back. - * The cmd->transport_lun_stopped_sem will be upped by - * transport_clear_lun_from_sessions() once the ConfigFS context caller - * has completed its operation on the struct se_cmd. - */ - if (cmd->transport_state & CMD_T_LUN_STOP) { - pr_debug("wait_for_tasks: Stopping" - " wait_for_completion(&cmd->t_tasktransport_lun_fe" - "_stop_comp); for ITT: 0x%08x\n", - cmd->se_tfo->get_task_tag(cmd)); - /* - * There is a special case for WRITES where a FE exception + - * LUN shutdown means ConfigFS context is still sleeping on - * transport_lun_stop_comp in transport_lun_wait_for_tasks(). - * We go ahead and up transport_lun_stop_comp just to be sure - * here. - */ - spin_unlock_irqrestore(&cmd->t_state_lock, flags); - complete(&cmd->transport_lun_stop_comp); - wait_for_completion(&cmd->transport_lun_fe_stop_comp); - spin_lock_irqsave(&cmd->t_state_lock, flags); - - target_remove_from_state_list(cmd); - /* - * At this point, the frontend who was the originator of this - * struct se_cmd, now owns the structure and can be released through - * normal means below. - */ - pr_debug("wait_for_tasks: Stopped" - " wait_for_completion(&cmd->t_tasktransport_lun_fe_" - "stop_comp); for ITT: 0x%08x\n", - cmd->se_tfo->get_task_tag(cmd)); - - cmd->transport_state &= ~CMD_T_LUN_STOP; - } if (!(cmd->transport_state & CMD_T_ACTIVE)) { spin_unlock_irqrestore(&cmd->t_state_lock, flags); @@ -4028,14 +2554,12 @@ bool transport_wait_for_tasks(struct se_cmd *cmd) spin_unlock_irqrestore(&cmd->t_state_lock, flags); - wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq); - wait_for_completion(&cmd->t_transport_stop_comp); spin_lock_irqsave(&cmd->t_state_lock, flags); cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP); - pr_debug("wait_for_tasks: Stopped wait_for_compltion(" + pr_debug("wait_for_tasks: Stopped wait_for_completion(" "&cmd->t_transport_stop_comp) for ITT: 0x%08x\n", cmd->se_tfo->get_task_tag(cmd)); @@ -4056,25 +2580,25 @@ static int transport_get_sense_codes( return 0; } -static int transport_set_sense_codes( - struct se_cmd *cmd, - u8 asc, - u8 ascq) +static +void transport_err_sector_info(unsigned char *buffer, sector_t bad_sector) { - cmd->scsi_asc = asc; - cmd->scsi_ascq = ascq; + /* Place failed LBA in sense data information descriptor 0. */ + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 0xc; + buffer[SPC_DESC_TYPE_OFFSET] = 0; /* Information */ + buffer[SPC_ADDITIONAL_DESC_LEN_OFFSET] = 0xa; + buffer[SPC_VALIDITY_OFFSET] = 0x80; - return 0; + /* Descriptor Information: failing sector */ + put_unaligned_be64(bad_sector, &buffer[12]); } -int transport_send_check_condition_and_sense( - struct se_cmd *cmd, - u8 reason, - int from_transport) +int +transport_send_check_condition_and_sense(struct se_cmd *cmd, + sense_reason_t reason, int from_transport) { unsigned char *buffer = cmd->sense_buffer; unsigned long flags; - int offset; u8 asc = 0, ascq = 0; spin_lock_irqsave(&cmd->t_state_lock, flags); @@ -4090,157 +2614,225 @@ int transport_send_check_condition_and_sense( if (!from_transport) cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE; - /* - * Data Segment and SenseLength of the fabric response PDU. - * - * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE - * from include/scsi/scsi_cmnd.h - */ - offset = cmd->se_tfo->set_fabric_sense_len(cmd, - TRANSPORT_SENSE_BUFFER); + /* * Actual SENSE DATA, see SPC-3 7.23.2 SPC_SENSE_KEY_OFFSET uses * SENSE KEY values from include/scsi/scsi.h */ switch (reason) { + case TCM_NO_SENSE: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* Not Ready */ + buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY; + /* NO ADDITIONAL SENSE INFORMATION */ + buffer[SPC_ASC_KEY_OFFSET] = 0; + buffer[SPC_ASCQ_KEY_OFFSET] = 0; + break; case TCM_NON_EXISTENT_LUN: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; /* LOGICAL UNIT NOT SUPPORTED */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x25; + buffer[SPC_ASC_KEY_OFFSET] = 0x25; break; case TCM_UNSUPPORTED_SCSI_OPCODE: case TCM_SECTOR_COUNT_TOO_MANY: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; /* INVALID COMMAND OPERATION CODE */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20; + buffer[SPC_ASC_KEY_OFFSET] = 0x20; break; case TCM_UNKNOWN_MODE_PAGE: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; /* INVALID FIELD IN CDB */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24; + buffer[SPC_ASC_KEY_OFFSET] = 0x24; break; case TCM_CHECK_CONDITION_ABORT_CMD: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ABORTED COMMAND */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; + buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; /* BUS DEVICE RESET FUNCTION OCCURRED */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29; - buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03; + buffer[SPC_ASC_KEY_OFFSET] = 0x29; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x03; break; case TCM_INCORRECT_AMOUNT_OF_DATA: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ABORTED COMMAND */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; + buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; /* WRITE ERROR */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c; + buffer[SPC_ASC_KEY_OFFSET] = 0x0c; /* NOT ENOUGH UNSOLICITED DATA */ - buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x0d; break; case TCM_INVALID_CDB_FIELD: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; /* INVALID FIELD IN CDB */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24; + buffer[SPC_ASC_KEY_OFFSET] = 0x24; break; case TCM_INVALID_PARAMETER_LIST: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; /* INVALID FIELD IN PARAMETER LIST */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26; + buffer[SPC_ASC_KEY_OFFSET] = 0x26; + break; + case TCM_PARAMETER_LIST_LENGTH_ERROR: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* ILLEGAL REQUEST */ + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* PARAMETER LIST LENGTH ERROR */ + buffer[SPC_ASC_KEY_OFFSET] = 0x1a; break; case TCM_UNEXPECTED_UNSOLICITED_DATA: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ABORTED COMMAND */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; + buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; /* WRITE ERROR */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c; + buffer[SPC_ASC_KEY_OFFSET] = 0x0c; /* UNEXPECTED_UNSOLICITED_DATA */ - buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x0c; break; case TCM_SERVICE_CRC_ERROR: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ABORTED COMMAND */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; + buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; /* PROTOCOL SERVICE CRC ERROR */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47; + buffer[SPC_ASC_KEY_OFFSET] = 0x47; /* N/A */ - buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x05; break; case TCM_SNACK_REJECTED: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* ABORTED COMMAND */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; + buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND; /* READ ERROR */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11; + buffer[SPC_ASC_KEY_OFFSET] = 0x11; /* FAILED RETRANSMISSION REQUEST */ - buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x13; break; case TCM_WRITE_PROTECTED: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* DATA PROTECT */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT; + buffer[SPC_SENSE_KEY_OFFSET] = DATA_PROTECT; /* WRITE PROTECTED */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27; + buffer[SPC_ASC_KEY_OFFSET] = 0x27; + break; + case TCM_ADDRESS_OUT_OF_RANGE: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* ILLEGAL REQUEST */ + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* LOGICAL BLOCK ADDRESS OUT OF RANGE */ + buffer[SPC_ASC_KEY_OFFSET] = 0x21; break; case TCM_CHECK_CONDITION_UNIT_ATTENTION: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* UNIT ATTENTION */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION; + buffer[SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION; core_scsi3_ua_for_check_condition(cmd, &asc, &ascq); - buffer[offset+SPC_ASC_KEY_OFFSET] = asc; - buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq; + buffer[SPC_ASC_KEY_OFFSET] = asc; + buffer[SPC_ASCQ_KEY_OFFSET] = ascq; break; case TCM_CHECK_CONDITION_NOT_READY: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; /* Not Ready */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY; + buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY; transport_get_sense_codes(cmd, &asc, &ascq); - buffer[offset+SPC_ASC_KEY_OFFSET] = asc; - buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq; + buffer[SPC_ASC_KEY_OFFSET] = asc; + buffer[SPC_ASCQ_KEY_OFFSET] = ascq; + break; + case TCM_MISCOMPARE_VERIFY: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + buffer[SPC_SENSE_KEY_OFFSET] = MISCOMPARE; + /* MISCOMPARE DURING VERIFY OPERATION */ + buffer[SPC_ASC_KEY_OFFSET] = 0x1d; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x00; + break; + case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* ILLEGAL REQUEST */ + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* LOGICAL BLOCK GUARD CHECK FAILED */ + buffer[SPC_ASC_KEY_OFFSET] = 0x10; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x01; + transport_err_sector_info(buffer, cmd->bad_sector); + break; + case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* ILLEGAL REQUEST */ + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */ + buffer[SPC_ASC_KEY_OFFSET] = 0x10; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x02; + transport_err_sector_info(buffer, cmd->bad_sector); + break; + case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED: + /* CURRENT ERROR */ + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* ILLEGAL REQUEST */ + buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */ + buffer[SPC_ASC_KEY_OFFSET] = 0x10; + buffer[SPC_ASCQ_KEY_OFFSET] = 0x03; + transport_err_sector_info(buffer, cmd->bad_sector); break; case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE: default: /* CURRENT ERROR */ - buffer[offset] = 0x70; - buffer[offset+SPC_ADD_SENSE_LEN_OFFSET] = 10; - /* ILLEGAL REQUEST */ - buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + buffer[0] = 0x70; + buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; + /* + * Returning ILLEGAL REQUEST would cause immediate IO errors on + * Solaris initiators. Returning NOT READY instead means the + * operations will be retried a finite number of times and we + * can survive intermittent errors. + */ + buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY; /* LOGICAL UNIT COMMUNICATION FAILURE */ - buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80; + buffer[SPC_ASC_KEY_OFFSET] = 0x08; break; } /* @@ -4251,32 +2843,35 @@ int transport_send_check_condition_and_sense( * Automatically padded, this value is encoded in the fabric's * data_length response PDU containing the SCSI defined sense data. */ - cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER + offset; + cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER; after_reason: + trace_target_cmd_complete(cmd); return cmd->se_tfo->queue_status(cmd); } EXPORT_SYMBOL(transport_send_check_condition_and_sense); int transport_check_aborted_status(struct se_cmd *cmd, int send_status) { - int ret = 0; + if (!(cmd->transport_state & CMD_T_ABORTED)) + return 0; + + /* + * If cmd has been aborted but either no status is to be sent or it has + * already been sent, just return + */ + if (!send_status || !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) + return 1; - if (cmd->transport_state & CMD_T_ABORTED) { - if (!send_status || - (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS)) - return 1; + pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB: 0x%02x ITT: 0x%08x\n", + cmd->t_task_cdb[0], cmd->se_tfo->get_task_tag(cmd)); - pr_debug("Sending delayed SAM_STAT_TASK_ABORTED" - " status for CDB: 0x%02x ITT: 0x%08x\n", - cmd->t_task_cdb[0], - cmd->se_tfo->get_task_tag(cmd)); + cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS; + cmd->scsi_status = SAM_STAT_TASK_ABORTED; + trace_target_cmd_complete(cmd); + cmd->se_tfo->queue_status(cmd); - cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS; - cmd->se_tfo->queue_status(cmd); - ret = 1; - } - return ret; + return 1; } EXPORT_SYMBOL(transport_check_aborted_status); @@ -4285,7 +2880,7 @@ void transport_send_task_abort(struct se_cmd *cmd) unsigned long flags; spin_lock_irqsave(&cmd->t_state_lock, flags); - if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) { + if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) { spin_unlock_irqrestore(&cmd->t_state_lock, flags); return; } @@ -4300,20 +2895,26 @@ void transport_send_task_abort(struct se_cmd *cmd) if (cmd->data_direction == DMA_TO_DEVICE) { if (cmd->se_tfo->write_pending_status(cmd) != 0) { cmd->transport_state |= CMD_T_ABORTED; - smp_mb__after_atomic_inc(); + cmd->se_cmd_flags |= SCF_SEND_DELAYED_TAS; + smp_mb__after_atomic(); + return; } } cmd->scsi_status = SAM_STAT_TASK_ABORTED; + transport_lun_remove_cmd(cmd); + pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x," " ITT: 0x%08x\n", cmd->t_task_cdb[0], cmd->se_tfo->get_task_tag(cmd)); + trace_target_cmd_complete(cmd); cmd->se_tfo->queue_status(cmd); } -static int transport_generic_do_tmr(struct se_cmd *cmd) +static void target_tmr_work(struct work_struct *work) { + struct se_cmd *cmd = container_of(work, struct se_cmd, work); struct se_device *dev = cmd->se_dev; struct se_tmr_req *tmr = cmd->se_tmr_req; int ret; @@ -4349,80 +2950,19 @@ static int transport_generic_do_tmr(struct se_cmd *cmd) cmd->se_tfo->queue_tm_rsp(cmd); transport_cmd_check_stop_to_fabric(cmd); - return 0; } -/* transport_processing_thread(): - * - * - */ -static int transport_processing_thread(void *param) +int transport_generic_handle_tmr( + struct se_cmd *cmd) { - int ret; - struct se_cmd *cmd; - struct se_device *dev = param; - - while (!kthread_should_stop()) { - ret = wait_event_interruptible(dev->dev_queue_obj.thread_wq, - atomic_read(&dev->dev_queue_obj.queue_cnt) || - kthread_should_stop()); - if (ret < 0) - goto out; - -get_cmd: - cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj); - if (!cmd) - continue; - - switch (cmd->t_state) { - case TRANSPORT_NEW_CMD: - BUG(); - break; - case TRANSPORT_NEW_CMD_MAP: - if (!cmd->se_tfo->new_cmd_map) { - pr_err("cmd->se_tfo->new_cmd_map is" - " NULL for TRANSPORT_NEW_CMD_MAP\n"); - BUG(); - } - ret = cmd->se_tfo->new_cmd_map(cmd); - if (ret < 0) { - transport_generic_request_failure(cmd); - break; - } - ret = transport_generic_new_cmd(cmd); - if (ret < 0) { - transport_generic_request_failure(cmd); - break; - } - break; - case TRANSPORT_PROCESS_WRITE: - transport_generic_process_write(cmd); - break; - case TRANSPORT_PROCESS_TMR: - transport_generic_do_tmr(cmd); - break; - case TRANSPORT_COMPLETE_QF_WP: - transport_write_pending_qf(cmd); - break; - case TRANSPORT_COMPLETE_QF_OK: - transport_complete_qf(cmd); - break; - default: - pr_err("Unknown t_state: %d for ITT: 0x%08x " - "i_state: %d on SE LUN: %u\n", - cmd->t_state, - cmd->se_tfo->get_task_tag(cmd), - cmd->se_tfo->get_cmd_state(cmd), - cmd->se_lun->unpacked_lun); - BUG(); - } + unsigned long flags; - goto get_cmd; - } + spin_lock_irqsave(&cmd->t_state_lock, flags); + cmd->transport_state |= CMD_T_ACTIVE; + spin_unlock_irqrestore(&cmd->t_state_lock, flags); -out: - WARN_ON(!list_empty(&dev->state_list)); - WARN_ON(!list_empty(&dev->dev_queue_obj.qobj_list)); - dev->process_thread = NULL; + INIT_WORK(&cmd->work, target_tmr_work); + queue_work(cmd->se_dev->tmr_wq, &cmd->work); return 0; } +EXPORT_SYMBOL(transport_generic_handle_tmr); |