aboutsummaryrefslogtreecommitdiff
path: root/block/blk-tag.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/blk-tag.c')
-rw-r--r--block/blk-tag.c107
1 files changed, 50 insertions, 57 deletions
diff --git a/block/blk-tag.c b/block/blk-tag.c
index 4780a46ce23..a185b86741e 100644
--- a/block/blk-tag.c
+++ b/block/blk-tag.c
@@ -5,6 +5,7 @@
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
+#include <linux/slab.h>
#include "blk.h"
@@ -26,19 +27,17 @@ struct request *blk_queue_find_tag(struct request_queue *q, int tag)
EXPORT_SYMBOL(blk_queue_find_tag);
/**
- * __blk_free_tags - release a given set of tag maintenance info
+ * blk_free_tags - release a given set of tag maintenance info
* @bqt: the tag map to free
*
- * Tries to free the specified @bqt@. Returns true if it was
- * actually freed and false if there are still references using it
+ * Drop the reference count on @bqt and frees it when the last reference
+ * is dropped.
*/
-static int __blk_free_tags(struct blk_queue_tag *bqt)
+void blk_free_tags(struct blk_queue_tag *bqt)
{
- int retval;
-
- retval = atomic_dec_and_test(&bqt->refcnt);
- if (retval) {
- BUG_ON(bqt->busy);
+ if (atomic_dec_and_test(&bqt->refcnt)) {
+ BUG_ON(find_first_bit(bqt->tag_map, bqt->max_depth) <
+ bqt->max_depth);
kfree(bqt->tag_index);
bqt->tag_index = NULL;
@@ -48,9 +47,8 @@ static int __blk_free_tags(struct blk_queue_tag *bqt)
kfree(bqt);
}
-
- return retval;
}
+EXPORT_SYMBOL(blk_free_tags);
/**
* __blk_queue_free_tags - release tag maintenance info
@@ -67,38 +65,23 @@ void __blk_queue_free_tags(struct request_queue *q)
if (!bqt)
return;
- __blk_free_tags(bqt);
+ blk_free_tags(bqt);
q->queue_tags = NULL;
- q->queue_flags &= ~(1 << QUEUE_FLAG_QUEUED);
+ queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED, q);
}
/**
- * blk_free_tags - release a given set of tag maintenance info
- * @bqt: the tag map to free
- *
- * For externally managed @bqt@ frees the map. Callers of this
- * function must guarantee to have released all the queues that
- * might have been using this tag map.
- */
-void blk_free_tags(struct blk_queue_tag *bqt)
-{
- if (unlikely(!__blk_free_tags(bqt)))
- BUG();
-}
-EXPORT_SYMBOL(blk_free_tags);
-
-/**
* blk_queue_free_tags - release tag maintenance info
* @q: the request queue for the device
*
* Notes:
- * This is used to disabled tagged queuing to a device, yet leave
+ * This is used to disable tagged queuing to a device, yet leave
* queue in function.
**/
void blk_queue_free_tags(struct request_queue *q)
{
- clear_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
+ queue_flag_clear_unlocked(QUEUE_FLAG_QUEUED, q);
}
EXPORT_SYMBOL(blk_queue_free_tags);
@@ -112,7 +95,7 @@ init_tag_map(struct request_queue *q, struct blk_queue_tag *tags, int depth)
if (q && depth > q->nr_requests * 2) {
depth = q->nr_requests * 2;
printk(KERN_ERR "%s: adjusted depth to %d\n",
- __FUNCTION__, depth);
+ __func__, depth);
}
tag_index = kzalloc(depth * sizeof(struct request *), GFP_ATOMIC);
@@ -147,7 +130,6 @@ static struct blk_queue_tag *__blk_queue_init_tags(struct request_queue *q,
if (init_tag_map(q, tags, depth))
goto fail;
- tags->busy = 0;
atomic_set(&tags->refcnt, 1);
return tags;
fail:
@@ -158,7 +140,6 @@ fail:
/**
* blk_init_tags - initialize the tag info for an external tag map
* @depth: the maximum queue depth supported
- * @tags: the tag to use
**/
struct blk_queue_tag *blk_init_tags(int depth)
{
@@ -171,6 +152,9 @@ EXPORT_SYMBOL(blk_init_tags);
* @q: the request queue for the device
* @depth: the maximum queue depth supported
* @tags: the tag to use
+ *
+ * Queue lock must be held here if the function is called to resize an
+ * existing map.
**/
int blk_queue_init_tags(struct request_queue *q, int depth,
struct blk_queue_tag *tags)
@@ -183,12 +167,13 @@ int blk_queue_init_tags(struct request_queue *q, int depth,
tags = __blk_queue_init_tags(q, depth);
if (!tags)
- goto fail;
+ return -ENOMEM;
+
} else if (q->queue_tags) {
rc = blk_queue_resize_tags(q, depth);
if (rc)
return rc;
- set_bit(QUEUE_FLAG_QUEUED, &q->queue_flags);
+ queue_flag_set(QUEUE_FLAG_QUEUED, q);
return 0;
} else
atomic_inc(&tags->refcnt);
@@ -197,12 +182,9 @@ int blk_queue_init_tags(struct request_queue *q, int depth,
* assign it, all done
*/
q->queue_tags = tags;
- q->queue_flags |= (1 << QUEUE_FLAG_QUEUED);
+ queue_flag_set_unlocked(QUEUE_FLAG_QUEUED, q);
INIT_LIST_HEAD(&q->tag_busy_list);
return 0;
-fail:
- kfree(tags);
- return -ENOMEM;
}
EXPORT_SYMBOL(blk_queue_init_tags);
@@ -268,7 +250,7 @@ EXPORT_SYMBOL(blk_queue_resize_tags);
* @rq: the request that has completed
*
* Description:
- * Typically called when end_that_request_first() returns 0, meaning
+ * Typically called when end_that_request_first() returns %0, meaning
* all transfers have been done for a request. It's important to call
* this function before end_that_request_last(), as that will put the
* request back on the free list thus corrupting the internal tag list.
@@ -279,16 +261,9 @@ EXPORT_SYMBOL(blk_queue_resize_tags);
void blk_queue_end_tag(struct request_queue *q, struct request *rq)
{
struct blk_queue_tag *bqt = q->queue_tags;
- int tag = rq->tag;
-
- BUG_ON(tag == -1);
+ unsigned tag = rq->tag; /* negative tags invalid */
- if (unlikely(tag >= bqt->real_max_depth))
- /*
- * This can happen after tag depth has been reduced.
- * FIXME: how about a warning or info message here?
- */
- return;
+ BUG_ON(tag >= bqt->real_max_depth);
list_del_init(&rq->queuelist);
rq->cmd_flags &= ~REQ_QUEUED;
@@ -296,13 +271,13 @@ void blk_queue_end_tag(struct request_queue *q, struct request *rq)
if (unlikely(bqt->tag_index[tag] == NULL))
printk(KERN_ERR "%s: tag %d is missing\n",
- __FUNCTION__, tag);
+ __func__, tag);
bqt->tag_index[tag] = NULL;
if (unlikely(!test_bit(tag, bqt->tag_map))) {
printk(KERN_ERR "%s: attempt to clear non-busy tag (%d)\n",
- __FUNCTION__, tag);
+ __func__, tag);
return;
}
/*
@@ -310,7 +285,6 @@ void blk_queue_end_tag(struct request_queue *q, struct request *rq)
* unlock memory barrier semantics.
*/
clear_bit_unlock(tag, bqt->tag_map);
- bqt->busy--;
}
EXPORT_SYMBOL(blk_queue_end_tag);
@@ -335,12 +309,13 @@ EXPORT_SYMBOL(blk_queue_end_tag);
int blk_queue_start_tag(struct request_queue *q, struct request *rq)
{
struct blk_queue_tag *bqt = q->queue_tags;
+ unsigned max_depth;
int tag;
if (unlikely((rq->cmd_flags & REQ_QUEUED))) {
printk(KERN_ERR
"%s: request %p for device [%s] already tagged %d",
- __FUNCTION__, rq,
+ __func__, rq,
rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->tag);
BUG();
}
@@ -348,10 +323,29 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
/*
* Protect against shared tag maps, as we may not have exclusive
* access to the tag map.
+ *
+ * We reserve a few tags just for sync IO, since we don't want
+ * to starve sync IO on behalf of flooding async IO.
*/
+ max_depth = bqt->max_depth;
+ if (!rq_is_sync(rq) && max_depth > 1) {
+ switch (max_depth) {
+ case 2:
+ max_depth = 1;
+ break;
+ case 3:
+ max_depth = 2;
+ break;
+ default:
+ max_depth -= 2;
+ }
+ if (q->in_flight[BLK_RW_ASYNC] > max_depth)
+ return 1;
+ }
+
do {
- tag = find_first_zero_bit(bqt->tag_map, bqt->max_depth);
- if (tag >= bqt->max_depth)
+ tag = find_first_zero_bit(bqt->tag_map, max_depth);
+ if (tag >= max_depth)
return 1;
} while (test_and_set_bit_lock(tag, bqt->tag_map));
@@ -363,9 +357,8 @@ int blk_queue_start_tag(struct request_queue *q, struct request *rq)
rq->cmd_flags |= REQ_QUEUED;
rq->tag = tag;
bqt->tag_index[tag] = rq;
- blkdev_dequeue_request(rq);
+ blk_start_request(rq);
list_add(&rq->queuelist, &q->tag_busy_list);
- bqt->busy++;
return 0;
}
EXPORT_SYMBOL(blk_queue_start_tag);
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-07 02:16:00 +0000