diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/drivers/block/ll_rw_blk.c linux/drivers/block/ll_rw_blk.c --- /opt/kernel/linux-2.4.19-pre8/drivers/block/ll_rw_blk.c 2002-05-03 08:42:13.000000000 +0200 +++ linux/drivers/block/ll_rw_blk.c 2002-05-06 15:30:16.000000000 +0200 @@ -178,6 +178,9 @@ if (count) printk("blk_cleanup_queue: leaked requests (%d)\n", count); + if (blk_queue_tagged(q)) + blk_queue_free_tags(q); + memset(q, 0, sizeof(*q)); } @@ -238,6 +241,205 @@ q->make_request_fn = mfn; } +/** + * blk_queue_free_tags - release tag maintenance info + * @q: the request queue for the device + * + * Notes: + * blk_cleanup_queue() will take care of calling this function, if tagging + * has been used. So there's usually no need to call this directly, unless + * tagging is just being disabled but the queue remains in function. + **/ +void blk_queue_free_tags(request_queue_t *q) +{ + struct blk_queue_tag *bqt = q->queue_tags; + + if (!bqt) + return; + + BUG_ON(bqt->busy); + BUG_ON(!list_empty(&bqt->busy_list)); + + kfree(bqt->tag_index); + bqt->tag_index = NULL; + + kfree(bqt->tag_map); + bqt->tag_map = NULL; + + kfree(bqt); + q->queue_tags = NULL; + q->tagged_queueing = 0; +} + +/** + * blk_queue_init_tags - initialize the queue tag info + * @q: the request queue for the device + * @depth: the maximum queue depth supported + **/ +int blk_queue_init_tags(request_queue_t *q, int depth) +{ + struct blk_queue_tag *tags; + int bits, i; + + if (depth > q->nr_requests) { + depth = q->nr_requests; + printk("blk_queue_init_tags: adjusted depth to %d\n", depth); + } + + tags = kmalloc(sizeof(struct blk_queue_tag), GFP_ATOMIC); + if (!tags) + goto fail; + + tags->tag_index = kmalloc(depth * sizeof(struct request *), GFP_ATOMIC); + if (!tags->tag_index) + goto fail_index; + + bits = (depth / BLK_TAGS_PER_LONG) + 1; + tags->tag_map = kmalloc(bits * sizeof(unsigned long), GFP_ATOMIC); + if (!tags->tag_map) + goto fail_map; + + memset(tags->tag_index, 0, depth * sizeof(struct request *)); + memset(tags->tag_map, 0, bits * sizeof(unsigned long)); + INIT_LIST_HEAD(&tags->busy_list); + tags->busy = 0; + tags->max_depth = depth; + + /* + * set the upper bits if the depth isn't a multiple of the word size + */ + for (i = depth; i < bits * BLK_TAGS_PER_LONG; i++) + __set_bit(i, tags->tag_map); + + /* + * assign it, all done + */ + q->queue_tags = tags; + q->tagged_queueing = 1; + return 0; + +fail_map: + kfree(tags->tag_index); +fail_index: + kfree(tags); +fail: + return -ENOMEM; +} + +/** + * blk_queue_end_tag - end tag operations for a request + * @q: the request queue for the device + * @tag: the tag that has completed + * + * Description: + * Typically called when end_that_request_first() returns 0, meaning + * all transfers have been done for a request. It's important to call + * this function before end_that_request_last(), as that will put the + * request back on the free list thus corrupting the internal tag list. + * + * Notes: + * io_request_lock must be held. + **/ +void blk_queue_end_tag(request_queue_t *q, struct request *rq) +{ + struct blk_queue_tag *bqt = q->queue_tags; + int tag = rq->tag; + + BUG_ON(tag == -1); + + if (unlikely(tag >= bqt->max_depth)) + return; + + if (unlikely(!__test_and_clear_bit(tag, bqt->tag_map))) { + printk("attempt to clear non-busy tag (%d)\n", tag); + return; + } + + list_del(&rq->queue); + rq->tag = -1; + rq->tagged = 0; + + if (unlikely(bqt->tag_index[tag] == NULL)) + printk("tag %d is missing\n", tag); + + bqt->tag_index[tag] = NULL; + bqt->busy--; +} + +/** + * blk_queue_start_tag - find a free tag and assign it + * @q: the request queue for the device + * @rq: the block request that needs tagging + * + * Description: + * Note that this function assumes that only regular read/write requests + * can be queued! The request will also be removed from the request queue, + * so it's the drivers responsibility to readd it if it should need to + * be restarted for some reason. + * + * Notes: + * io_request_lock must be held + **/ +int blk_queue_start_tag(request_queue_t *q, struct request *rq) +{ + struct blk_queue_tag *bqt = q->queue_tags; + unsigned long *map = bqt->tag_map; + int tag = 0; + + if (rq->cmd != READ && rq->cmd != WRITE) + return 1; + + for (map = bqt->tag_map; *map == -1UL; map++) { + tag += BLK_TAGS_PER_LONG; + + if (tag >= bqt->max_depth) + return 1; + } + + tag += ffz(*map); + __set_bit(tag, bqt->tag_map); + + rq->tag = tag; + rq->tagged = 1; + bqt->tag_index[tag] = rq; + blkdev_dequeue_request(rq); + list_add(&rq->queue, &bqt->busy_list); + bqt->busy++; + return 0; +} + +/** + * blk_queue_invalidate_tags - invalidate all pending tags + * @q: the request queue for the device + * + * Description: + * Hardware conditions may dictate a need to stop all pending requests. + * In this case, we will safely clear the block side of the tag queue and + * readd all requests to the request queue in the right order. + * + * Notes: + * io_request_lock must be held. + **/ +void blk_queue_invalidate_tags(request_queue_t *q) +{ + struct blk_queue_tag *bqt = q->queue_tags; + struct list_head *tmp; + struct request *rq; + + list_for_each(tmp, &bqt->busy_list) { + rq = list_entry(tmp, struct request, queue); + + if (rq->tag == -1) { + printk("bad tag found on list\n"); + list_del(&rq->queue); + rq->tagged = 0; + } else + blk_queue_end_tag(q, rq); + + list_add(&rq->queue, &q->queue_head); + } +} + static inline int ll_new_segment(request_queue_t *q, struct request *req, int max_segments) { if (req->nr_segments < max_segments) { @@ -667,6 +869,7 @@ hd->wr_sectors += sectors; break; default: + ; } if (!merge) up_ios(hd); @@ -685,6 +888,7 @@ hd->wr_ios++; break; default: + ; } down_ios(hd); } @@ -1441,3 +1645,9 @@ EXPORT_SYMBOL(blkdev_release_request); EXPORT_SYMBOL(req_finished_io); EXPORT_SYMBOL(generic_unplug_device); + +EXPORT_SYMBOL(blk_queue_init_tags); +EXPORT_SYMBOL(blk_queue_free_tags); +EXPORT_SYMBOL(blk_queue_start_tag); +EXPORT_SYMBOL(blk_queue_end_tag); +EXPORT_SYMBOL(blk_queue_invalidate_tags); diff -urzN -X /home/axboe/cdrom/exclude /opt/kernel/linux-2.4.19-pre8/include/linux/blkdev.h linux/include/linux/blkdev.h --- /opt/kernel/linux-2.4.19-pre8/include/linux/blkdev.h 2002-05-03 08:42:22.000000000 +0200 +++ linux/include/linux/blkdev.h 2002-05-06 13:24:29.000000000 +0200 @@ -37,12 +37,14 @@ unsigned int nr_segments; unsigned int nr_hw_segments; unsigned long current_nr_sectors; + int tag; void * special; char * buffer; struct completion * waiting; struct buffer_head * bh; struct buffer_head * bhtail; request_queue_t *q; + char tagged; }; #include @@ -72,6 +74,18 @@ struct list_head free; }; +#define BLK_TAGS_PER_LONG (sizeof(unsigned long) * 8) +#define BLK_TAGS_MASK (BLK_TAGS_PER_LONG - 1) + +struct blk_queue_tag { + struct request **tag_index; /* map of busy tags */ + unsigned long *tag_map; /* bit map of free/busy tags */ + struct list_head busy_list; /* fifo list of busy tags */ + int busy; /* current depth */ + int max_depth; +}; + + struct request_queue { /* @@ -123,6 +137,8 @@ */ char head_active; + char tagged_queueing; + /* * Is meant to protect the queue in the future instead of * io_request_lock @@ -133,8 +149,12 @@ * Tasks wait here for free read and write requests */ wait_queue_head_t wait_for_requests[2]; + + struct blk_queue_tag *queue_tags; }; +#define blk_queue_tagged(q) (q)->tagged_queueing + struct blk_dev_struct { /* * queue_proc has to be atomic @@ -175,6 +195,19 @@ extern void blk_queue_make_request(request_queue_t *, make_request_fn *); extern void generic_unplug_device(void *); +/* + * tag stuff + */ +#define blk_queue_tag_request(q, tag) ((q)->queue_tags->tag_index[(tag)]) +#define blk_queue_tag_depth(q) ((q)->queue_tags->busy) +#define blk_queue_tag_queue(q) ((q)->queue_tags->busy < (q)->queue_tags->max_depth) +#define blk_rq_tagged(rq) ((rq)->tagged) +extern int blk_queue_start_tag(request_queue_t *, struct request *); +extern void blk_queue_end_tag(request_queue_t *, struct request *); +extern int blk_queue_init_tags(request_queue_t *, int); +extern void blk_queue_free_tags(request_queue_t *); +extern void blk_queue_invalidate_tags(request_queue_t *); + extern int * blk_size[MAX_BLKDEV]; extern int * blksize_size[MAX_BLKDEV];