diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c index c4d991d..5222b61 100644 --- a/block/deadline-iosched.c +++ b/block/deadline-iosched.c @@ -17,11 +17,12 @@ /* * See Documentation/block/deadline-iosched.txt */ -static const int read_expire = HZ / 2; /* max time before a read is submitted. */ -static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */ -static const int writes_starved = 2; /* max times reads can starve a write */ -static const int fifo_batch = 16; /* # of sequential requests treated as one - by the above parameters. For throughput. */ +static const int rt_sync_expire = HZ / 8; /* max time before a real-time sync operation (e.g. read for RT class process) is submitted. */ +static const int sync_expire = HZ / 2; /* max time before a sync operation (e.g. read) is submitted. */ +static const int async_expire = 5 * HZ; /* ditto for async operations (e.g. writes), these limits are SOFT! */ +static const int async_starved = 2; /* max times SYNC can starve ASYNC requests */ +static const int fifo_batch = 16; /* min # of sequential requests treated as one by the above parameters. For throughput. */ +static const int time_quantum = HZ / 10; /* min duration for a batch */ struct deadline_data { /* @@ -31,27 +32,33 @@ struct deadline_data { /* * requests (deadline_rq s) are present on both sort_list and fifo_list */ - struct rb_root sort_list[2]; - struct list_head fifo_list[2]; + struct rb_root sort_list[2]; /* READ, WRITE */ + struct list_head fifo_list[3]; /* 0=ASYNC (or IDLE), 1=SYNC (or RT ASYNC), 2=RT SYNC */ /* - * next in sort order. read, write or both are NULL + * next in sort order. */ - struct request *next_rq[2]; + struct request *next_rq; unsigned int batching; /* number of sequential requests made */ - sector_t last_sector; /* head position */ unsigned int starved; /* times reads have starved writes */ /* * settings that change how the i/o scheduler behaves */ - int fifo_expire[2]; + int fifo_expire[3]; + int time_quantum; int fifo_batch; - int writes_starved; + int async_starved; int front_merges; + + /* + current batch data & stats + */ + int cur_batch_prio; + unsigned long cur_batch_start; }; -static void deadline_move_request(struct deadline_data *, struct request *); +static void deadline_move_request(struct deadline_data *, struct request *, int nonrot); static inline struct rb_root * deadline_rb_root(struct deadline_data *dd, struct request *rq) @@ -63,7 +70,7 @@ deadline_rb_root(struct deadline_data *dd, struct request *rq) * get the request after `rq' in sector-sorted order */ static inline struct request * -deadline_latter_request(struct request *rq) +deadline_next_request(struct request *rq) { struct rb_node *node = rb_next(&rq->rb_node); @@ -73,27 +80,118 @@ deadline_latter_request(struct request *rq) return NULL; } +/* + * get the request before `rq' in sector-sorted order + */ +static inline struct request * +deadline_prev_request(struct request *rq) +{ + struct rb_node *node = rb_prev(&rq->rb_node); + + if (node) + return rb_entry_rq(node); + + return NULL; +} + static void -deadline_add_rq_rb(struct deadline_data *dd, struct request *rq) +deadline_add_rq_rb(struct deadline_data *dd, struct request *rq, int nonrot) { struct rb_root *root = deadline_rb_root(dd, rq); struct request *__alias; while (unlikely(__alias = elv_rb_add(root, rq))) - deadline_move_request(dd, __alias); + deadline_move_request(dd, __alias, nonrot); } static inline void deadline_del_rq_rb(struct deadline_data *dd, struct request *rq) { - const int data_dir = rq_data_dir(rq); - - if (dd->next_rq[data_dir] == rq) - dd->next_rq[data_dir] = deadline_latter_request(rq); + if (dd->next_rq == rq) + dd->next_rq = deadline_next_request(rq); elv_rb_del(deadline_rb_root(dd, rq), rq); } +static void +list_add_timesorted(struct list_head *q, struct request *rq) +{ + struct list_head *entry; + int stop_flags = REQ_SOFTBARRIER | REQ_HARDBARRIER | REQ_STARTED; + list_for_each_prev(entry, q) { + struct request *pos = list_entry_rq(entry); + if (pos->cmd_flags & stop_flags) + break; + if (rq_fifo_time(rq) > rq_fifo_time(pos)) + break; + if (rq_fifo_time(rq) == rq_fifo_time(pos) && + rq->sector >= pos->sector) + break; + } + list_add(&rq->queuelist, entry); +} + +static int ioprio_lub(unsigned short aprio, unsigned short bprio) +{ + unsigned short aclass = IOPRIO_PRIO_CLASS(aprio); + unsigned short bclass = IOPRIO_PRIO_CLASS(bprio); + + if (aclass == IOPRIO_CLASS_NONE) + return bprio; + if (bclass == IOPRIO_CLASS_NONE) + return aprio; + + if (aclass == bclass) + return min(aprio, bprio); + if (aclass > bclass) + return bprio; + else + return aprio; +} + +static void +deadline_merge_prio_data(struct request_queue *q, struct request *rq) +{ + struct task_struct *tsk = current; + struct io_context *ioc = get_io_context(GFP_ATOMIC,q->node); + int ioprio_class = IOPRIO_CLASS_NONE; + int ioprio = IOPRIO_NORM; + + if(ioc) { + ioprio_class = task_ioprio_class(ioc); + } + + switch (ioprio_class) { + default: + printk(KERN_ERR "deadline: bad prio %x\n", ioprio_class); + case IOPRIO_CLASS_NONE: + /* + * no prio set, inherit CPU scheduling settings + */ + ioprio = task_nice_ioprio(tsk); + ioprio_class = task_nice_ioclass(tsk); + break; + case IOPRIO_CLASS_RT: + case IOPRIO_CLASS_BE: + ioprio = task_ioprio(ioc); + break; + case IOPRIO_CLASS_IDLE: + ioprio = 7; + break; + } + + ioprio=IOPRIO_PRIO_VALUE(ioprio_class,ioprio); + rq->ioprio=ioprio_lub(rq->ioprio,ioprio); +} + +static int +deadline_compute_request_priority(struct request *req) +{ + unsigned short ioprio_class=IOPRIO_PRIO_CLASS(req_get_ioprio(req)); + return (ioprio_class!=IOPRIO_CLASS_IDLE)* + (!!rq_is_sync(req) + (rq_data_dir(req)==READ)*(ioprio_class==IOPRIO_CLASS_RT)); +} + /* * add rq to rbtree and fifo */ @@ -101,15 +199,17 @@ static void deadline_add_request(struct request_queue *q, struct request *rq) { struct deadline_data *dd = q->elevator->elevator_data; - const int data_dir = rq_data_dir(rq); - deadline_add_rq_rb(dd, rq); + deadline_merge_prio_data(q,rq); + deadline_add_rq_rb(dd, rq, blk_queue_nonrot(q)); /* - * set expire time and add to fifo list + * set request creation time and add to fifo list */ - rq_set_fifo_time(rq, jiffies + dd->fifo_expire[data_dir]); - list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]); + + rq_set_fifo_time(rq, jiffies); + + list_add_timesorted(&dd->fifo_list[deadline_compute_request_priority(rq)],rq); } /* @@ -157,14 +257,16 @@ static void deadline_merged_request(struct request_queue *q, struct request *req, int type) { struct deadline_data *dd = q->elevator->elevator_data; - /* * if the merge was a front merge, we need to reposition request */ if (type == ELEVATOR_FRONT_MERGE) { elv_rb_del(deadline_rb_root(dd, req), req); - deadline_add_rq_rb(dd, req); + deadline_add_rq_rb(dd, req, blk_queue_nonrot(q)); } + + deadline_merge_prio_data(q,req); + } static void @@ -172,7 +274,7 @@ deadline_merged_requests(struct request_queue *q, struct request *req, struct request *next) { /* - * if next expires before rq, assign its expire time to rq + * request that cannot idle. if next expires before rq, assign its expire time to rq * and move into next position (next will be deleted) in fifo */ if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) { @@ -204,15 +306,33 @@ deadline_move_to_dispatch(struct deadline_data *dd, struct request *rq) * move an entry to dispatch queue */ static void -deadline_move_request(struct deadline_data *dd, struct request *rq) +deadline_move_request(struct deadline_data *dd, struct request *rq, int nonrot) { const int data_dir = rq_data_dir(rq); + dd->next_rq = NULL; + + if(data_dir != READ || !nonrot) { + int max_search = dd->fifo_batch; + /* for rot devices, or writes on non-rot, requests are dispatched in disk order */ + dd->next_rq = rq; + /* try to get requests of at least the same priority as current one */ + while(max_search-- && (dd->next_rq = deadline_next_request(dd->next_rq)) && dd->cur_batch_prio>deadline_compute_request_priority(dd->next_rq)); + if(!max_search || !dd->next_rq) { // did not get a next of the same priority, demote batch to lower, and continue in disk order + dd->next_rq = deadline_next_request(rq); + if(dd->next_rq) dd->cur_batch_prio = deadline_compute_request_priority(dd->next_rq); + } - dd->next_rq[READ] = NULL; - dd->next_rq[WRITE] = NULL; - dd->next_rq[data_dir] = deadline_latter_request(rq); - - dd->last_sector = rq_end_sector(rq); + } else { /* nonrot && data_dir==READ : requests are dispatched in deadline order */ + struct list_head *entry; + list_for_each(entry, &dd->fifo_list[dd->cur_batch_prio]) { + struct request *pos = list_entry_rq(entry); + if(pos==rq) continue; + if(rq_data_dir(pos)==data_dir) { /* find same direction (always READ) */ + dd->next_rq = pos; + break; + } + } + } /* * take it off the sort and fifo list, move @@ -222,20 +342,16 @@ deadline_move_request(struct deadline_data *dd, struct request *rq) } /* - * deadline_check_fifo returns 0 if there are no expired requests on the fifo, - * 1 otherwise. Requires !list_empty(&dd->fifo_list[data_dir]) + * deadline_check_fifo returns 0 if there are no expired requests on the fifo for given priority, + * 1 otherwise. Requires !list_empty(&dd->fifo_list[prio]) */ -static inline int deadline_check_fifo(struct deadline_data *dd, int ddir) +static inline int deadline_check_request(struct deadline_data *dd, unsigned prio) { - struct request *rq = rq_entry_fifo(dd->fifo_list[ddir].next); - + BUG_ON(list_empty(&dd->fifo_list[prio])); /* - * rq is expired! + * deadline is expired! */ - if (time_after(jiffies, rq_fifo_time(rq))) - return 1; - - return 0; + return time_after(jiffies, dd->fifo_expire[prio] + rq_fifo_time(rq_entry_fifo(dd->fifo_list[prio].next))); } /* @@ -245,36 +361,31 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir) static int deadline_dispatch_requests(struct request_queue *q, int force) { struct deadline_data *dd = q->elevator->elevator_data; - const int reads = !list_empty(&dd->fifo_list[READ]); - const int writes = !list_empty(&dd->fifo_list[WRITE]); - struct request *rq; - int data_dir; + int rt_ = !list_empty(&dd->fifo_list[2]); + int sync_ = !list_empty(&dd->fifo_list[1]); + int async_ = !list_empty(&dd->fifo_list[0]); + struct request *rq = dd->next_rq; + int request_prio = dd->cur_batch_prio; - /* - * batches are currently reads XOR writes - */ - if (dd->next_rq[WRITE]) - rq = dd->next_rq[WRITE]; - else - rq = dd->next_rq[READ]; - - if (rq && dd->batching < dd->fifo_batch) + if (rq && (dd->batching < dd->fifo_batch || jiffies-dd->cur_batch_start < dd->time_quantum)) { /* we have a next request are still entitled to batch */ goto dispatch_request; + } /* * at this point we are not running a batch. select the appropriate * data direction (read / write) */ - if (reads) { - BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[READ])); - - if (writes && (dd->starved++ >= dd->writes_starved)) - goto dispatch_writes; - - data_dir = READ; + if (rt_) { + request_prio = 2; + goto dispatch_find_request; + } + if (sync_) { + if (async_ && (dd->starved++ >= dd->async_starved)) + goto dispatch_async; + request_prio = 1; goto dispatch_find_request; } @@ -282,37 +393,44 @@ static int deadline_dispatch_requests(struct request_queue *q, int force) * there are either no reads or writes have been starved */ - if (writes) { -dispatch_writes: - BUG_ON(RB_EMPTY_ROOT(&dd->sort_list[WRITE])); - + if (async_) { +dispatch_async: dd->starved = 0; - - data_dir = WRITE; - + request_prio = 0; goto dispatch_find_request; } + dd->cur_batch_start=jiffies; + dd->batching = 0; return 0; dispatch_find_request: + /* - * we are not running a batch, find best request for selected data_dir + * we are not running a batch, find best request for selected request_prio */ - if (deadline_check_fifo(dd, data_dir) || !dd->next_rq[data_dir]) { + if (!dd->next_rq || + dd->cur_batch_prio < request_prio || + deadline_check_request(dd, request_prio)) { /* - * A deadline has expired, the last request was in the other - * direction, or we have run out of higher-sectored requests. - * Start again from the request with the earliest expiry time. + * A deadline has expired, the previous batch had a lower priority, + * or we have run out of higher-sectored requests. + * Start again (a bit before) the request with the earliest expiry time. */ - rq = rq_entry_fifo(dd->fifo_list[data_dir].next); + struct request * nrq = rq_entry_fifo(dd->fifo_list[request_prio].next); + int batch = dd->fifo_batch/3; + rq=nrq; + while(batch-- && (nrq = deadline_prev_request(nrq))) + if(request_prio<=deadline_compute_request_priority(nrq)) rq = nrq; } else { /* - * The last req was the same dir and we have a next request in + * The last batch was same or higher priority and we have a next request in * sort order. No expired requests so continue on from here. */ - rq = dd->next_rq[data_dir]; + rq = dd->next_rq; } + dd->cur_batch_prio = request_prio; + dd->cur_batch_start = jiffies; dd->batching = 0; @@ -320,26 +438,29 @@ dispatch_request: /* * rq is the selected appropriate request. */ - dd->batching++; - deadline_move_request(dd, rq); + dd->batching++; + deadline_move_request(dd, rq, blk_queue_nonrot(q)); return 1; } static int deadline_queue_empty(struct request_queue *q) { struct deadline_data *dd = q->elevator->elevator_data; - - return list_empty(&dd->fifo_list[WRITE]) - && list_empty(&dd->fifo_list[READ]); + return list_empty(&dd->fifo_list[0]) + && list_empty(&dd->fifo_list[1]) + && list_empty(&dd->fifo_list[2]); } + + static void deadline_exit_queue(struct elevator_queue *e) { struct deadline_data *dd = e->elevator_data; - BUG_ON(!list_empty(&dd->fifo_list[READ])); - BUG_ON(!list_empty(&dd->fifo_list[WRITE])); + BUG_ON(!list_empty(&dd->fifo_list[0])); + BUG_ON(!list_empty(&dd->fifo_list[1])); + BUG_ON(!list_empty(&dd->fifo_list[2])); kfree(dd); } @@ -355,13 +476,16 @@ static void *deadline_init_queue(struct request_queue *q) if (!dd) return NULL; - INIT_LIST_HEAD(&dd->fifo_list[READ]); - INIT_LIST_HEAD(&dd->fifo_list[WRITE]); + INIT_LIST_HEAD(&dd->fifo_list[0]); + INIT_LIST_HEAD(&dd->fifo_list[1]); + INIT_LIST_HEAD(&dd->fifo_list[2]); dd->sort_list[READ] = RB_ROOT; dd->sort_list[WRITE] = RB_ROOT; - dd->fifo_expire[READ] = read_expire; - dd->fifo_expire[WRITE] = write_expire; - dd->writes_starved = writes_starved; + dd->fifo_expire[0] = async_expire; + dd->fifo_expire[1] = sync_expire; + dd->fifo_expire[2] = rt_sync_expire; + dd->time_quantum = time_quantum; + dd->async_starved = async_starved; dd->front_merges = 1; dd->fifo_batch = fifo_batch; return dd; @@ -395,9 +519,12 @@ static ssize_t __FUNC(struct elevator_queue *e, char *page) \ __data = jiffies_to_msecs(__data); \ return deadline_var_show(__data, (page)); \ } -SHOW_FUNCTION(deadline_read_expire_show, dd->fifo_expire[READ], 1); -SHOW_FUNCTION(deadline_write_expire_show, dd->fifo_expire[WRITE], 1); -SHOW_FUNCTION(deadline_writes_starved_show, dd->writes_starved, 0); +SHOW_FUNCTION(deadline_async_expire_show, dd->fifo_expire[0], 1); +SHOW_FUNCTION(deadline_sync_expire_show, dd->fifo_expire[1], 1); +SHOW_FUNCTION(deadline_rt_sync_expire_show, dd->fifo_expire[2], 1); +SHOW_FUNCTION(deadline_time_quantum_show, dd->time_quantum, 1); + +SHOW_FUNCTION(deadline_async_starved_show, dd->async_starved, 0); SHOW_FUNCTION(deadline_front_merges_show, dd->front_merges, 0); SHOW_FUNCTION(deadline_fifo_batch_show, dd->fifo_batch, 0); #undef SHOW_FUNCTION @@ -418,9 +545,12 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) *(__PTR) = __data; \ return ret; \ } -STORE_FUNCTION(deadline_read_expire_store, &dd->fifo_expire[READ], 0, INT_MAX, 1); -STORE_FUNCTION(deadline_write_expire_store, &dd->fifo_expire[WRITE], 0, INT_MAX, 1); -STORE_FUNCTION(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX, 0); +STORE_FUNCTION(deadline_async_expire_store, &dd->fifo_expire[0], 0, INT_MAX, 1); +STORE_FUNCTION(deadline_sync_expire_store, &dd->fifo_expire[1], 0, INT_MAX, 1); +STORE_FUNCTION(deadline_rt_sync_expire_store, &dd->fifo_expire[2], 0, INT_MAX, 1); +STORE_FUNCTION(deadline_time_quantum_store, &dd->time_quantum, 0, INT_MAX, 1); + +STORE_FUNCTION(deadline_async_starved_store, &dd->async_starved, INT_MIN, INT_MAX, 0); STORE_FUNCTION(deadline_front_merges_store, &dd->front_merges, 0, 1, 0); STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0); #undef STORE_FUNCTION @@ -430,9 +560,11 @@ STORE_FUNCTION(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX, 0); deadline_##name##_store) static struct elv_fs_entry deadline_attrs[] = { - DD_ATTR(read_expire), - DD_ATTR(write_expire), - DD_ATTR(writes_starved), + DD_ATTR(async_expire), + DD_ATTR(sync_expire), + DD_ATTR(rt_sync_expire), + DD_ATTR(time_quantum), + DD_ATTR(async_starved), DD_ATTR(front_merges), DD_ATTR(fifo_batch), __ATTR_NULL