Commit 8e296755 authored by Jens Axboe's avatar Jens Axboe

cfq-iosched: implement slower async initiate and queue ramp up

This slowly ramps up the async queue depth based on the time
passed since the sync IO, and doesn't allow async at all until
a sync slice period has passed.
Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
parent 365722bb
...@@ -2492,6 +2492,14 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) ...@@ -2492,6 +2492,14 @@ int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
} }
EXPORT_SYMBOL(kblockd_schedule_work); EXPORT_SYMBOL(kblockd_schedule_work);
int kblockd_schedule_delayed_work(struct request_queue *q,
struct delayed_work *work,
unsigned long delay)
{
return queue_delayed_work(kblockd_workqueue, work, delay);
}
EXPORT_SYMBOL(kblockd_schedule_delayed_work);
int __init blk_dev_init(void) int __init blk_dev_init(void)
{ {
BUILD_BUG_ON(__REQ_NR_BITS > 8 * BUILD_BUG_ON(__REQ_NR_BITS > 8 *
......
...@@ -150,7 +150,7 @@ struct cfq_data { ...@@ -150,7 +150,7 @@ struct cfq_data {
* idle window management * idle window management
*/ */
struct timer_list idle_slice_timer; struct timer_list idle_slice_timer;
struct work_struct unplug_work; struct delayed_work unplug_work;
struct cfq_queue *active_queue; struct cfq_queue *active_queue;
struct cfq_io_context *active_cic; struct cfq_io_context *active_cic;
...@@ -268,11 +268,13 @@ static inline int cfq_bio_sync(struct bio *bio) ...@@ -268,11 +268,13 @@ static inline int cfq_bio_sync(struct bio *bio)
* scheduler run of queue, if there are requests pending and no one in the * scheduler run of queue, if there are requests pending and no one in the
* driver that will restart queueing * driver that will restart queueing
*/ */
static inline void cfq_schedule_dispatch(struct cfq_data *cfqd) static inline void cfq_schedule_dispatch(struct cfq_data *cfqd,
unsigned long delay)
{ {
if (cfqd->busy_queues) { if (cfqd->busy_queues) {
cfq_log(cfqd, "schedule dispatch"); cfq_log(cfqd, "schedule dispatch");
kblockd_schedule_work(cfqd->queue, &cfqd->unplug_work); kblockd_schedule_delayed_work(cfqd->queue, &cfqd->unplug_work,
delay);
} }
} }
...@@ -1316,8 +1318,6 @@ static int cfq_dispatch_requests(struct request_queue *q, int force) ...@@ -1316,8 +1318,6 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
* Does this cfqq already have too much IO in flight? * Does this cfqq already have too much IO in flight?
*/ */
if (cfqq->dispatched >= max_dispatch) { if (cfqq->dispatched >= max_dispatch) {
unsigned long load_at = cfqd->last_end_sync_rq + cfq_slice_sync;
/* /*
* idle queue must always only have a single IO in flight * idle queue must always only have a single IO in flight
*/ */
...@@ -1331,20 +1331,36 @@ static int cfq_dispatch_requests(struct request_queue *q, int force) ...@@ -1331,20 +1331,36 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
return 0; return 0;
/* /*
* If a sync request has completed recently, don't overload * Sole queue user, allow bigger slice
* the dispatch queue yet with async requests.
*/ */
if (cfqd->cfq_desktop && !cfq_cfqq_sync(cfqq) max_dispatch *= 4;
&& time_before(jiffies, load_at)) }
return 0;
/*
* Async queues must wait a bit before being allowed dispatch.
* We also ramp up the dispatch depth gradually for async IO,
* based on the last sync IO we serviced
*/
if (!cfq_cfqq_sync(cfqq) && cfqd->cfq_desktop) {
unsigned long last_sync = jiffies - cfqd->last_end_sync_rq;
unsigned int depth;
/* /*
* we are the only queue, allow up to 4 times of 'quantum' * must wait a bit longer
*/ */
if (cfqq->dispatched >= 4 * max_dispatch) if (last_sync < cfq_slice_sync) {
cfq_schedule_dispatch(cfqd, cfq_slice_sync - last_sync);
return 0; return 0;
}
depth = last_sync / cfq_slice_sync;
if (depth < max_dispatch)
max_dispatch = depth;
} }
if (cfqq->dispatched >= max_dispatch)
return 0;
/* /*
* Dispatch a request from this cfqq * Dispatch a request from this cfqq
*/ */
...@@ -1389,7 +1405,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq) ...@@ -1389,7 +1405,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
if (unlikely(cfqd->active_queue == cfqq)) { if (unlikely(cfqd->active_queue == cfqq)) {
__cfq_slice_expired(cfqd, cfqq, 0); __cfq_slice_expired(cfqd, cfqq, 0);
cfq_schedule_dispatch(cfqd); cfq_schedule_dispatch(cfqd, 0);
} }
kmem_cache_free(cfq_pool, cfqq); kmem_cache_free(cfq_pool, cfqq);
...@@ -1484,7 +1500,7 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq) ...@@ -1484,7 +1500,7 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
if (unlikely(cfqq == cfqd->active_queue)) { if (unlikely(cfqq == cfqd->active_queue)) {
__cfq_slice_expired(cfqd, cfqq, 0); __cfq_slice_expired(cfqd, cfqq, 0);
cfq_schedule_dispatch(cfqd); cfq_schedule_dispatch(cfqd, 0);
} }
cfq_put_queue(cfqq); cfq_put_queue(cfqq);
...@@ -2201,7 +2217,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) ...@@ -2201,7 +2217,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
} }
if (!rq_in_driver(cfqd)) if (!rq_in_driver(cfqd))
cfq_schedule_dispatch(cfqd); cfq_schedule_dispatch(cfqd, 0);
} }
/* /*
...@@ -2331,7 +2347,7 @@ queue_fail: ...@@ -2331,7 +2347,7 @@ queue_fail:
if (cic) if (cic)
put_io_context(cic->ioc); put_io_context(cic->ioc);
cfq_schedule_dispatch(cfqd); cfq_schedule_dispatch(cfqd, 0);
spin_unlock_irqrestore(q->queue_lock, flags); spin_unlock_irqrestore(q->queue_lock, flags);
cfq_log(cfqd, "set_request fail"); cfq_log(cfqd, "set_request fail");
return 1; return 1;
...@@ -2340,7 +2356,7 @@ queue_fail: ...@@ -2340,7 +2356,7 @@ queue_fail:
static void cfq_kick_queue(struct work_struct *work) static void cfq_kick_queue(struct work_struct *work)
{ {
struct cfq_data *cfqd = struct cfq_data *cfqd =
container_of(work, struct cfq_data, unplug_work); container_of(work, struct cfq_data, unplug_work.work);
struct request_queue *q = cfqd->queue; struct request_queue *q = cfqd->queue;
spin_lock_irq(q->queue_lock); spin_lock_irq(q->queue_lock);
...@@ -2394,7 +2410,7 @@ static void cfq_idle_slice_timer(unsigned long data) ...@@ -2394,7 +2410,7 @@ static void cfq_idle_slice_timer(unsigned long data)
expire: expire:
cfq_slice_expired(cfqd, timed_out); cfq_slice_expired(cfqd, timed_out);
out_kick: out_kick:
cfq_schedule_dispatch(cfqd); cfq_schedule_dispatch(cfqd, 0);
out_cont: out_cont:
spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); spin_unlock_irqrestore(cfqd->queue->queue_lock, flags);
} }
...@@ -2402,7 +2418,7 @@ out_cont: ...@@ -2402,7 +2418,7 @@ out_cont:
static void cfq_shutdown_timer_wq(struct cfq_data *cfqd) static void cfq_shutdown_timer_wq(struct cfq_data *cfqd)
{ {
del_timer_sync(&cfqd->idle_slice_timer); del_timer_sync(&cfqd->idle_slice_timer);
cancel_work_sync(&cfqd->unplug_work); cancel_delayed_work_sync(&cfqd->unplug_work);
} }
static void cfq_put_async_queues(struct cfq_data *cfqd) static void cfq_put_async_queues(struct cfq_data *cfqd)
...@@ -2484,7 +2500,7 @@ static void *cfq_init_queue(struct request_queue *q) ...@@ -2484,7 +2500,7 @@ static void *cfq_init_queue(struct request_queue *q)
cfqd->idle_slice_timer.function = cfq_idle_slice_timer; cfqd->idle_slice_timer.function = cfq_idle_slice_timer;
cfqd->idle_slice_timer.data = (unsigned long) cfqd; cfqd->idle_slice_timer.data = (unsigned long) cfqd;
INIT_WORK(&cfqd->unplug_work, cfq_kick_queue); INIT_DELAYED_WORK(&cfqd->unplug_work, cfq_kick_queue);
cfqd->cfq_quantum = cfq_quantum; cfqd->cfq_quantum = cfq_quantum;
cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0]; cfqd->cfq_fifo_expire[0] = cfq_fifo_expire[0];
......
...@@ -1147,7 +1147,11 @@ static inline void put_dev_sector(Sector p) ...@@ -1147,7 +1147,11 @@ static inline void put_dev_sector(Sector p)
} }
struct work_struct; struct work_struct;
struct delayed_work;
int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
int kblockd_schedule_delayed_work(struct request_queue *q,
struct delayed_work *work,
unsigned long delay);
#define MODULE_ALIAS_BLOCKDEV(major,minor) \ #define MODULE_ALIAS_BLOCKDEV(major,minor) \
MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment