Commit df02202c authored by Jens Axboe's avatar Jens Axboe Committed by Linus Torvalds

[PATCH] switchable and modular io schedulers

This patch modularizes the io schedulers completely, allowing them to be
modular.  Additionally it enables online switching of io schedulers.  See
also http://lwn.net/Articles/102593/ .


There's a scheduler file in the sysfs directory for the block device
queue:

axboe@router:/sys/block/hda/queue> ls
iosched            max_sectors_kb  read_ahead_kb
max_hw_sectors_kb  nr_requests     scheduler

If you list the contents of the file, it will show available schedulers
and the active one:

axboe@router:/sys/block/hda/queue> cat scheduler
[cfq]

Lets load a few more.

router:/sys/block/hda/queue # modprobe deadline-iosched
router:/sys/block/hda/queue # modprobe as-iosched
router:/sys/block/hda/queue # cat scheduler
[cfq] deadline anticipatory

Changing is done with

router:/sys/block/hda/queue # echo deadline > scheduler
router:/sys/block/hda/queue # cat scheduler
cfq [deadline] anticipatory

deadline is now the new active io scheduler for hda.
Signed-off-by: default avatarJens Axboe <axboe@suse.de>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 3d3d8747
config IOSCHED_NOOP config IOSCHED_NOOP
bool "No-op I/O scheduler" if EMBEDDED bool
default y default y
---help--- ---help---
The no-op I/O scheduler is a minimal scheduler that does basic merging The no-op I/O scheduler is a minimal scheduler that does basic merging
...@@ -9,7 +9,7 @@ config IOSCHED_NOOP ...@@ -9,7 +9,7 @@ config IOSCHED_NOOP
the kernel. the kernel.
config IOSCHED_AS config IOSCHED_AS
bool "Anticipatory I/O scheduler" if EMBEDDED tristate "Anticipatory I/O scheduler"
default y default y
---help--- ---help---
The anticipatory I/O scheduler is the default disk scheduler. It is The anticipatory I/O scheduler is the default disk scheduler. It is
...@@ -18,7 +18,7 @@ config IOSCHED_AS ...@@ -18,7 +18,7 @@ config IOSCHED_AS
slower in some cases especially some database loads. slower in some cases especially some database loads.
config IOSCHED_DEADLINE config IOSCHED_DEADLINE
bool "Deadline I/O scheduler" if EMBEDDED tristate "Deadline I/O scheduler"
default y default y
---help--- ---help---
The deadline I/O scheduler is simple and compact, and is often as The deadline I/O scheduler is simple and compact, and is often as
...@@ -28,7 +28,7 @@ config IOSCHED_DEADLINE ...@@ -28,7 +28,7 @@ config IOSCHED_DEADLINE
anticipatory I/O scheduler and so is a good choice. anticipatory I/O scheduler and so is a good choice.
config IOSCHED_CFQ config IOSCHED_CFQ
bool "CFQ I/O scheduler" if EMBEDDED tristate "CFQ I/O scheduler"
default y default y
---help--- ---help---
The CFQ I/O scheduler tries to distribute bandwidth equally The CFQ I/O scheduler tries to distribute bandwidth equally
......
...@@ -614,7 +614,7 @@ static void as_antic_stop(struct as_data *ad) ...@@ -614,7 +614,7 @@ static void as_antic_stop(struct as_data *ad)
static void as_antic_timeout(unsigned long data) static void as_antic_timeout(unsigned long data)
{ {
struct request_queue *q = (struct request_queue *)data; struct request_queue *q = (struct request_queue *)data;
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(q->queue_lock, flags); spin_lock_irqsave(q->queue_lock, flags);
...@@ -945,7 +945,7 @@ static void update_write_batch(struct as_data *ad) ...@@ -945,7 +945,7 @@ static void update_write_batch(struct as_data *ad)
*/ */
static void as_completed_request(request_queue_t *q, struct request *rq) static void as_completed_request(request_queue_t *q, struct request *rq)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(rq); struct as_rq *arq = RQ_DATA(rq);
WARN_ON(!list_empty(&rq->queuelist)); WARN_ON(!list_empty(&rq->queuelist));
...@@ -1030,7 +1030,7 @@ static void as_remove_queued_request(request_queue_t *q, struct request *rq) ...@@ -1030,7 +1030,7 @@ static void as_remove_queued_request(request_queue_t *q, struct request *rq)
{ {
struct as_rq *arq = RQ_DATA(rq); struct as_rq *arq = RQ_DATA(rq);
const int data_dir = arq->is_sync; const int data_dir = arq->is_sync;
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
WARN_ON(arq->state != AS_RQ_QUEUED); WARN_ON(arq->state != AS_RQ_QUEUED);
...@@ -1361,7 +1361,7 @@ static int as_dispatch_request(struct as_data *ad) ...@@ -1361,7 +1361,7 @@ static int as_dispatch_request(struct as_data *ad)
static struct request *as_next_request(request_queue_t *q) static struct request *as_next_request(request_queue_t *q)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct request *rq = NULL; struct request *rq = NULL;
/* /*
...@@ -1469,7 +1469,7 @@ static void as_add_request(struct as_data *ad, struct as_rq *arq) ...@@ -1469,7 +1469,7 @@ static void as_add_request(struct as_data *ad, struct as_rq *arq)
*/ */
static void as_requeue_request(request_queue_t *q, struct request *rq) static void as_requeue_request(request_queue_t *q, struct request *rq)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(rq); struct as_rq *arq = RQ_DATA(rq);
if (arq) { if (arq) {
...@@ -1509,7 +1509,7 @@ static void as_account_queued_request(struct as_data *ad, struct request *rq) ...@@ -1509,7 +1509,7 @@ static void as_account_queued_request(struct as_data *ad, struct request *rq)
static void static void
as_insert_request(request_queue_t *q, struct request *rq, int where) as_insert_request(request_queue_t *q, struct request *rq, int where)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(rq); struct as_rq *arq = RQ_DATA(rq);
if (arq) { if (arq) {
...@@ -1562,7 +1562,7 @@ as_insert_request(request_queue_t *q, struct request *rq, int where) ...@@ -1562,7 +1562,7 @@ as_insert_request(request_queue_t *q, struct request *rq, int where)
*/ */
static int as_queue_empty(request_queue_t *q) static int as_queue_empty(request_queue_t *q)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
if (!list_empty(&ad->fifo_list[REQ_ASYNC]) if (!list_empty(&ad->fifo_list[REQ_ASYNC])
|| !list_empty(&ad->fifo_list[REQ_SYNC]) || !list_empty(&ad->fifo_list[REQ_SYNC])
...@@ -1601,7 +1601,7 @@ as_latter_request(request_queue_t *q, struct request *rq) ...@@ -1601,7 +1601,7 @@ as_latter_request(request_queue_t *q, struct request *rq)
static int static int
as_merge(request_queue_t *q, struct request **req, struct bio *bio) as_merge(request_queue_t *q, struct request **req, struct bio *bio)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
sector_t rb_key = bio->bi_sector + bio_sectors(bio); sector_t rb_key = bio->bi_sector + bio_sectors(bio);
struct request *__rq; struct request *__rq;
int ret; int ret;
...@@ -1656,7 +1656,7 @@ as_merge(request_queue_t *q, struct request **req, struct bio *bio) ...@@ -1656,7 +1656,7 @@ as_merge(request_queue_t *q, struct request **req, struct bio *bio)
static void as_merged_request(request_queue_t *q, struct request *req) static void as_merged_request(request_queue_t *q, struct request *req)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(req); struct as_rq *arq = RQ_DATA(req);
/* /*
...@@ -1701,7 +1701,7 @@ static void ...@@ -1701,7 +1701,7 @@ static void
as_merged_requests(request_queue_t *q, struct request *req, as_merged_requests(request_queue_t *q, struct request *req,
struct request *next) struct request *next)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(req); struct as_rq *arq = RQ_DATA(req);
struct as_rq *anext = RQ_DATA(next); struct as_rq *anext = RQ_DATA(next);
...@@ -1788,7 +1788,7 @@ static void as_work_handler(void *data) ...@@ -1788,7 +1788,7 @@ static void as_work_handler(void *data)
static void as_put_request(request_queue_t *q, struct request *rq) static void as_put_request(request_queue_t *q, struct request *rq)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = RQ_DATA(rq); struct as_rq *arq = RQ_DATA(rq);
if (!arq) { if (!arq) {
...@@ -1807,7 +1807,7 @@ static void as_put_request(request_queue_t *q, struct request *rq) ...@@ -1807,7 +1807,7 @@ static void as_put_request(request_queue_t *q, struct request *rq)
static int as_set_request(request_queue_t *q, struct request *rq, int gfp_mask) static int as_set_request(request_queue_t *q, struct request *rq, int gfp_mask)
{ {
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct as_rq *arq = mempool_alloc(ad->arq_pool, gfp_mask); struct as_rq *arq = mempool_alloc(ad->arq_pool, gfp_mask);
if (arq) { if (arq) {
...@@ -1829,7 +1829,7 @@ static int as_set_request(request_queue_t *q, struct request *rq, int gfp_mask) ...@@ -1829,7 +1829,7 @@ static int as_set_request(request_queue_t *q, struct request *rq, int gfp_mask)
static int as_may_queue(request_queue_t *q, int rw) static int as_may_queue(request_queue_t *q, int rw)
{ {
int ret = 0; int ret = 0;
struct as_data *ad = q->elevator.elevator_data; struct as_data *ad = q->elevator->elevator_data;
struct io_context *ioc; struct io_context *ioc;
if (ad->antic_status == ANTIC_WAIT_REQ || if (ad->antic_status == ANTIC_WAIT_REQ ||
ad->antic_status == ANTIC_WAIT_NEXT) { ad->antic_status == ANTIC_WAIT_NEXT) {
...@@ -1842,7 +1842,7 @@ static int as_may_queue(request_queue_t *q, int rw) ...@@ -1842,7 +1842,7 @@ static int as_may_queue(request_queue_t *q, int rw)
return ret; return ret;
} }
static void as_exit(request_queue_t *q, elevator_t *e) static void as_exit_queue(elevator_t *e)
{ {
struct as_data *ad = e->elevator_data; struct as_data *ad = e->elevator_data;
...@@ -1862,7 +1862,7 @@ static void as_exit(request_queue_t *q, elevator_t *e) ...@@ -1862,7 +1862,7 @@ static void as_exit(request_queue_t *q, elevator_t *e)
* initialize elevator private data (as_data), and alloc a arq for * initialize elevator private data (as_data), and alloc a arq for
* each request on the free lists * each request on the free lists
*/ */
static int as_init(request_queue_t *q, elevator_t *e) static int as_init_queue(request_queue_t *q, elevator_t *e)
{ {
struct as_data *ad; struct as_data *ad;
int i; int i;
...@@ -2070,39 +2070,64 @@ static struct kobj_type as_ktype = { ...@@ -2070,39 +2070,64 @@ static struct kobj_type as_ktype = {
.default_attrs = default_attrs, .default_attrs = default_attrs,
}; };
static int __init as_slab_setup(void) static struct elevator_type iosched_as = {
.ops = {
.elevator_merge_fn = as_merge,
.elevator_merged_fn = as_merged_request,
.elevator_merge_req_fn = as_merged_requests,
.elevator_next_req_fn = as_next_request,
.elevator_add_req_fn = as_insert_request,
.elevator_remove_req_fn = as_remove_request,
.elevator_requeue_req_fn = as_requeue_request,
.elevator_queue_empty_fn = as_queue_empty,
.elevator_completed_req_fn = as_completed_request,
.elevator_former_req_fn = as_former_request,
.elevator_latter_req_fn = as_latter_request,
.elevator_set_req_fn = as_set_request,
.elevator_put_req_fn = as_put_request,
.elevator_may_queue_fn = as_may_queue,
.elevator_init_fn = as_init_queue,
.elevator_exit_fn = as_exit_queue,
},
.elevator_ktype = &as_ktype,
.elevator_name = "anticipatory",
.elevator_owner = THIS_MODULE,
};
int as_init(void)
{ {
int ret;
arq_pool = kmem_cache_create("as_arq", sizeof(struct as_rq), arq_pool = kmem_cache_create("as_arq", sizeof(struct as_rq),
0, 0, NULL, NULL); 0, 0, NULL, NULL);
if (!arq_pool) if (!arq_pool)
panic("as: can't init slab pool\n"); return -ENOMEM;
return 0; ret = elv_register(&iosched_as);
if (!ret) {
/*
* don't allow AS to get unregistered, since we would have
* to browse all tasks in the system and release their
* as_io_context first
*/
__module_get(THIS_MODULE);
return 0;
}
kmem_cache_destroy(arq_pool);
return ret;
} }
subsys_initcall(as_slab_setup); void as_exit(void)
{
elevator_t iosched_as = { kmem_cache_destroy(arq_pool);
.elevator_merge_fn = as_merge, elv_unregister(&iosched_as);
.elevator_merged_fn = as_merged_request, }
.elevator_merge_req_fn = as_merged_requests,
.elevator_next_req_fn = as_next_request, module_init(as_init);
.elevator_add_req_fn = as_insert_request, module_exit(as_exit);
.elevator_remove_req_fn = as_remove_request,
.elevator_requeue_req_fn = as_requeue_request,
.elevator_queue_empty_fn = as_queue_empty,
.elevator_completed_req_fn = as_completed_request,
.elevator_former_req_fn = as_former_request,
.elevator_latter_req_fn = as_latter_request,
.elevator_set_req_fn = as_set_request,
.elevator_put_req_fn = as_put_request,
.elevator_may_queue_fn = as_may_queue,
.elevator_init_fn = as_init,
.elevator_exit_fn = as_exit,
.elevator_ktype = &as_ktype,
.elevator_name = "anticipatory",
};
EXPORT_SYMBOL(iosched_as); MODULE_AUTHOR("Nick Piggin");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("anticipatory IO scheduler");
...@@ -246,7 +246,7 @@ cfq_find_rq_rb(struct cfq_data *cfqd, sector_t sector) ...@@ -246,7 +246,7 @@ cfq_find_rq_rb(struct cfq_data *cfqd, sector_t sector)
static void cfq_remove_request(request_queue_t *q, struct request *rq) static void cfq_remove_request(request_queue_t *q, struct request *rq)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_rq *crq = RQ_DATA(rq); struct cfq_rq *crq = RQ_DATA(rq);
if (crq) { if (crq) {
...@@ -267,7 +267,7 @@ static void cfq_remove_request(request_queue_t *q, struct request *rq) ...@@ -267,7 +267,7 @@ static void cfq_remove_request(request_queue_t *q, struct request *rq)
static int static int
cfq_merge(request_queue_t *q, struct request **req, struct bio *bio) cfq_merge(request_queue_t *q, struct request **req, struct bio *bio)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct request *__rq; struct request *__rq;
int ret; int ret;
...@@ -305,7 +305,7 @@ cfq_merge(request_queue_t *q, struct request **req, struct bio *bio) ...@@ -305,7 +305,7 @@ cfq_merge(request_queue_t *q, struct request **req, struct bio *bio)
static void cfq_merged_request(request_queue_t *q, struct request *req) static void cfq_merged_request(request_queue_t *q, struct request *req)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_rq *crq = RQ_DATA(req); struct cfq_rq *crq = RQ_DATA(req);
cfq_del_crq_hash(crq); cfq_del_crq_hash(crq);
...@@ -404,7 +404,7 @@ static int cfq_dispatch_requests(request_queue_t *q, struct cfq_data *cfqd) ...@@ -404,7 +404,7 @@ static int cfq_dispatch_requests(request_queue_t *q, struct cfq_data *cfqd)
static struct request *cfq_next_request(request_queue_t *q) static struct request *cfq_next_request(request_queue_t *q)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct request *rq; struct request *rq;
if (!list_empty(cfqd->dispatch)) { if (!list_empty(cfqd->dispatch)) {
...@@ -531,7 +531,7 @@ static void cfq_enqueue(struct cfq_data *cfqd, struct cfq_rq *crq) ...@@ -531,7 +531,7 @@ static void cfq_enqueue(struct cfq_data *cfqd, struct cfq_rq *crq)
static void static void
cfq_insert_request(request_queue_t *q, struct request *rq, int where) cfq_insert_request(request_queue_t *q, struct request *rq, int where)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_rq *crq = RQ_DATA(rq); struct cfq_rq *crq = RQ_DATA(rq);
switch (where) { switch (where) {
...@@ -562,7 +562,7 @@ cfq_insert_request(request_queue_t *q, struct request *rq, int where) ...@@ -562,7 +562,7 @@ cfq_insert_request(request_queue_t *q, struct request *rq, int where)
static int cfq_queue_empty(request_queue_t *q) static int cfq_queue_empty(request_queue_t *q)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
if (list_empty(cfqd->dispatch) && list_empty(&cfqd->rr_list)) if (list_empty(cfqd->dispatch) && list_empty(&cfqd->rr_list))
return 1; return 1;
...@@ -596,7 +596,7 @@ cfq_latter_request(request_queue_t *q, struct request *rq) ...@@ -596,7 +596,7 @@ cfq_latter_request(request_queue_t *q, struct request *rq)
static int cfq_may_queue(request_queue_t *q, int rw) static int cfq_may_queue(request_queue_t *q, int rw)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_queue *cfqq; struct cfq_queue *cfqq;
int ret = 1; int ret = 1;
...@@ -621,7 +621,7 @@ static int cfq_may_queue(request_queue_t *q, int rw) ...@@ -621,7 +621,7 @@ static int cfq_may_queue(request_queue_t *q, int rw)
static void cfq_put_request(request_queue_t *q, struct request *rq) static void cfq_put_request(request_queue_t *q, struct request *rq)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_rq *crq = RQ_DATA(rq); struct cfq_rq *crq = RQ_DATA(rq);
struct request_list *rl; struct request_list *rl;
int other_rw; int other_rw;
...@@ -654,7 +654,7 @@ static void cfq_put_request(request_queue_t *q, struct request *rq) ...@@ -654,7 +654,7 @@ static void cfq_put_request(request_queue_t *q, struct request *rq)
static int cfq_set_request(request_queue_t *q, struct request *rq, int gfp_mask) static int cfq_set_request(request_queue_t *q, struct request *rq, int gfp_mask)
{ {
struct cfq_data *cfqd = q->elevator.elevator_data; struct cfq_data *cfqd = q->elevator->elevator_data;
struct cfq_queue *cfqq; struct cfq_queue *cfqq;
struct cfq_rq *crq; struct cfq_rq *crq;
...@@ -679,7 +679,7 @@ static int cfq_set_request(request_queue_t *q, struct request *rq, int gfp_mask) ...@@ -679,7 +679,7 @@ static int cfq_set_request(request_queue_t *q, struct request *rq, int gfp_mask)
return 1; return 1;
} }
static void cfq_exit(request_queue_t *q, elevator_t *e) static void cfq_exit_queue(elevator_t *e)
{ {
struct cfq_data *cfqd = e->elevator_data; struct cfq_data *cfqd = e->elevator_data;
...@@ -690,7 +690,7 @@ static void cfq_exit(request_queue_t *q, elevator_t *e) ...@@ -690,7 +690,7 @@ static void cfq_exit(request_queue_t *q, elevator_t *e)
kfree(cfqd); kfree(cfqd);
} }
static int cfq_init(request_queue_t *q, elevator_t *e) static int cfq_init_queue(request_queue_t *q, elevator_t *e)
{ {
struct cfq_data *cfqd; struct cfq_data *cfqd;
int i; int i;
...@@ -732,7 +732,6 @@ static int cfq_init(request_queue_t *q, elevator_t *e) ...@@ -732,7 +732,6 @@ static int cfq_init(request_queue_t *q, elevator_t *e)
cfqd->cfq_queued = cfq_queued; cfqd->cfq_queued = cfq_queued;
cfqd->cfq_quantum = cfq_quantum; cfqd->cfq_quantum = cfq_quantum;
return 0; return 0;
out_crqpool: out_crqpool:
kfree(cfqd->cfq_hash); kfree(cfqd->cfq_hash);
...@@ -743,30 +742,38 @@ static int cfq_init(request_queue_t *q, elevator_t *e) ...@@ -743,30 +742,38 @@ static int cfq_init(request_queue_t *q, elevator_t *e)
return -ENOMEM; return -ENOMEM;
} }
static int __init cfq_slab_setup(void) static void cfq_slab_kill(void)
{
if (crq_pool)
kmem_cache_destroy(crq_pool);
if (cfq_mpool)
mempool_destroy(cfq_mpool);
if (cfq_pool)
kmem_cache_destroy(cfq_pool);
}
static int cfq_slab_setup(void)
{ {
crq_pool = kmem_cache_create("crq_pool", sizeof(struct cfq_rq), 0, 0, crq_pool = kmem_cache_create("crq_pool", sizeof(struct cfq_rq), 0, 0,
NULL, NULL); NULL, NULL);
if (!crq_pool) if (!crq_pool)
panic("cfq_iosched: can't init crq pool\n"); goto fail;
cfq_pool = kmem_cache_create("cfq_pool", sizeof(struct cfq_queue), 0, 0, cfq_pool = kmem_cache_create("cfq_pool", sizeof(struct cfq_queue), 0, 0,
NULL, NULL); NULL, NULL);
if (!cfq_pool) if (!cfq_pool)
panic("cfq_iosched: can't init cfq pool\n"); goto fail;
cfq_mpool = mempool_create(64, mempool_alloc_slab, mempool_free_slab, cfq_pool); cfq_mpool = mempool_create(64, mempool_alloc_slab, mempool_free_slab, cfq_pool);
if (!cfq_mpool) if (!cfq_mpool)
panic("cfq_iosched: can't init cfq mpool\n"); goto fail;
return 0; return 0;
fail:
cfq_slab_kill();
return -ENOMEM;
} }
subsys_initcall(cfq_slab_setup);
/* /*
* sysfs parts below --> * sysfs parts below -->
*/ */
...@@ -868,23 +875,51 @@ struct kobj_type cfq_ktype = { ...@@ -868,23 +875,51 @@ struct kobj_type cfq_ktype = {
.default_attrs = default_attrs, .default_attrs = default_attrs,
}; };
elevator_t iosched_cfq = { static struct elevator_type iosched_cfq = {
.elevator_name = "cfq", .ops = {
.elevator_ktype = &cfq_ktype, .elevator_merge_fn = cfq_merge,
.elevator_merge_fn = cfq_merge, .elevator_merged_fn = cfq_merged_request,
.elevator_merged_fn = cfq_merged_request, .elevator_merge_req_fn = cfq_merged_requests,
.elevator_merge_req_fn = cfq_merged_requests, .elevator_next_req_fn = cfq_next_request,
.elevator_next_req_fn = cfq_next_request, .elevator_add_req_fn = cfq_insert_request,
.elevator_add_req_fn = cfq_insert_request, .elevator_remove_req_fn = cfq_remove_request,
.elevator_remove_req_fn = cfq_remove_request, .elevator_queue_empty_fn = cfq_queue_empty,
.elevator_queue_empty_fn = cfq_queue_empty, .elevator_former_req_fn = cfq_former_request,
.elevator_former_req_fn = cfq_former_request, .elevator_latter_req_fn = cfq_latter_request,
.elevator_latter_req_fn = cfq_latter_request, .elevator_set_req_fn = cfq_set_request,
.elevator_set_req_fn = cfq_set_request, .elevator_put_req_fn = cfq_put_request,
.elevator_put_req_fn = cfq_put_request, .elevator_may_queue_fn = cfq_may_queue,
.elevator_may_queue_fn = cfq_may_queue, .elevator_init_fn = cfq_init_queue,
.elevator_init_fn = cfq_init, .elevator_exit_fn = cfq_exit_queue,
.elevator_exit_fn = cfq_exit, },
.elevator_ktype = &cfq_ktype,
.elevator_name = "cfq",
.elevator_owner = THIS_MODULE,
}; };
EXPORT_SYMBOL(iosched_cfq); int cfq_init(void)
{
int ret;
if (cfq_slab_setup())
return -ENOMEM;
ret = elv_register(&iosched_cfq);
if (ret)
cfq_slab_kill();
return ret;
}
void cfq_exit(void)
{
cfq_slab_kill();
elv_unregister(&iosched_cfq);
}
module_init(cfq_init);
module_exit(cfq_exit);
MODULE_AUTHOR("Jens Axboe");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Completely Fair Queueing IO scheduler");
...@@ -289,7 +289,7 @@ deadline_find_first_drq(struct deadline_data *dd, int data_dir) ...@@ -289,7 +289,7 @@ deadline_find_first_drq(struct deadline_data *dd, int data_dir)
static inline void static inline void
deadline_add_request(struct request_queue *q, struct request *rq) deadline_add_request(struct request_queue *q, struct request *rq)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct deadline_rq *drq = RQ_DATA(rq); struct deadline_rq *drq = RQ_DATA(rq);
const int data_dir = rq_data_dir(drq->request); const int data_dir = rq_data_dir(drq->request);
...@@ -317,7 +317,7 @@ static void deadline_remove_request(request_queue_t *q, struct request *rq) ...@@ -317,7 +317,7 @@ static void deadline_remove_request(request_queue_t *q, struct request *rq)
struct deadline_rq *drq = RQ_DATA(rq); struct deadline_rq *drq = RQ_DATA(rq);
if (drq) { if (drq) {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
list_del_init(&drq->fifo); list_del_init(&drq->fifo);
deadline_remove_merge_hints(q, drq); deadline_remove_merge_hints(q, drq);
...@@ -328,7 +328,7 @@ static void deadline_remove_request(request_queue_t *q, struct request *rq) ...@@ -328,7 +328,7 @@ static void deadline_remove_request(request_queue_t *q, struct request *rq)
static int static int
deadline_merge(request_queue_t *q, struct request **req, struct bio *bio) deadline_merge(request_queue_t *q, struct request **req, struct bio *bio)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct request *__rq; struct request *__rq;
int ret; int ret;
...@@ -383,7 +383,7 @@ deadline_merge(request_queue_t *q, struct request **req, struct bio *bio) ...@@ -383,7 +383,7 @@ deadline_merge(request_queue_t *q, struct request **req, struct bio *bio)
static void deadline_merged_request(request_queue_t *q, struct request *req) static void deadline_merged_request(request_queue_t *q, struct request *req)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct deadline_rq *drq = RQ_DATA(req); struct deadline_rq *drq = RQ_DATA(req);
/* /*
...@@ -407,7 +407,7 @@ static void ...@@ -407,7 +407,7 @@ static void
deadline_merged_requests(request_queue_t *q, struct request *req, deadline_merged_requests(request_queue_t *q, struct request *req,
struct request *next) struct request *next)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct deadline_rq *drq = RQ_DATA(req); struct deadline_rq *drq = RQ_DATA(req);
struct deadline_rq *dnext = RQ_DATA(next); struct deadline_rq *dnext = RQ_DATA(next);
...@@ -604,7 +604,7 @@ static int deadline_dispatch_requests(struct deadline_data *dd) ...@@ -604,7 +604,7 @@ static int deadline_dispatch_requests(struct deadline_data *dd)
static struct request *deadline_next_request(request_queue_t *q) static struct request *deadline_next_request(request_queue_t *q)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct request *rq; struct request *rq;
/* /*
...@@ -625,7 +625,7 @@ static struct request *deadline_next_request(request_queue_t *q) ...@@ -625,7 +625,7 @@ static struct request *deadline_next_request(request_queue_t *q)
static void static void
deadline_insert_request(request_queue_t *q, struct request *rq, int where) deadline_insert_request(request_queue_t *q, struct request *rq, int where)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
/* barriers must flush the reorder queue */ /* barriers must flush the reorder queue */
if (unlikely(rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER) if (unlikely(rq->flags & (REQ_SOFTBARRIER | REQ_HARDBARRIER)
...@@ -653,7 +653,7 @@ deadline_insert_request(request_queue_t *q, struct request *rq, int where) ...@@ -653,7 +653,7 @@ deadline_insert_request(request_queue_t *q, struct request *rq, int where)
static int deadline_queue_empty(request_queue_t *q) static int deadline_queue_empty(request_queue_t *q)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
if (!list_empty(&dd->fifo_list[WRITE]) if (!list_empty(&dd->fifo_list[WRITE])
|| !list_empty(&dd->fifo_list[READ]) || !list_empty(&dd->fifo_list[READ])
...@@ -687,7 +687,7 @@ deadline_latter_request(request_queue_t *q, struct request *rq) ...@@ -687,7 +687,7 @@ deadline_latter_request(request_queue_t *q, struct request *rq)
return NULL; return NULL;
} }
static void deadline_exit(request_queue_t *q, elevator_t *e) static void deadline_exit_queue(elevator_t *e)
{ {
struct deadline_data *dd = e->elevator_data; struct deadline_data *dd = e->elevator_data;
...@@ -703,7 +703,7 @@ static void deadline_exit(request_queue_t *q, elevator_t *e) ...@@ -703,7 +703,7 @@ static void deadline_exit(request_queue_t *q, elevator_t *e)
* initialize elevator private data (deadline_data), and alloc a drq for * initialize elevator private data (deadline_data), and alloc a drq for
* each request on the free lists * each request on the free lists
*/ */
static int deadline_init(request_queue_t *q, elevator_t *e) static int deadline_init_queue(request_queue_t *q, elevator_t *e)
{ {
struct deadline_data *dd; struct deadline_data *dd;
int i; int i;
...@@ -748,7 +748,7 @@ static int deadline_init(request_queue_t *q, elevator_t *e) ...@@ -748,7 +748,7 @@ static int deadline_init(request_queue_t *q, elevator_t *e)
static void deadline_put_request(request_queue_t *q, struct request *rq) static void deadline_put_request(request_queue_t *q, struct request *rq)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct deadline_rq *drq = RQ_DATA(rq); struct deadline_rq *drq = RQ_DATA(rq);
if (drq) { if (drq) {
...@@ -760,7 +760,7 @@ static void deadline_put_request(request_queue_t *q, struct request *rq) ...@@ -760,7 +760,7 @@ static void deadline_put_request(request_queue_t *q, struct request *rq)
static int static int
deadline_set_request(request_queue_t *q, struct request *rq, int gfp_mask) deadline_set_request(request_queue_t *q, struct request *rq, int gfp_mask)
{ {
struct deadline_data *dd = q->elevator.elevator_data; struct deadline_data *dd = q->elevator->elevator_data;
struct deadline_rq *drq; struct deadline_rq *drq;
drq = mempool_alloc(dd->drq_pool, gfp_mask); drq = mempool_alloc(dd->drq_pool, gfp_mask);
...@@ -906,36 +906,54 @@ struct kobj_type deadline_ktype = { ...@@ -906,36 +906,54 @@ struct kobj_type deadline_ktype = {
.default_attrs = default_attrs, .default_attrs = default_attrs,
}; };
static int __init deadline_slab_setup(void) static struct elevator_type iosched_deadline = {
.ops = {
.elevator_merge_fn = deadline_merge,
.elevator_merged_fn = deadline_merged_request,
.elevator_merge_req_fn = deadline_merged_requests,
.elevator_next_req_fn = deadline_next_request,
.elevator_add_req_fn = deadline_insert_request,
.elevator_remove_req_fn = deadline_remove_request,
.elevator_queue_empty_fn = deadline_queue_empty,
.elevator_former_req_fn = deadline_former_request,
.elevator_latter_req_fn = deadline_latter_request,
.elevator_set_req_fn = deadline_set_request,
.elevator_put_req_fn = deadline_put_request,
.elevator_init_fn = deadline_init_queue,
.elevator_exit_fn = deadline_exit_queue,
},
.elevator_ktype = &deadline_ktype,
.elevator_name = "deadline",
.elevator_owner = THIS_MODULE,
};
int deadline_init(void)
{ {
int ret;
drq_pool = kmem_cache_create("deadline_drq", sizeof(struct deadline_rq), drq_pool = kmem_cache_create("deadline_drq", sizeof(struct deadline_rq),
0, 0, NULL, NULL); 0, 0, NULL, NULL);
if (!drq_pool) if (!drq_pool)
panic("deadline: can't init slab pool\n"); return -ENOMEM;
return 0; ret = elv_register(&iosched_deadline);
if (ret)
kmem_cache_destroy(drq_pool);
return ret;
} }
subsys_initcall(deadline_slab_setup); void deadline_exit(void)
{
elevator_t iosched_deadline = { kmem_cache_destroy(drq_pool);
.elevator_merge_fn = deadline_merge, elv_unregister(&iosched_deadline);
.elevator_merged_fn = deadline_merged_request, }
.elevator_merge_req_fn = deadline_merged_requests,
.elevator_next_req_fn = deadline_next_request, module_init(deadline_init);
.elevator_add_req_fn = deadline_insert_request, module_exit(deadline_exit);
.elevator_remove_req_fn = deadline_remove_request,
.elevator_queue_empty_fn = deadline_queue_empty,
.elevator_former_req_fn = deadline_former_request,
.elevator_latter_req_fn = deadline_latter_request,
.elevator_set_req_fn = deadline_set_request,
.elevator_put_req_fn = deadline_put_request,
.elevator_init_fn = deadline_init,
.elevator_exit_fn = deadline_exit,
.elevator_ktype = &deadline_ktype,
.elevator_name = "deadline",
};
EXPORT_SYMBOL(iosched_deadline); MODULE_AUTHOR("Jens Axboe");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("deadline IO scheduler");
This diff is collapsed.
...@@ -1395,7 +1395,8 @@ void blk_cleanup_queue(request_queue_t * q) ...@@ -1395,7 +1395,8 @@ void blk_cleanup_queue(request_queue_t * q)
if (!atomic_dec_and_test(&q->refcnt)) if (!atomic_dec_and_test(&q->refcnt))
return; return;
elevator_exit(q); if (q->elevator)
elevator_exit(q->elevator);
del_timer_sync(&q->unplug_timer); del_timer_sync(&q->unplug_timer);
kblockd_flush(); kblockd_flush();
...@@ -1418,6 +1419,7 @@ static int blk_init_free_list(request_queue_t *q) ...@@ -1418,6 +1419,7 @@ static int blk_init_free_list(request_queue_t *q)
rl->count[READ] = rl->count[WRITE] = 0; rl->count[READ] = rl->count[WRITE] = 0;
init_waitqueue_head(&rl->wait[READ]); init_waitqueue_head(&rl->wait[READ]);
init_waitqueue_head(&rl->wait[WRITE]); init_waitqueue_head(&rl->wait[WRITE]);
init_waitqueue_head(&rl->drain);
rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep); rl->rq_pool = mempool_create(BLKDEV_MIN_RQ, mempool_alloc_slab, mempool_free_slab, request_cachep);
...@@ -1429,45 +1431,6 @@ static int blk_init_free_list(request_queue_t *q) ...@@ -1429,45 +1431,6 @@ static int blk_init_free_list(request_queue_t *q)
static int __make_request(request_queue_t *, struct bio *); static int __make_request(request_queue_t *, struct bio *);
static elevator_t *chosen_elevator =
#if defined(CONFIG_IOSCHED_AS)
&iosched_as;
#elif defined(CONFIG_IOSCHED_DEADLINE)
&iosched_deadline;
#elif defined(CONFIG_IOSCHED_CFQ)
&iosched_cfq;
#elif defined(CONFIG_IOSCHED_NOOP)
&elevator_noop;
#else
NULL;
#error "You must have at least 1 I/O scheduler selected"
#endif
#if defined(CONFIG_IOSCHED_AS) || defined(CONFIG_IOSCHED_DEADLINE) || defined (CONFIG_IOSCHED_NOOP)
static int __init elevator_setup(char *str)
{
#ifdef CONFIG_IOSCHED_DEADLINE
if (!strcmp(str, "deadline"))
chosen_elevator = &iosched_deadline;
#endif
#ifdef CONFIG_IOSCHED_AS
if (!strcmp(str, "as"))
chosen_elevator = &iosched_as;
#endif
#ifdef CONFIG_IOSCHED_CFQ
if (!strcmp(str, "cfq"))
chosen_elevator = &iosched_cfq;
#endif
#ifdef CONFIG_IOSCHED_NOOP
if (!strcmp(str, "noop"))
chosen_elevator = &elevator_noop;
#endif
return 1;
}
__setup("elevator=", elevator_setup);
#endif /* CONFIG_IOSCHED_AS || CONFIG_IOSCHED_DEADLINE || CONFIG_IOSCHED_NOOP */
request_queue_t *blk_alloc_queue(int gfp_mask) request_queue_t *blk_alloc_queue(int gfp_mask)
{ {
request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask); request_queue_t *q = kmem_cache_alloc(requestq_cachep, gfp_mask);
...@@ -1520,21 +1483,14 @@ EXPORT_SYMBOL(blk_alloc_queue); ...@@ -1520,21 +1483,14 @@ EXPORT_SYMBOL(blk_alloc_queue);
**/ **/
request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
{ {
request_queue_t *q; request_queue_t *q = blk_alloc_queue(GFP_KERNEL);
static int printed;
q = blk_alloc_queue(GFP_KERNEL);
if (!q) if (!q)
return NULL; return NULL;
if (blk_init_free_list(q)) if (blk_init_free_list(q))
goto out_init; goto out_init;
if (!printed) {
printed = 1;
printk("Using %s io scheduler\n", chosen_elevator->elevator_name);
}
q->request_fn = rfn; q->request_fn = rfn;
q->back_merge_fn = ll_back_merge_fn; q->back_merge_fn = ll_back_merge_fn;
q->front_merge_fn = ll_front_merge_fn; q->front_merge_fn = ll_front_merge_fn;
...@@ -1555,7 +1511,7 @@ request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) ...@@ -1555,7 +1511,7 @@ request_queue_t *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
/* /*
* all done * all done
*/ */
if (!elevator_init(q, chosen_elevator)) if (!elevator_init(q, NULL))
return q; return q;
blk_cleanup_queue(q); blk_cleanup_queue(q);
...@@ -1649,6 +1605,9 @@ static void freed_request(request_queue_t *q, int rw) ...@@ -1649,6 +1605,9 @@ static void freed_request(request_queue_t *q, int rw)
if (!waitqueue_active(&rl->wait[rw])) if (!waitqueue_active(&rl->wait[rw]))
blk_clear_queue_full(q, rw); blk_clear_queue_full(q, rw);
} }
if (unlikely(waitqueue_active(&rl->drain)) &&
!rl->count[READ] && !rl->count[WRITE])
wake_up(&rl->drain);
} }
#define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist) #define blkdev_free_rq(list) list_entry((list)->next, struct request, queuelist)
...@@ -1661,6 +1620,9 @@ static struct request *get_request(request_queue_t *q, int rw, int gfp_mask) ...@@ -1661,6 +1620,9 @@ static struct request *get_request(request_queue_t *q, int rw, int gfp_mask)
struct request_list *rl = &q->rq; struct request_list *rl = &q->rq;
struct io_context *ioc = get_io_context(gfp_mask); struct io_context *ioc = get_io_context(gfp_mask);
if (unlikely(test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)))
return NULL;
spin_lock_irq(q->queue_lock); spin_lock_irq(q->queue_lock);
if (rl->count[rw]+1 >= q->nr_requests) { if (rl->count[rw]+1 >= q->nr_requests) {
/* /*
...@@ -2506,6 +2468,70 @@ static inline void blk_partition_remap(struct bio *bio) ...@@ -2506,6 +2468,70 @@ static inline void blk_partition_remap(struct bio *bio)
} }
} }
void blk_finish_queue_drain(request_queue_t *q)
{
struct request_list *rl = &q->rq;
clear_bit(QUEUE_FLAG_DRAIN, &q->queue_flags);
wake_up(&rl->wait[0]);
wake_up(&rl->wait[1]);
wake_up(&rl->drain);
}
/*
* We rely on the fact that only requests allocated through blk_alloc_request()
* have io scheduler private data structures associated with them. Any other
* type of request (allocated on stack or through kmalloc()) should not go
* to the io scheduler core, but be attached to the queue head instead.
*/
void blk_wait_queue_drained(request_queue_t *q)
{
struct request_list *rl = &q->rq;
DEFINE_WAIT(wait);
spin_lock_irq(q->queue_lock);
set_bit(QUEUE_FLAG_DRAIN, &q->queue_flags);
while (rl->count[READ] || rl->count[WRITE]) {
prepare_to_wait(&rl->drain, &wait, TASK_UNINTERRUPTIBLE);
if (rl->count[READ] || rl->count[WRITE]) {
__generic_unplug_device(q);
spin_unlock_irq(q->queue_lock);
io_schedule();
spin_lock_irq(q->queue_lock);
}
finish_wait(&rl->drain, &wait);
}
spin_unlock_irq(q->queue_lock);
}
/*
* block waiting for the io scheduler being started again.
*/
static inline void block_wait_queue_running(request_queue_t *q)
{
DEFINE_WAIT(wait);
while (test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags)) {
struct request_list *rl = &q->rq;
prepare_to_wait_exclusive(&rl->drain, &wait,
TASK_UNINTERRUPTIBLE);
/*
* re-check the condition. avoids using prepare_to_wait()
* in the fast path (queue is running)
*/
if (test_bit(QUEUE_FLAG_DRAIN, &q->queue_flags))
io_schedule();
finish_wait(&rl->drain, &wait);
}
}
/** /**
* generic_make_request: hand a buffer to its device driver for I/O * generic_make_request: hand a buffer to its device driver for I/O
* @bio: The bio describing the location in memory and on the device. * @bio: The bio describing the location in memory and on the device.
...@@ -2595,6 +2621,8 @@ void generic_make_request(struct bio *bio) ...@@ -2595,6 +2621,8 @@ void generic_make_request(struct bio *bio)
if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags)) if (test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))
goto end_io; goto end_io;
block_wait_queue_running(q);
/* /*
* If this device has partitions, remap block n * If this device has partitions, remap block n
* of partition p to block n+start(p) of the disk. * of partition p to block n+start(p) of the disk.
...@@ -3018,6 +3046,7 @@ void kblockd_flush(void) ...@@ -3018,6 +3046,7 @@ void kblockd_flush(void)
{ {
flush_workqueue(kblockd_workqueue); flush_workqueue(kblockd_workqueue);
} }
EXPORT_SYMBOL(kblockd_flush);
int __init blk_dev_init(void) int __init blk_dev_init(void)
{ {
...@@ -3036,6 +3065,7 @@ int __init blk_dev_init(void) ...@@ -3036,6 +3065,7 @@ int __init blk_dev_init(void)
blk_max_low_pfn = max_low_pfn; blk_max_low_pfn = max_low_pfn;
blk_max_pfn = max_pfn; blk_max_pfn = max_pfn;
return 0; return 0;
} }
...@@ -3055,6 +3085,7 @@ void put_io_context(struct io_context *ioc) ...@@ -3055,6 +3085,7 @@ void put_io_context(struct io_context *ioc)
kmem_cache_free(iocontext_cachep, ioc); kmem_cache_free(iocontext_cachep, ioc);
} }
} }
EXPORT_SYMBOL(put_io_context);
/* Called by the exitting task */ /* Called by the exitting task */
void exit_io_context(void) void exit_io_context(void)
...@@ -3106,6 +3137,7 @@ struct io_context *get_io_context(int gfp_flags) ...@@ -3106,6 +3137,7 @@ struct io_context *get_io_context(int gfp_flags)
local_irq_restore(flags); local_irq_restore(flags);
return ret; return ret;
} }
EXPORT_SYMBOL(get_io_context);
void copy_io_context(struct io_context **pdst, struct io_context **psrc) void copy_io_context(struct io_context **pdst, struct io_context **psrc)
{ {
...@@ -3119,6 +3151,7 @@ void copy_io_context(struct io_context **pdst, struct io_context **psrc) ...@@ -3119,6 +3151,7 @@ void copy_io_context(struct io_context **pdst, struct io_context **psrc)
*pdst = src; *pdst = src;
} }
} }
EXPORT_SYMBOL(copy_io_context);
void swap_io_context(struct io_context **ioc1, struct io_context **ioc2) void swap_io_context(struct io_context **ioc1, struct io_context **ioc2)
{ {
...@@ -3127,7 +3160,7 @@ void swap_io_context(struct io_context **ioc1, struct io_context **ioc2) ...@@ -3127,7 +3160,7 @@ void swap_io_context(struct io_context **ioc1, struct io_context **ioc2)
*ioc1 = *ioc2; *ioc1 = *ioc2;
*ioc2 = temp; *ioc2 = temp;
} }
EXPORT_SYMBOL(swap_io_context);
/* /*
* sysfs parts below * sysfs parts below
...@@ -3285,11 +3318,18 @@ static struct queue_sysfs_entry queue_max_hw_sectors_entry = { ...@@ -3285,11 +3318,18 @@ static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
.show = queue_max_hw_sectors_show, .show = queue_max_hw_sectors_show,
}; };
static struct queue_sysfs_entry queue_iosched_entry = {
.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
.show = elv_iosched_show,
.store = elv_iosched_store,
};
static struct attribute *default_attrs[] = { static struct attribute *default_attrs[] = {
&queue_requests_entry.attr, &queue_requests_entry.attr,
&queue_ra_entry.attr, &queue_ra_entry.attr,
&queue_max_hw_sectors_entry.attr, &queue_max_hw_sectors_entry.attr,
&queue_max_sectors_entry.attr, &queue_max_sectors_entry.attr,
&queue_iosched_entry.attr,
NULL, NULL,
}; };
......
...@@ -83,12 +83,31 @@ struct request *elevator_noop_next_request(request_queue_t *q) ...@@ -83,12 +83,31 @@ struct request *elevator_noop_next_request(request_queue_t *q)
return NULL; return NULL;
} }
elevator_t elevator_noop = { static struct elevator_type elevator_noop = {
.elevator_merge_fn = elevator_noop_merge, .ops = {
.elevator_merge_req_fn = elevator_noop_merge_requests, .elevator_merge_fn = elevator_noop_merge,
.elevator_next_req_fn = elevator_noop_next_request, .elevator_merge_req_fn = elevator_noop_merge_requests,
.elevator_add_req_fn = elevator_noop_add_request, .elevator_next_req_fn = elevator_noop_next_request,
.elevator_name = "noop", .elevator_add_req_fn = elevator_noop_add_request,
},
.elevator_name = "noop",
.elevator_owner = THIS_MODULE,
}; };
EXPORT_SYMBOL(elevator_noop); int noop_init(void)
{
return elv_register(&elevator_noop);
}
void noop_exit(void)
{
elv_unregister(&elevator_noop);
}
module_init(noop_init);
module_exit(noop_exit);
MODULE_AUTHOR("Jens Axboe");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("No-op IO scheduler");
...@@ -1595,8 +1595,8 @@ dasd_alloc_queue(struct dasd_device * device) ...@@ -1595,8 +1595,8 @@ dasd_alloc_queue(struct dasd_device * device)
device->request_queue->queuedata = device; device->request_queue->queuedata = device;
#if 0 #if 0
elevator_exit(device->request_queue); elevator_exit(device->request_queue->elevator);
rc = elevator_init(device->request_queue, &elevator_noop); rc = elevator_init(device->request_queue, "noop");
if (rc) { if (rc) {
blk_cleanup_queue(device->request_queue); blk_cleanup_queue(device->request_queue);
return rc; return rc;
......
...@@ -225,8 +225,8 @@ tapeblock_setup_device(struct tape_device * device) ...@@ -225,8 +225,8 @@ tapeblock_setup_device(struct tape_device * device)
if (!blkdat->request_queue) if (!blkdat->request_queue)
return -ENOMEM; return -ENOMEM;
elevator_exit(blkdat->request_queue); elevator_exit(blkdat->request_queue->elevator);
rc = elevator_init(blkdat->request_queue, &elevator_noop); rc = elevator_init(blkdat->request_queue, "noop");
if (rc) if (rc)
goto cleanup_queue; goto cleanup_queue;
......
...@@ -19,8 +19,8 @@ ...@@ -19,8 +19,8 @@
struct request_queue; struct request_queue;
typedef struct request_queue request_queue_t; typedef struct request_queue request_queue_t;
struct elevator_s; struct elevator_queue;
typedef struct elevator_s elevator_t; typedef struct elevator_queue elevator_t;
struct request_pm_state; struct request_pm_state;
#define BLKDEV_MIN_RQ 4 #define BLKDEV_MIN_RQ 4
...@@ -80,6 +80,7 @@ struct request_list { ...@@ -80,6 +80,7 @@ struct request_list {
int count[2]; int count[2];
mempool_t *rq_pool; mempool_t *rq_pool;
wait_queue_head_t wait[2]; wait_queue_head_t wait[2];
wait_queue_head_t drain;
}; };
#define BLK_MAX_CDB 16 #define BLK_MAX_CDB 16
...@@ -279,7 +280,7 @@ struct request_queue ...@@ -279,7 +280,7 @@ struct request_queue
*/ */
struct list_head queue_head; struct list_head queue_head;
struct request *last_merge; struct request *last_merge;
elevator_t elevator; elevator_t *elevator;
/* /*
* the queue request freelist, one for reads and one for writes * the queue request freelist, one for reads and one for writes
...@@ -381,6 +382,7 @@ struct request_queue ...@@ -381,6 +382,7 @@ struct request_queue
#define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */ #define QUEUE_FLAG_REENTER 6 /* Re-entrancy avoidance */
#define QUEUE_FLAG_PLUGGED 7 /* queue is plugged */ #define QUEUE_FLAG_PLUGGED 7 /* queue is plugged */
#define QUEUE_FLAG_ORDERED 8 /* supports ordered writes */ #define QUEUE_FLAG_ORDERED 8 /* supports ordered writes */
#define QUEUE_FLAG_DRAIN 9 /* draining queue for sched switch */
#define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags) #define blk_queue_plugged(q) test_bit(QUEUE_FLAG_PLUGGED, &(q)->queue_flags)
#define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
...@@ -617,6 +619,8 @@ extern void blk_dump_rq_flags(struct request *, char *); ...@@ -617,6 +619,8 @@ extern void blk_dump_rq_flags(struct request *, char *);
extern void generic_unplug_device(request_queue_t *); extern void generic_unplug_device(request_queue_t *);
extern void __generic_unplug_device(request_queue_t *); extern void __generic_unplug_device(request_queue_t *);
extern long nr_blockdev_pages(void); extern long nr_blockdev_pages(void);
extern void blk_wait_queue_drained(request_queue_t *);
extern void blk_finish_queue_drain(request_queue_t *);
int blk_get_queue(request_queue_t *); int blk_get_queue(request_queue_t *);
request_queue_t *blk_alloc_queue(int); request_queue_t *blk_alloc_queue(int);
......
...@@ -22,9 +22,9 @@ typedef int (elevator_set_req_fn) (request_queue_t *, struct request *, int); ...@@ -22,9 +22,9 @@ typedef int (elevator_set_req_fn) (request_queue_t *, struct request *, int);
typedef void (elevator_put_req_fn) (request_queue_t *, struct request *); typedef void (elevator_put_req_fn) (request_queue_t *, struct request *);
typedef int (elevator_init_fn) (request_queue_t *, elevator_t *); typedef int (elevator_init_fn) (request_queue_t *, elevator_t *);
typedef void (elevator_exit_fn) (request_queue_t *, elevator_t *); typedef void (elevator_exit_fn) (elevator_t *);
struct elevator_s struct elevator_ops
{ {
elevator_merge_fn *elevator_merge_fn; elevator_merge_fn *elevator_merge_fn;
elevator_merged_fn *elevator_merged_fn; elevator_merged_fn *elevator_merged_fn;
...@@ -48,12 +48,32 @@ struct elevator_s ...@@ -48,12 +48,32 @@ struct elevator_s
elevator_init_fn *elevator_init_fn; elevator_init_fn *elevator_init_fn;
elevator_exit_fn *elevator_exit_fn; elevator_exit_fn *elevator_exit_fn;
};
void *elevator_data; #define ELV_NAME_MAX (16)
struct kobject kobj; /*
* identifies an elevator type, such as AS or deadline
*/
struct elevator_type
{
struct list_head list;
struct elevator_ops ops;
struct elevator_type *elevator_type;
struct kobj_type *elevator_ktype; struct kobj_type *elevator_ktype;
const char *elevator_name; char elevator_name[ELV_NAME_MAX];
struct module *elevator_owner;
};
/*
* each queue has an elevator_queue assoicated with it
*/
struct elevator_queue
{
struct elevator_ops *ops;
void *elevator_data;
struct kobject kobj;
struct elevator_type *elevator_type;
}; };
/* /*
...@@ -79,28 +99,19 @@ extern int elv_set_request(request_queue_t *, struct request *, int); ...@@ -79,28 +99,19 @@ extern int elv_set_request(request_queue_t *, struct request *, int);
extern void elv_put_request(request_queue_t *, struct request *); extern void elv_put_request(request_queue_t *, struct request *);
/* /*
* noop I/O scheduler. always merges, always inserts new request at tail * io scheduler registration
*/
extern elevator_t elevator_noop;
/*
* deadline i/o scheduler. uses request time outs to prevent indefinite
* starvation
*/
extern elevator_t iosched_deadline;
/*
* anticipatory I/O scheduler
*/ */
extern elevator_t iosched_as; extern int elv_register(struct elevator_type *);
extern void elv_unregister(struct elevator_type *);
/* /*
* completely fair queueing I/O scheduler * io scheduler sysfs switching
*/ */
extern elevator_t iosched_cfq; extern ssize_t elv_iosched_show(request_queue_t *, char *);
extern ssize_t elv_iosched_store(request_queue_t *, const char *, size_t);
extern int elevator_init(request_queue_t *, elevator_t *); extern int elevator_init(request_queue_t *, char *);
extern void elevator_exit(request_queue_t *); extern void elevator_exit(elevator_t *);
extern int elv_rq_merge_ok(struct request *, struct bio *); extern int elv_rq_merge_ok(struct request *, struct bio *);
extern int elv_try_merge(struct request *, struct bio *); extern int elv_try_merge(struct request *, struct bio *);
extern int elv_try_last_merge(request_queue_t *, struct bio *); extern int elv_try_last_merge(request_queue_t *, struct bio *);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment