Commit f2d1f0ae authored by Jens Axboe's avatar Jens Axboe

cfq-iosched: cache prio_tree root in cfqq->p_root

Currently we look it up from ->ioprio, but ->ioprio can change if
either the process gets its IO priority changed explicitly, or if
cfq decides to temporarily boost it. So if we are unlucky, we can
end up attempting to remove a node from a different rbtree root than
where it was added.

Fix this by using ->org_ioprio as the prio_tree index, since that
will only change for explicit IO priority settings (not for a boost).
Additionally cache the rbtree root inside the cfqq, then we don't have
to add code to reinsert the cfqq in the prio_tree if IO priority changes.
Signed-off-by: default avatarJens Axboe <jens.axboe@oracle.com>
parent 3ac6c9f8
...@@ -154,6 +154,8 @@ struct cfq_queue { ...@@ -154,6 +154,8 @@ struct cfq_queue {
unsigned long rb_key; unsigned long rb_key;
/* prio tree member */ /* prio tree member */
struct rb_node p_node; struct rb_node p_node;
/* prio tree root we belong to, if any */
struct rb_root *p_root;
/* sorted list of pending requests */ /* sorted list of pending requests */
struct rb_root sort_list; struct rb_root sort_list;
/* if fifo isn't expired, next request to serve */ /* if fifo isn't expired, next request to serve */
...@@ -558,10 +560,10 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq, ...@@ -558,10 +560,10 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
} }
static struct cfq_queue * static struct cfq_queue *
cfq_prio_tree_lookup(struct cfq_data *cfqd, int ioprio, sector_t sector, cfq_prio_tree_lookup(struct cfq_data *cfqd, struct rb_root *root,
struct rb_node **ret_parent, struct rb_node ***rb_link) sector_t sector, struct rb_node **ret_parent,
struct rb_node ***rb_link)
{ {
struct rb_root *root = &cfqd->prio_trees[ioprio];
struct rb_node **p, *parent; struct rb_node **p, *parent;
struct cfq_queue *cfqq = NULL; struct cfq_queue *cfqq = NULL;
...@@ -595,24 +597,27 @@ cfq_prio_tree_lookup(struct cfq_data *cfqd, int ioprio, sector_t sector, ...@@ -595,24 +597,27 @@ cfq_prio_tree_lookup(struct cfq_data *cfqd, int ioprio, sector_t sector,
static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq) static void cfq_prio_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq)
{ {
struct rb_root *root = &cfqd->prio_trees[cfqq->ioprio];
struct rb_node **p, *parent; struct rb_node **p, *parent;
struct cfq_queue *__cfqq; struct cfq_queue *__cfqq;
if (!RB_EMPTY_NODE(&cfqq->p_node)) if (cfqq->p_root) {
rb_erase_init(&cfqq->p_node, root); rb_erase(&cfqq->p_node, cfqq->p_root);
cfqq->p_root = NULL;
}
if (cfq_class_idle(cfqq)) if (cfq_class_idle(cfqq))
return; return;
if (!cfqq->next_rq) if (!cfqq->next_rq)
return; return;
__cfqq = cfq_prio_tree_lookup(cfqd, cfqq->ioprio, cfqq->next_rq->sector, cfqq->p_root = &cfqd->prio_trees[cfqq->org_ioprio];
__cfqq = cfq_prio_tree_lookup(cfqd, cfqq->p_root, cfqq->next_rq->sector,
&parent, &p); &parent, &p);
if (!__cfqq) { if (!__cfqq) {
rb_link_node(&cfqq->p_node, parent, p); rb_link_node(&cfqq->p_node, parent, p);
rb_insert_color(&cfqq->p_node, root); rb_insert_color(&cfqq->p_node, cfqq->p_root);
} } else
cfqq->p_root = NULL;
} }
/* /*
...@@ -657,8 +662,10 @@ static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq) ...@@ -657,8 +662,10 @@ static void cfq_del_cfqq_rr(struct cfq_data *cfqd, struct cfq_queue *cfqq)
if (!RB_EMPTY_NODE(&cfqq->rb_node)) if (!RB_EMPTY_NODE(&cfqq->rb_node))
cfq_rb_erase(&cfqq->rb_node, &cfqd->service_tree); cfq_rb_erase(&cfqq->rb_node, &cfqd->service_tree);
if (!RB_EMPTY_NODE(&cfqq->p_node)) if (cfqq->p_root) {
rb_erase_init(&cfqq->p_node, &cfqd->prio_trees[cfqq->ioprio]); rb_erase(&cfqq->p_node, cfqq->p_root);
cfqq->p_root = NULL;
}
BUG_ON(!cfqd->busy_queues); BUG_ON(!cfqd->busy_queues);
cfqd->busy_queues--; cfqd->busy_queues--;
...@@ -965,7 +972,7 @@ static inline int cfq_rq_close(struct cfq_data *cfqd, struct request *rq) ...@@ -965,7 +972,7 @@ static inline int cfq_rq_close(struct cfq_data *cfqd, struct request *rq)
static struct cfq_queue *cfqq_close(struct cfq_data *cfqd, static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
struct cfq_queue *cur_cfqq) struct cfq_queue *cur_cfqq)
{ {
struct rb_root *root = &cfqd->prio_trees[cur_cfqq->ioprio]; struct rb_root *root = &cfqd->prio_trees[cur_cfqq->org_ioprio];
struct rb_node *parent, *node; struct rb_node *parent, *node;
struct cfq_queue *__cfqq; struct cfq_queue *__cfqq;
sector_t sector = cfqd->last_position; sector_t sector = cfqd->last_position;
...@@ -977,8 +984,7 @@ static struct cfq_queue *cfqq_close(struct cfq_data *cfqd, ...@@ -977,8 +984,7 @@ static struct cfq_queue *cfqq_close(struct cfq_data *cfqd,
* First, if we find a request starting at the end of the last * First, if we find a request starting at the end of the last
* request, choose it. * request, choose it.
*/ */
__cfqq = cfq_prio_tree_lookup(cfqd, cur_cfqq->ioprio, __cfqq = cfq_prio_tree_lookup(cfqd, root, sector, &parent, NULL);
sector, &parent, NULL);
if (__cfqq) if (__cfqq)
return __cfqq; return __cfqq;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment