Commit 88ec22d3 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

sched: Remove the cfs_rq dependency from set_task_cpu()

In order to remove the cfs_rq dependency from set_task_cpu() we
need to ensure the task is cfs_rq invariant for all callsites.

The simple approach is to substract cfs_rq->min_vruntime from
se->vruntime on dequeue, and add cfs_rq->min_vruntime on
enqueue.

However, this has the downside of breaking FAIR_SLEEPERS since
we loose the old vruntime as we only maintain the relative
position.

To solve this, we observe that we only migrate runnable tasks,
we do this using deactivate_task(.sleep=0) and
activate_task(.wakeup=0), therefore we can restrain the
min_vruntime invariance to that state.

The only other case is wakeup balancing, since we want to
maintain the old vruntime we cannot make it relative on dequeue,
but since we don't migrate inactive tasks, we can do so right
before we activate it again.

This is where we need the new pre-wakeup hook, we need to call
this while still holding the old rq->lock. We could fold it into
->select_task_rq(), but since that has multiple callsites and
would obfuscate the locking requirements, that seems like a
fudge.

This leaves the fork() case, simply make sure that ->task_fork()
leaves the ->vruntime in a relative state.

This covers all cases where set_task_cpu() gets called, and
ensures it sees a relative vruntime.
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
LKML-Reference: <20091216170518.191697025@chello.nl>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent efbbd05a
...@@ -1116,7 +1116,7 @@ struct sched_class { ...@@ -1116,7 +1116,7 @@ struct sched_class {
struct task_struct *task); struct task_struct *task);
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
void (*moved_group) (struct task_struct *p); void (*moved_group) (struct task_struct *p, int on_rq);
#endif #endif
}; };
......
...@@ -2038,8 +2038,6 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) ...@@ -2038,8 +2038,6 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
void set_task_cpu(struct task_struct *p, unsigned int new_cpu) void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
{ {
int old_cpu = task_cpu(p); int old_cpu = task_cpu(p);
struct cfs_rq *old_cfsrq = task_cfs_rq(p),
*new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu);
#ifdef CONFIG_SCHED_DEBUG #ifdef CONFIG_SCHED_DEBUG
/* /*
...@@ -2056,8 +2054,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) ...@@ -2056,8 +2054,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS,
1, 1, NULL, 0); 1, 1, NULL, 0);
} }
p->se.vruntime -= old_cfsrq->min_vruntime -
new_cfsrq->min_vruntime;
__set_task_cpu(p, new_cpu); __set_task_cpu(p, new_cpu);
} }
...@@ -10102,7 +10098,7 @@ void sched_move_task(struct task_struct *tsk) ...@@ -10102,7 +10098,7 @@ void sched_move_task(struct task_struct *tsk)
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
if (tsk->sched_class->moved_group) if (tsk->sched_class->moved_group)
tsk->sched_class->moved_group(tsk); tsk->sched_class->moved_group(tsk, on_rq);
#endif #endif
if (unlikely(running)) if (unlikely(running))
......
...@@ -510,6 +510,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, ...@@ -510,6 +510,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
curr->sum_exec_runtime += delta_exec; curr->sum_exec_runtime += delta_exec;
schedstat_add(cfs_rq, exec_clock, delta_exec); schedstat_add(cfs_rq, exec_clock, delta_exec);
delta_exec_weighted = calc_delta_fair(delta_exec, curr); delta_exec_weighted = calc_delta_fair(delta_exec, curr);
curr->vruntime += delta_exec_weighted; curr->vruntime += delta_exec_weighted;
update_min_vruntime(cfs_rq); update_min_vruntime(cfs_rq);
} }
...@@ -765,16 +766,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) ...@@ -765,16 +766,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
se->vruntime = vruntime; se->vruntime = vruntime;
} }
#define ENQUEUE_WAKEUP 1
#define ENQUEUE_MIGRATE 2
static void static void
enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
{ {
/*
* Update the normalized vruntime before updating min_vruntime
* through callig update_curr().
*/
if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATE))
se->vruntime += cfs_rq->min_vruntime;
/* /*
* Update run-time statistics of the 'current'. * Update run-time statistics of the 'current'.
*/ */
update_curr(cfs_rq); update_curr(cfs_rq);
account_entity_enqueue(cfs_rq, se); account_entity_enqueue(cfs_rq, se);
if (wakeup) { if (flags & ENQUEUE_WAKEUP) {
place_entity(cfs_rq, se, 0); place_entity(cfs_rq, se, 0);
enqueue_sleeper(cfs_rq, se); enqueue_sleeper(cfs_rq, se);
} }
...@@ -828,6 +839,14 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) ...@@ -828,6 +839,14 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
__dequeue_entity(cfs_rq, se); __dequeue_entity(cfs_rq, se);
account_entity_dequeue(cfs_rq, se); account_entity_dequeue(cfs_rq, se);
update_min_vruntime(cfs_rq); update_min_vruntime(cfs_rq);
/*
* Normalize the entity after updating the min_vruntime because the
* update can refer to the ->curr item and we need to reflect this
* movement in our normalized position.
*/
if (!sleep)
se->vruntime -= cfs_rq->min_vruntime;
} }
/* /*
...@@ -1038,13 +1057,19 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup) ...@@ -1038,13 +1057,19 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
{ {
struct cfs_rq *cfs_rq; struct cfs_rq *cfs_rq;
struct sched_entity *se = &p->se; struct sched_entity *se = &p->se;
int flags = 0;
if (wakeup)
flags |= ENQUEUE_WAKEUP;
if (p->state == TASK_WAKING)
flags |= ENQUEUE_MIGRATE;
for_each_sched_entity(se) { for_each_sched_entity(se) {
if (se->on_rq) if (se->on_rq)
break; break;
cfs_rq = cfs_rq_of(se); cfs_rq = cfs_rq_of(se);
enqueue_entity(cfs_rq, se, wakeup); enqueue_entity(cfs_rq, se, flags);
wakeup = 1; flags = ENQUEUE_WAKEUP;
} }
hrtick_update(rq); hrtick_update(rq);
...@@ -1120,6 +1145,14 @@ static void yield_task_fair(struct rq *rq) ...@@ -1120,6 +1145,14 @@ static void yield_task_fair(struct rq *rq)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static void task_waking_fair(struct rq *rq, struct task_struct *p)
{
struct sched_entity *se = &p->se;
struct cfs_rq *cfs_rq = cfs_rq_of(se);
se->vruntime -= cfs_rq->min_vruntime;
}
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
/* /*
* effective_load() calculates the load change as seen from the root_task_group * effective_load() calculates the load change as seen from the root_task_group
...@@ -1978,6 +2011,8 @@ static void task_fork_fair(struct task_struct *p) ...@@ -1978,6 +2011,8 @@ static void task_fork_fair(struct task_struct *p)
resched_task(rq->curr); resched_task(rq->curr);
} }
se->vruntime -= cfs_rq->min_vruntime;
raw_spin_unlock_irqrestore(&rq->lock, flags); raw_spin_unlock_irqrestore(&rq->lock, flags);
} }
...@@ -2031,12 +2066,13 @@ static void set_curr_task_fair(struct rq *rq) ...@@ -2031,12 +2066,13 @@ static void set_curr_task_fair(struct rq *rq)
} }
#ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_FAIR_GROUP_SCHED
static void moved_group_fair(struct task_struct *p) static void moved_group_fair(struct task_struct *p, int on_rq)
{ {
struct cfs_rq *cfs_rq = task_cfs_rq(p); struct cfs_rq *cfs_rq = task_cfs_rq(p);
update_curr(cfs_rq); update_curr(cfs_rq);
place_entity(cfs_rq, &p->se, 1); if (!on_rq)
place_entity(cfs_rq, &p->se, 1);
} }
#endif #endif
...@@ -2076,6 +2112,8 @@ static const struct sched_class fair_sched_class = { ...@@ -2076,6 +2112,8 @@ static const struct sched_class fair_sched_class = {
.move_one_task = move_one_task_fair, .move_one_task = move_one_task_fair,
.rq_online = rq_online_fair, .rq_online = rq_online_fair,
.rq_offline = rq_offline_fair, .rq_offline = rq_offline_fair,
.task_waking = task_waking_fair,
#endif #endif
.set_curr_task = set_curr_task_fair, .set_curr_task = set_curr_task_fair,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment