Commit eea08f32 authored by Arun R Bharadwaj's avatar Arun R Bharadwaj Committed by Thomas Gleixner

timers: Logic to move non pinned timers

* Arun R Bharadwaj <arun@linux.vnet.ibm.com> [2009-04-16 12:11:36]:

This patch migrates all non pinned timers and hrtimers to the current
idle load balancer, from all the idle CPUs. Timers firing on busy CPUs
are not migrated.

While migrating hrtimers, care should be taken to check if migrating
a hrtimer would result in a latency or not. So we compare the expiry of the
hrtimer with the next timer interrupt on the target cpu and migrate the
hrtimer only if it expires *after* the next interrupt on the target cpu.
So, added a clockevents_get_next_event() helper function to return the
next_event on the target cpu's clock_event_device.

[ tglx: cleanups and simplifications ]
Signed-off-by: default avatarArun R Bharadwaj <arun@linux.vnet.ibm.com>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent cd1bb94b
...@@ -143,3 +143,12 @@ extern void clockevents_notify(unsigned long reason, void *arg); ...@@ -143,3 +143,12 @@ extern void clockevents_notify(unsigned long reason, void *arg);
#endif #endif
#endif #endif
#ifdef CONFIG_GENERIC_CLOCKEVENTS
extern ktime_t clockevents_get_next_event(int cpu);
#else
static inline ktime_t clockevents_get_next_event(int cpu)
{
return (ktime_t) { .tv64 = KTIME_MAX };
}
#endif
...@@ -257,6 +257,7 @@ extern void task_rq_unlock_wait(struct task_struct *p); ...@@ -257,6 +257,7 @@ extern void task_rq_unlock_wait(struct task_struct *p);
extern cpumask_var_t nohz_cpu_mask; extern cpumask_var_t nohz_cpu_mask;
#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ) #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ)
extern int select_nohz_load_balancer(int cpu); extern int select_nohz_load_balancer(int cpu);
extern int get_nohz_load_balancer(void);
#else #else
static inline int select_nohz_load_balancer(int cpu) static inline int select_nohz_load_balancer(int cpu)
{ {
...@@ -1772,6 +1773,17 @@ int sched_nr_latency_handler(struct ctl_table *table, int write, ...@@ -1772,6 +1773,17 @@ int sched_nr_latency_handler(struct ctl_table *table, int write,
struct file *file, void __user *buffer, size_t *length, struct file *file, void __user *buffer, size_t *length,
loff_t *ppos); loff_t *ppos);
#endif #endif
#ifdef CONFIG_SCHED_DEBUG
static inline unsigned int get_sysctl_timer_migration(void)
{
return sysctl_timer_migration;
}
#else
static inline unsigned int get_sysctl_timer_migration(void)
{
return 1;
}
#endif
extern unsigned int sysctl_sched_rt_period; extern unsigned int sysctl_sched_rt_period;
extern int sysctl_sched_rt_runtime; extern int sysctl_sched_rt_runtime;
......
...@@ -43,6 +43,8 @@ ...@@ -43,6 +43,8 @@
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/debugobjects.h> #include <linux/debugobjects.h>
#include <linux/sched.h>
#include <linux/timer.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
...@@ -198,8 +200,19 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, ...@@ -198,8 +200,19 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
{ {
struct hrtimer_clock_base *new_base; struct hrtimer_clock_base *new_base;
struct hrtimer_cpu_base *new_cpu_base; struct hrtimer_cpu_base *new_cpu_base;
int cpu, preferred_cpu = -1;
cpu = smp_processor_id();
#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
preferred_cpu = get_nohz_load_balancer();
if (preferred_cpu >= 0)
cpu = preferred_cpu;
}
#endif
new_cpu_base = &__get_cpu_var(hrtimer_bases); again:
new_cpu_base = &per_cpu(hrtimer_bases, cpu);
new_base = &new_cpu_base->clock_base[base->index]; new_base = &new_cpu_base->clock_base[base->index];
if (base != new_base) { if (base != new_base) {
...@@ -219,6 +232,40 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, ...@@ -219,6 +232,40 @@ switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
timer->base = NULL; timer->base = NULL;
spin_unlock(&base->cpu_base->lock); spin_unlock(&base->cpu_base->lock);
spin_lock(&new_base->cpu_base->lock); spin_lock(&new_base->cpu_base->lock);
/* Optimized away for NOHZ=n SMP=n */
if (cpu == preferred_cpu) {
/* Calculate clock monotonic expiry time */
#ifdef CONFIG_HIGH_RES_TIMERS
ktime_t expires = ktime_sub(hrtimer_get_expires(timer),
new_base->offset);
#else
ktime_t expires = hrtimer_get_expires(timer);
#endif
/*
* Get the next event on target cpu from the
* clock events layer.
* This covers the highres=off nohz=on case as well.
*/
ktime_t next = clockevents_get_next_event(cpu);
ktime_t delta = ktime_sub(expires, next);
/*
* We do not migrate the timer when it is expiring
* before the next event on the target cpu because
* we cannot reprogram the target cpu hardware and
* we would cause it to fire late.
*/
if (delta.tv64 < 0) {
cpu = smp_processor_id();
spin_unlock(&new_base->cpu_base->lock);
spin_lock(&base->cpu_base->lock);
timer->base = base;
goto again;
}
}
timer->base = new_base; timer->base = new_base;
} }
return new_base; return new_base;
...@@ -236,7 +283,7 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) ...@@ -236,7 +283,7 @@ lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
return base; return base;
} }
# define switch_hrtimer_base(t, b) (b) # define switch_hrtimer_base(t, b, p) (b)
#endif /* !CONFIG_SMP */ #endif /* !CONFIG_SMP */
......
...@@ -4244,6 +4244,11 @@ static struct { ...@@ -4244,6 +4244,11 @@ static struct {
.load_balancer = ATOMIC_INIT(-1), .load_balancer = ATOMIC_INIT(-1),
}; };
int get_nohz_load_balancer(void)
{
return atomic_read(&nohz.load_balancer);
}
/* /*
* This routine will try to nominate the ilb (idle load balancing) * This routine will try to nominate the ilb (idle load balancing)
* owner among the cpus whose ticks are stopped. ilb owner will do the idle * owner among the cpus whose ticks are stopped. ilb owner will do the idle
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <linux/notifier.h> #include <linux/notifier.h>
#include <linux/smp.h> #include <linux/smp.h>
#include <linux/sysdev.h> #include <linux/sysdev.h>
#include <linux/tick.h>
/* The registered clock event devices */ /* The registered clock event devices */
static LIST_HEAD(clockevent_devices); static LIST_HEAD(clockevent_devices);
...@@ -251,4 +252,15 @@ void clockevents_notify(unsigned long reason, void *arg) ...@@ -251,4 +252,15 @@ void clockevents_notify(unsigned long reason, void *arg)
spin_unlock(&clockevents_lock); spin_unlock(&clockevents_lock);
} }
EXPORT_SYMBOL_GPL(clockevents_notify); EXPORT_SYMBOL_GPL(clockevents_notify);
ktime_t clockevents_get_next_event(int cpu)
{
struct tick_device *td;
struct clock_event_device *dev;
td = &per_cpu(tick_cpu_device, cpu);
dev = td->evtdev;
return dev->next_event;
}
#endif #endif
...@@ -37,6 +37,7 @@ ...@@ -37,6 +37,7 @@
#include <linux/delay.h> #include <linux/delay.h>
#include <linux/tick.h> #include <linux/tick.h>
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/sched.h>
#include <asm/uaccess.h> #include <asm/uaccess.h>
#include <asm/unistd.h> #include <asm/unistd.h>
...@@ -609,9 +610,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, ...@@ -609,9 +610,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
{ {
struct tvec_base *base, *new_base; struct tvec_base *base, *new_base;
unsigned long flags; unsigned long flags;
int ret; int ret = 0 , cpu;
ret = 0;
timer_stats_timer_set_start_info(timer); timer_stats_timer_set_start_info(timer);
BUG_ON(!timer->function); BUG_ON(!timer->function);
...@@ -630,6 +629,18 @@ __mod_timer(struct timer_list *timer, unsigned long expires, ...@@ -630,6 +629,18 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
new_base = __get_cpu_var(tvec_bases); new_base = __get_cpu_var(tvec_bases);
cpu = smp_processor_id();
#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
if (!pinned && get_sysctl_timer_migration() && idle_cpu(cpu)) {
int preferred_cpu = get_nohz_load_balancer();
if (preferred_cpu >= 0)
cpu = preferred_cpu;
}
#endif
new_base = per_cpu(tvec_bases, cpu);
if (base != new_base) { if (base != new_base) {
/* /*
* We are trying to schedule the timer on the local CPU. * We are trying to schedule the timer on the local CPU.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment