Skip to content

L
linux-davinci
Commit dcf39949 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files
Options

rcu: Convert rcupreempt locks to atomic_spinlock 

Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 609368d8
Show whitespace changes
Inline Side-by-side
Showing with 30 additions and 30 deletions
kernel/rcupreempt.c
View file @ dcf39949
...@@ -71,7 +71,7 @@ ...@@ -71,7 +71,7 @@
*/ */
#define GP_STAGES 2 #define GP_STAGES 2
struct rcu_data { struct rcu_data {
spinlock_t lock; /* Protect rcu_data fields. */ atomic_spinlock_t lock; /* Protect rcu_data fields. */
long completed; /* Number of last completed batch. */ long completed; /* Number of last completed batch. */
int waitlistcount; int waitlistcount;
struct rcu_head *nextlist; struct rcu_head *nextlist;
...@@ -138,7 +138,7 @@ enum rcu_sched_sleep_states { ...@@ -138,7 +138,7 @@ enum rcu_sched_sleep_states {
}; };
struct rcu_ctrlblk { struct rcu_ctrlblk {
spinlock_t fliplock; /* Protect state-machine transitions. */ atomic_spinlock_t fliplock; /* Protect state-machine transitions. */
long completed; /* Number of last completed batch. */ long completed; /* Number of last completed batch. */
enum rcu_try_flip_states rcu_try_flip_state; /* The current state of enum rcu_try_flip_states rcu_try_flip_state; /* The current state of
the rcu state machine */ the rcu state machine */
...@@ -193,7 +193,7 @@ void rcu_exit_nohz(void) ...@@ -193,7 +193,7 @@ void rcu_exit_nohz(void)
static DEFINE_PER_CPU(struct rcu_data, rcu_data); static DEFINE_PER_CPU(struct rcu_data, rcu_data);
static struct rcu_ctrlblk rcu_ctrlblk = { static struct rcu_ctrlblk rcu_ctrlblk = {
.fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock), .fliplock = __ATOMIC_SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock),
.completed = 0, .completed = 0,
.rcu_try_flip_state = rcu_try_flip_idle_state, .rcu_try_flip_state = rcu_try_flip_idle_state,
.schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock), .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock),
...@@ -910,7 +910,7 @@ static void rcu_try_flip(void) ...@@ -910,7 +910,7 @@ static void rcu_try_flip(void)
unsigned long flags; unsigned long flags;
RCU_TRACE_ME(rcupreempt_trace_try_flip_1); RCU_TRACE_ME(rcupreempt_trace_try_flip_1);
if (unlikely(!spin_trylock_irqsave(&rcu_ctrlblk.fliplock, flags))) { if (unlikely(!atomic_spin_trylock_irqsave(&rcu_ctrlblk.fliplock, flags))) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_e1); RCU_TRACE_ME(rcupreempt_trace_try_flip_e1);
return; return;
} }
...@@ -941,7 +941,7 @@ static void rcu_try_flip(void) ...@@ -941,7 +941,7 @@ static void rcu_try_flip(void)
rcu_ctrlblk.rcu_try_flip_state = rcu_ctrlblk.rcu_try_flip_state =
rcu_try_flip_idle_state; rcu_try_flip_idle_state;
} }
spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); atomic_spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
} }
/* /*
...@@ -986,13 +986,13 @@ void rcu_check_callbacks(int cpu, int user) ...@@ -986,13 +986,13 @@ void rcu_check_callbacks(int cpu, int user)
rcu_check_mb(cpu); rcu_check_mb(cpu);
if (rcu_ctrlblk.completed == rdp->completed) if (rcu_ctrlblk.completed == rdp->completed)
rcu_try_flip(); rcu_try_flip();
spin_lock_irqsave(&rdp->lock, flags); atomic_spin_lock_irqsave(&rdp->lock, flags);
RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp); RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
__rcu_advance_callbacks(rdp); __rcu_advance_callbacks(rdp);
if (rdp->donelist == NULL) { if (rdp->donelist == NULL) {
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} else { } else {
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
raise_softirq(RCU_SOFTIRQ); raise_softirq(RCU_SOFTIRQ);
} }
} }
...@@ -1011,10 +1011,10 @@ void rcu_advance_callbacks(int cpu, int user) ...@@ -1011,10 +1011,10 @@ void rcu_advance_callbacks(int cpu, int user)
if (rcu_ctrlblk.completed == rdp->completed) if (rcu_ctrlblk.completed == rdp->completed)
return; return;
} }
spin_lock_irqsave(&rdp->lock, flags); atomic_spin_lock_irqsave(&rdp->lock, flags);
RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp); RCU_TRACE_RDP(rcupreempt_trace_check_callbacks, rdp);
__rcu_advance_callbacks(rdp); __rcu_advance_callbacks(rdp);
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} }
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU
...@@ -1042,7 +1042,7 @@ void rcu_offline_cpu(int cpu) ...@@ -1042,7 +1042,7 @@ void rcu_offline_cpu(int cpu)
* Otherwise rcu_barrier() will fail * Otherwise rcu_barrier() will fail
*/ */
spin_lock_irqsave(&rdp->lock, flags); atomic_spin_lock_irqsave(&rdp->lock, flags);
rcu_offline_cpu_enqueue(rdp->donelist, rdp->donetail, list, tail); rcu_offline_cpu_enqueue(rdp->donelist, rdp->donetail, list, tail);
for (i = GP_STAGES - 1; i >= 0; i--) for (i = GP_STAGES - 1; i >= 0; i--)
rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i], rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i],
...@@ -1053,12 +1053,12 @@ void rcu_offline_cpu(int cpu) ...@@ -1053,12 +1053,12 @@ void rcu_offline_cpu(int cpu)
rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail, rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail,
schedlist, schedtail); schedlist, schedtail);
rdp->rcu_sched_sleeping = 0; rdp->rcu_sched_sleeping = 0;
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
rdp->waitlistcount = 0; rdp->waitlistcount = 0;
/* Disengage the newly dead CPU from the grace-period computation. */ /* Disengage the newly dead CPU from the grace-period computation. */
spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); atomic_spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
rcu_check_mb(cpu); rcu_check_mb(cpu);
if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) { if (per_cpu(rcu_flip_flag, cpu) == rcu_flipped) {
smp_mb(); /* Subsequent counter accesses must see new value */ smp_mb(); /* Subsequent counter accesses must see new value */
...@@ -1075,7 +1075,7 @@ void rcu_offline_cpu(int cpu) ...@@ -1075,7 +1075,7 @@ void rcu_offline_cpu(int cpu)
cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map)); cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); atomic_spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
/* /*
* Place the removed callbacks on the current CPU's queue. * Place the removed callbacks on the current CPU's queue.
...@@ -1089,14 +1089,14 @@ void rcu_offline_cpu(int cpu) ...@@ -1089,14 +1089,14 @@ void rcu_offline_cpu(int cpu)
local_irq_save(flags); /* disable preempt till we know what lock. */ local_irq_save(flags); /* disable preempt till we know what lock. */
rdp = RCU_DATA_ME(); rdp = RCU_DATA_ME();
spin_lock(&rdp->lock); atomic_spin_lock(&rdp->lock);
*rdp->nexttail = list; *rdp->nexttail = list;
if (list) if (list)
rdp->nexttail = tail; rdp->nexttail = tail;
*rdp->nextschedtail = schedlist; *rdp->nextschedtail = schedlist;
if (schedlist) if (schedlist)
rdp->nextschedtail = schedtail; rdp->nextschedtail = schedtail;
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} }
#else /* #ifdef CONFIG_HOTPLUG_CPU */ #else /* #ifdef CONFIG_HOTPLUG_CPU */
...@@ -1112,9 +1112,9 @@ void __cpuinit rcu_online_cpu(int cpu) ...@@ -1112,9 +1112,9 @@ void __cpuinit rcu_online_cpu(int cpu)
unsigned long flags; unsigned long flags;
struct rcu_data *rdp; struct rcu_data *rdp;
spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); atomic_spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map)); cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); atomic_spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
/* /*
* The rcu_sched grace-period processing might have bypassed * The rcu_sched grace-period processing might have bypassed
...@@ -1126,9 +1126,9 @@ void __cpuinit rcu_online_cpu(int cpu) ...@@ -1126,9 +1126,9 @@ void __cpuinit rcu_online_cpu(int cpu)
*/ */
rdp = RCU_DATA_CPU(cpu); rdp = RCU_DATA_CPU(cpu);
spin_lock_irqsave(&rdp->lock, flags); atomic_spin_lock_irqsave(&rdp->lock, flags);
rdp->rcu_sched_sleeping = 1; rdp->rcu_sched_sleeping = 1;
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} }
static void rcu_process_callbacks(struct softirq_action *unused) static void rcu_process_callbacks(struct softirq_action *unused)
...@@ -1139,16 +1139,16 @@ static void rcu_process_callbacks(struct softirq_action *unused) ...@@ -1139,16 +1139,16 @@ static void rcu_process_callbacks(struct softirq_action *unused)
local_irq_save(flags); local_irq_save(flags);
rdp = RCU_DATA_ME(); rdp = RCU_DATA_ME();
spin_lock(&rdp->lock); atomic_spin_lock(&rdp->lock);
list = rdp->donelist; list = rdp->donelist;
if (list == NULL) { if (list == NULL) {
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
return; return;
} }
rdp->donelist = NULL; rdp->donelist = NULL;
rdp->donetail = &rdp->donelist; rdp->donetail = &rdp->donelist;
RCU_TRACE_RDP(rcupreempt_trace_done_remove, rdp); RCU_TRACE_RDP(rcupreempt_trace_done_remove, rdp);
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
while (list) { while (list) {
next = list->next; next = list->next;
list->func(list); list->func(list);
...@@ -1166,12 +1166,12 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) ...@@ -1166,12 +1166,12 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
head->next = NULL; head->next = NULL;
local_irq_save(flags); local_irq_save(flags);
rdp = RCU_DATA_ME(); rdp = RCU_DATA_ME();
spin_lock(&rdp->lock); atomic_spin_lock(&rdp->lock);
__rcu_advance_callbacks(rdp); __rcu_advance_callbacks(rdp);
*rdp->nexttail = head; *rdp->nexttail = head;
rdp->nexttail = &head->next; rdp->nexttail = &head->next;
RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp); RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp);
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} }
EXPORT_SYMBOL_GPL(call_rcu); EXPORT_SYMBOL_GPL(call_rcu);
...@@ -1185,7 +1185,7 @@ void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) ...@@ -1185,7 +1185,7 @@ void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
head->next = NULL; head->next = NULL;
local_irq_save(flags); local_irq_save(flags);
rdp = RCU_DATA_ME(); rdp = RCU_DATA_ME();
spin_lock(&rdp->lock); atomic_spin_lock(&rdp->lock);
*rdp->nextschedtail = head; *rdp->nextschedtail = head;
rdp->nextschedtail = &head->next; rdp->nextschedtail = &head->next;
if (rdp->rcu_sched_sleeping) { if (rdp->rcu_sched_sleeping) {
...@@ -1195,7 +1195,7 @@ void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) ...@@ -1195,7 +1195,7 @@ void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
rdp->rcu_sched_sleeping = 0; rdp->rcu_sched_sleeping = 0;
wake_gp = 1; wake_gp = 1;
} }
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
if (wake_gp) { if (wake_gp) {
/* Wake up grace-period processing, unless someone beat us. */ /* Wake up grace-period processing, unless someone beat us. */
...@@ -1291,7 +1291,7 @@ static int rcu_sched_grace_period(void *arg) ...@@ -1291,7 +1291,7 @@ static int rcu_sched_grace_period(void *arg)
for_each_online_cpu(cpu) { for_each_online_cpu(cpu) {
rdp = RCU_DATA_CPU(cpu); rdp = RCU_DATA_CPU(cpu);
spin_lock_irqsave(&rdp->lock, flags); atomic_spin_lock_irqsave(&rdp->lock, flags);
/* /*
* We are running on this CPU irq-disabled, so no * We are running on this CPU irq-disabled, so no
...@@ -1330,7 +1330,7 @@ static int rcu_sched_grace_period(void *arg) ...@@ -1330,7 +1330,7 @@ static int rcu_sched_grace_period(void *arg)
rdp->rcu_sched_sleeping = couldsleep; rdp->rcu_sched_sleeping = couldsleep;
spin_unlock_irqrestore(&rdp->lock, flags); atomic_spin_unlock_irqrestore(&rdp->lock, flags);
} }
/* If we saw callbacks on the last scan, go deal with them. */ /* If we saw callbacks on the last scan, go deal with them. */
...@@ -1452,7 +1452,7 @@ void __init __rcu_init(void) ...@@ -1452,7 +1452,7 @@ void __init __rcu_init(void)
printk(KERN_NOTICE "Preemptible RCU implementation.\n"); printk(KERN_NOTICE "Preemptible RCU implementation.\n");
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
rdp = RCU_DATA_CPU(cpu); rdp = RCU_DATA_CPU(cpu);
spin_lock_init(&rdp->lock); atomic_spin_lock_init(&rdp->lock);
rdp->completed = 0; rdp->completed = 0;
rdp->waitlistcount = 0; rdp->waitlistcount = 0;
rdp->nextlist = NULL; rdp->nextlist = NULL;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment