Skip to content

L
linux-davinci
Commit d227cf76 authored by Thomas Gleixner's avatar Thomas Gleixner
Browse files
Options

rtmutex: Convert wait_lock and pi_lock to atomic_spinlock 

Fix up all users.
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent 0835f1f8
Hide whitespace changes
Inline Side-by-side
Showing with 92 additions and 92 deletions
include/linux/init_task.h
View file @ d227cf76
...@@ -159,7 +159,7 @@ extern struct cred init_cred; ...@@ -159,7 +159,7 @@ extern struct cred init_cred;
.journal_info = NULL, \ .journal_info = NULL, \
.cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
.fs_excl = ATOMIC_INIT(0), \ .fs_excl = ATOMIC_INIT(0), \
.pi_lock = __SPIN_LOCK_UNLOCKED(tsk.pi_lock), \ .pi_lock = __ATOMIC_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \
.timer_slack_ns = 50000, /* 50 usec default slack */ \ .timer_slack_ns = 50000, /* 50 usec default slack */ \
.pids = { \ .pids = { \
[PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \
......
include/linux/rtmutex.h
View file @ d227cf76
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
* @owner: the mutex owner * @owner: the mutex owner
*/ */
struct rt_mutex { struct rt_mutex {
spinlock_t wait_lock; atomic_spinlock_t wait_lock;
struct plist_head wait_list; struct plist_head wait_list;
struct task_struct *owner; struct task_struct *owner;
#ifdef CONFIG_DEBUG_RT_MUTEXES #ifdef CONFIG_DEBUG_RT_MUTEXES
...@@ -63,8 +63,8 @@ struct hrtimer_sleeper; ...@@ -63,8 +63,8 @@ struct hrtimer_sleeper;
#endif #endif
#define __RT_MUTEX_INITIALIZER(mutexname) \ #define __RT_MUTEX_INITIALIZER(mutexname) \
{ .wait_lock = __SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ { .wait_lock = __ATOMIC_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \
, .wait_list = PLIST_HEAD_INIT(mutexname.wait_list, mutexname.wait_lock) \ , .wait_list = PLIST_HEAD_INIT_ATOMIC(mutexname.wait_list, mutexname.wait_lock) \
, .owner = NULL \ , .owner = NULL \
__DEBUG_RT_MUTEX_INITIALIZER(mutexname)} __DEBUG_RT_MUTEX_INITIALIZER(mutexname)}
...@@ -98,7 +98,7 @@ extern void rt_mutex_unlock(struct rt_mutex *lock); ...@@ -98,7 +98,7 @@ extern void rt_mutex_unlock(struct rt_mutex *lock);
#ifdef CONFIG_RT_MUTEXES #ifdef CONFIG_RT_MUTEXES
# define INIT_RT_MUTEXES(tsk) \ # define INIT_RT_MUTEXES(tsk) \
.pi_waiters = PLIST_HEAD_INIT(tsk.pi_waiters, tsk.pi_lock), \ .pi_waiters = PLIST_HEAD_INIT_ATOMIC(tsk.pi_waiters, tsk.pi_lock), \
INIT_RT_MUTEX_DEBUG(tsk) INIT_RT_MUTEX_DEBUG(tsk)
#else #else
# define INIT_RT_MUTEXES(tsk) # define INIT_RT_MUTEXES(tsk)
......
include/linux/sched.h
View file @ d227cf76
...@@ -1347,7 +1347,7 @@ struct task_struct { ...@@ -1347,7 +1347,7 @@ struct task_struct {
#endif #endif
/* Protection of the PI data structures: */ /* Protection of the PI data structures: */
spinlock_t pi_lock; atomic_spinlock_t pi_lock;
#ifdef CONFIG_RT_MUTEXES #ifdef CONFIG_RT_MUTEXES
/* PI waiters blocked on a rt_mutex held by this task */ /* PI waiters blocked on a rt_mutex held by this task */
......
kernel/exit.c
View file @ d227cf76
...@@ -930,7 +930,7 @@ NORET_TYPE void do_exit(long code) ...@@ -930,7 +930,7 @@ NORET_TYPE void do_exit(long code)
* an exiting task cleaning up the robust pi futexes. * an exiting task cleaning up the robust pi futexes.
*/ */
smp_mb(); smp_mb();
spin_unlock_wait(&tsk->pi_lock); atomic_spin_unlock_wait(&tsk->pi_lock);
if (unlikely(in_atomic())) if (unlikely(in_atomic()))
printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n", printk(KERN_INFO "note: %s[%d] exited with preempt_count %d\n",
......
kernel/fork.c
View file @ d227cf76
...@@ -907,9 +907,9 @@ SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr) ...@@ -907,9 +907,9 @@ SYSCALL_DEFINE1(set_tid_address, int __user *, tidptr)
static void rt_mutex_init_task(struct task_struct *p) static void rt_mutex_init_task(struct task_struct *p)
{ {
spin_lock_init(&p->pi_lock); atomic_spin_lock_init(&p->pi_lock);
#ifdef CONFIG_RT_MUTEXES #ifdef CONFIG_RT_MUTEXES
plist_head_init(&p->pi_waiters, &p->pi_lock); plist_head_init_atomic(&p->pi_waiters, &p->pi_lock);
p->pi_blocked_on = NULL; p->pi_blocked_on = NULL;
#endif #endif
} }
......
kernel/futex.c
View file @ d227cf76
...@@ -392,9 +392,9 @@ static void free_pi_state(struct futex_pi_state *pi_state) ...@@ -392,9 +392,9 @@ static void free_pi_state(struct futex_pi_state *pi_state)
* and has cleaned up the pi_state already * and has cleaned up the pi_state already
*/ */
if (pi_state->owner) { if (pi_state->owner) {
spin_lock_irq(&pi_state->owner->pi_lock); atomic_spin_lock_irq(&pi_state->owner->pi_lock);
list_del_init(&pi_state->list); list_del_init(&pi_state->list);
spin_unlock_irq(&pi_state->owner->pi_lock); atomic_spin_unlock_irq(&pi_state->owner->pi_lock);
rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner); rt_mutex_proxy_unlock(&pi_state->pi_mutex, pi_state->owner);
} }
...@@ -459,18 +459,18 @@ void exit_pi_state_list(struct task_struct *curr) ...@@ -459,18 +459,18 @@ void exit_pi_state_list(struct task_struct *curr)
* pi_state_list anymore, but we have to be careful * pi_state_list anymore, but we have to be careful
* versus waiters unqueueing themselves: * versus waiters unqueueing themselves:
*/ */
spin_lock_irq(&curr->pi_lock); atomic_spin_lock_irq(&curr->pi_lock);
while (!list_empty(head)) { while (!list_empty(head)) {
next = head->next; next = head->next;
pi_state = list_entry(next, struct futex_pi_state, list); pi_state = list_entry(next, struct futex_pi_state, list);
key = pi_state->key; key = pi_state->key;
hb = hash_futex(&key); hb = hash_futex(&key);
spin_unlock_irq(&curr->pi_lock); atomic_spin_unlock_irq(&curr->pi_lock);
spin_lock(&hb->lock); spin_lock(&hb->lock);
spin_lock_irq(&curr->pi_lock); atomic_spin_lock_irq(&curr->pi_lock);
/* /*
* We dropped the pi-lock, so re-check whether this * We dropped the pi-lock, so re-check whether this
* task still owns the PI-state: * task still owns the PI-state:
...@@ -484,15 +484,15 @@ void exit_pi_state_list(struct task_struct *curr) ...@@ -484,15 +484,15 @@ void exit_pi_state_list(struct task_struct *curr)
WARN_ON(list_empty(&pi_state->list)); WARN_ON(list_empty(&pi_state->list));
list_del_init(&pi_state->list); list_del_init(&pi_state->list);
pi_state->owner = NULL; pi_state->owner = NULL;
spin_unlock_irq(&curr->pi_lock); atomic_spin_unlock_irq(&curr->pi_lock);
rt_mutex_unlock(&pi_state->pi_mutex); rt_mutex_unlock(&pi_state->pi_mutex);
spin_unlock(&hb->lock); spin_unlock(&hb->lock);
spin_lock_irq(&curr->pi_lock); atomic_spin_lock_irq(&curr->pi_lock);
} }
spin_unlock_irq(&curr->pi_lock); atomic_spin_unlock_irq(&curr->pi_lock);
} }
static int static int
...@@ -547,7 +547,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, ...@@ -547,7 +547,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
* change of the task flags, we do this protected by * change of the task flags, we do this protected by
* p->pi_lock: * p->pi_lock:
*/ */
spin_lock_irq(&p->pi_lock); atomic_spin_lock_irq(&p->pi_lock);
if (unlikely(p->flags & PF_EXITING)) { if (unlikely(p->flags & PF_EXITING)) {
/* /*
* The task is on the way out. When PF_EXITPIDONE is * The task is on the way out. When PF_EXITPIDONE is
...@@ -556,7 +556,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, ...@@ -556,7 +556,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
*/ */
int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN; int ret = (p->flags & PF_EXITPIDONE) ? -ESRCH : -EAGAIN;
spin_unlock_irq(&p->pi_lock); atomic_spin_unlock_irq(&p->pi_lock);
put_task_struct(p); put_task_struct(p);
return ret; return ret;
} }
...@@ -575,7 +575,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb, ...@@ -575,7 +575,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
WARN_ON(!list_empty(&pi_state->list)); WARN_ON(!list_empty(&pi_state->list));
list_add(&pi_state->list, &p->pi_state_list); list_add(&pi_state->list, &p->pi_state_list);
pi_state->owner = p; pi_state->owner = p;
spin_unlock_irq(&p->pi_lock); atomic_spin_unlock_irq(&p->pi_lock);
put_task_struct(p); put_task_struct(p);
...@@ -749,7 +749,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) ...@@ -749,7 +749,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
if (!pi_state) if (!pi_state)
return -EINVAL; return -EINVAL;
spin_lock(&pi_state->pi_mutex.wait_lock); atomic_spin_lock(&pi_state->pi_mutex.wait_lock);
new_owner = rt_mutex_next_owner(&pi_state->pi_mutex); new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
/* /*
...@@ -778,23 +778,23 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) ...@@ -778,23 +778,23 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
else if (curval != uval) else if (curval != uval)
ret = -EINVAL; ret = -EINVAL;
if (ret) { if (ret) {
spin_unlock(&pi_state->pi_mutex.wait_lock); atomic_spin_unlock(&pi_state->pi_mutex.wait_lock);
return ret; return ret;
} }
} }
spin_lock_irq(&pi_state->owner->pi_lock); atomic_spin_lock_irq(&pi_state->owner->pi_lock);
WARN_ON(list_empty(&pi_state->list)); WARN_ON(list_empty(&pi_state->list));
list_del_init(&pi_state->list); list_del_init(&pi_state->list);
spin_unlock_irq(&pi_state->owner->pi_lock); atomic_spin_unlock_irq(&pi_state->owner->pi_lock);
spin_lock_irq(&new_owner->pi_lock); atomic_spin_lock_irq(&new_owner->pi_lock);
WARN_ON(!list_empty(&pi_state->list)); WARN_ON(!list_empty(&pi_state->list));
list_add(&pi_state->list, &new_owner->pi_state_list); list_add(&pi_state->list, &new_owner->pi_state_list);
pi_state->owner = new_owner; pi_state->owner = new_owner;
spin_unlock_irq(&new_owner->pi_lock); atomic_spin_unlock_irq(&new_owner->pi_lock);
spin_unlock(&pi_state->pi_mutex.wait_lock); atomic_spin_unlock(&pi_state->pi_mutex.wait_lock);
rt_mutex_unlock(&pi_state->pi_mutex); rt_mutex_unlock(&pi_state->pi_mutex);
return 0; return 0;
...@@ -1474,18 +1474,18 @@ retry: ...@@ -1474,18 +1474,18 @@ retry:
* itself. * itself.
*/ */
if (pi_state->owner != NULL) { if (pi_state->owner != NULL) {
spin_lock_irq(&pi_state->owner->pi_lock); atomic_spin_lock_irq(&pi_state->owner->pi_lock);
WARN_ON(list_empty(&pi_state->list)); WARN_ON(list_empty(&pi_state->list));
list_del_init(&pi_state->list); list_del_init(&pi_state->list);
spin_unlock_irq(&pi_state->owner->pi_lock); atomic_spin_unlock_irq(&pi_state->owner->pi_lock);
} }
pi_state->owner = newowner; pi_state->owner = newowner;
spin_lock_irq(&newowner->pi_lock); atomic_spin_lock_irq(&newowner->pi_lock);
WARN_ON(!list_empty(&pi_state->list)); WARN_ON(!list_empty(&pi_state->list));
list_add(&pi_state->list, &newowner->pi_state_list); list_add(&pi_state->list, &newowner->pi_state_list);
spin_unlock_irq(&newowner->pi_lock); atomic_spin_unlock_irq(&newowner->pi_lock);
return 0; return 0;
/* /*
......
kernel/rtmutex-debug.c
View file @ d227cf76
...@@ -37,8 +37,8 @@ do { \ ...@@ -37,8 +37,8 @@ do { \
if (rt_trace_on) { \ if (rt_trace_on) { \
rt_trace_on = 0; \ rt_trace_on = 0; \
console_verbose(); \ console_verbose(); \
if (spin_is_locked(&current->pi_lock)) \ if (atomic_spin_is_locked(&current->pi_lock)) \
spin_unlock(&current->pi_lock); \ atomic_spin_unlock(&current->pi_lock); \
} \ } \
} while (0) } while (0)
......
kernel/rtmutex.c
View file @ d227cf76
...@@ -131,16 +131,16 @@ static void __rt_mutex_adjust_prio(struct task_struct *task) ...@@ -131,16 +131,16 @@ static void __rt_mutex_adjust_prio(struct task_struct *task)
* *
* (Note: We do this outside of the protection of lock->wait_lock to * (Note: We do this outside of the protection of lock->wait_lock to
* allow the lock to be taken while or before we readjust the priority * allow the lock to be taken while or before we readjust the priority
* of task. We do not use the spin_xx_mutex() variants here as we are * of task. We do not use the atomic_spin_xx_mutex() variants here as we are
* outside of the debug path.) * outside of the debug path.)
*/ */
static void rt_mutex_adjust_prio(struct task_struct *task) static void rt_mutex_adjust_prio(struct task_struct *task)
{ {
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
__rt_mutex_adjust_prio(task); __rt_mutex_adjust_prio(task);
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
} }
/* /*
...@@ -195,7 +195,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -195,7 +195,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* /*
* Task can not go away as we did a get_task() before ! * Task can not go away as we did a get_task() before !
*/ */
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
waiter = task->pi_blocked_on; waiter = task->pi_blocked_on;
/* /*
...@@ -231,8 +231,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -231,8 +231,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
goto out_unlock_pi; goto out_unlock_pi;
lock = waiter->lock; lock = waiter->lock;
if (!spin_trylock(&lock->wait_lock)) { if (!atomic_spin_trylock(&lock->wait_lock)) {
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
cpu_relax(); cpu_relax();
goto retry; goto retry;
} }
...@@ -240,7 +240,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -240,7 +240,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* Deadlock detection */ /* Deadlock detection */
if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock); debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
ret = deadlock_detect ? -EDEADLK : 0; ret = deadlock_detect ? -EDEADLK : 0;
goto out_unlock_pi; goto out_unlock_pi;
} }
...@@ -253,13 +253,13 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -253,13 +253,13 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
plist_add(&waiter->list_entry, &lock->wait_list); plist_add(&waiter->list_entry, &lock->wait_list);
/* Release the task */ /* Release the task */
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
put_task_struct(task); put_task_struct(task);
/* Grab the next task */ /* Grab the next task */
task = rt_mutex_owner(lock); task = rt_mutex_owner(lock);
get_task_struct(task); get_task_struct(task);
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
if (waiter == rt_mutex_top_waiter(lock)) { if (waiter == rt_mutex_top_waiter(lock)) {
/* Boost the owner */ /* Boost the owner */
...@@ -277,10 +277,10 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -277,10 +277,10 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
__rt_mutex_adjust_prio(task); __rt_mutex_adjust_prio(task);
} }
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
top_waiter = rt_mutex_top_waiter(lock); top_waiter = rt_mutex_top_waiter(lock);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
if (!detect_deadlock && waiter != top_waiter) if (!detect_deadlock && waiter != top_waiter)
goto out_put_task; goto out_put_task;
...@@ -288,7 +288,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, ...@@ -288,7 +288,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
goto again; goto again;
out_unlock_pi: out_unlock_pi:
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
out_put_task: out_put_task:
put_task_struct(task); put_task_struct(task);
...@@ -313,9 +313,9 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, ...@@ -313,9 +313,9 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
if (pendowner == task) if (pendowner == task)
return 1; return 1;
spin_lock_irqsave(&pendowner->pi_lock, flags); atomic_spin_lock_irqsave(&pendowner->pi_lock, flags);
if (task->prio >= pendowner->prio) { if (task->prio >= pendowner->prio) {
spin_unlock_irqrestore(&pendowner->pi_lock, flags); atomic_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
return 0; return 0;
} }
...@@ -325,7 +325,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, ...@@ -325,7 +325,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
* priority. * priority.
*/ */
if (likely(!rt_mutex_has_waiters(lock))) { if (likely(!rt_mutex_has_waiters(lock))) {
spin_unlock_irqrestore(&pendowner->pi_lock, flags); atomic_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
return 1; return 1;
} }
...@@ -333,7 +333,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, ...@@ -333,7 +333,7 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
next = rt_mutex_top_waiter(lock); next = rt_mutex_top_waiter(lock);
plist_del(&next->pi_list_entry, &pendowner->pi_waiters); plist_del(&next->pi_list_entry, &pendowner->pi_waiters);
__rt_mutex_adjust_prio(pendowner); __rt_mutex_adjust_prio(pendowner);
spin_unlock_irqrestore(&pendowner->pi_lock, flags); atomic_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
/* /*
* We are going to steal the lock and a waiter was * We are going to steal the lock and a waiter was
...@@ -350,10 +350,10 @@ static inline int try_to_steal_lock(struct rt_mutex *lock, ...@@ -350,10 +350,10 @@ static inline int try_to_steal_lock(struct rt_mutex *lock,
* might be task: * might be task:
*/ */
if (likely(next->task != task)) { if (likely(next->task != task)) {
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
plist_add(&next->pi_list_entry, &task->pi_waiters); plist_add(&next->pi_list_entry, &task->pi_waiters);
__rt_mutex_adjust_prio(task); __rt_mutex_adjust_prio(task);
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
} }
return 1; return 1;
} }
...@@ -420,7 +420,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, ...@@ -420,7 +420,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
unsigned long flags; unsigned long flags;
int chain_walk = 0, res; int chain_walk = 0, res;
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
__rt_mutex_adjust_prio(task); __rt_mutex_adjust_prio(task);
waiter->task = task; waiter->task = task;
waiter->lock = lock; waiter->lock = lock;
...@@ -434,17 +434,17 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, ...@@ -434,17 +434,17 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
task->pi_blocked_on = waiter; task->pi_blocked_on = waiter;
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
if (waiter == rt_mutex_top_waiter(lock)) { if (waiter == rt_mutex_top_waiter(lock)) {
spin_lock_irqsave(&owner->pi_lock, flags); atomic_spin_lock_irqsave(&owner->pi_lock, flags);
plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters); plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
plist_add(&waiter->pi_list_entry, &owner->pi_waiters); plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
__rt_mutex_adjust_prio(owner); __rt_mutex_adjust_prio(owner);
if (owner->pi_blocked_on) if (owner->pi_blocked_on)
chain_walk = 1; chain_walk = 1;
spin_unlock_irqrestore(&owner->pi_lock, flags); atomic_spin_unlock_irqrestore(&owner->pi_lock, flags);
} }
else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
chain_walk = 1; chain_walk = 1;
...@@ -459,12 +459,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, ...@@ -459,12 +459,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
*/ */
get_task_struct(owner); get_task_struct(owner);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter, res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
task); task);
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
return res; return res;
} }
...@@ -483,7 +483,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) ...@@ -483,7 +483,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
struct task_struct *pendowner; struct task_struct *pendowner;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&current->pi_lock, flags); atomic_spin_lock_irqsave(&current->pi_lock, flags);
waiter = rt_mutex_top_waiter(lock); waiter = rt_mutex_top_waiter(lock);
plist_del(&waiter->list_entry, &lock->wait_list); plist_del(&waiter->list_entry, &lock->wait_list);
...@@ -500,7 +500,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) ...@@ -500,7 +500,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING); rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);
spin_unlock_irqrestore(&current->pi_lock, flags); atomic_spin_unlock_irqrestore(&current->pi_lock, flags);
/* /*
* Clear the pi_blocked_on variable and enqueue a possible * Clear the pi_blocked_on variable and enqueue a possible
...@@ -509,7 +509,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) ...@@ -509,7 +509,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
* waiter with higher priority than pending-owner->normal_prio * waiter with higher priority than pending-owner->normal_prio
* is blocked on the unboosted (pending) owner. * is blocked on the unboosted (pending) owner.
*/ */
spin_lock_irqsave(&pendowner->pi_lock, flags); atomic_spin_lock_irqsave(&pendowner->pi_lock, flags);
WARN_ON(!pendowner->pi_blocked_on); WARN_ON(!pendowner->pi_blocked_on);
WARN_ON(pendowner->pi_blocked_on != waiter); WARN_ON(pendowner->pi_blocked_on != waiter);
...@@ -523,7 +523,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock) ...@@ -523,7 +523,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
next = rt_mutex_top_waiter(lock); next = rt_mutex_top_waiter(lock);
plist_add(&next->pi_list_entry, &pendowner->pi_waiters); plist_add(&next->pi_list_entry, &pendowner->pi_waiters);
} }
spin_unlock_irqrestore(&pendowner->pi_lock, flags); atomic_spin_unlock_irqrestore(&pendowner->pi_lock, flags);
wake_up_process(pendowner); wake_up_process(pendowner);
} }
...@@ -541,15 +541,15 @@ static void remove_waiter(struct rt_mutex *lock, ...@@ -541,15 +541,15 @@ static void remove_waiter(struct rt_mutex *lock,
unsigned long flags; unsigned long flags;
int chain_walk = 0; int chain_walk = 0;
spin_lock_irqsave(&current->pi_lock, flags); atomic_spin_lock_irqsave(&current->pi_lock, flags);
plist_del(&waiter->list_entry, &lock->wait_list); plist_del(&waiter->list_entry, &lock->wait_list);
waiter->task = NULL; waiter->task = NULL;
current->pi_blocked_on = NULL; current->pi_blocked_on = NULL;
spin_unlock_irqrestore(&current->pi_lock, flags); atomic_spin_unlock_irqrestore(&current->pi_lock, flags);
if (first && owner != current) { if (first && owner != current) {
spin_lock_irqsave(&owner->pi_lock, flags); atomic_spin_lock_irqsave(&owner->pi_lock, flags);
plist_del(&waiter->pi_list_entry, &owner->pi_waiters); plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
...@@ -564,7 +564,7 @@ static void remove_waiter(struct rt_mutex *lock, ...@@ -564,7 +564,7 @@ static void remove_waiter(struct rt_mutex *lock,
if (owner->pi_blocked_on) if (owner->pi_blocked_on)
chain_walk = 1; chain_walk = 1;
spin_unlock_irqrestore(&owner->pi_lock, flags); atomic_spin_unlock_irqrestore(&owner->pi_lock, flags);
} }
WARN_ON(!plist_node_empty(&waiter->pi_list_entry)); WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
...@@ -575,11 +575,11 @@ static void remove_waiter(struct rt_mutex *lock, ...@@ -575,11 +575,11 @@ static void remove_waiter(struct rt_mutex *lock,
/* gets dropped in rt_mutex_adjust_prio_chain()! */ /* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(owner); get_task_struct(owner);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current); rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
} }
/* /*
...@@ -592,15 +592,15 @@ void rt_mutex_adjust_pi(struct task_struct *task) ...@@ -592,15 +592,15 @@ void rt_mutex_adjust_pi(struct task_struct *task)
struct rt_mutex_waiter *waiter; struct rt_mutex_waiter *waiter;
unsigned long flags; unsigned long flags;
spin_lock_irqsave(&task->pi_lock, flags); atomic_spin_lock_irqsave(&task->pi_lock, flags);
waiter = task->pi_blocked_on; waiter = task->pi_blocked_on;
if (!waiter || waiter->list_entry.prio == task->prio) { if (!waiter || waiter->list_entry.prio == task->prio) {
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
return; return;
} }
spin_unlock_irqrestore(&task->pi_lock, flags); atomic_spin_unlock_irqrestore(&task->pi_lock, flags);
/* gets dropped in rt_mutex_adjust_prio_chain()! */ /* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(task); get_task_struct(task);
...@@ -672,14 +672,14 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, ...@@ -672,14 +672,14 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state,
break; break;
} }
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
debug_rt_mutex_print_deadlock(waiter); debug_rt_mutex_print_deadlock(waiter);
if (waiter->task) if (waiter->task)
schedule_rt_mutex(lock); schedule_rt_mutex(lock);
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
set_current_state(state); set_current_state(state);
} }
...@@ -700,11 +700,11 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, ...@@ -700,11 +700,11 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
debug_rt_mutex_init_waiter(&waiter); debug_rt_mutex_init_waiter(&waiter);
waiter.task = NULL; waiter.task = NULL;
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
/* Try to acquire the lock again: */ /* Try to acquire the lock again: */
if (try_to_take_rt_mutex(lock)) { if (try_to_take_rt_mutex(lock)) {
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
return 0; return 0;
} }
...@@ -731,7 +731,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, ...@@ -731,7 +731,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
*/ */
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
/* Remove pending timer: */ /* Remove pending timer: */
if (unlikely(timeout)) if (unlikely(timeout))
...@@ -758,7 +758,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) ...@@ -758,7 +758,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock)
{ {
int ret = 0; int ret = 0;
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
if (likely(rt_mutex_owner(lock) != current)) { if (likely(rt_mutex_owner(lock) != current)) {
...@@ -770,7 +770,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) ...@@ -770,7 +770,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock)
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
} }
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
return ret; return ret;
} }
...@@ -781,7 +781,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock) ...@@ -781,7 +781,7 @@ rt_mutex_slowtrylock(struct rt_mutex *lock)
static void __sched static void __sched
rt_mutex_slowunlock(struct rt_mutex *lock) rt_mutex_slowunlock(struct rt_mutex *lock)
{ {
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
debug_rt_mutex_unlock(lock); debug_rt_mutex_unlock(lock);
...@@ -789,13 +789,13 @@ rt_mutex_slowunlock(struct rt_mutex *lock) ...@@ -789,13 +789,13 @@ rt_mutex_slowunlock(struct rt_mutex *lock)
if (!rt_mutex_has_waiters(lock)) { if (!rt_mutex_has_waiters(lock)) {
lock->owner = NULL; lock->owner = NULL;
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
return; return;
} }
wakeup_next_waiter(lock); wakeup_next_waiter(lock);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
/* Undo pi boosting if necessary: */ /* Undo pi boosting if necessary: */
rt_mutex_adjust_prio(current); rt_mutex_adjust_prio(current);
...@@ -970,8 +970,8 @@ EXPORT_SYMBOL_GPL(rt_mutex_destroy); ...@@ -970,8 +970,8 @@ EXPORT_SYMBOL_GPL(rt_mutex_destroy);
void __rt_mutex_init(struct rt_mutex *lock, const char *name) void __rt_mutex_init(struct rt_mutex *lock, const char *name)
{ {
lock->owner = NULL; lock->owner = NULL;
spin_lock_init(&lock->wait_lock); atomic_spin_lock_init(&lock->wait_lock);
plist_head_init(&lock->wait_list, &lock->wait_lock); plist_head_init_atomic(&lock->wait_list, &lock->wait_lock);
debug_rt_mutex_init(lock, name); debug_rt_mutex_init(lock, name);
} }
...@@ -1032,7 +1032,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, ...@@ -1032,7 +1032,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
{ {
int ret; int ret;
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
mark_rt_mutex_waiters(lock); mark_rt_mutex_waiters(lock);
...@@ -1058,7 +1058,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, ...@@ -1058,7 +1058,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
*/ */
ret = 0; ret = 0;
} }
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
debug_rt_mutex_print_deadlock(waiter); debug_rt_mutex_print_deadlock(waiter);
...@@ -1108,7 +1108,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, ...@@ -1108,7 +1108,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
{ {
int ret; int ret;
spin_lock(&lock->wait_lock); atomic_spin_lock(&lock->wait_lock);
set_current_state(TASK_INTERRUPTIBLE); set_current_state(TASK_INTERRUPTIBLE);
...@@ -1126,7 +1126,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, ...@@ -1126,7 +1126,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
*/ */
fixup_rt_mutex_waiters(lock); fixup_rt_mutex_waiters(lock);
spin_unlock(&lock->wait_lock); atomic_spin_unlock(&lock->wait_lock);
/* /*
* Readjust priority, when we did not get the lock. We might have been * Readjust priority, when we did not get the lock. We might have been
......
kernel/sched.c
View file @ d227cf76
...@@ -6202,7 +6202,7 @@ recheck: ...@@ -6202,7 +6202,7 @@ recheck:
* make sure no PI-waiters arrive (or leave) while we are * make sure no PI-waiters arrive (or leave) while we are
* changing the priority of the task: * changing the priority of the task:
*/ */
spin_lock_irqsave(&p->pi_lock, flags); atomic_spin_lock_irqsave(&p->pi_lock, flags);
/* /*
* To be able to change p->policy safely, the apropriate * To be able to change p->policy safely, the apropriate
* runqueue lock must be held. * runqueue lock must be held.
...@@ -6212,7 +6212,7 @@ recheck: ...@@ -6212,7 +6212,7 @@ recheck:
if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
policy = oldpolicy = -1; policy = oldpolicy = -1;
__task_rq_unlock(rq); __task_rq_unlock(rq);
spin_unlock_irqrestore(&p->pi_lock, flags); atomic_spin_unlock_irqrestore(&p->pi_lock, flags);
goto recheck; goto recheck;
} }
update_rq_clock(rq); update_rq_clock(rq);
...@@ -6234,7 +6234,7 @@ recheck: ...@@ -6234,7 +6234,7 @@ recheck:
check_class_changed(rq, p, prev_class, oldprio, running); check_class_changed(rq, p, prev_class, oldprio, running);
} }
__task_rq_unlock(rq); __task_rq_unlock(rq);
spin_unlock_irqrestore(&p->pi_lock, flags); atomic_spin_unlock_irqrestore(&p->pi_lock, flags);
rt_mutex_adjust_pi(p); rt_mutex_adjust_pi(p);
...@@ -9361,7 +9361,7 @@ void __init sched_init(void) ...@@ -9361,7 +9361,7 @@ void __init sched_init(void)
#endif #endif
#ifdef CONFIG_RT_MUTEXES #ifdef CONFIG_RT_MUTEXES
plist_head_init(&init_task.pi_waiters, &init_task.pi_lock); plist_head_init_atomic(&init_task.pi_waiters, &init_task.pi_lock);
#endif #endif
/* /*
...@@ -9477,13 +9477,13 @@ void normalize_rt_tasks(void) ...@@ -9477,13 +9477,13 @@ void normalize_rt_tasks(void)
continue; continue;
} }
spin_lock(&p->pi_lock); atomic_spin_lock(&p->pi_lock);
rq = __task_rq_lock(p); rq = __task_rq_lock(p);
normalize_task(rq, p); normalize_task(rq, p);
__task_rq_unlock(rq); __task_rq_unlock(rq);
spin_unlock(&p->pi_lock); atomic_spin_unlock(&p->pi_lock);
} while_each_thread(g, p); } while_each_thread(g, p);
read_unlock_irqrestore(&tasklist_lock, flags); read_unlock_irqrestore(&tasklist_lock, flags);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment