diff --git a/include/asm-alpha/spinlock.h b/include/asm-alpha/spinlock.h index 0c294c9b0c55..aeeb125f6851 100644 --- a/include/asm-alpha/spinlock.h +++ b/include/asm-alpha/spinlock.h @@ -166,4 +166,8 @@ static inline void __raw_write_unlock(raw_rwlock_t * lock) lock->lock = 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* _ALPHA_SPINLOCK_H */ diff --git a/include/asm-arm/spinlock.h b/include/asm-arm/spinlock.h index 01b7c26a3038..861092fbaa53 100644 --- a/include/asm-arm/spinlock.h +++ b/include/asm-arm/spinlock.h @@ -218,4 +218,8 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw) /* read_can_lock - would read_trylock() succeed? */ #define __raw_read_can_lock(x) ((x)->lock < 0x80000000) +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-cris/arch-v32/spinlock.h b/include/asm-cris/arch-v32/spinlock.h index 52df72a62232..5f43df0a5fb4 100644 --- a/include/asm-cris/arch-v32/spinlock.h +++ b/include/asm-cris/arch-v32/spinlock.h @@ -160,4 +160,8 @@ static __inline__ int is_write_locked(rwlock_t *rw) return rw->counter < 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_ARCH_SPINLOCK_H */ diff --git a/include/asm-i386/spinlock.h b/include/asm-i386/spinlock.h index b0b3043f05e1..c18b71fae6b3 100644 --- a/include/asm-i386/spinlock.h +++ b/include/asm-i386/spinlock.h @@ -205,4 +205,8 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw) : "+m" (rw->lock) : : "memory"); } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-ia64/spinlock.h b/include/asm-ia64/spinlock.h index 9e83210dc312..ff857e31738a 100644 --- a/include/asm-ia64/spinlock.h +++ b/include/asm-ia64/spinlock.h @@ -213,4 +213,8 @@ static inline int __raw_read_trylock(raw_rwlock_t *x) return (u32)ia64_cmpxchg4_acq((__u32 *)(x), new.word, old.word) == old.word; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* _ASM_IA64_SPINLOCK_H */ diff --git a/include/asm-m32r/spinlock.h b/include/asm-m32r/spinlock.h index f9f90727a4a1..f5cfba81ee10 100644 --- a/include/asm-m32r/spinlock.h +++ b/include/asm-m32r/spinlock.h @@ -316,4 +316,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *lock) return 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* _ASM_M32R_SPINLOCK_H */ diff --git a/include/asm-mips/spinlock.h b/include/asm-mips/spinlock.h index 4c1a1b53aeaf..c8d5587467bb 100644 --- a/include/asm-mips/spinlock.h +++ b/include/asm-mips/spinlock.h @@ -328,4 +328,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* _ASM_SPINLOCK_H */ diff --git a/include/asm-parisc/spinlock.h b/include/asm-parisc/spinlock.h index a93960e232cf..e1825530365d 100644 --- a/include/asm-parisc/spinlock.h +++ b/include/asm-parisc/spinlock.h @@ -152,4 +152,8 @@ static __inline__ int __raw_write_can_lock(raw_rwlock_t *rw) return !rw->counter; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-powerpc/spinlock.h b/include/asm-powerpc/spinlock.h index c31e4382a775..eaccac8327f2 100644 --- a/include/asm-powerpc/spinlock.h +++ b/include/asm-powerpc/spinlock.h @@ -285,5 +285,9 @@ static __inline__ void __raw_write_unlock(raw_rwlock_t *rw) rw->lock = 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __KERNEL__ */ #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-ppc/spinlock.h b/include/asm-ppc/spinlock.h index 5c64b75f0295..fccaf5531e57 100644 --- a/include/asm-ppc/spinlock.h +++ b/include/asm-ppc/spinlock.h @@ -161,4 +161,8 @@ static __inline__ void __raw_write_unlock(raw_rwlock_t *rw) rw->lock = 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-s390/spinlock.h b/include/asm-s390/spinlock.h index ce3edf6d63b3..5f00feaf1be6 100644 --- a/include/asm-s390/spinlock.h +++ b/include/asm-s390/spinlock.h @@ -154,4 +154,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) return _raw_write_trylock_retry(rw); } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/include/asm-sh/spinlock.h b/include/asm-sh/spinlock.h index 846322d4c35d..54458fd24981 100644 --- a/include/asm-sh/spinlock.h +++ b/include/asm-sh/spinlock.h @@ -100,4 +100,8 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) return 0; } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SH_SPINLOCK_H */ diff --git a/include/asm-sparc/spinlock.h b/include/asm-sparc/spinlock.h index 1c75474ba1df..557d08959d2f 100644 --- a/include/asm-sparc/spinlock.h +++ b/include/asm-sparc/spinlock.h @@ -154,6 +154,10 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) #define __raw_read_trylock(lock) generic__raw_read_trylock(lock) +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #define __raw_read_can_lock(rw) (!((rw)->lock & 0xff)) #define __raw_write_can_lock(rw) (!(rw)->lock) diff --git a/include/asm-sparc64/spinlock.h b/include/asm-sparc64/spinlock.h index bd5ffc76bc7e..0006fe9f8c7a 100644 --- a/include/asm-sparc64/spinlock.h +++ b/include/asm-sparc64/spinlock.h @@ -241,6 +241,10 @@ static int inline __write_trylock(raw_rwlock_t *lock) #define __raw_read_can_lock(rw) (!((rw)->lock & 0x80000000UL)) #define __raw_write_can_lock(rw) (!(rw)->lock) +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* !(__ASSEMBLY__) */ #endif /* !(__SPARC64_SPINLOCK_H) */ diff --git a/include/asm-x86_64/spinlock.h b/include/asm-x86_64/spinlock.h index 3daf5b005905..05ef097ba55b 100644 --- a/include/asm-x86_64/spinlock.h +++ b/include/asm-x86_64/spinlock.h @@ -133,4 +133,8 @@ static inline void __raw_write_unlock(raw_rwlock_t *rw) : "=m" (rw->lock) : : "memory"); } +#define _raw_spin_relax(lock) cpu_relax() +#define _raw_read_relax(lock) cpu_relax() +#define _raw_write_relax(lock) cpu_relax() + #endif /* __ASM_SPINLOCK_H */ diff --git a/kernel/spinlock.c b/kernel/spinlock.c index d48143eafbfd..476c3741511b 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c @@ -215,7 +215,7 @@ void __lockfunc _##op##_lock(locktype##_t *lock) \ if (!(lock)->break_lock) \ (lock)->break_lock = 1; \ while (!op##_can_lock(lock) && (lock)->break_lock) \ - cpu_relax(); \ + _raw_##op##_relax(&lock->raw_lock); \ } \ (lock)->break_lock = 0; \ } \ @@ -237,7 +237,7 @@ unsigned long __lockfunc _##op##_lock_irqsave(locktype##_t *lock) \ if (!(lock)->break_lock) \ (lock)->break_lock = 1; \ while (!op##_can_lock(lock) && (lock)->break_lock) \ - cpu_relax(); \ + _raw_##op##_relax(&lock->raw_lock); \ } \ (lock)->break_lock = 0; \ return flags; \