Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-28 05:41:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

locking: Add __lockfunc to slow path functions

Browse Source 
So that we can skip the functions in the perf lock contention and other
places like /proc/PID/wchan.

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Waiman Long <longman@redhat.com>
Link: https://lore.kernel.org/r/20220810220346.1919485-1-namhyung@kernel.org
This commit is contained in:
Namhyung Kim 2022-08-10 15:03:46 -07:00 committed by Peter Zijlstra
parent 9aeaf5bc4e
commit 501f7f69bc
4 changed files with 12 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
arch/x86/include/asm/qspinlock_paravirt.h
View File

@ -12,7 +12,7 @@
*/ */
#ifdef CONFIG_64BIT #ifdef CONFIG_64BIT
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath); __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath, ".spinlock.text");
#define __pv_queued_spin_unlock __pv_queued_spin_unlock #define __pv_queued_spin_unlock __pv_queued_spin_unlock
#define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock" #define PV_UNLOCK "__raw_callee_save___pv_queued_spin_unlock"
#define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath" #define PV_UNLOCK_SLOWPATH "__raw_callee_save___pv_queued_spin_unlock_slowpath"
@ -20,9 +20,10 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
/* /*
* Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock * Optimized assembly version of __raw_callee_save___pv_queued_spin_unlock
* which combines the registers saving trunk and the body of the following * which combines the registers saving trunk and the body of the following
* C code: * C code. Note that it puts the code in the .spinlock.text section which
* is equivalent to adding __lockfunc in the C code:
* *
* void __pv_queued_spin_unlock(struct qspinlock *lock) * void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock)
* { * {
* u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0); * u8 lockval = cmpxchg(&lock->locked, _Q_LOCKED_VAL, 0);
* *
@ -36,7 +37,7 @@ PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock_slowpath);
* rsi = lockval (second argument) * rsi = lockval (second argument)
* rdx = internal variable (set to 0) * rdx = internal variable (set to 0)
*/ */
asm (".pushsection .text;" asm (".pushsection .spinlock.text;"
".globl " PV_UNLOCK ";" ".globl " PV_UNLOCK ";"
".type " PV_UNLOCK ", @function;" ".type " PV_UNLOCK ", @function;"
".align 4,0x90;" ".align 4,0x90;"
@ -65,8 +66,8 @@ asm (".pushsection .text;"
#else /* CONFIG_64BIT */ #else /* CONFIG_64BIT */
extern void __pv_queued_spin_unlock(struct qspinlock *lock); extern void __lockfunc __pv_queued_spin_unlock(struct qspinlock *lock);
PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock); __PV_CALLEE_SAVE_REGS_THUNK(__pv_queued_spin_unlock, ".spinlock.text");
#endif /* CONFIG_64BIT */ #endif /* CONFIG_64BIT */
#endif #endif

4
kernel/locking/qrwlock.c
View File

@ -18,7 +18,7 @@
* queued_read_lock_slowpath - acquire read lock of a queued rwlock * queued_read_lock_slowpath - acquire read lock of a queued rwlock
* @lock: Pointer to queued rwlock structure * @lock: Pointer to queued rwlock structure
*/ */
void queued_read_lock_slowpath(struct qrwlock *lock) void __lockfunc queued_read_lock_slowpath(struct qrwlock *lock)
{ {
/* /*
* Readers come here when they cannot get the lock without waiting * Readers come here when they cannot get the lock without waiting
@ -63,7 +63,7 @@ EXPORT_SYMBOL(queued_read_lock_slowpath);
* queued_write_lock_slowpath - acquire write lock of a queued rwlock * queued_write_lock_slowpath - acquire write lock of a queued rwlock
* @lock : Pointer to queued rwlock structure * @lock : Pointer to queued rwlock structure
*/ */
void queued_write_lock_slowpath(struct qrwlock *lock) void __lockfunc queued_write_lock_slowpath(struct qrwlock *lock)
{ {
int cnts; int cnts;

2
kernel/locking/qspinlock.c
View File

@ -313,7 +313,7 @@ static __always_inline u32 __pv_wait_head_or_lock(struct qspinlock *lock,
* contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' : * contended : (*,x,y) +--> (*,0,0) ---> (*,0,1) -' :
* queue : ^--' : * queue : ^--' :
*/ */
void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val) void __lockfunc queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
{ {
struct mcs_spinlock *prev, *next, *node; struct mcs_spinlock *prev, *next, *node;
u32 old, tail; u32 old, tail;

4
kernel/locking/qspinlock_paravirt.h
View File

@ -489,7 +489,7 @@ gotlock:
* PV versions of the unlock fastpath and slowpath functions to be used * PV versions of the unlock fastpath and slowpath functions to be used
* instead of queued_spin_unlock(). * instead of queued_spin_unlock().
*/ */
__visible void __visible __lockfunc void
__pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked) __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
{ {
struct pv_node *node; struct pv_node *node;
@ -544,7 +544,7 @@ __pv_queued_spin_unlock_slowpath(struct qspinlock *lock, u8 locked)
#include <asm/qspinlock_paravirt.h> #include <asm/qspinlock_paravirt.h>
#ifndef __pv_queued_spin_unlock #ifndef __pv_queued_spin_unlock
__visible void __pv_queued_spin_unlock(struct qspinlock *lock) __visible __lockfunc void __pv_queued_spin_unlock(struct qspinlock *lock)
{ {
u8 locked; u8 locked;