Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-10 06:01:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

smp: Change function signatures to use call_single_data_t

Browse Source 
call_single_data_t is a size-aligned typedef of struct __call_single_data.

This alignment is desirable in order to have smp_call_function*() avoid
bouncing an extra cacheline in case of an unaligned csd, given this
would hurt performance.

Since the removal of struct request->csd in commit 660e802c76
("blk-mq: use percpu csd to remote complete instead of per-rq csd") there
are no current users of smp_call_function*() with unaligned csd.

Change every 'struct __call_single_data' function parameter to
'call_single_data_t', so we have warnings if any new code tries to
introduce an smp_call_function*() call with unaligned csd.

Signed-off-by: Leonardo Bras <leobras@redhat.com>
Reviewed-by: Guo Ren <guoren@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20230831063129.335425-1-leobras@redhat.com
This commit is contained in:
Leonardo Bras 2023-08-31 03:31:28 -03:00 committed by Ingo Molnar
parent e0a99a839f
commit d090ec0df8
4 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/smp.h
View File

@ -53,7 +53,7 @@ int smp_call_function_single(int cpuid, smp_call_func_t func, void *info,
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
void *info, bool wait, const struct cpumask *mask); void *info, bool wait, const struct cpumask *mask);
int smp_call_function_single_async(int cpu, struct __call_single_data *csd); int smp_call_function_single_async(int cpu, call_single_data_t *csd);
/* /*
* Cpus stopping functions in panic. All have default weak definitions. * Cpus stopping functions in panic. All have default weak definitions.

8
include/trace/events/csd.h
View File

@ -12,7 +12,7 @@ TRACE_EVENT(csd_queue_cpu,
TP_PROTO(const unsigned int cpu, TP_PROTO(const unsigned int cpu,
unsigned long callsite, unsigned long callsite,
smp_call_func_t func, smp_call_func_t func,
struct __call_single_data *csd), call_single_data_t *csd),
TP_ARGS(cpu, callsite, func, csd), TP_ARGS(cpu, callsite, func, csd),
@ -39,7 +39,7 @@ TRACE_EVENT(csd_queue_cpu,
*/ */
DECLARE_EVENT_CLASS(csd_function, DECLARE_EVENT_CLASS(csd_function,
TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
TP_ARGS(func, csd), TP_ARGS(func, csd),
@ -57,12 +57,12 @@ DECLARE_EVENT_CLASS(csd_function,
); );
DEFINE_EVENT(csd_function, csd_function_entry, DEFINE_EVENT(csd_function, csd_function_entry,
TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
TP_ARGS(func, csd) TP_ARGS(func, csd)
); );
DEFINE_EVENT(csd_function, csd_function_exit, DEFINE_EVENT(csd_function, csd_function_exit,
TP_PROTO(smp_call_func_t func, struct __call_single_data *csd), TP_PROTO(smp_call_func_t func, call_single_data_t *csd),
TP_ARGS(func, csd) TP_ARGS(func, csd)
); );

26
kernel/smp.c
View File

@ -125,7 +125,7 @@ send_call_function_ipi_mask(struct cpumask *mask)
} }
static __always_inline void static __always_inline void
csd_do_func(smp_call_func_t func, void *info, struct __call_single_data *csd) csd_do_func(smp_call_func_t func, void *info, call_single_data_t *csd)
{ {
trace_csd_function_entry(func, csd); trace_csd_function_entry(func, csd);
func(info); func(info);
@ -172,7 +172,7 @@ module_param(csd_lock_timeout, ulong, 0444);
static atomic_t csd_bug_count = ATOMIC_INIT(0); static atomic_t csd_bug_count = ATOMIC_INIT(0);
/* Record current CSD work for current CPU, NULL to erase. */ /* Record current CSD work for current CPU, NULL to erase. */
static void __csd_lock_record(struct __call_single_data *csd) static void __csd_lock_record(call_single_data_t *csd)
{ {
if (!csd) { if (!csd) {
smp_mb(); /* NULL cur_csd after unlock. */ smp_mb(); /* NULL cur_csd after unlock. */
@ -187,13 +187,13 @@ static void __csd_lock_record(struct __call_single_data *csd)
/* Or before unlock, as the case may be. */ /* Or before unlock, as the case may be. */
} }
static __always_inline void csd_lock_record(struct __call_single_data *csd) static __always_inline void csd_lock_record(call_single_data_t *csd)
{ {
if (static_branch_unlikely(&csdlock_debug_enabled)) if (static_branch_unlikely(&csdlock_debug_enabled))
__csd_lock_record(csd); __csd_lock_record(csd);
} }
static int csd_lock_wait_getcpu(struct __call_single_data *csd) static int csd_lock_wait_getcpu(call_single_data_t *csd)
{ {
unsigned int csd_type; unsigned int csd_type;
@ -208,7 +208,7 @@ static int csd_lock_wait_getcpu(struct __call_single_data *csd)
* the CSD_TYPE_SYNC/ASYNC types provide the destination CPU, * the CSD_TYPE_SYNC/ASYNC types provide the destination CPU,
* so waiting on other types gets much less information. * so waiting on other types gets much less information.
*/ */
static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id) static bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *bug_id)
{ {
int cpu = -1; int cpu = -1;
int cpux; int cpux;
@ -272,7 +272,7 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *
* previous function call. For multi-cpu calls its even more interesting * previous function call. For multi-cpu calls its even more interesting
* as we'll have to ensure no other cpu is observing our csd. * as we'll have to ensure no other cpu is observing our csd.
*/ */
static void __csd_lock_wait(struct __call_single_data *csd) static void __csd_lock_wait(call_single_data_t *csd)
{ {
int bug_id = 0; int bug_id = 0;
u64 ts0, ts1; u64 ts0, ts1;
@ -286,7 +286,7 @@ static void __csd_lock_wait(struct __call_single_data *csd)
smp_acquire__after_ctrl_dep(); smp_acquire__after_ctrl_dep();
} }
static __always_inline void csd_lock_wait(struct __call_single_data *csd) static __always_inline void csd_lock_wait(call_single_data_t *csd)
{ {
if (static_branch_unlikely(&csdlock_debug_enabled)) { if (static_branch_unlikely(&csdlock_debug_enabled)) {
__csd_lock_wait(csd); __csd_lock_wait(csd);
@ -296,17 +296,17 @@ static __always_inline void csd_lock_wait(struct __call_single_data *csd)
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
} }
#else #else
static void csd_lock_record(struct __call_single_data *csd) static void csd_lock_record(call_single_data_t *csd)
{ {
} }
static __always_inline void csd_lock_wait(struct __call_single_data *csd) static __always_inline void csd_lock_wait(call_single_data_t *csd)
{ {
smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK)); smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));
} }
#endif #endif
static __always_inline void csd_lock(struct __call_single_data *csd) static __always_inline void csd_lock(call_single_data_t *csd)
{ {
csd_lock_wait(csd); csd_lock_wait(csd);
csd->node.u_flags |= CSD_FLAG_LOCK; csd->node.u_flags |= CSD_FLAG_LOCK;
@ -319,7 +319,7 @@ static __always_inline void csd_lock(struct __call_single_data *csd)
smp_wmb(); smp_wmb();
} }
static __always_inline void csd_unlock(struct __call_single_data *csd) static __always_inline void csd_unlock(call_single_data_t *csd)
{ {
WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK)); WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));
@ -372,7 +372,7 @@ void __smp_call_single_queue(int cpu, struct llist_node *node)
* for execution on the given CPU. data must already have * for execution on the given CPU. data must already have
* ->func, ->info, and ->flags set. * ->func, ->info, and ->flags set.
*/ */
static int generic_exec_single(int cpu, struct __call_single_data *csd) static int generic_exec_single(int cpu, call_single_data_t *csd)
{ {
if (cpu == smp_processor_id()) { if (cpu == smp_processor_id()) {
smp_call_func_t func = csd->func; smp_call_func_t func = csd->func;
@ -658,7 +658,7 @@ EXPORT_SYMBOL(smp_call_function_single);
* *
* Return: %0 on success or negative errno value on error * Return: %0 on success or negative errno value on error
*/ */
int smp_call_function_single_async(int cpu, struct __call_single_data *csd) int smp_call_function_single_async(int cpu, call_single_data_t *csd)
{ {
int err = 0; int err = 0;

2
kernel/up.c
View File

@ -25,7 +25,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
} }
EXPORT_SYMBOL(smp_call_function_single); EXPORT_SYMBOL(smp_call_function_single);
int smp_call_function_single_async(int cpu, struct __call_single_data *csd) int smp_call_function_single_async(int cpu, call_single_data_t *csd)
{ {
unsigned long flags; unsigned long flags;