mirror of
https://github.com/torvalds/linux.git
synced 2024-09-20 23:13:00 +00:00
22e4ebb975
Implement MEMBARRIER_CMD_PRIVATE_EXPEDITED with IPIs using cpumask built from all runqueues for which current thread's mm is the same as the thread calling sys_membarrier. It executes faster than the non-expedited variant (no blocking). It also works on NOHZ_FULL configurations. Scheduler-wise, it requires a memory barrier before and after context switching between processes (which have different mm). The memory barrier before context switch is already present. For the barrier after context switch: * Our TSO archs can do RELEASE without being a full barrier. Look at x86 spin_unlock() being a regular STORE for example. But for those archs, all atomics imply smp_mb and all of them have atomic ops in switch_mm() for mm_cpumask(), and on x86 the CR3 load acts as a full barrier. * From all weakly ordered machines, only ARM64 and PPC can do RELEASE, the rest does indeed do smp_mb(), so there the spin_unlock() is a full barrier and we're good. * ARM64 has a very heavy barrier in switch_to(), which suffices. * PPC just removed its barrier from switch_to(), but appears to be talking about adding something to switch_mm(). So add a smp_mb__after_unlock_lock() for now, until this is settled on the PPC side. Changes since v3: - Properly document the memory barriers provided by each architecture. Changes since v2: - Address comments from Peter Zijlstra, - Add smp_mb__after_unlock_lock() after finish_lock_switch() in finish_task_switch() to add the memory barrier we need after storing to rq->curr. This is much simpler than the previous approach relying on atomic_dec_and_test() in mmdrop(), which actually added a memory barrier in the common case of switching between userspace processes. - Return -EINVAL when MEMBARRIER_CMD_SHARED is used on a nohz_full kernel, rather than having the whole membarrier system call returning -ENOSYS. Indeed, CMD_PRIVATE_EXPEDITED is compatible with nohz_full. Adapt the CMD_QUERY mask accordingly. Changes since v1: - move membarrier code under kernel/sched/ because it uses the scheduler runqueue, - only add the barrier when we switch from a kernel thread. The case where we switch from a user-space thread is already handled by the atomic_dec_and_test() in mmdrop(). - add a comment to mmdrop() documenting the requirement on the implicit memory barrier. CC: Peter Zijlstra <peterz@infradead.org> CC: Paul E. McKenney <paulmck@linux.vnet.ibm.com> CC: Boqun Feng <boqun.feng@gmail.com> CC: Andrew Hunter <ahh@google.com> CC: Maged Michael <maged.michael@gmail.com> CC: gromer@google.com CC: Avi Kivity <avi@scylladb.com> CC: Benjamin Herrenschmidt <benh@kernel.crashing.org> CC: Paul Mackerras <paulus@samba.org> CC: Michael Ellerman <mpe@ellerman.id.au> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Tested-by: Dave Watson <davejwatson@fb.com>
124 lines
3.9 KiB
Makefile
124 lines
3.9 KiB
Makefile
#
|
|
# Makefile for the linux kernel.
|
|
#
|
|
|
|
obj-y = fork.o exec_domain.o panic.o \
|
|
cpu.o exit.o softirq.o resource.o \
|
|
sysctl.o sysctl_binary.o capability.o ptrace.o user.o \
|
|
signal.o sys.o kmod.o workqueue.o pid.o task_work.o \
|
|
extable.o params.o \
|
|
kthread.o sys_ni.o nsproxy.o \
|
|
notifier.o ksysfs.o cred.o reboot.o \
|
|
async.o range.o smpboot.o ucount.o
|
|
|
|
obj-$(CONFIG_MULTIUSER) += groups.o
|
|
|
|
ifdef CONFIG_FUNCTION_TRACER
|
|
# Do not trace internal ftrace files
|
|
CFLAGS_REMOVE_irq_work.o = $(CC_FLAGS_FTRACE)
|
|
endif
|
|
|
|
# Prevents flicker of uninteresting __do_softirq()/__local_bh_disable_ip()
|
|
# in coverage traces.
|
|
KCOV_INSTRUMENT_softirq.o := n
|
|
# These are called from save_stack_trace() on slub debug path,
|
|
# and produce insane amounts of uninteresting coverage.
|
|
KCOV_INSTRUMENT_module.o := n
|
|
KCOV_INSTRUMENT_extable.o := n
|
|
# Don't self-instrument.
|
|
KCOV_INSTRUMENT_kcov.o := n
|
|
KASAN_SANITIZE_kcov.o := n
|
|
|
|
# cond_syscall is currently not LTO compatible
|
|
CFLAGS_sys_ni.o = $(DISABLE_LTO)
|
|
|
|
obj-y += sched/
|
|
obj-y += locking/
|
|
obj-y += power/
|
|
obj-y += printk/
|
|
obj-y += irq/
|
|
obj-y += rcu/
|
|
obj-y += livepatch/
|
|
|
|
obj-$(CONFIG_CHECKPOINT_RESTORE) += kcmp.o
|
|
obj-$(CONFIG_FREEZER) += freezer.o
|
|
obj-$(CONFIG_PROFILING) += profile.o
|
|
obj-$(CONFIG_STACKTRACE) += stacktrace.o
|
|
obj-y += time/
|
|
obj-$(CONFIG_FUTEX) += futex.o
|
|
ifeq ($(CONFIG_COMPAT),y)
|
|
obj-$(CONFIG_FUTEX) += futex_compat.o
|
|
endif
|
|
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
|
|
obj-$(CONFIG_SMP) += smp.o
|
|
ifneq ($(CONFIG_SMP),y)
|
|
obj-y += up.o
|
|
endif
|
|
obj-$(CONFIG_UID16) += uid16.o
|
|
obj-$(CONFIG_MODULES) += module.o
|
|
obj-$(CONFIG_MODULE_SIG) += module_signing.o
|
|
obj-$(CONFIG_KALLSYMS) += kallsyms.o
|
|
obj-$(CONFIG_BSD_PROCESS_ACCT) += acct.o
|
|
obj-$(CONFIG_CRASH_CORE) += crash_core.o
|
|
obj-$(CONFIG_KEXEC_CORE) += kexec_core.o
|
|
obj-$(CONFIG_KEXEC) += kexec.o
|
|
obj-$(CONFIG_KEXEC_FILE) += kexec_file.o
|
|
obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o
|
|
obj-$(CONFIG_COMPAT) += compat.o
|
|
obj-$(CONFIG_CGROUPS) += cgroup/
|
|
obj-$(CONFIG_UTS_NS) += utsname.o
|
|
obj-$(CONFIG_USER_NS) += user_namespace.o
|
|
obj-$(CONFIG_PID_NS) += pid_namespace.o
|
|
obj-$(CONFIG_IKCONFIG) += configs.o
|
|
obj-$(CONFIG_SMP) += stop_machine.o
|
|
obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o
|
|
obj-$(CONFIG_AUDIT) += audit.o auditfilter.o
|
|
obj-$(CONFIG_AUDITSYSCALL) += auditsc.o
|
|
obj-$(CONFIG_AUDIT_WATCH) += audit_watch.o audit_fsnotify.o
|
|
obj-$(CONFIG_AUDIT_TREE) += audit_tree.o
|
|
obj-$(CONFIG_GCOV_KERNEL) += gcov/
|
|
obj-$(CONFIG_KCOV) += kcov.o
|
|
obj-$(CONFIG_KPROBES) += kprobes.o
|
|
obj-$(CONFIG_KGDB) += debug/
|
|
obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o
|
|
obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o
|
|
obj-$(CONFIG_HARDLOCKUP_DETECTOR_PERF) += watchdog_hld.o
|
|
obj-$(CONFIG_SECCOMP) += seccomp.o
|
|
obj-$(CONFIG_RELAY) += relay.o
|
|
obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
|
|
obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
|
|
obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
|
|
obj-$(CONFIG_TRACEPOINTS) += tracepoint.o
|
|
obj-$(CONFIG_LATENCYTOP) += latencytop.o
|
|
obj-$(CONFIG_ELFCORE) += elfcore.o
|
|
obj-$(CONFIG_FUNCTION_TRACER) += trace/
|
|
obj-$(CONFIG_TRACING) += trace/
|
|
obj-$(CONFIG_TRACE_CLOCK) += trace/
|
|
obj-$(CONFIG_RING_BUFFER) += trace/
|
|
obj-$(CONFIG_TRACEPOINTS) += trace/
|
|
obj-$(CONFIG_IRQ_WORK) += irq_work.o
|
|
obj-$(CONFIG_CPU_PM) += cpu_pm.o
|
|
obj-$(CONFIG_BPF) += bpf/
|
|
|
|
obj-$(CONFIG_PERF_EVENTS) += events/
|
|
|
|
obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
|
|
obj-$(CONFIG_PADATA) += padata.o
|
|
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
|
|
obj-$(CONFIG_JUMP_LABEL) += jump_label.o
|
|
obj-$(CONFIG_CONTEXT_TRACKING) += context_tracking.o
|
|
obj-$(CONFIG_TORTURE_TEST) += torture.o
|
|
|
|
obj-$(CONFIG_HAS_IOMEM) += memremap.o
|
|
|
|
$(obj)/configs.o: $(obj)/config_data.h
|
|
|
|
targets += config_data.gz
|
|
$(obj)/config_data.gz: $(KCONFIG_CONFIG) FORCE
|
|
$(call if_changed,gzip)
|
|
|
|
filechk_ikconfiggz = (echo "static const char kernel_config_data[] __used = MAGIC_START"; cat $< | scripts/basic/bin2c; echo "MAGIC_END;")
|
|
targets += config_data.h
|
|
$(obj)/config_data.h: $(obj)/config_data.gz FORCE
|
|
$(call filechk,ikconfiggz)
|