2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* mm/mmap.c
|
|
|
|
|
*
|
|
|
|
|
* Written by obz.
|
|
|
|
|
*
|
2009-01-05 14:06:29 +00:00
|
|
|
* Address space accounting code <alan@lxorguk.ukuu.org.uk>
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
|
2014-06-06 21:38:30 +00:00
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
|
2013-04-29 22:08:33 +00:00
|
|
|
#include <linux/kernel.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/slab.h>
|
2007-10-17 06:29:23 +00:00
|
|
|
#include <linux/backing-dev.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/mm.h>
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
#include <linux/vmacache.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/shm.h>
|
|
|
|
|
#include <linux/mman.h>
|
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
#include <linux/swap.h>
|
|
|
|
|
#include <linux/syscalls.h>
|
2006-01-11 20:17:46 +00:00
|
|
|
#include <linux/capability.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/init.h>
|
|
|
|
|
#include <linux/file.h>
|
|
|
|
|
#include <linux/fs.h>
|
|
|
|
|
#include <linux/personality.h>
|
|
|
|
|
#include <linux/security.h>
|
|
|
|
|
#include <linux/hugetlb.h>
|
2016-07-26 22:26:15 +00:00
|
|
|
#include <linux/shmem_fs.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/profile.h>
|
2011-10-16 06:01:52 +00:00
|
|
|
#include <linux/export.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/mount.h>
|
|
|
|
|
#include <linux/mempolicy.h>
|
|
|
|
|
#include <linux/rmap.h>
|
mmu-notifiers: core
With KVM/GFP/XPMEM there isn't just the primary CPU MMU pointing to pages.
There are secondary MMUs (with secondary sptes and secondary tlbs) too.
sptes in the kvm case are shadow pagetables, but when I say spte in
mmu-notifier context, I mean "secondary pte". In GRU case there's no
actual secondary pte and there's only a secondary tlb because the GRU
secondary MMU has no knowledge about sptes and every secondary tlb miss
event in the MMU always generates a page fault that has to be resolved by
the CPU (this is not the case of KVM where the a secondary tlb miss will
walk sptes in hardware and it will refill the secondary tlb transparently
to software if the corresponding spte is present). The same way
zap_page_range has to invalidate the pte before freeing the page, the spte
(and secondary tlb) must also be invalidated before any page is freed and
reused.
Currently we take a page_count pin on every page mapped by sptes, but that
means the pages can't be swapped whenever they're mapped by any spte
because they're part of the guest working set. Furthermore a spte unmap
event can immediately lead to a page to be freed when the pin is released
(so requiring the same complex and relatively slow tlb_gather smp safe
logic we have in zap_page_range and that can be avoided completely if the
spte unmap event doesn't require an unpin of the page previously mapped in
the secondary MMU).
The mmu notifiers allow kvm/GRU/XPMEM to attach to the tsk->mm and know
when the VM is swapping or freeing or doing anything on the primary MMU so
that the secondary MMU code can drop sptes before the pages are freed,
avoiding all page pinning and allowing 100% reliable swapping of guest
physical address space. Furthermore it avoids the code that teardown the
mappings of the secondary MMU, to implement a logic like tlb_gather in
zap_page_range that would require many IPI to flush other cpu tlbs, for
each fixed number of spte unmapped.
To make an example: if what happens on the primary MMU is a protection
downgrade (from writeable to wrprotect) the secondary MMU mappings will be
invalidated, and the next secondary-mmu-page-fault will call
get_user_pages and trigger a do_wp_page through get_user_pages if it
called get_user_pages with write=1, and it'll re-establishing an updated
spte or secondary-tlb-mapping on the copied page. Or it will setup a
readonly spte or readonly tlb mapping if it's a guest-read, if it calls
get_user_pages with write=0. This is just an example.
This allows to map any page pointed by any pte (and in turn visible in the
primary CPU MMU), into a secondary MMU (be it a pure tlb like GRU, or an
full MMU with both sptes and secondary-tlb like the shadow-pagetable layer
with kvm), or a remote DMA in software like XPMEM (hence needing of
schedule in XPMEM code to send the invalidate to the remote node, while no
need to schedule in kvm/gru as it's an immediate event like invalidating
primary-mmu pte).
At least for KVM without this patch it's impossible to swap guests
reliably. And having this feature and removing the page pin allows
several other optimizations that simplify life considerably.
Dependencies:
1) mm_take_all_locks() to register the mmu notifier when the whole VM
isn't doing anything with "mm". This allows mmu notifier users to keep
track if the VM is in the middle of the invalidate_range_begin/end
critical section with an atomic counter incraese in range_begin and
decreased in range_end. No secondary MMU page fault is allowed to map
any spte or secondary tlb reference, while the VM is in the middle of
range_begin/end as any page returned by get_user_pages in that critical
section could later immediately be freed without any further
->invalidate_page notification (invalidate_range_begin/end works on
ranges and ->invalidate_page isn't called immediately before freeing
the page). To stop all page freeing and pagetable overwrites the
mmap_sem must be taken in write mode and all other anon_vma/i_mmap
locks must be taken too.
2) It'd be a waste to add branches in the VM if nobody could possibly
run KVM/GRU/XPMEM on the kernel, so mmu notifiers will only enabled if
CONFIG_KVM=m/y. In the current kernel kvm won't yet take advantage of
mmu notifiers, but this already allows to compile a KVM external module
against a kernel with mmu notifiers enabled and from the next pull from
kvm.git we'll start using them. And GRU/XPMEM will also be able to
continue the development by enabling KVM=m in their config, until they
submit all GRU/XPMEM GPLv2 code to the mainline kernel. Then they can
also enable MMU_NOTIFIERS in the same way KVM does it (even if KVM=n).
This guarantees nobody selects MMU_NOTIFIER=y if KVM and GRU and XPMEM
are all =n.
The mmu_notifier_register call can fail because mm_take_all_locks may be
interrupted by a signal and return -EINTR. Because mmu_notifier_reigster
is used when a driver startup, a failure can be gracefully handled. Here
an example of the change applied to kvm to register the mmu notifiers.
Usually when a driver startups other allocations are required anyway and
-ENOMEM failure paths exists already.
struct kvm *kvm_arch_create_vm(void)
{
struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
+ int err;
if (!kvm)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
+ kvm->arch.mmu_notifier.ops = &kvm_mmu_notifier_ops;
+ err = mmu_notifier_register(&kvm->arch.mmu_notifier, current->mm);
+ if (err) {
+ kfree(kvm);
+ return ERR_PTR(err);
+ }
+
return kvm;
}
mmu_notifier_unregister returns void and it's reliable.
The patch also adds a few needed but missing includes that would prevent
kernel to compile after these changes on non-x86 archs (x86 didn't need
them by luck).
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: fix mm/filemap_xip.c build]
[akpm@linux-foundation.org: fix mm/mmu_notifier.c build]
Signed-off-by: Andrea Arcangeli <andrea@qumranet.com>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Jack Steiner <steiner@sgi.com>
Cc: Robin Holt <holt@sgi.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Kanoj Sarcar <kanojsarcar@yahoo.com>
Cc: Roland Dreier <rdreier@cisco.com>
Cc: Steve Wise <swise@opengridcomputing.com>
Cc: Avi Kivity <avi@qumranet.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Anthony Liguori <aliguori@us.ibm.com>
Cc: Chris Wright <chrisw@redhat.com>
Cc: Marcelo Tosatti <marcelo@kvack.org>
Cc: Eric Dumazet <dada1@cosmosbay.com>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Cc: Izik Eidus <izike@qumranet.com>
Cc: Anthony Liguori <aliguori@us.ibm.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 22:46:29 +00:00
|
|
|
#include <linux/mmu_notifier.h>
|
2014-08-06 23:06:36 +00:00
|
|
|
#include <linux/mmdebug.h>
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 10:02:48 +00:00
|
|
|
#include <linux/perf_event.h>
|
2010-10-30 06:54:44 +00:00
|
|
|
#include <linux/audit.h>
|
2011-01-13 23:46:59 +00:00
|
|
|
#include <linux/khugepaged.h>
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
#include <linux/uprobes.h>
|
2012-12-12 00:01:38 +00:00
|
|
|
#include <linux/rbtree_augmented.h>
|
2013-04-29 22:08:12 +00:00
|
|
|
#include <linux/notifier.h>
|
|
|
|
|
#include <linux/memory.h>
|
2014-06-06 21:38:30 +00:00
|
|
|
#include <linux/printk.h>
|
2015-09-04 22:46:24 +00:00
|
|
|
#include <linux/userfaultfd_k.h>
|
2016-02-03 00:57:43 +00:00
|
|
|
#include <linux/moduleparam.h>
|
mm/core, x86/mm/pkeys: Add execute-only protection keys support
Protection keys provide new page-based protection in hardware.
But, they have an interesting attribute: they only affect data
accesses and never affect instruction fetches. That means that
if we set up some memory which is set as "access-disabled" via
protection keys, we can still execute from it.
This patch uses protection keys to set up mappings to do just that.
If a user calls:
mmap(..., PROT_EXEC);
or
mprotect(ptr, sz, PROT_EXEC);
(note PROT_EXEC-only without PROT_READ/WRITE), the kernel will
notice this, and set a special protection key on the memory. It
also sets the appropriate bits in the Protection Keys User Rights
(PKRU) register so that the memory becomes unreadable and
unwritable.
I haven't found any userspace that does this today. With this
facility in place, we expect userspace to move to use it
eventually. Userspace _could_ start doing this today. Any
PROT_EXEC calls get converted to PROT_READ inside the kernel, and
would transparently be upgraded to "true" PROT_EXEC with this
code. IOW, userspace never has to do any PROT_EXEC runtime
detection.
This feature provides enhanced protection against leaking
executable memory contents. This helps thwart attacks which are
attempting to find ROP gadgets on the fly.
But, the security provided by this approach is not comprehensive.
The PKRU register which controls access permissions is a normal
user register writable from unprivileged userspace. An attacker
who can execute the 'wrpkru' instruction can easily disable the
protection provided by this feature.
The protection key that is used for execute-only support is
permanently dedicated at compile time. This is fine for now
because there is currently no API to set a protection key other
than this one.
Despite there being a constant PKRU value across the entire
system, we do not set it unless this feature is in use in a
process. That is to preserve the PKRU XSAVE 'init state',
which can lead to faster context switches.
PKRU *is* a user register and the kernel is modifying it. That
means that code doing:
pkru = rdpkru()
pkru |= 0x100;
mmap(..., PROT_EXEC);
wrpkru(pkru);
could lose the bits in PKRU that enforce execute-only
permissions. To avoid this, we suggest avoiding ever calling
mmap() or mprotect() when the PKRU value is expected to be
unstable.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chen Gang <gang.chen.5i5j@gmail.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: David Hildenbrand <dahi@linux.vnet.ibm.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Piotr Kwapulinski <kwapulinski.piotr@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Stephen Smalley <sds@tycho.nsa.gov>
Cc: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: keescook@google.com
Cc: linux-kernel@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210240.CB4BB5CA@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-12 21:02:40 +00:00
|
|
|
#include <linux/pkeys.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
#include <asm/uaccess.h>
|
|
|
|
|
#include <asm/cacheflush.h>
|
|
|
|
|
#include <asm/tlb.h>
|
2007-05-02 17:27:14 +00:00
|
|
|
#include <asm/mmu_context.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-24 04:27:10 +00:00
|
|
|
#include "internal.h"
|
|
|
|
|
|
2006-09-07 10:17:04 +00:00
|
|
|
#ifndef arch_mmap_check
|
|
|
|
|
#define arch_mmap_check(addr, len, flags) (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
mm: mmap: add new /proc tunable for mmap_base ASLR
Address Space Layout Randomization (ASLR) provides a barrier to
exploitation of user-space processes in the presence of security
vulnerabilities by making it more difficult to find desired code/data
which could help an attack. This is done by adding a random offset to
the location of regions in the process address space, with a greater
range of potential offset values corresponding to better protection/a
larger search-space for brute force, but also to greater potential for
fragmentation.
The offset added to the mmap_base address, which provides the basis for
the majority of the mappings for a process, is set once on process exec
in arch_pick_mmap_layout() and is done via hard-coded per-arch values,
which reflect, hopefully, the best compromise for all systems. The
trade-off between increased entropy in the offset value generation and
the corresponding increased variability in address space fragmentation
is not absolute, however, and some platforms may tolerate higher amounts
of entropy. This patch introduces both new Kconfig values and a sysctl
interface which may be used to change the amount of entropy used for
offset generation on a system.
The direct motivation for this change was in response to the
libstagefright vulnerabilities that affected Android, specifically to
information provided by Google's project zero at:
http://googleprojectzero.blogspot.com/2015/09/stagefrightened.html
The attack presented therein, by Google's project zero, specifically
targeted the limited randomness used to generate the offset added to the
mmap_base address in order to craft a brute-force-based attack.
Concretely, the attack was against the mediaserver process, which was
limited to respawning every 5 seconds, on an arm device. The hard-coded
8 bits used resulted in an average expected success rate of defeating
the mmap ASLR after just over 10 minutes (128 tries at 5 seconds a
piece). With this patch, and an accompanying increase in the entropy
value to 16 bits, the same attack would take an average expected time of
over 45 hours (32768 tries), which makes it both less feasible and more
likely to be noticed.
The introduced Kconfig and sysctl options are limited by per-arch
minimum and maximum values, the minimum of which was chosen to match the
current hard-coded value and the maximum of which was chosen so as to
give the greatest flexibility without generating an invalid mmap_base
address, generally a 3-4 bits less than the number of bits in the
user-space accessible virtual address space.
When decided whether or not to change the default value, a system
developer should consider that mmap_base address could be placed
anywhere up to 2^(value) bits away from the non-randomized location,
which would introduce variable-sized areas above and below the mmap_base
address such that the maximum vm_area_struct size may be reduced,
preventing very large allocations.
This patch (of 4):
ASLR only uses as few as 8 bits to generate the random offset for the
mmap base address on 32 bit architectures. This value was chosen to
prevent a poorly chosen value from dividing the address space in such a
way as to prevent large allocations. This may not be an issue on all
platforms. Allow the specification of a minimum number of bits so that
platforms desiring greater ASLR protection may determine where to place
the trade-off.
Signed-off-by: Daniel Cashman <dcashman@google.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Kees Cook <keescook@chromium.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Don Zickus <dzickus@redhat.com>
Cc: Eric W. Biederman <ebiederm@xmission.com>
Cc: Heinrich Schuchardt <xypron.glpk@gmx.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Mark Salyzyn <salyzyn@android.com>
Cc: Jeff Vander Stoep <jeffv@google.com>
Cc: Nick Kralevich <nnk@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Hector Marco-Gisbert <hecmargi@upv.es>
Cc: Borislav Petkov <bp@suse.de>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 23:19:53 +00:00
|
|
|
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
|
|
|
|
|
const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
|
|
|
|
|
const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX;
|
|
|
|
|
int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
|
|
|
|
|
const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
|
|
|
|
|
const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
|
|
|
|
|
int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
|
|
|
|
|
#endif
|
|
|
|
|
|
2016-05-20 23:57:45 +00:00
|
|
|
static bool ignore_rlimit_data;
|
2016-02-03 00:57:43 +00:00
|
|
|
core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
|
mm: mmap: add new /proc tunable for mmap_base ASLR
Address Space Layout Randomization (ASLR) provides a barrier to
exploitation of user-space processes in the presence of security
vulnerabilities by making it more difficult to find desired code/data
which could help an attack. This is done by adding a random offset to
the location of regions in the process address space, with a greater
range of potential offset values corresponding to better protection/a
larger search-space for brute force, but also to greater potential for
fragmentation.
The offset added to the mmap_base address, which provides the basis for
the majority of the mappings for a process, is set once on process exec
in arch_pick_mmap_layout() and is done via hard-coded per-arch values,
which reflect, hopefully, the best compromise for all systems. The
trade-off between increased entropy in the offset value generation and
the corresponding increased variability in address space fragmentation
is not absolute, however, and some platforms may tolerate higher amounts
of entropy. This patch introduces both new Kconfig values and a sysctl
interface which may be used to change the amount of entropy used for
offset generation on a system.
The direct motivation for this change was in response to the
libstagefright vulnerabilities that affected Android, specifically to
information provided by Google's project zero at:
http://googleprojectzero.blogspot.com/2015/09/stagefrightened.html
The attack presented therein, by Google's project zero, specifically
targeted the limited randomness used to generate the offset added to the
mmap_base address in order to craft a brute-force-based attack.
Concretely, the attack was against the mediaserver process, which was
limited to respawning every 5 seconds, on an arm device. The hard-coded
8 bits used resulted in an average expected success rate of defeating
the mmap ASLR after just over 10 minutes (128 tries at 5 seconds a
piece). With this patch, and an accompanying increase in the entropy
value to 16 bits, the same attack would take an average expected time of
over 45 hours (32768 tries), which makes it both less feasible and more
likely to be noticed.
The introduced Kconfig and sysctl options are limited by per-arch
minimum and maximum values, the minimum of which was chosen to match the
current hard-coded value and the maximum of which was chosen so as to
give the greatest flexibility without generating an invalid mmap_base
address, generally a 3-4 bits less than the number of bits in the
user-space accessible virtual address space.
When decided whether or not to change the default value, a system
developer should consider that mmap_base address could be placed
anywhere up to 2^(value) bits away from the non-randomized location,
which would introduce variable-sized areas above and below the mmap_base
address such that the maximum vm_area_struct size may be reduced,
preventing very large allocations.
This patch (of 4):
ASLR only uses as few as 8 bits to generate the random offset for the
mmap base address on 32 bit architectures. This value was chosen to
prevent a poorly chosen value from dividing the address space in such a
way as to prevent large allocations. This may not be an issue on all
platforms. Allow the specification of a minimum number of bits so that
platforms desiring greater ASLR protection may determine where to place
the trade-off.
Signed-off-by: Daniel Cashman <dcashman@google.com>
Cc: Russell King <linux@arm.linux.org.uk>
Acked-by: Kees Cook <keescook@chromium.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: Don Zickus <dzickus@redhat.com>
Cc: Eric W. Biederman <ebiederm@xmission.com>
Cc: Heinrich Schuchardt <xypron.glpk@gmx.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Mark Salyzyn <salyzyn@android.com>
Cc: Jeff Vander Stoep <jeffv@google.com>
Cc: Nick Kralevich <nnk@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Hector Marco-Gisbert <hecmargi@upv.es>
Cc: Borislav Petkov <bp@suse.de>
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-01-14 23:19:53 +00:00
|
|
|
|
[PATCH] freepgt: free_pgtables use vma list
Recent woes with some arches needing their own pgd_addr_end macro; and 4-level
clear_page_range regression since 2.6.10's clear_page_tables; and its
long-standing well-known inefficiency in searching throughout the higher-level
page tables for those few entries to clear and free: all can be blamed on
ignoring the list of vmas when we free page tables.
Replace exit_mmap's clear_page_range of the total user address space by
free_pgtables operating on the mm's vma list; unmap_region use it in the same
way, giving floor and ceiling beyond which it may not free tables. This
brings lmbench fork/exec/sh numbers back to 2.6.10 (unless preempt is enabled,
in which case latency fixes spoil unmap_vmas throughput).
Beware: the do_mmap_pgoff driver failure case must now use unmap_region
instead of zap_page_range, since a page table might have been allocated, and
can only be freed while it is touched by some vma.
Move free_pgtables from mmap.c to memory.c, where its lower levels are adapted
from the clear_page_range levels. (Most of free_pgtables' old code was
actually for a non-existent case, prev not properly set up, dating from before
hch gave us split_vma.) Pass mmu_gather** in the public interfaces, since we
might want to add latency lockdrops later; but no attempt to do so yet, going
by vma should itself reduce latency.
But what if is_hugepage_only_range? Those ia64 and ppc64 cases need careful
examination: put that off until a later patch of the series.
What of x86_64's 32bit vdso page __map_syscall32 maps outside any vma?
And the range to sparc64's flush_tlb_pgtables? It's less clear to me now that
we need to do more than is done here - every PMD_SIZE ever occupied will be
flushed, do we really have to flush every PGDIR_SIZE ever partially occupied?
A shame to complicate it unnecessarily.
Special thanks to David Miller for time spent repairing my ceilings.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-04-19 20:29:15 +00:00
|
|
|
static void unmap_region(struct mm_struct *mm,
|
|
|
|
|
struct vm_area_struct *vma, struct vm_area_struct *prev,
|
|
|
|
|
unsigned long start, unsigned long end);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* description of effects of mapping type and prot in current implementation.
|
|
|
|
|
* this is due to the limited x86 page protection hardware. The expected
|
|
|
|
|
* behavior is in parens:
|
|
|
|
|
*
|
|
|
|
|
* map_type prot
|
|
|
|
|
* PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
|
|
|
|
|
* MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
|
|
|
|
|
* w: (no) no w: (no) no w: (yes) yes w: (no) no
|
|
|
|
|
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
|
2014-10-09 22:26:29 +00:00
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
|
|
|
|
|
* w: (no) no w: (no) no w: (copy) copy w: (no) no
|
|
|
|
|
* x: (no) no x: (no) yes x: (no) yes x: (yes) yes
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
pgprot_t protection_map[16] = {
|
|
|
|
|
__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
|
|
|
|
|
__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
|
|
|
|
|
};
|
|
|
|
|
|
2006-07-26 20:39:49 +00:00
|
|
|
pgprot_t vm_get_page_prot(unsigned long vm_flags)
|
|
|
|
|
{
|
2008-07-07 14:28:51 +00:00
|
|
|
return __pgprot(pgprot_val(protection_map[vm_flags &
|
|
|
|
|
(VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
|
|
|
|
|
pgprot_val(arch_vm_get_page_prot(vm_flags)));
|
2006-07-26 20:39:49 +00:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(vm_get_page_prot);
|
|
|
|
|
|
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set. If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared. Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect. An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").
Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:46 +00:00
|
|
|
static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
|
|
|
|
|
{
|
|
|
|
|
return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Update vma->vm_page_prot to reflect vma->vm_flags. */
|
|
|
|
|
void vma_set_page_prot(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
unsigned long vm_flags = vma->vm_flags;
|
|
|
|
|
|
|
|
|
|
vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
|
|
|
|
|
if (vma_wants_writenotify(vma)) {
|
|
|
|
|
vm_flags &= ~VM_SHARED;
|
|
|
|
|
vma->vm_page_prot = vm_pgprot_modify(vma->vm_page_prot,
|
|
|
|
|
vm_flags);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2014-12-13 00:54:24 +00:00
|
|
|
* Requires inode->i_mapping->i_mmap_rwsem
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
static void __remove_shared_vm_struct(struct vm_area_struct *vma,
|
|
|
|
|
struct file *file, struct address_space *mapping)
|
|
|
|
|
{
|
|
|
|
|
if (vma->vm_flags & VM_DENYWRITE)
|
2013-01-23 22:07:38 +00:00
|
|
|
atomic_inc(&file_inode(file)->i_writecount);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vma->vm_flags & VM_SHARED)
|
2014-08-08 21:25:25 +00:00
|
|
|
mapping_unmap_writable(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
flush_dcache_mmap_lock(mapping);
|
2015-02-10 22:09:59 +00:00
|
|
|
vma_interval_tree_remove(vma, &mapping->i_mmap);
|
2005-04-16 22:20:36 +00:00
|
|
|
flush_dcache_mmap_unlock(mapping);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
2012-10-08 23:31:25 +00:00
|
|
|
* Unlink a file-based vm structure from its interval tree, to hide
|
2005-10-30 01:15:57 +00:00
|
|
|
* vma from rmap and vmtruncate before freeing its page tables.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2005-10-30 01:15:57 +00:00
|
|
|
void unlink_file_vma(struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
struct file *file = vma->vm_file;
|
|
|
|
|
|
|
|
|
|
if (file) {
|
|
|
|
|
struct address_space *mapping = file->f_mapping;
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_lock_write(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
__remove_shared_vm_struct(vma, file, mapping);
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_unlock_write(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-10-30 01:15:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Close a vm structure and free it, returning the next.
|
|
|
|
|
*/
|
|
|
|
|
static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *next = vma->vm_next;
|
|
|
|
|
|
|
|
|
|
might_sleep();
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vma->vm_ops && vma->vm_ops->close)
|
|
|
|
|
vma->vm_ops->close(vma);
|
2012-10-08 23:28:54 +00:00
|
|
|
if (vma->vm_file)
|
2005-10-30 01:15:57 +00:00
|
|
|
fput(vma->vm_file);
|
2008-04-28 09:13:08 +00:00
|
|
|
mpol_put(vma_policy(vma));
|
2005-04-16 22:20:36 +00:00
|
|
|
kmem_cache_free(vm_area_cachep, vma);
|
2005-10-30 01:15:57 +00:00
|
|
|
return next;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2016-05-27 22:57:31 +00:00
|
|
|
static int do_brk(unsigned long addr, unsigned long len);
|
2012-04-20 22:35:40 +00:00
|
|
|
|
2009-01-14 13:14:15 +00:00
|
|
|
SYSCALL_DEFINE1(brk, unsigned long, brk)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:27:32 +00:00
|
|
|
unsigned long retval;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long newbrk, oldbrk;
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
2008-06-06 05:46:05 +00:00
|
|
|
unsigned long min_brk;
|
2013-02-23 00:32:40 +00:00
|
|
|
bool populate;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-23 23:25:27 +00:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
|
return -EINTR;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-06-06 05:46:05 +00:00
|
|
|
#ifdef CONFIG_COMPAT_BRK
|
2011-01-13 23:47:23 +00:00
|
|
|
/*
|
|
|
|
|
* CONFIG_COMPAT_BRK can still be overridden by setting
|
|
|
|
|
* randomize_va_space to 2, which will still cause mm->start_brk
|
|
|
|
|
* to be arbitrarily shifted
|
|
|
|
|
*/
|
2011-04-14 22:22:09 +00:00
|
|
|
if (current->brk_randomized)
|
2011-01-13 23:47:23 +00:00
|
|
|
min_brk = mm->start_brk;
|
|
|
|
|
else
|
|
|
|
|
min_brk = mm->end_data;
|
2008-06-06 05:46:05 +00:00
|
|
|
#else
|
|
|
|
|
min_brk = mm->start_brk;
|
|
|
|
|
#endif
|
|
|
|
|
if (brk < min_brk)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
2006-04-11 05:52:57 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Check against rlimit here. If this check is done later after the test
|
|
|
|
|
* of oldbrk with newbrk then it can escape the test and let the data
|
|
|
|
|
* segment grow beyond its set limit the in case where the limit is
|
|
|
|
|
* not page aligned -Ram Gupta
|
|
|
|
|
*/
|
2014-10-09 22:27:32 +00:00
|
|
|
if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
|
|
|
|
|
mm->end_data, mm->start_data))
|
2006-04-11 05:52:57 +00:00
|
|
|
goto out;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
newbrk = PAGE_ALIGN(brk);
|
|
|
|
|
oldbrk = PAGE_ALIGN(mm->brk);
|
|
|
|
|
if (oldbrk == newbrk)
|
|
|
|
|
goto set_brk;
|
|
|
|
|
|
|
|
|
|
/* Always allow shrinking brk. */
|
|
|
|
|
if (brk <= mm->brk) {
|
|
|
|
|
if (!do_munmap(mm, newbrk, oldbrk-newbrk))
|
|
|
|
|
goto set_brk;
|
|
|
|
|
goto out;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Check against existing mmap mappings. */
|
|
|
|
|
if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
/* Ok, looks good - let it rip. */
|
2016-05-27 22:57:31 +00:00
|
|
|
if (do_brk(oldbrk, newbrk-oldbrk) < 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
2013-02-23 00:32:40 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
set_brk:
|
|
|
|
|
mm->brk = brk;
|
2013-02-23 00:32:40 +00:00
|
|
|
populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
|
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
if (populate)
|
|
|
|
|
mm_populate(oldbrk, newbrk - oldbrk);
|
|
|
|
|
return brk;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
out:
|
|
|
|
|
retval = mm->brk;
|
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
return retval;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:38 +00:00
|
|
|
static long vma_compute_subtree_gap(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
unsigned long max, subtree_gap;
|
|
|
|
|
max = vma->vm_start;
|
|
|
|
|
if (vma->vm_prev)
|
|
|
|
|
max -= vma->vm_prev->vm_end;
|
|
|
|
|
if (vma->vm_rb.rb_left) {
|
|
|
|
|
subtree_gap = rb_entry(vma->vm_rb.rb_left,
|
|
|
|
|
struct vm_area_struct, vm_rb)->rb_subtree_gap;
|
|
|
|
|
if (subtree_gap > max)
|
|
|
|
|
max = subtree_gap;
|
|
|
|
|
}
|
|
|
|
|
if (vma->vm_rb.rb_right) {
|
|
|
|
|
subtree_gap = rb_entry(vma->vm_rb.rb_right,
|
|
|
|
|
struct vm_area_struct, vm_rb)->rb_subtree_gap;
|
|
|
|
|
if (subtree_gap > max)
|
|
|
|
|
max = subtree_gap;
|
|
|
|
|
}
|
|
|
|
|
return max;
|
|
|
|
|
}
|
|
|
|
|
|
2012-10-08 23:31:45 +00:00
|
|
|
#ifdef CONFIG_DEBUG_VM_RB
|
2016-02-05 23:36:13 +00:00
|
|
|
static int browse_rb(struct mm_struct *mm)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-02-05 23:36:13 +00:00
|
|
|
struct rb_root *root = &mm->mm_rb;
|
2012-12-12 00:01:42 +00:00
|
|
|
int i = 0, j, bug = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct rb_node *nd, *pn = NULL;
|
|
|
|
|
unsigned long prev = 0, pend = 0;
|
|
|
|
|
|
|
|
|
|
for (nd = rb_first(root); nd; nd = rb_next(nd)) {
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
vma = rb_entry(nd, struct vm_area_struct, vm_rb);
|
2012-12-12 00:01:42 +00:00
|
|
|
if (vma->vm_start < prev) {
|
2014-10-09 22:28:19 +00:00
|
|
|
pr_emerg("vm_start %lx < prev %lx\n",
|
|
|
|
|
vma->vm_start, prev);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
|
|
|
|
if (vma->vm_start < pend) {
|
2014-10-09 22:28:19 +00:00
|
|
|
pr_emerg("vm_start %lx < pend %lx\n",
|
|
|
|
|
vma->vm_start, pend);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
|
|
|
|
if (vma->vm_start > vma->vm_end) {
|
2014-10-09 22:28:19 +00:00
|
|
|
pr_emerg("vm_start %lx > vm_end %lx\n",
|
|
|
|
|
vma->vm_start, vma->vm_end);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
2016-02-05 23:36:13 +00:00
|
|
|
spin_lock(&mm->page_table_lock);
|
2012-12-12 00:01:42 +00:00
|
|
|
if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
|
2014-09-09 21:50:59 +00:00
|
|
|
pr_emerg("free gap %lx, correct %lx\n",
|
2012-12-12 00:01:42 +00:00
|
|
|
vma->rb_subtree_gap,
|
|
|
|
|
vma_compute_subtree_gap(vma));
|
|
|
|
|
bug = 1;
|
|
|
|
|
}
|
2016-02-05 23:36:13 +00:00
|
|
|
spin_unlock(&mm->page_table_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
i++;
|
|
|
|
|
pn = nd;
|
2007-03-01 04:13:13 +00:00
|
|
|
prev = vma->vm_start;
|
|
|
|
|
pend = vma->vm_end;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
j = 0;
|
2012-12-12 00:01:42 +00:00
|
|
|
for (nd = pn; nd; nd = rb_prev(nd))
|
2005-04-16 22:20:36 +00:00
|
|
|
j++;
|
2012-12-12 00:01:42 +00:00
|
|
|
if (i != j) {
|
2014-09-09 21:50:59 +00:00
|
|
|
pr_emerg("backwards %d, forwards %d\n", j, i);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2012-12-12 00:01:42 +00:00
|
|
|
return bug ? -1 : i;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:38 +00:00
|
|
|
static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
|
|
|
|
|
{
|
|
|
|
|
struct rb_node *nd;
|
|
|
|
|
|
|
|
|
|
for (nd = rb_first(root); nd; nd = rb_next(nd)) {
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
vma = rb_entry(nd, struct vm_area_struct, vm_rb);
|
2014-10-09 22:28:39 +00:00
|
|
|
VM_BUG_ON_VMA(vma != ignore &&
|
|
|
|
|
vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
|
|
|
|
|
vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-04-03 21:48:03 +00:00
|
|
|
static void validate_mm(struct mm_struct *mm)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
int bug = 0;
|
|
|
|
|
int i = 0;
|
2012-12-12 00:01:42 +00:00
|
|
|
unsigned long highest_address = 0;
|
2012-10-08 23:31:45 +00:00
|
|
|
struct vm_area_struct *vma = mm->mmap;
|
2014-10-09 22:28:19 +00:00
|
|
|
|
2012-10-08 23:31:45 +00:00
|
|
|
while (vma) {
|
2016-02-05 23:36:50 +00:00
|
|
|
struct anon_vma *anon_vma = vma->anon_vma;
|
2012-10-08 23:31:45 +00:00
|
|
|
struct anon_vma_chain *avc;
|
2014-10-09 22:28:19 +00:00
|
|
|
|
2016-02-05 23:36:50 +00:00
|
|
|
if (anon_vma) {
|
|
|
|
|
anon_vma_lock_read(anon_vma);
|
|
|
|
|
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
|
|
|
|
anon_vma_interval_tree_verify(avc);
|
|
|
|
|
anon_vma_unlock_read(anon_vma);
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:42 +00:00
|
|
|
highest_address = vma->vm_end;
|
2012-10-08 23:31:45 +00:00
|
|
|
vma = vma->vm_next;
|
2005-04-16 22:20:36 +00:00
|
|
|
i++;
|
|
|
|
|
}
|
2012-12-12 00:01:42 +00:00
|
|
|
if (i != mm->map_count) {
|
2014-09-09 21:50:59 +00:00
|
|
|
pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
|
|
|
|
if (highest_address != mm->highest_vm_end) {
|
2014-09-09 21:50:59 +00:00
|
|
|
pr_emerg("mm->highest_vm_end %lx, found %lx\n",
|
2014-10-09 22:28:19 +00:00
|
|
|
mm->highest_vm_end, highest_address);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
2016-02-05 23:36:13 +00:00
|
|
|
i = browse_rb(mm);
|
2012-12-12 00:01:42 +00:00
|
|
|
if (i != mm->map_count) {
|
2014-10-09 22:28:19 +00:00
|
|
|
if (i != -1)
|
|
|
|
|
pr_emerg("map_count %d rb %d\n", mm->map_count, i);
|
2012-12-12 00:01:42 +00:00
|
|
|
bug = 1;
|
|
|
|
|
}
|
2014-10-09 22:28:39 +00:00
|
|
|
VM_BUG_ON_MM(bug, mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
#else
|
2012-12-12 00:01:38 +00:00
|
|
|
#define validate_mm_rb(root, ignore) do { } while (0)
|
2005-04-16 22:20:36 +00:00
|
|
|
#define validate_mm(mm) do { } while (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
2012-12-12 00:01:38 +00:00
|
|
|
RB_DECLARE_CALLBACKS(static, vma_gap_callbacks, struct vm_area_struct, vm_rb,
|
|
|
|
|
unsigned long, rb_subtree_gap, vma_compute_subtree_gap)
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Update augmented rbtree rb_subtree_gap values after vma->vm_start or
|
|
|
|
|
* vma->vm_prev->vm_end values changed, without modifying the vma's position
|
|
|
|
|
* in the rbtree.
|
|
|
|
|
*/
|
|
|
|
|
static void vma_gap_update(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* As it turns out, RB_DECLARE_CALLBACKS() already created a callback
|
|
|
|
|
* function that does exacltly what we want.
|
|
|
|
|
*/
|
|
|
|
|
vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void vma_rb_insert(struct vm_area_struct *vma,
|
|
|
|
|
struct rb_root *root)
|
|
|
|
|
{
|
|
|
|
|
/* All rb_subtree_gap values must be consistent prior to insertion */
|
|
|
|
|
validate_mm_rb(root, NULL);
|
|
|
|
|
|
|
|
|
|
rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
|
|
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* All rb_subtree_gap values must be consistent prior to erase,
|
|
|
|
|
* with the possible exception of the vma being erased.
|
|
|
|
|
*/
|
|
|
|
|
validate_mm_rb(root, vma);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Note rb_erase_augmented is a fairly large inline function,
|
|
|
|
|
* so make sure we instantiate it only once with our desired
|
|
|
|
|
* augmented rbtree callbacks.
|
|
|
|
|
*/
|
|
|
|
|
rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
|
|
|
|
|
}
|
|
|
|
|
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
/*
|
|
|
|
|
* vma has some anon_vma assigned, and is already inserted on that
|
|
|
|
|
* anon_vma's interval trees.
|
|
|
|
|
*
|
|
|
|
|
* Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
|
|
|
|
|
* vma must be removed from the anon_vma's interval trees using
|
|
|
|
|
* anon_vma_interval_tree_pre_update_vma().
|
|
|
|
|
*
|
|
|
|
|
* After the update, the vma will be reinserted using
|
|
|
|
|
* anon_vma_interval_tree_post_update_vma().
|
|
|
|
|
*
|
|
|
|
|
* The entire update must be protected by exclusive mmap_sem and by
|
|
|
|
|
* the root anon_vma's mutex.
|
|
|
|
|
*/
|
|
|
|
|
static inline void
|
|
|
|
|
anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
struct anon_vma_chain *avc;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
|
|
|
|
anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
|
anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
struct anon_vma_chain *avc;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
|
|
|
|
anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
|
|
|
|
|
}
|
|
|
|
|
|
2012-10-08 23:29:07 +00:00
|
|
|
static int find_vma_links(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
unsigned long end, struct vm_area_struct **pprev,
|
|
|
|
|
struct rb_node ***rb_link, struct rb_node **rb_parent)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-10-08 23:29:07 +00:00
|
|
|
struct rb_node **__rb_link, *__rb_parent, *rb_prev;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
__rb_link = &mm->mm_rb.rb_node;
|
|
|
|
|
rb_prev = __rb_parent = NULL;
|
|
|
|
|
|
|
|
|
|
while (*__rb_link) {
|
|
|
|
|
struct vm_area_struct *vma_tmp;
|
|
|
|
|
|
|
|
|
|
__rb_parent = *__rb_link;
|
|
|
|
|
vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
|
|
|
|
|
|
|
|
|
|
if (vma_tmp->vm_end > addr) {
|
2012-10-08 23:29:07 +00:00
|
|
|
/* Fail if an existing vma overlaps the area */
|
|
|
|
|
if (vma_tmp->vm_start < end)
|
|
|
|
|
return -ENOMEM;
|
2005-04-16 22:20:36 +00:00
|
|
|
__rb_link = &__rb_parent->rb_left;
|
|
|
|
|
} else {
|
|
|
|
|
rb_prev = __rb_parent;
|
|
|
|
|
__rb_link = &__rb_parent->rb_right;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
*pprev = NULL;
|
|
|
|
|
if (rb_prev)
|
|
|
|
|
*pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
|
|
|
|
|
*rb_link = __rb_link;
|
|
|
|
|
*rb_parent = __rb_parent;
|
2012-10-08 23:29:07 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2013-04-29 22:08:33 +00:00
|
|
|
static unsigned long count_vma_pages_range(struct mm_struct *mm,
|
|
|
|
|
unsigned long addr, unsigned long end)
|
|
|
|
|
{
|
|
|
|
|
unsigned long nr_pages = 0;
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
|
|
|
|
|
/* Find first overlaping mapping */
|
|
|
|
|
vma = find_vma_intersection(mm, addr, end);
|
|
|
|
|
if (!vma)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
nr_pages = (min(end, vma->vm_end) -
|
|
|
|
|
max(addr, vma->vm_start)) >> PAGE_SHIFT;
|
|
|
|
|
|
|
|
|
|
/* Iterate over the rest of the overlaps */
|
|
|
|
|
for (vma = vma->vm_next; vma; vma = vma->vm_next) {
|
|
|
|
|
unsigned long overlap_len;
|
|
|
|
|
|
|
|
|
|
if (vma->vm_start > end)
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
overlap_len = min(end, vma->vm_end) - vma->vm_start;
|
|
|
|
|
nr_pages += overlap_len >> PAGE_SHIFT;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return nr_pages;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
struct rb_node **rb_link, struct rb_node *rb_parent)
|
|
|
|
|
{
|
2012-12-12 00:01:38 +00:00
|
|
|
/* Update tracking information for the gap following the new vma. */
|
|
|
|
|
if (vma->vm_next)
|
|
|
|
|
vma_gap_update(vma->vm_next);
|
|
|
|
|
else
|
|
|
|
|
mm->highest_vm_end = vma->vm_end;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* vma->vm_prev wasn't known when we followed the rbtree to find the
|
|
|
|
|
* correct insertion point for that vma. As a result, we could not
|
|
|
|
|
* update the vma vm_rb parents rb_subtree_gap values on the way down.
|
|
|
|
|
* So, we first insert the vma with a zero rb_subtree_gap value
|
|
|
|
|
* (to be consistent with what we did on the way down), and then
|
|
|
|
|
* immediately update the gap to the correct value. Finally we
|
|
|
|
|
* rebalance the rbtree after all augmented values have been set.
|
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
rb_link_node(&vma->vm_rb, rb_parent, rb_link);
|
2012-12-12 00:01:38 +00:00
|
|
|
vma->rb_subtree_gap = 0;
|
|
|
|
|
vma_gap_update(vma);
|
|
|
|
|
vma_rb_insert(vma, &mm->mm_rb);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2008-10-19 03:27:01 +00:00
|
|
|
static void __vma_link_file(struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-01-06 22:40:21 +00:00
|
|
|
struct file *file;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
file = vma->vm_file;
|
|
|
|
|
if (file) {
|
|
|
|
|
struct address_space *mapping = file->f_mapping;
|
|
|
|
|
|
|
|
|
|
if (vma->vm_flags & VM_DENYWRITE)
|
2013-01-23 22:07:38 +00:00
|
|
|
atomic_dec(&file_inode(file)->i_writecount);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vma->vm_flags & VM_SHARED)
|
2014-08-08 21:25:25 +00:00
|
|
|
atomic_inc(&mapping->i_mmap_writable);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
flush_dcache_mmap_lock(mapping);
|
2015-02-10 22:09:59 +00:00
|
|
|
vma_interval_tree_insert(vma, &mapping->i_mmap);
|
2005-04-16 22:20:36 +00:00
|
|
|
flush_dcache_mmap_unlock(mapping);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void
|
|
|
|
|
__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
struct vm_area_struct *prev, struct rb_node **rb_link,
|
|
|
|
|
struct rb_node *rb_parent)
|
|
|
|
|
{
|
|
|
|
|
__vma_link_list(mm, vma, prev, rb_parent);
|
|
|
|
|
__vma_link_rb(mm, vma, rb_link, rb_parent);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
struct vm_area_struct *prev, struct rb_node **rb_link,
|
|
|
|
|
struct rb_node *rb_parent)
|
|
|
|
|
{
|
|
|
|
|
struct address_space *mapping = NULL;
|
|
|
|
|
|
2014-06-04 23:07:33 +00:00
|
|
|
if (vma->vm_file) {
|
2005-04-16 22:20:36 +00:00
|
|
|
mapping = vma->vm_file->f_mapping;
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_lock_write(mapping);
|
2014-06-04 23:07:33 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
__vma_link(mm, vma, prev, rb_link, rb_parent);
|
|
|
|
|
__vma_link_file(vma);
|
|
|
|
|
|
|
|
|
|
if (mapping)
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_unlock_write(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
mm->map_count++;
|
|
|
|
|
validate_mm(mm);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
2012-03-21 23:34:16 +00:00
|
|
|
* Helper for vma_adjust() in the split_vma insert case: insert a vma into the
|
2012-10-08 23:31:25 +00:00
|
|
|
* mm's list and rbtree. It has already been inserted into the interval tree.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2009-01-06 22:40:21 +00:00
|
|
|
static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-10-08 23:29:07 +00:00
|
|
|
struct vm_area_struct *prev;
|
2009-01-06 22:40:21 +00:00
|
|
|
struct rb_node **rb_link, *rb_parent;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-10-08 23:29:07 +00:00
|
|
|
if (find_vma_links(mm, vma->vm_start, vma->vm_end,
|
|
|
|
|
&prev, &rb_link, &rb_parent))
|
|
|
|
|
BUG();
|
2005-04-16 22:20:36 +00:00
|
|
|
__vma_link(mm, vma, prev, rb_link, rb_parent);
|
|
|
|
|
mm->map_count++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
|
__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
struct vm_area_struct *prev)
|
|
|
|
|
{
|
2012-12-12 00:01:38 +00:00
|
|
|
struct vm_area_struct *next;
|
2010-08-20 23:24:55 +00:00
|
|
|
|
2012-12-12 00:01:38 +00:00
|
|
|
vma_rb_erase(vma, &mm->mm_rb);
|
|
|
|
|
prev->vm_next = next = vma->vm_next;
|
2010-08-20 23:24:55 +00:00
|
|
|
if (next)
|
|
|
|
|
next->vm_prev = prev;
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
|
|
|
|
|
/* Kill the cache */
|
|
|
|
|
vmacache_invalidate(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
|
|
|
|
|
* is already present in an i_mmap tree without adjusting the tree.
|
|
|
|
|
* The following helper function should be used when such adjustments
|
|
|
|
|
* are necessary. The "insert" vma (if any) is to be inserted
|
|
|
|
|
* before we drop the necessary locks.
|
|
|
|
|
*/
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
int vma_adjust(struct vm_area_struct *vma, unsigned long start,
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
|
|
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
|
|
|
struct vm_area_struct *next = vma->vm_next;
|
|
|
|
|
struct vm_area_struct *importer = NULL;
|
|
|
|
|
struct address_space *mapping = NULL;
|
2012-10-08 23:31:25 +00:00
|
|
|
struct rb_root *root = NULL;
|
2010-08-10 00:18:40 +00:00
|
|
|
struct anon_vma *anon_vma = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct file *file = vma->vm_file;
|
2012-12-12 00:01:38 +00:00
|
|
|
bool start_changed = false, end_changed = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
long adjust_next = 0;
|
|
|
|
|
int remove_next = 0;
|
|
|
|
|
|
|
|
|
|
if (next && !insert) {
|
2010-04-10 22:22:30 +00:00
|
|
|
struct vm_area_struct *exporter = NULL;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (end >= next->vm_end) {
|
|
|
|
|
/*
|
|
|
|
|
* vma expands, overlapping all the next, and
|
|
|
|
|
* perhaps the one after too (mprotect case 6).
|
|
|
|
|
*/
|
|
|
|
|
again: remove_next = 1 + (end > next->vm_end);
|
|
|
|
|
end = next->vm_end;
|
2010-04-10 22:22:30 +00:00
|
|
|
exporter = next;
|
2005-04-16 22:20:36 +00:00
|
|
|
importer = vma;
|
|
|
|
|
} else if (end > next->vm_start) {
|
|
|
|
|
/*
|
|
|
|
|
* vma expands, overlapping part of the next:
|
|
|
|
|
* mprotect case 5 shifting the boundary up.
|
|
|
|
|
*/
|
|
|
|
|
adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
|
2010-04-10 22:22:30 +00:00
|
|
|
exporter = next;
|
2005-04-16 22:20:36 +00:00
|
|
|
importer = vma;
|
|
|
|
|
} else if (end < vma->vm_end) {
|
|
|
|
|
/*
|
|
|
|
|
* vma shrinks, and !insert tells it's not
|
|
|
|
|
* split_vma inserting another: so it must be
|
|
|
|
|
* mprotect case 4 shifting the boundary down.
|
|
|
|
|
*/
|
2014-10-09 22:26:29 +00:00
|
|
|
adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
|
2010-04-10 22:22:30 +00:00
|
|
|
exporter = vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
importer = next;
|
|
|
|
|
}
|
|
|
|
|
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
/*
|
|
|
|
|
* Easily overlooked: when mprotect shifts the boundary,
|
|
|
|
|
* make sure the expanding vma has anon_vma set if the
|
|
|
|
|
* shrinking vma had, to cover any anon pages imported.
|
|
|
|
|
*/
|
2010-04-10 22:22:30 +00:00
|
|
|
if (exporter && exporter->anon_vma && !importer->anon_vma) {
|
2014-12-02 23:59:42 +00:00
|
|
|
int error;
|
|
|
|
|
|
2015-01-11 13:54:06 +00:00
|
|
|
importer->anon_vma = exporter->anon_vma;
|
2014-12-02 23:59:42 +00:00
|
|
|
error = anon_vma_clone(importer, exporter);
|
2015-03-25 22:55:11 +00:00
|
|
|
if (error)
|
2014-12-02 23:59:42 +00:00
|
|
|
return error;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-07-26 22:25:48 +00:00
|
|
|
vma_adjust_trans_huge(vma, start, end, adjust_next);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (file) {
|
|
|
|
|
mapping = file->f_mapping;
|
2015-02-10 22:09:59 +00:00
|
|
|
root = &mapping->i_mmap;
|
|
|
|
|
uprobe_munmap(vma, vma->vm_start, vma->vm_end);
|
2012-03-30 18:26:46 +00:00
|
|
|
|
2015-02-10 22:09:59 +00:00
|
|
|
if (adjust_next)
|
|
|
|
|
uprobe_munmap(next, next->vm_start, next->vm_end);
|
2012-03-30 18:26:46 +00:00
|
|
|
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_lock_write(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (insert) {
|
|
|
|
|
/*
|
2012-10-08 23:31:25 +00:00
|
|
|
* Put into interval tree now, so instantiated pages
|
2005-04-16 22:20:36 +00:00
|
|
|
* are visible to arm/parisc __flush_dcache_page
|
|
|
|
|
* throughout; but we cannot insert into address
|
|
|
|
|
* space until vma start or end is updated.
|
|
|
|
|
*/
|
|
|
|
|
__vma_link_file(insert);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma = vma->anon_vma;
|
|
|
|
|
if (!anon_vma && adjust_next)
|
|
|
|
|
anon_vma = next->anon_vma;
|
|
|
|
|
if (anon_vma) {
|
2014-10-09 22:28:10 +00:00
|
|
|
VM_BUG_ON_VMA(adjust_next && next->anon_vma &&
|
|
|
|
|
anon_vma != next->anon_vma, next);
|
2012-12-02 19:56:50 +00:00
|
|
|
anon_vma_lock_write(anon_vma);
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma_interval_tree_pre_update_vma(vma);
|
|
|
|
|
if (adjust_next)
|
|
|
|
|
anon_vma_interval_tree_pre_update_vma(next);
|
|
|
|
|
}
|
2010-08-10 00:18:40 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (root) {
|
|
|
|
|
flush_dcache_mmap_lock(mapping);
|
2012-10-08 23:31:25 +00:00
|
|
|
vma_interval_tree_remove(vma, root);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (adjust_next)
|
2012-10-08 23:31:25 +00:00
|
|
|
vma_interval_tree_remove(next, root);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:38 +00:00
|
|
|
if (start != vma->vm_start) {
|
|
|
|
|
vma->vm_start = start;
|
|
|
|
|
start_changed = true;
|
|
|
|
|
}
|
|
|
|
|
if (end != vma->vm_end) {
|
|
|
|
|
vma->vm_end = end;
|
|
|
|
|
end_changed = true;
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
vma->vm_pgoff = pgoff;
|
|
|
|
|
if (adjust_next) {
|
|
|
|
|
next->vm_start += adjust_next << PAGE_SHIFT;
|
|
|
|
|
next->vm_pgoff += adjust_next;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (root) {
|
|
|
|
|
if (adjust_next)
|
2012-10-08 23:31:25 +00:00
|
|
|
vma_interval_tree_insert(next, root);
|
|
|
|
|
vma_interval_tree_insert(vma, root);
|
2005-04-16 22:20:36 +00:00
|
|
|
flush_dcache_mmap_unlock(mapping);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (remove_next) {
|
|
|
|
|
/*
|
|
|
|
|
* vma_merge has merged next into vma, and needs
|
|
|
|
|
* us to remove next before dropping the locks.
|
|
|
|
|
*/
|
|
|
|
|
__vma_unlink(mm, next, vma);
|
|
|
|
|
if (file)
|
|
|
|
|
__remove_shared_vm_struct(next, file, mapping);
|
|
|
|
|
} else if (insert) {
|
|
|
|
|
/*
|
|
|
|
|
* split_vma has split insert from vma, and needs
|
|
|
|
|
* us to insert it before dropping the locks
|
|
|
|
|
* (it may either follow vma or precede it).
|
|
|
|
|
*/
|
|
|
|
|
__insert_vm_struct(mm, insert);
|
2012-12-12 00:01:38 +00:00
|
|
|
} else {
|
|
|
|
|
if (start_changed)
|
|
|
|
|
vma_gap_update(vma);
|
|
|
|
|
if (end_changed) {
|
|
|
|
|
if (!next)
|
|
|
|
|
mm->highest_vm_end = end;
|
|
|
|
|
else if (!adjust_next)
|
|
|
|
|
vma_gap_update(next);
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
if (anon_vma) {
|
|
|
|
|
anon_vma_interval_tree_post_update_vma(vma);
|
|
|
|
|
if (adjust_next)
|
|
|
|
|
anon_vma_interval_tree_post_update_vma(next);
|
2013-02-23 00:34:40 +00:00
|
|
|
anon_vma_unlock_write(anon_vma);
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
if (mapping)
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_unlock_write(mapping);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
if (root) {
|
2012-02-17 08:27:41 +00:00
|
|
|
uprobe_mmap(vma);
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
|
|
|
|
|
if (adjust_next)
|
2012-02-17 08:27:41 +00:00
|
|
|
uprobe_mmap(next);
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (remove_next) {
|
2008-04-29 08:01:36 +00:00
|
|
|
if (file) {
|
2012-04-11 10:35:27 +00:00
|
|
|
uprobe_munmap(next, next->vm_start, next->vm_end);
|
2005-04-16 22:20:36 +00:00
|
|
|
fput(file);
|
2008-04-29 08:01:36 +00:00
|
|
|
}
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
if (next->anon_vma)
|
|
|
|
|
anon_vma_merge(vma, next);
|
2005-04-16 22:20:36 +00:00
|
|
|
mm->map_count--;
|
2013-07-31 20:53:28 +00:00
|
|
|
mpol_put(vma_policy(next));
|
2005-04-16 22:20:36 +00:00
|
|
|
kmem_cache_free(vm_area_cachep, next);
|
|
|
|
|
/*
|
|
|
|
|
* In mprotect's case 6 (see comments on vma_merge),
|
|
|
|
|
* we must remove another next too. It would clutter
|
|
|
|
|
* up the code too much to do both in one go.
|
|
|
|
|
*/
|
2012-12-12 00:01:38 +00:00
|
|
|
next = vma->vm_next;
|
|
|
|
|
if (remove_next == 2)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto again;
|
2012-12-12 00:01:38 +00:00
|
|
|
else if (next)
|
|
|
|
|
vma_gap_update(next);
|
|
|
|
|
else
|
|
|
|
|
mm->highest_vm_end = end;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
if (insert && file)
|
2012-02-17 08:27:41 +00:00
|
|
|
uprobe_mmap(insert);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
validate_mm(mm);
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
|
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If the vma has a ->close operation then the driver probably needs to release
|
|
|
|
|
* per-vma resources, so we don't attempt to merge those.
|
|
|
|
|
*/
|
|
|
|
|
static inline int is_mergeable_vma(struct vm_area_struct *vma,
|
2015-09-04 22:46:24 +00:00
|
|
|
struct file *file, unsigned long vm_flags,
|
|
|
|
|
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-01-23 23:53:42 +00:00
|
|
|
/*
|
|
|
|
|
* VM_SOFTDIRTY should not prevent from VMA merging, if we
|
|
|
|
|
* match the flags but dirty bit -- the caller should mark
|
|
|
|
|
* merged VMA as dirty. If dirty bit won't be excluded from
|
|
|
|
|
* comparison, we increase pressue on the memory system forcing
|
|
|
|
|
* the kernel to generate new VMAs when old one could be
|
|
|
|
|
* extended instead.
|
|
|
|
|
*/
|
|
|
|
|
if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
if (vma->vm_file != file)
|
|
|
|
|
return 0;
|
|
|
|
|
if (vma->vm_ops && vma->vm_ops->close)
|
|
|
|
|
return 0;
|
2015-09-04 22:46:24 +00:00
|
|
|
if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
|
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
|
2011-05-25 00:11:20 +00:00
|
|
|
struct anon_vma *anon_vma2,
|
|
|
|
|
struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-05-25 00:11:20 +00:00
|
|
|
/*
|
|
|
|
|
* The list_is_singular() test is to avoid merging VMA cloned from
|
|
|
|
|
* parents. This can improve scalability caused by anon_vma lock.
|
|
|
|
|
*/
|
|
|
|
|
if ((!anon_vma1 || !anon_vma2) && (!vma ||
|
|
|
|
|
list_is_singular(&vma->anon_vma_chain)))
|
|
|
|
|
return 1;
|
|
|
|
|
return anon_vma1 == anon_vma2;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
|
|
|
|
|
* in front of (at a lower virtual address and file offset than) the vma.
|
|
|
|
|
*
|
|
|
|
|
* We cannot merge two vmas if they have differently assigned (non-NULL)
|
|
|
|
|
* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
|
|
|
|
|
*
|
|
|
|
|
* We don't check here for the merged mmap wrapping around the end of pagecache
|
|
|
|
|
* indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
|
|
|
|
|
* wrap, nor mmaps which cover the final page at index -1UL.
|
|
|
|
|
*/
|
|
|
|
|
static int
|
|
|
|
|
can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
struct anon_vma *anon_vma, struct file *file,
|
|
|
|
|
pgoff_t vm_pgoff,
|
|
|
|
|
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-09-04 22:46:24 +00:00
|
|
|
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
|
2011-05-25 00:11:20 +00:00
|
|
|
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vma->vm_pgoff == vm_pgoff)
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
|
|
|
|
|
* beyond (at a higher virtual address and file offset than) the vma.
|
|
|
|
|
*
|
|
|
|
|
* We cannot merge two vmas if they have differently assigned (non-NULL)
|
|
|
|
|
* anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
|
|
|
|
|
*/
|
|
|
|
|
static int
|
|
|
|
|
can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
struct anon_vma *anon_vma, struct file *file,
|
|
|
|
|
pgoff_t vm_pgoff,
|
|
|
|
|
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-09-04 22:46:24 +00:00
|
|
|
if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
|
2011-05-25 00:11:20 +00:00
|
|
|
is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
pgoff_t vm_pglen;
|
2013-07-03 22:01:26 +00:00
|
|
|
vm_pglen = vma_pages(vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vma->vm_pgoff + vm_pglen == vm_pgoff)
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
|
|
|
|
|
* whether that can be merged with its predecessor or its successor.
|
|
|
|
|
* Or both (it neatly fills a hole).
|
|
|
|
|
*
|
|
|
|
|
* In most cases - when called for mmap, brk or mremap - [addr,end) is
|
|
|
|
|
* certain not to be mapped by the time vma_merge is called; but when
|
|
|
|
|
* called for mprotect, it is certain to be already mapped (either at
|
|
|
|
|
* an offset within prev, or at the start of next), and the flags of
|
|
|
|
|
* this area are about to be changed to vm_flags - and the no-change
|
|
|
|
|
* case has already been eliminated.
|
|
|
|
|
*
|
|
|
|
|
* The following mprotect cases have to be considered, where AAAA is
|
|
|
|
|
* the area passed down from mprotect_fixup, never extending beyond one
|
|
|
|
|
* vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
|
|
|
|
|
*
|
|
|
|
|
* AAAA AAAA AAAA AAAA
|
|
|
|
|
* PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
|
|
|
|
|
* cannot merge might become might become might become
|
|
|
|
|
* PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
|
|
|
|
|
* mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
|
|
|
|
|
* mremap move: PPPPNNNNNNNN 8
|
|
|
|
|
* AAAA
|
|
|
|
|
* PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
|
|
|
|
|
* might become case 1 below case 2 below case 3 below
|
|
|
|
|
*
|
|
|
|
|
* Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
|
|
|
|
|
* mprotect_fixup updates vm_flags & vm_page_prot on successful return.
|
|
|
|
|
*/
|
|
|
|
|
struct vm_area_struct *vma_merge(struct mm_struct *mm,
|
|
|
|
|
struct vm_area_struct *prev, unsigned long addr,
|
|
|
|
|
unsigned long end, unsigned long vm_flags,
|
2014-10-09 22:26:29 +00:00
|
|
|
struct anon_vma *anon_vma, struct file *file,
|
2015-09-04 22:46:24 +00:00
|
|
|
pgoff_t pgoff, struct mempolicy *policy,
|
|
|
|
|
struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
|
|
|
|
|
struct vm_area_struct *area, *next;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We later require that vma->vm_flags == vm_flags,
|
|
|
|
|
* so this tests vma->vm_flags & VM_SPECIAL, too.
|
|
|
|
|
*/
|
|
|
|
|
if (vm_flags & VM_SPECIAL)
|
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
|
|
if (prev)
|
|
|
|
|
next = prev->vm_next;
|
|
|
|
|
else
|
|
|
|
|
next = mm->mmap;
|
|
|
|
|
area = next;
|
|
|
|
|
if (next && next->vm_end == end) /* cases 6, 7, 8 */
|
|
|
|
|
next = next->vm_next;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Can it merge with the predecessor?
|
|
|
|
|
*/
|
|
|
|
|
if (prev && prev->vm_end == addr &&
|
2014-10-09 22:26:29 +00:00
|
|
|
mpol_equal(vma_policy(prev), policy) &&
|
2005-04-16 22:20:36 +00:00
|
|
|
can_vma_merge_after(prev, vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
anon_vma, file, pgoff,
|
|
|
|
|
vm_userfaultfd_ctx)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* OK, it can. Can we now merge in the successor as well?
|
|
|
|
|
*/
|
|
|
|
|
if (next && end == next->vm_start &&
|
|
|
|
|
mpol_equal(policy, vma_policy(next)) &&
|
|
|
|
|
can_vma_merge_before(next, vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
anon_vma, file,
|
|
|
|
|
pgoff+pglen,
|
|
|
|
|
vm_userfaultfd_ctx) &&
|
2005-04-16 22:20:36 +00:00
|
|
|
is_mergeable_anon_vma(prev->anon_vma,
|
2011-05-25 00:11:20 +00:00
|
|
|
next->anon_vma, NULL)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* cases 1, 6 */
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(prev, prev->vm_start,
|
2005-04-16 22:20:36 +00:00
|
|
|
next->vm_end, prev->vm_pgoff, NULL);
|
|
|
|
|
} else /* cases 2, 5, 7 */
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(prev, prev->vm_start,
|
2005-04-16 22:20:36 +00:00
|
|
|
end, prev->vm_pgoff, NULL);
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
if (err)
|
|
|
|
|
return NULL;
|
2014-10-29 21:50:31 +00:00
|
|
|
khugepaged_enter_vma_merge(prev, vm_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
return prev;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Can this new request be merged in front of next?
|
|
|
|
|
*/
|
|
|
|
|
if (next && end == next->vm_start &&
|
2014-10-09 22:26:29 +00:00
|
|
|
mpol_equal(policy, vma_policy(next)) &&
|
2005-04-16 22:20:36 +00:00
|
|
|
can_vma_merge_before(next, vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
anon_vma, file, pgoff+pglen,
|
|
|
|
|
vm_userfaultfd_ctx)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (prev && addr < prev->vm_end) /* case 4 */
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(prev, prev->vm_start,
|
2005-04-16 22:20:36 +00:00
|
|
|
addr, prev->vm_pgoff, NULL);
|
|
|
|
|
else /* cases 3, 8 */
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(area, addr, next->vm_end,
|
2005-04-16 22:20:36 +00:00
|
|
|
next->vm_pgoff - pglen, NULL);
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
if (err)
|
|
|
|
|
return NULL;
|
2014-10-29 21:50:31 +00:00
|
|
|
khugepaged_enter_vma_merge(area, vm_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
return area;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2010-04-10 17:36:19 +00:00
|
|
|
/*
|
|
|
|
|
* Rough compatbility check to quickly see if it's even worth looking
|
|
|
|
|
* at sharing an anon_vma.
|
|
|
|
|
*
|
|
|
|
|
* They need to have the same vm_file, and the flags can only differ
|
|
|
|
|
* in things that mprotect may change.
|
|
|
|
|
*
|
|
|
|
|
* NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
|
|
|
|
|
* we can merge the two vma's. For example, we refuse to merge a vma if
|
|
|
|
|
* there is a vm_ops->close() function, because that indicates that the
|
|
|
|
|
* driver is doing some kind of reference counting. But that doesn't
|
|
|
|
|
* really matter for the anon_vma sharing case.
|
|
|
|
|
*/
|
|
|
|
|
static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
|
|
|
|
|
{
|
|
|
|
|
return a->vm_end == b->vm_start &&
|
|
|
|
|
mpol_equal(vma_policy(a), vma_policy(b)) &&
|
|
|
|
|
a->vm_file == b->vm_file &&
|
2014-01-23 23:53:42 +00:00
|
|
|
!((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
|
2010-04-10 17:36:19 +00:00
|
|
|
b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Do some basic sanity checking to see if we can re-use the anon_vma
|
|
|
|
|
* from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
|
|
|
|
|
* the same as 'old', the other will be the new one that is trying
|
|
|
|
|
* to share the anon_vma.
|
|
|
|
|
*
|
|
|
|
|
* NOTE! This runs with mm_sem held for reading, so it is possible that
|
|
|
|
|
* the anon_vma of 'old' is concurrently in the process of being set up
|
|
|
|
|
* by another page fault trying to merge _that_. But that's ok: if it
|
|
|
|
|
* is being set up, that automatically means that it will be a singleton
|
|
|
|
|
* acceptable for merging, so we can do all of this optimistically. But
|
2015-04-15 23:14:08 +00:00
|
|
|
* we do that READ_ONCE() to make sure that we never re-load the pointer.
|
2010-04-10 17:36:19 +00:00
|
|
|
*
|
|
|
|
|
* IOW: that the "list_is_singular()" test on the anon_vma_chain only
|
|
|
|
|
* matters for the 'stable anon_vma' case (ie the thing we want to avoid
|
|
|
|
|
* is to return an anon_vma that is "complex" due to having gone through
|
|
|
|
|
* a fork).
|
|
|
|
|
*
|
|
|
|
|
* We also make sure that the two vma's are compatible (adjacent,
|
|
|
|
|
* and with the same memory policies). That's all stable, even with just
|
|
|
|
|
* a read lock on the mm_sem.
|
|
|
|
|
*/
|
|
|
|
|
static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
|
|
|
|
|
{
|
|
|
|
|
if (anon_vma_compatible(a, b)) {
|
2015-04-15 23:14:08 +00:00
|
|
|
struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
|
2010-04-10 17:36:19 +00:00
|
|
|
|
|
|
|
|
if (anon_vma && list_is_singular(&old->anon_vma_chain))
|
|
|
|
|
return anon_vma;
|
|
|
|
|
}
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* find_mergeable_anon_vma is used by anon_vma_prepare, to check
|
|
|
|
|
* neighbouring vmas for a suitable anon_vma, before it goes off
|
|
|
|
|
* to allocate a new anon_vma. It checks because a repetitive
|
|
|
|
|
* sequence of mprotects and faults may otherwise lead to distinct
|
|
|
|
|
* anon_vmas being allocated, preventing vma merge in subsequent
|
|
|
|
|
* mprotect.
|
|
|
|
|
*/
|
|
|
|
|
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
|
|
|
|
|
{
|
2010-04-10 17:36:19 +00:00
|
|
|
struct anon_vma *anon_vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct vm_area_struct *near;
|
|
|
|
|
|
|
|
|
|
near = vma->vm_next;
|
|
|
|
|
if (!near)
|
|
|
|
|
goto try_prev;
|
|
|
|
|
|
2010-04-10 17:36:19 +00:00
|
|
|
anon_vma = reusable_anon_vma(near, vma, near);
|
|
|
|
|
if (anon_vma)
|
|
|
|
|
return anon_vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
try_prev:
|
2011-06-16 07:35:09 +00:00
|
|
|
near = vma->vm_prev;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!near)
|
|
|
|
|
goto none;
|
|
|
|
|
|
2010-04-10 17:36:19 +00:00
|
|
|
anon_vma = reusable_anon_vma(near, near, vma);
|
|
|
|
|
if (anon_vma)
|
|
|
|
|
return anon_vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
none:
|
|
|
|
|
/*
|
|
|
|
|
* There's no absolute need to look only at touching neighbours:
|
|
|
|
|
* we could search further afield for "compatible" anon_vmas.
|
|
|
|
|
* But it would probably just be a waste of time searching,
|
|
|
|
|
* or lead to too many vmas hanging off the same anon_vma.
|
|
|
|
|
* We're trying to allow mprotect remerging later on,
|
|
|
|
|
* not trying to minimize memory used for anon_vmas.
|
|
|
|
|
*/
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2012-02-13 03:58:52 +00:00
|
|
|
/*
|
|
|
|
|
* If a hint addr is less than mmap_min_addr change hint to be as
|
|
|
|
|
* low as possible but still greater than mmap_min_addr
|
|
|
|
|
*/
|
|
|
|
|
static inline unsigned long round_hint_to_min(unsigned long hint)
|
|
|
|
|
{
|
|
|
|
|
hint &= PAGE_MASK;
|
|
|
|
|
if (((void *)hint != NULL) &&
|
|
|
|
|
(hint < mmap_min_addr))
|
|
|
|
|
return PAGE_ALIGN(mmap_min_addr);
|
|
|
|
|
return hint;
|
|
|
|
|
}
|
|
|
|
|
|
2014-01-21 23:49:15 +00:00
|
|
|
static inline int mlock_future_check(struct mm_struct *mm,
|
|
|
|
|
unsigned long flags,
|
|
|
|
|
unsigned long len)
|
|
|
|
|
{
|
|
|
|
|
unsigned long locked, lock_limit;
|
|
|
|
|
|
|
|
|
|
/* mlock MCL_FUTURE? */
|
|
|
|
|
if (flags & VM_LOCKED) {
|
|
|
|
|
locked = len >> PAGE_SHIFT;
|
|
|
|
|
locked += mm->locked_vm;
|
|
|
|
|
lock_limit = rlimit(RLIMIT_MEMLOCK);
|
|
|
|
|
lock_limit >>= PAGE_SHIFT;
|
|
|
|
|
if (locked > lock_limit && !capable(CAP_IPC_LOCK))
|
|
|
|
|
return -EAGAIN;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2009-09-18 02:26:26 +00:00
|
|
|
* The caller must hold down_write(¤t->mm->mmap_sem).
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2015-09-09 22:39:29 +00:00
|
|
|
unsigned long do_mmap(struct file *file, unsigned long addr,
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long len, unsigned long prot,
|
2015-09-09 22:39:29 +00:00
|
|
|
unsigned long flags, vm_flags_t vm_flags,
|
|
|
|
|
unsigned long pgoff, unsigned long *populate)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:29 +00:00
|
|
|
struct mm_struct *mm = current->mm;
|
mm/core, x86/mm/pkeys: Add execute-only protection keys support
Protection keys provide new page-based protection in hardware.
But, they have an interesting attribute: they only affect data
accesses and never affect instruction fetches. That means that
if we set up some memory which is set as "access-disabled" via
protection keys, we can still execute from it.
This patch uses protection keys to set up mappings to do just that.
If a user calls:
mmap(..., PROT_EXEC);
or
mprotect(ptr, sz, PROT_EXEC);
(note PROT_EXEC-only without PROT_READ/WRITE), the kernel will
notice this, and set a special protection key on the memory. It
also sets the appropriate bits in the Protection Keys User Rights
(PKRU) register so that the memory becomes unreadable and
unwritable.
I haven't found any userspace that does this today. With this
facility in place, we expect userspace to move to use it
eventually. Userspace _could_ start doing this today. Any
PROT_EXEC calls get converted to PROT_READ inside the kernel, and
would transparently be upgraded to "true" PROT_EXEC with this
code. IOW, userspace never has to do any PROT_EXEC runtime
detection.
This feature provides enhanced protection against leaking
executable memory contents. This helps thwart attacks which are
attempting to find ROP gadgets on the fly.
But, the security provided by this approach is not comprehensive.
The PKRU register which controls access permissions is a normal
user register writable from unprivileged userspace. An attacker
who can execute the 'wrpkru' instruction can easily disable the
protection provided by this feature.
The protection key that is used for execute-only support is
permanently dedicated at compile time. This is fine for now
because there is currently no API to set a protection key other
than this one.
Despite there being a constant PKRU value across the entire
system, we do not set it unless this feature is in use in a
process. That is to preserve the PKRU XSAVE 'init state',
which can lead to faster context switches.
PKRU *is* a user register and the kernel is modifying it. That
means that code doing:
pkru = rdpkru()
pkru |= 0x100;
mmap(..., PROT_EXEC);
wrpkru(pkru);
could lose the bits in PKRU that enforce execute-only
permissions. To avoid this, we suggest avoiding ever calling
mmap() or mprotect() when the PKRU value is expected to be
unstable.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chen Gang <gang.chen.5i5j@gmail.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: David Hildenbrand <dahi@linux.vnet.ibm.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Piotr Kwapulinski <kwapulinski.piotr@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Stephen Smalley <sds@tycho.nsa.gov>
Cc: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: keescook@google.com
Cc: linux-kernel@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210240.CB4BB5CA@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-12 21:02:40 +00:00
|
|
|
int pkey = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-02-23 00:32:47 +00:00
|
|
|
*populate = 0;
|
2013-02-23 00:32:37 +00:00
|
|
|
|
2015-06-24 23:58:39 +00:00
|
|
|
if (!len)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* Does the application expect PROT_READ to imply PROT_EXEC?
|
|
|
|
|
*
|
|
|
|
|
* (the exception is when the underlying filesystem is noexec
|
|
|
|
|
* mounted, in which case we dont add PROT_EXEC.)
|
|
|
|
|
*/
|
|
|
|
|
if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
|
2015-06-29 19:42:03 +00:00
|
|
|
if (!(file && path_noexec(&file->f_path)))
|
2005-04-16 22:20:36 +00:00
|
|
|
prot |= PROT_EXEC;
|
|
|
|
|
|
2007-11-26 23:47:40 +00:00
|
|
|
if (!(flags & MAP_FIXED))
|
|
|
|
|
addr = round_hint_to_min(addr);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Careful about overflows.. */
|
|
|
|
|
len = PAGE_ALIGN(len);
|
2009-12-03 20:23:11 +00:00
|
|
|
if (!len)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* offset overflow? */
|
|
|
|
|
if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
|
2014-10-09 22:26:29 +00:00
|
|
|
return -EOVERFLOW;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* Too many mappings? */
|
|
|
|
|
if (mm->map_count > sysctl_max_map_count)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* Obtain the address to map to. we verify (or select) it and ensure
|
|
|
|
|
* that it represents a valid section of the address space.
|
|
|
|
|
*/
|
|
|
|
|
addr = get_unmapped_area(file, addr, len, pgoff, flags);
|
2015-11-06 02:46:54 +00:00
|
|
|
if (offset_in_page(addr))
|
2005-04-16 22:20:36 +00:00
|
|
|
return addr;
|
|
|
|
|
|
mm/core, x86/mm/pkeys: Add execute-only protection keys support
Protection keys provide new page-based protection in hardware.
But, they have an interesting attribute: they only affect data
accesses and never affect instruction fetches. That means that
if we set up some memory which is set as "access-disabled" via
protection keys, we can still execute from it.
This patch uses protection keys to set up mappings to do just that.
If a user calls:
mmap(..., PROT_EXEC);
or
mprotect(ptr, sz, PROT_EXEC);
(note PROT_EXEC-only without PROT_READ/WRITE), the kernel will
notice this, and set a special protection key on the memory. It
also sets the appropriate bits in the Protection Keys User Rights
(PKRU) register so that the memory becomes unreadable and
unwritable.
I haven't found any userspace that does this today. With this
facility in place, we expect userspace to move to use it
eventually. Userspace _could_ start doing this today. Any
PROT_EXEC calls get converted to PROT_READ inside the kernel, and
would transparently be upgraded to "true" PROT_EXEC with this
code. IOW, userspace never has to do any PROT_EXEC runtime
detection.
This feature provides enhanced protection against leaking
executable memory contents. This helps thwart attacks which are
attempting to find ROP gadgets on the fly.
But, the security provided by this approach is not comprehensive.
The PKRU register which controls access permissions is a normal
user register writable from unprivileged userspace. An attacker
who can execute the 'wrpkru' instruction can easily disable the
protection provided by this feature.
The protection key that is used for execute-only support is
permanently dedicated at compile time. This is fine for now
because there is currently no API to set a protection key other
than this one.
Despite there being a constant PKRU value across the entire
system, we do not set it unless this feature is in use in a
process. That is to preserve the PKRU XSAVE 'init state',
which can lead to faster context switches.
PKRU *is* a user register and the kernel is modifying it. That
means that code doing:
pkru = rdpkru()
pkru |= 0x100;
mmap(..., PROT_EXEC);
wrpkru(pkru);
could lose the bits in PKRU that enforce execute-only
permissions. To avoid this, we suggest avoiding ever calling
mmap() or mprotect() when the PKRU value is expected to be
unstable.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chen Gang <gang.chen.5i5j@gmail.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: David Hildenbrand <dahi@linux.vnet.ibm.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Piotr Kwapulinski <kwapulinski.piotr@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Stephen Smalley <sds@tycho.nsa.gov>
Cc: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: keescook@google.com
Cc: linux-kernel@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210240.CB4BB5CA@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-12 21:02:40 +00:00
|
|
|
if (prot == PROT_EXEC) {
|
|
|
|
|
pkey = execute_only_pkey(mm);
|
|
|
|
|
if (pkey < 0)
|
|
|
|
|
pkey = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Do simple checking here so the lower-level routines won't have
|
|
|
|
|
* to. we assume access permissions have been handled by the open
|
|
|
|
|
* of the memory object, so we don't do any here.
|
|
|
|
|
*/
|
mm/core, x86/mm/pkeys: Add execute-only protection keys support
Protection keys provide new page-based protection in hardware.
But, they have an interesting attribute: they only affect data
accesses and never affect instruction fetches. That means that
if we set up some memory which is set as "access-disabled" via
protection keys, we can still execute from it.
This patch uses protection keys to set up mappings to do just that.
If a user calls:
mmap(..., PROT_EXEC);
or
mprotect(ptr, sz, PROT_EXEC);
(note PROT_EXEC-only without PROT_READ/WRITE), the kernel will
notice this, and set a special protection key on the memory. It
also sets the appropriate bits in the Protection Keys User Rights
(PKRU) register so that the memory becomes unreadable and
unwritable.
I haven't found any userspace that does this today. With this
facility in place, we expect userspace to move to use it
eventually. Userspace _could_ start doing this today. Any
PROT_EXEC calls get converted to PROT_READ inside the kernel, and
would transparently be upgraded to "true" PROT_EXEC with this
code. IOW, userspace never has to do any PROT_EXEC runtime
detection.
This feature provides enhanced protection against leaking
executable memory contents. This helps thwart attacks which are
attempting to find ROP gadgets on the fly.
But, the security provided by this approach is not comprehensive.
The PKRU register which controls access permissions is a normal
user register writable from unprivileged userspace. An attacker
who can execute the 'wrpkru' instruction can easily disable the
protection provided by this feature.
The protection key that is used for execute-only support is
permanently dedicated at compile time. This is fine for now
because there is currently no API to set a protection key other
than this one.
Despite there being a constant PKRU value across the entire
system, we do not set it unless this feature is in use in a
process. That is to preserve the PKRU XSAVE 'init state',
which can lead to faster context switches.
PKRU *is* a user register and the kernel is modifying it. That
means that code doing:
pkru = rdpkru()
pkru |= 0x100;
mmap(..., PROT_EXEC);
wrpkru(pkru);
could lose the bits in PKRU that enforce execute-only
permissions. To avoid this, we suggest avoiding ever calling
mmap() or mprotect() when the PKRU value is expected to be
unstable.
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bp@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Chen Gang <gang.chen.5i5j@gmail.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dave Hansen <dave@sr71.net>
Cc: David Hildenbrand <dahi@linux.vnet.ibm.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Piotr Kwapulinski <kwapulinski.piotr@gmail.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Stephen Smalley <sds@tycho.nsa.gov>
Cc: Vladimir Murzin <vladimir.murzin@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: keescook@google.com
Cc: linux-kernel@vger.kernel.org
Cc: linux-mm@kvack.org
Link: http://lkml.kernel.org/r/20160212210240.CB4BB5CA@viggo.jf.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2016-02-12 21:02:40 +00:00
|
|
|
vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
|
2005-04-16 22:20:36 +00:00
|
|
|
mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
|
|
|
|
|
|
2009-09-22 00:03:36 +00:00
|
|
|
if (flags & MAP_LOCKED)
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!can_do_mlock())
|
|
|
|
|
return -EPERM;
|
2008-10-19 03:26:50 +00:00
|
|
|
|
2014-01-21 23:49:15 +00:00
|
|
|
if (mlock_future_check(mm, vm_flags, len))
|
|
|
|
|
return -EAGAIN;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
if (file) {
|
2013-09-11 21:20:19 +00:00
|
|
|
struct inode *inode = file_inode(file);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
switch (flags & MAP_TYPE) {
|
|
|
|
|
case MAP_SHARED:
|
|
|
|
|
if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
|
|
|
|
|
return -EACCES;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Make sure we don't allow writing to an append-only
|
|
|
|
|
* file..
|
|
|
|
|
*/
|
|
|
|
|
if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
|
|
|
|
|
return -EACCES;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Make sure there are no mandatory locks on the file.
|
|
|
|
|
*/
|
2014-03-10 13:54:15 +00:00
|
|
|
if (locks_verify_locked(file))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EAGAIN;
|
|
|
|
|
|
|
|
|
|
vm_flags |= VM_SHARED | VM_MAYSHARE;
|
|
|
|
|
if (!(file->f_mode & FMODE_WRITE))
|
|
|
|
|
vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
|
|
|
|
|
|
|
|
|
|
/* fall through */
|
|
|
|
|
case MAP_PRIVATE:
|
|
|
|
|
if (!(file->f_mode & FMODE_READ))
|
|
|
|
|
return -EACCES;
|
2015-06-29 19:42:03 +00:00
|
|
|
if (path_noexec(&file->f_path)) {
|
2006-10-15 21:09:55 +00:00
|
|
|
if (vm_flags & VM_EXEC)
|
|
|
|
|
return -EPERM;
|
|
|
|
|
vm_flags &= ~VM_MAYEXEC;
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-22 20:27:52 +00:00
|
|
|
if (!file->f_op->mmap)
|
2006-10-15 21:09:55 +00:00
|
|
|
return -ENODEV;
|
2013-09-11 21:20:18 +00:00
|
|
|
if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
|
|
|
|
|
return -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
switch (flags & MAP_TYPE) {
|
|
|
|
|
case MAP_SHARED:
|
2013-09-11 21:20:18 +00:00
|
|
|
if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
|
|
|
|
|
return -EINVAL;
|
2008-09-03 14:09:47 +00:00
|
|
|
/*
|
|
|
|
|
* Ignore pgoff.
|
|
|
|
|
*/
|
|
|
|
|
pgoff = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
vm_flags |= VM_SHARED | VM_MAYSHARE;
|
|
|
|
|
break;
|
|
|
|
|
case MAP_PRIVATE:
|
|
|
|
|
/*
|
|
|
|
|
* Set pgoff according to addr for anon_vma.
|
|
|
|
|
*/
|
|
|
|
|
pgoff = addr >> PAGE_SHIFT;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2013-02-23 00:32:43 +00:00
|
|
|
/*
|
|
|
|
|
* Set 'VM_NORESERVE' if we should not account for the
|
|
|
|
|
* memory use of this mapping.
|
|
|
|
|
*/
|
|
|
|
|
if (flags & MAP_NORESERVE) {
|
|
|
|
|
/* We honor MAP_NORESERVE if allowed to overcommit */
|
|
|
|
|
if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
|
|
|
|
|
vm_flags |= VM_NORESERVE;
|
|
|
|
|
|
|
|
|
|
/* hugetlb applies strict overcommit unless MAP_NORESERVE */
|
|
|
|
|
if (file && is_file_hugepages(file))
|
|
|
|
|
vm_flags |= VM_NORESERVE;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
addr = mmap_region(file, addr, len, vm_flags, pgoff);
|
2013-03-28 23:26:23 +00:00
|
|
|
if (!IS_ERR_VALUE(addr) &&
|
|
|
|
|
((vm_flags & VM_LOCKED) ||
|
|
|
|
|
(flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
|
2013-02-23 00:32:47 +00:00
|
|
|
*populate = len;
|
2013-02-23 00:32:37 +00:00
|
|
|
return addr;
|
2007-07-16 06:38:26 +00:00
|
|
|
}
|
2012-04-21 00:13:58 +00:00
|
|
|
|
2009-12-30 20:17:34 +00:00
|
|
|
SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
|
|
|
|
|
unsigned long, prot, unsigned long, flags,
|
|
|
|
|
unsigned long, fd, unsigned long, pgoff)
|
|
|
|
|
{
|
|
|
|
|
struct file *file = NULL;
|
2015-11-06 02:48:35 +00:00
|
|
|
unsigned long retval;
|
2009-12-30 20:17:34 +00:00
|
|
|
|
|
|
|
|
if (!(flags & MAP_ANONYMOUS)) {
|
2010-10-30 06:54:44 +00:00
|
|
|
audit_mmap_fd(fd, flags);
|
2009-12-30 20:17:34 +00:00
|
|
|
file = fget(fd);
|
|
|
|
|
if (!file)
|
2015-11-06 02:48:35 +00:00
|
|
|
return -EBADF;
|
2013-05-07 23:18:13 +00:00
|
|
|
if (is_file_hugepages(file))
|
|
|
|
|
len = ALIGN(len, huge_page_size(hstate_file(file)));
|
2013-07-08 23:00:26 +00:00
|
|
|
retval = -EINVAL;
|
|
|
|
|
if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
|
|
|
|
|
goto out_fput;
|
2009-12-30 20:17:34 +00:00
|
|
|
} else if (flags & MAP_HUGETLB) {
|
|
|
|
|
struct user_struct *user = NULL;
|
2013-07-08 23:01:08 +00:00
|
|
|
struct hstate *hs;
|
2013-05-07 23:18:13 +00:00
|
|
|
|
2013-07-08 23:01:08 +00:00
|
|
|
hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & SHM_HUGE_MASK);
|
2013-05-09 07:08:15 +00:00
|
|
|
if (!hs)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
len = ALIGN(len, huge_page_size(hs));
|
2009-12-30 20:17:34 +00:00
|
|
|
/*
|
|
|
|
|
* VM_NORESERVE is used because the reservations will be
|
|
|
|
|
* taken when vm_ops->mmap() is called
|
|
|
|
|
* A dummy user value is used because we are not locking
|
|
|
|
|
* memory so no accounting is necessary
|
|
|
|
|
*/
|
2013-05-07 23:18:13 +00:00
|
|
|
file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
|
2012-12-12 00:01:34 +00:00
|
|
|
VM_NORESERVE,
|
|
|
|
|
&user, HUGETLB_ANONHUGE_INODE,
|
|
|
|
|
(flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
|
2009-12-30 20:17:34 +00:00
|
|
|
if (IS_ERR(file))
|
|
|
|
|
return PTR_ERR(file);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
|
|
|
|
|
|
2016-05-23 23:25:30 +00:00
|
|
|
retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
|
2013-07-08 23:00:26 +00:00
|
|
|
out_fput:
|
2009-12-30 20:17:34 +00:00
|
|
|
if (file)
|
|
|
|
|
fput(file);
|
|
|
|
|
return retval;
|
|
|
|
|
}
|
|
|
|
|
|
2010-03-10 23:21:15 +00:00
|
|
|
#ifdef __ARCH_WANT_SYS_OLD_MMAP
|
|
|
|
|
struct mmap_arg_struct {
|
|
|
|
|
unsigned long addr;
|
|
|
|
|
unsigned long len;
|
|
|
|
|
unsigned long prot;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
unsigned long fd;
|
|
|
|
|
unsigned long offset;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
|
|
|
|
|
{
|
|
|
|
|
struct mmap_arg_struct a;
|
|
|
|
|
|
|
|
|
|
if (copy_from_user(&a, arg, sizeof(a)))
|
|
|
|
|
return -EFAULT;
|
2015-11-06 02:46:54 +00:00
|
|
|
if (offset_in_page(a.offset))
|
2010-03-10 23:21:15 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
|
|
|
|
|
a.offset >> PAGE_SHIFT);
|
|
|
|
|
}
|
|
|
|
|
#endif /* __ARCH_WANT_SYS_OLD_MMAP */
|
|
|
|
|
|
2007-07-29 22:36:13 +00:00
|
|
|
/*
|
|
|
|
|
* Some shared mappigns will want the pages marked read-only
|
|
|
|
|
* to track write events. If so, we'll downgrade vm_page_prot
|
|
|
|
|
* to the private version (using protection_map[] without the
|
|
|
|
|
* VM_SHARED bit).
|
|
|
|
|
*/
|
|
|
|
|
int vma_wants_writenotify(struct vm_area_struct *vma)
|
|
|
|
|
{
|
2011-05-26 10:16:19 +00:00
|
|
|
vm_flags_t vm_flags = vma->vm_flags;
|
2015-09-22 21:59:12 +00:00
|
|
|
const struct vm_operations_struct *vm_ops = vma->vm_ops;
|
2007-07-29 22:36:13 +00:00
|
|
|
|
|
|
|
|
/* If it was private or non-writable, the write bit is already clear */
|
|
|
|
|
if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/* The backer wishes to know when pages are first written to? */
|
2015-09-22 21:59:12 +00:00
|
|
|
if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
|
2007-07-29 22:36:13 +00:00
|
|
|
return 1;
|
|
|
|
|
|
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set. If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared. Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect. An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").
Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:46 +00:00
|
|
|
/* The open routine did something to the protections that pgprot_modify
|
|
|
|
|
* won't preserve? */
|
2007-07-29 22:36:13 +00:00
|
|
|
if (pgprot_val(vma->vm_page_prot) !=
|
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set. If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared. Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect. An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").
Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:46 +00:00
|
|
|
pgprot_val(vm_pgprot_modify(vma->vm_page_prot, vm_flags)))
|
2007-07-29 22:36:13 +00:00
|
|
|
return 0;
|
|
|
|
|
|
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set. If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared. Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect. An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").
Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:46 +00:00
|
|
|
/* Do we need to track softdirty? */
|
|
|
|
|
if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
|
|
|
|
|
return 1;
|
|
|
|
|
|
2007-07-29 22:36:13 +00:00
|
|
|
/* Specialty mapping? */
|
2012-10-08 23:28:40 +00:00
|
|
|
if (vm_flags & VM_PFNMAP)
|
2007-07-29 22:36:13 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/* Can the mapping track the dirty pages? */
|
|
|
|
|
return vma->vm_file && vma->vm_file->f_mapping &&
|
|
|
|
|
mapping_cap_account_dirty(vma->vm_file->f_mapping);
|
|
|
|
|
}
|
|
|
|
|
|
2009-01-31 23:08:56 +00:00
|
|
|
/*
|
|
|
|
|
* We account for memory if it's a private writeable mapping,
|
2009-02-10 14:02:27 +00:00
|
|
|
* not hugepages and VM_NORESERVE wasn't set.
|
2009-01-31 23:08:56 +00:00
|
|
|
*/
|
2011-05-26 10:16:19 +00:00
|
|
|
static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
|
2009-01-31 23:08:56 +00:00
|
|
|
{
|
2009-02-10 14:02:27 +00:00
|
|
|
/*
|
|
|
|
|
* hugetlb has its own accounting separate from the core VM
|
|
|
|
|
* VM_HUGETLB may not be set yet so we cannot check for that flag.
|
|
|
|
|
*/
|
|
|
|
|
if (file && is_file_hugepages(file))
|
|
|
|
|
return 0;
|
|
|
|
|
|
2009-01-31 23:08:56 +00:00
|
|
|
return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
|
|
|
|
|
}
|
|
|
|
|
|
2007-07-16 06:38:26 +00:00
|
|
|
unsigned long mmap_region(struct file *file, unsigned long addr,
|
2013-02-23 00:32:43 +00:00
|
|
|
unsigned long len, vm_flags_t vm_flags, unsigned long pgoff)
|
2007-07-16 06:38:26 +00:00
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma, *prev;
|
|
|
|
|
int error;
|
|
|
|
|
struct rb_node **rb_link, *rb_parent;
|
|
|
|
|
unsigned long charged = 0;
|
|
|
|
|
|
2013-04-29 22:08:33 +00:00
|
|
|
/* Check against address space limit. */
|
2016-01-14 23:22:07 +00:00
|
|
|
if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
|
2013-04-29 22:08:33 +00:00
|
|
|
unsigned long nr_pages;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* MAP_FIXED may remove pages of mappings that intersects with
|
|
|
|
|
* requested mapping. Account for the pages it would unmap.
|
|
|
|
|
*/
|
|
|
|
|
nr_pages = count_vma_pages_range(mm, addr, addr + len);
|
|
|
|
|
|
2016-01-14 23:22:07 +00:00
|
|
|
if (!may_expand_vm(mm, vm_flags,
|
|
|
|
|
(len >> PAGE_SHIFT) - nr_pages))
|
2013-04-29 22:08:33 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Clear old maps */
|
2015-04-15 23:14:32 +00:00
|
|
|
while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
|
|
|
|
|
&rb_parent)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (do_munmap(mm, addr, len))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
|
2009-01-31 23:08:56 +00:00
|
|
|
/*
|
|
|
|
|
* Private writable mapping: check memory availability
|
|
|
|
|
*/
|
2009-02-10 14:02:27 +00:00
|
|
|
if (accountable_mapping(file, vm_flags)) {
|
2009-01-31 23:08:56 +00:00
|
|
|
charged = len >> PAGE_SHIFT;
|
2012-02-13 03:58:52 +00:00
|
|
|
if (security_vm_enough_memory_mm(mm, charged))
|
2009-01-31 23:08:56 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
vm_flags |= VM_ACCOUNT;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
2009-01-30 01:46:42 +00:00
|
|
|
* Can we just expand an old mapping?
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2015-09-04 22:46:24 +00:00
|
|
|
vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
|
|
|
|
|
NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
|
2009-01-30 01:46:42 +00:00
|
|
|
if (vma)
|
|
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Determine the object being mapped and call the appropriate
|
|
|
|
|
* specific mapper. the address has already been validated, but
|
|
|
|
|
* not unmapped, but the maps are removed from the list.
|
|
|
|
|
*/
|
2006-03-25 11:06:43 +00:00
|
|
|
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!vma) {
|
|
|
|
|
error = -ENOMEM;
|
|
|
|
|
goto unacct_error;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
vma->vm_mm = mm;
|
|
|
|
|
vma->vm_start = addr;
|
|
|
|
|
vma->vm_end = addr + len;
|
|
|
|
|
vma->vm_flags = vm_flags;
|
2007-10-19 06:39:15 +00:00
|
|
|
vma->vm_page_prot = vm_get_page_prot(vm_flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
vma->vm_pgoff = pgoff;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
INIT_LIST_HEAD(&vma->anon_vma_chain);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
if (file) {
|
|
|
|
|
if (vm_flags & VM_DENYWRITE) {
|
|
|
|
|
error = deny_write_access(file);
|
|
|
|
|
if (error)
|
|
|
|
|
goto free_vma;
|
|
|
|
|
}
|
2014-08-08 21:25:25 +00:00
|
|
|
if (vm_flags & VM_SHARED) {
|
|
|
|
|
error = mapping_map_writable(file->f_mapping);
|
|
|
|
|
if (error)
|
|
|
|
|
goto allow_write_and_free_vma;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* ->mmap() can change vma->vm_file, but must guarantee that
|
|
|
|
|
* vma_link() below can deny write-access if VM_DENYWRITE is set
|
|
|
|
|
* and map writably if VM_SHARED is set. This usually means the
|
|
|
|
|
* new file must not have been exposed to user-space, yet.
|
|
|
|
|
*/
|
2012-08-27 18:48:26 +00:00
|
|
|
vma->vm_file = get_file(file);
|
2005-04-16 22:20:36 +00:00
|
|
|
error = file->f_op->mmap(file, vma);
|
|
|
|
|
if (error)
|
|
|
|
|
goto unmap_and_free_vma;
|
2009-09-22 00:03:41 +00:00
|
|
|
|
|
|
|
|
/* Can addr have changed??
|
|
|
|
|
*
|
|
|
|
|
* Answer: Yes, several device drivers can do it in their
|
|
|
|
|
* f_op->mmap method. -DaveM
|
2012-12-12 21:51:53 +00:00
|
|
|
* Bug: If addr is changed, prev, rb_link, rb_parent should
|
|
|
|
|
* be updated for vma_link()
|
2009-09-22 00:03:41 +00:00
|
|
|
*/
|
2012-12-12 21:51:53 +00:00
|
|
|
WARN_ON_ONCE(addr != vma->vm_start);
|
|
|
|
|
|
2009-09-22 00:03:41 +00:00
|
|
|
addr = vma->vm_start;
|
|
|
|
|
vm_flags = vma->vm_flags;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else if (vm_flags & VM_SHARED) {
|
|
|
|
|
error = shmem_zero_setup(vma);
|
|
|
|
|
if (error)
|
|
|
|
|
goto free_vma;
|
|
|
|
|
}
|
|
|
|
|
|
2009-01-30 01:46:42 +00:00
|
|
|
vma_link(mm, vma, prev, rb_link, rb_parent);
|
2008-04-28 09:12:10 +00:00
|
|
|
/* Once vma denies write, undo our temporary denial count */
|
2014-08-08 21:25:25 +00:00
|
|
|
if (file) {
|
|
|
|
|
if (vm_flags & VM_SHARED)
|
|
|
|
|
mapping_unmap_writable(file->f_mapping);
|
|
|
|
|
if (vm_flags & VM_DENYWRITE)
|
|
|
|
|
allow_write_access(file);
|
|
|
|
|
}
|
2013-09-11 21:20:20 +00:00
|
|
|
file = vma->vm_file;
|
2008-04-28 09:12:10 +00:00
|
|
|
out:
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 10:02:48 +00:00
|
|
|
perf_event_mmap(vma);
|
2009-03-30 17:07:05 +00:00
|
|
|
|
2016-01-14 23:22:07 +00:00
|
|
|
vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (vm_flags & VM_LOCKED) {
|
2013-02-23 00:32:37 +00:00
|
|
|
if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
|
|
|
|
|
vma == get_gate_vma(current->mm)))
|
2010-03-05 21:41:43 +00:00
|
|
|
mm->locked_vm += (len >> PAGE_SHIFT);
|
2013-02-23 00:32:37 +00:00
|
|
|
else
|
2015-11-06 02:51:36 +00:00
|
|
|
vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
|
2013-02-23 00:32:37 +00:00
|
|
|
}
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
|
2012-08-19 17:10:42 +00:00
|
|
|
if (file)
|
|
|
|
|
uprobe_mmap(vma);
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
|
2013-09-11 21:22:24 +00:00
|
|
|
/*
|
|
|
|
|
* New (or expanded) vma always get soft dirty status.
|
|
|
|
|
* Otherwise user-space soft-dirty page tracker won't
|
|
|
|
|
* be able to distinguish situation when vma area unmapped,
|
|
|
|
|
* then new mapped in-place (which must be aimed as
|
|
|
|
|
* a completely new data area).
|
|
|
|
|
*/
|
|
|
|
|
vma->vm_flags |= VM_SOFTDIRTY;
|
|
|
|
|
|
mm: softdirty: enable write notifications on VMAs after VM_SOFTDIRTY cleared
For VMAs that don't want write notifications, PTEs created for read faults
have their write bit set. If the read fault happens after VM_SOFTDIRTY is
cleared, then the PTE's softdirty bit will remain clear after subsequent
writes.
Here's a simple code snippet to demonstrate the bug:
char* m = mmap(NULL, getpagesize(), PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_SHARED, -1, 0);
system("echo 4 > /proc/$PPID/clear_refs"); /* clear VM_SOFTDIRTY */
assert(*m == '\0'); /* new PTE allows write access */
assert(!soft_dirty(x));
*m = 'x'; /* should dirty the page */
assert(soft_dirty(x)); /* fails */
With this patch, write notifications are enabled when VM_SOFTDIRTY is
cleared. Furthermore, to avoid unnecessary faults, write notifications
are disabled when VM_SOFTDIRTY is set.
As a side effect of enabling and disabling write notifications with
care, this patch fixes a bug in mprotect where vm_page_prot bits set by
drivers were zapped on mprotect. An analogous bug was fixed in mmap by
commit c9d0bf241451 ("mm: uncached vma support with writenotify").
Signed-off-by: Peter Feiner <pfeiner@google.com>
Reported-by: Peter Feiner <pfeiner@google.com>
Suggested-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Cyrill Gorcunov <gorcunov@openvz.org>
Cc: Pavel Emelyanov <xemul@parallels.com>
Cc: Jamie Liu <jamieliu@google.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-10-13 22:55:46 +00:00
|
|
|
vma_set_page_prot(vma);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
return addr;
|
|
|
|
|
|
|
|
|
|
unmap_and_free_vma:
|
|
|
|
|
vma->vm_file = NULL;
|
|
|
|
|
fput(file);
|
|
|
|
|
|
|
|
|
|
/* Undo any partial mapping done by a device driver. */
|
[PATCH] freepgt: free_pgtables use vma list
Recent woes with some arches needing their own pgd_addr_end macro; and 4-level
clear_page_range regression since 2.6.10's clear_page_tables; and its
long-standing well-known inefficiency in searching throughout the higher-level
page tables for those few entries to clear and free: all can be blamed on
ignoring the list of vmas when we free page tables.
Replace exit_mmap's clear_page_range of the total user address space by
free_pgtables operating on the mm's vma list; unmap_region use it in the same
way, giving floor and ceiling beyond which it may not free tables. This
brings lmbench fork/exec/sh numbers back to 2.6.10 (unless preempt is enabled,
in which case latency fixes spoil unmap_vmas throughput).
Beware: the do_mmap_pgoff driver failure case must now use unmap_region
instead of zap_page_range, since a page table might have been allocated, and
can only be freed while it is touched by some vma.
Move free_pgtables from mmap.c to memory.c, where its lower levels are adapted
from the clear_page_range levels. (Most of free_pgtables' old code was
actually for a non-existent case, prev not properly set up, dating from before
hch gave us split_vma.) Pass mmu_gather** in the public interfaces, since we
might want to add latency lockdrops later; but no attempt to do so yet, going
by vma should itself reduce latency.
But what if is_hugepage_only_range? Those ia64 and ppc64 cases need careful
examination: put that off until a later patch of the series.
What of x86_64's 32bit vdso page __map_syscall32 maps outside any vma?
And the range to sparc64's flush_tlb_pgtables? It's less clear to me now that
we need to do more than is done here - every PMD_SIZE ever occupied will be
flushed, do we really have to flush every PGDIR_SIZE ever partially occupied?
A shame to complicate it unnecessarily.
Special thanks to David Miller for time spent repairing my ceilings.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-04-19 20:29:15 +00:00
|
|
|
unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
|
|
|
|
|
charged = 0;
|
2014-08-08 21:25:25 +00:00
|
|
|
if (vm_flags & VM_SHARED)
|
|
|
|
|
mapping_unmap_writable(file->f_mapping);
|
|
|
|
|
allow_write_and_free_vma:
|
|
|
|
|
if (vm_flags & VM_DENYWRITE)
|
|
|
|
|
allow_write_access(file);
|
2005-04-16 22:20:36 +00:00
|
|
|
free_vma:
|
|
|
|
|
kmem_cache_free(vm_area_cachep, vma);
|
|
|
|
|
unacct_error:
|
|
|
|
|
if (charged)
|
|
|
|
|
vm_unacct_memory(charged);
|
|
|
|
|
return error;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:49 +00:00
|
|
|
unsigned long unmapped_area(struct vm_unmapped_area_info *info)
|
|
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* We implement the search by looking for an rbtree node that
|
|
|
|
|
* immediately follows a suitable gap. That is,
|
|
|
|
|
* - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
|
|
|
|
|
* - gap_end = vma->vm_start >= info->low_limit + length;
|
|
|
|
|
* - gap_end - gap_start >= length
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
unsigned long length, low_limit, high_limit, gap_start, gap_end;
|
|
|
|
|
|
|
|
|
|
/* Adjust search length to account for worst case alignment overhead */
|
|
|
|
|
length = info->length + info->align_mask;
|
|
|
|
|
if (length < info->length)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* Adjust search limits by the desired length */
|
|
|
|
|
if (info->high_limit < length)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
high_limit = info->high_limit - length;
|
|
|
|
|
|
|
|
|
|
if (info->low_limit > high_limit)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
low_limit = info->low_limit + length;
|
|
|
|
|
|
|
|
|
|
/* Check if rbtree root looks promising */
|
|
|
|
|
if (RB_EMPTY_ROOT(&mm->mm_rb))
|
|
|
|
|
goto check_highest;
|
|
|
|
|
vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
|
|
|
|
|
if (vma->rb_subtree_gap < length)
|
|
|
|
|
goto check_highest;
|
|
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
|
/* Visit left subtree if it looks promising */
|
|
|
|
|
gap_end = vma->vm_start;
|
|
|
|
|
if (gap_end >= low_limit && vma->vm_rb.rb_left) {
|
|
|
|
|
struct vm_area_struct *left =
|
|
|
|
|
rb_entry(vma->vm_rb.rb_left,
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (left->rb_subtree_gap >= length) {
|
|
|
|
|
vma = left;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
|
|
|
|
|
check_current:
|
|
|
|
|
/* Check if current node has a suitable gap */
|
|
|
|
|
if (gap_start > high_limit)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
if (gap_end >= low_limit && gap_end - gap_start >= length)
|
|
|
|
|
goto found;
|
|
|
|
|
|
|
|
|
|
/* Visit right subtree if it looks promising */
|
|
|
|
|
if (vma->vm_rb.rb_right) {
|
|
|
|
|
struct vm_area_struct *right =
|
|
|
|
|
rb_entry(vma->vm_rb.rb_right,
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (right->rb_subtree_gap >= length) {
|
|
|
|
|
vma = right;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Go back up the rbtree to find next candidate node */
|
|
|
|
|
while (true) {
|
|
|
|
|
struct rb_node *prev = &vma->vm_rb;
|
|
|
|
|
if (!rb_parent(prev))
|
|
|
|
|
goto check_highest;
|
|
|
|
|
vma = rb_entry(rb_parent(prev),
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (prev == vma->vm_rb.rb_left) {
|
|
|
|
|
gap_start = vma->vm_prev->vm_end;
|
|
|
|
|
gap_end = vma->vm_start;
|
|
|
|
|
goto check_current;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
check_highest:
|
|
|
|
|
/* Check highest gap, which does not precede any rbtree node */
|
|
|
|
|
gap_start = mm->highest_vm_end;
|
|
|
|
|
gap_end = ULONG_MAX; /* Only for VM_BUG_ON below */
|
|
|
|
|
if (gap_start > high_limit)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
found:
|
|
|
|
|
/* We found a suitable gap. Clip it with the original low_limit. */
|
|
|
|
|
if (gap_start < info->low_limit)
|
|
|
|
|
gap_start = info->low_limit;
|
|
|
|
|
|
|
|
|
|
/* Adjust gap address to the desired alignment */
|
|
|
|
|
gap_start += (info->align_offset - gap_start) & info->align_mask;
|
|
|
|
|
|
|
|
|
|
VM_BUG_ON(gap_start + info->length > info->high_limit);
|
|
|
|
|
VM_BUG_ON(gap_start + info->length > gap_end);
|
|
|
|
|
return gap_start;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
|
|
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
unsigned long length, low_limit, high_limit, gap_start, gap_end;
|
|
|
|
|
|
|
|
|
|
/* Adjust search length to account for worst case alignment overhead */
|
|
|
|
|
length = info->length + info->align_mask;
|
|
|
|
|
if (length < info->length)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Adjust search limits by the desired length.
|
|
|
|
|
* See implementation comment at top of unmapped_area().
|
|
|
|
|
*/
|
|
|
|
|
gap_end = info->high_limit;
|
|
|
|
|
if (gap_end < length)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
high_limit = gap_end - length;
|
|
|
|
|
|
|
|
|
|
if (info->low_limit > high_limit)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
low_limit = info->low_limit + length;
|
|
|
|
|
|
|
|
|
|
/* Check highest gap, which does not precede any rbtree node */
|
|
|
|
|
gap_start = mm->highest_vm_end;
|
|
|
|
|
if (gap_start <= high_limit)
|
|
|
|
|
goto found_highest;
|
|
|
|
|
|
|
|
|
|
/* Check if rbtree root looks promising */
|
|
|
|
|
if (RB_EMPTY_ROOT(&mm->mm_rb))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
|
|
|
|
|
if (vma->rb_subtree_gap < length)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
|
/* Visit right subtree if it looks promising */
|
|
|
|
|
gap_start = vma->vm_prev ? vma->vm_prev->vm_end : 0;
|
|
|
|
|
if (gap_start <= high_limit && vma->vm_rb.rb_right) {
|
|
|
|
|
struct vm_area_struct *right =
|
|
|
|
|
rb_entry(vma->vm_rb.rb_right,
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (right->rb_subtree_gap >= length) {
|
|
|
|
|
vma = right;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
check_current:
|
|
|
|
|
/* Check if current node has a suitable gap */
|
|
|
|
|
gap_end = vma->vm_start;
|
|
|
|
|
if (gap_end < low_limit)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
if (gap_start <= high_limit && gap_end - gap_start >= length)
|
|
|
|
|
goto found;
|
|
|
|
|
|
|
|
|
|
/* Visit left subtree if it looks promising */
|
|
|
|
|
if (vma->vm_rb.rb_left) {
|
|
|
|
|
struct vm_area_struct *left =
|
|
|
|
|
rb_entry(vma->vm_rb.rb_left,
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (left->rb_subtree_gap >= length) {
|
|
|
|
|
vma = left;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Go back up the rbtree to find next candidate node */
|
|
|
|
|
while (true) {
|
|
|
|
|
struct rb_node *prev = &vma->vm_rb;
|
|
|
|
|
if (!rb_parent(prev))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
vma = rb_entry(rb_parent(prev),
|
|
|
|
|
struct vm_area_struct, vm_rb);
|
|
|
|
|
if (prev == vma->vm_rb.rb_right) {
|
|
|
|
|
gap_start = vma->vm_prev ?
|
|
|
|
|
vma->vm_prev->vm_end : 0;
|
|
|
|
|
goto check_current;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
found:
|
|
|
|
|
/* We found a suitable gap. Clip it with the original high_limit. */
|
|
|
|
|
if (gap_end > info->high_limit)
|
|
|
|
|
gap_end = info->high_limit;
|
|
|
|
|
|
|
|
|
|
found_highest:
|
|
|
|
|
/* Compute highest gap address at the desired alignment */
|
|
|
|
|
gap_end -= info->length;
|
|
|
|
|
gap_end -= (gap_end - info->align_offset) & info->align_mask;
|
|
|
|
|
|
|
|
|
|
VM_BUG_ON(gap_end < info->low_limit);
|
|
|
|
|
VM_BUG_ON(gap_end < gap_start);
|
|
|
|
|
return gap_end;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Get an address range which is currently unmapped.
|
|
|
|
|
* For shmat() with addr=0.
|
|
|
|
|
*
|
|
|
|
|
* Ugly calling convention alert:
|
|
|
|
|
* Return value with the low bits set means error value,
|
|
|
|
|
* ie
|
|
|
|
|
* if (ret & ~PAGE_MASK)
|
|
|
|
|
* error = ret;
|
|
|
|
|
*
|
|
|
|
|
* This function "knows" that -ENOMEM has the bits set.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef HAVE_ARCH_UNMAPPED_AREA
|
|
|
|
|
unsigned long
|
|
|
|
|
arch_get_unmapped_area(struct file *filp, unsigned long addr,
|
|
|
|
|
unsigned long len, unsigned long pgoff, unsigned long flags)
|
|
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma;
|
2012-12-12 00:01:49 +00:00
|
|
|
struct vm_unmapped_area_info info;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm: ensure get_unmapped_area() returns higher address than mmap_min_addr
This patch fixes the problem that get_unmapped_area() can return illegal
address and result in failing mmap(2) etc.
In case that the address higher than PAGE_SIZE is set to
/proc/sys/vm/mmap_min_addr, the address lower than mmap_min_addr can be
returned by get_unmapped_area(), even if you do not pass any virtual
address hint (i.e. the second argument).
This is because the current get_unmapped_area() code does not take into
account mmap_min_addr.
This leads to two actual problems as follows:
1. mmap(2) can fail with EPERM on the process without CAP_SYS_RAWIO,
although any illegal parameter is not passed.
2. The bottom-up search path after the top-down search might not work in
arch_get_unmapped_area_topdown().
Note: The first and third chunk of my patch, which changes "len" check,
are for more precise check using mmap_min_addr, and not for solving the
above problem.
[How to reproduce]
--- test.c -------------------------------------------------
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/errno.h>
int main(int argc, char *argv[])
{
void *ret = NULL, *last_map;
size_t pagesize = sysconf(_SC_PAGESIZE);
do {
last_map = ret;
ret = mmap(0, pagesize, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
// printf("ret=%p\n", ret);
} while (ret != MAP_FAILED);
if (errno != ENOMEM) {
printf("ERR: unexpected errno: %d (last map=%p)\n",
errno, last_map);
}
return 0;
}
---------------------------------------------------------------
$ gcc -m32 -o test test.c
$ sudo sysctl -w vm.mmap_min_addr=65536
vm.mmap_min_addr = 65536
$ ./test (run as non-priviledge user)
ERR: unexpected errno: 1 (last map=0x10000)
Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:21 +00:00
|
|
|
if (len > TASK_SIZE - mmap_min_addr)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2007-05-06 21:50:13 +00:00
|
|
|
if (flags & MAP_FIXED)
|
|
|
|
|
return addr;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (addr) {
|
|
|
|
|
addr = PAGE_ALIGN(addr);
|
|
|
|
|
vma = find_vma(mm, addr);
|
mm: ensure get_unmapped_area() returns higher address than mmap_min_addr
This patch fixes the problem that get_unmapped_area() can return illegal
address and result in failing mmap(2) etc.
In case that the address higher than PAGE_SIZE is set to
/proc/sys/vm/mmap_min_addr, the address lower than mmap_min_addr can be
returned by get_unmapped_area(), even if you do not pass any virtual
address hint (i.e. the second argument).
This is because the current get_unmapped_area() code does not take into
account mmap_min_addr.
This leads to two actual problems as follows:
1. mmap(2) can fail with EPERM on the process without CAP_SYS_RAWIO,
although any illegal parameter is not passed.
2. The bottom-up search path after the top-down search might not work in
arch_get_unmapped_area_topdown().
Note: The first and third chunk of my patch, which changes "len" check,
are for more precise check using mmap_min_addr, and not for solving the
above problem.
[How to reproduce]
--- test.c -------------------------------------------------
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/errno.h>
int main(int argc, char *argv[])
{
void *ret = NULL, *last_map;
size_t pagesize = sysconf(_SC_PAGESIZE);
do {
last_map = ret;
ret = mmap(0, pagesize, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
// printf("ret=%p\n", ret);
} while (ret != MAP_FAILED);
if (errno != ENOMEM) {
printf("ERR: unexpected errno: %d (last map=%p)\n",
errno, last_map);
}
return 0;
}
---------------------------------------------------------------
$ gcc -m32 -o test test.c
$ sudo sysctl -w vm.mmap_min_addr=65536
vm.mmap_min_addr = 65536
$ ./test (run as non-priviledge user)
ERR: unexpected errno: 1 (last map=0x10000)
Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:21 +00:00
|
|
|
if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
|
2005-04-16 22:20:36 +00:00
|
|
|
(!vma || addr + len <= vma->vm_start))
|
|
|
|
|
return addr;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:49 +00:00
|
|
|
info.flags = 0;
|
|
|
|
|
info.length = len;
|
2013-11-12 23:07:54 +00:00
|
|
|
info.low_limit = mm->mmap_base;
|
2012-12-12 00:01:49 +00:00
|
|
|
info.high_limit = TASK_SIZE;
|
|
|
|
|
info.align_mask = 0;
|
|
|
|
|
return vm_unmapped_area(&info);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2014-10-09 22:26:29 +00:00
|
|
|
#endif
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This mmap-allocator allocates new areas top-down from below the
|
|
|
|
|
* stack's low limit (the base):
|
|
|
|
|
*/
|
|
|
|
|
#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
|
|
|
|
|
unsigned long
|
|
|
|
|
arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
|
|
|
|
|
const unsigned long len, const unsigned long pgoff,
|
|
|
|
|
const unsigned long flags)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
2012-12-12 00:01:49 +00:00
|
|
|
unsigned long addr = addr0;
|
|
|
|
|
struct vm_unmapped_area_info info;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* requested length too big for entire address space */
|
mm: ensure get_unmapped_area() returns higher address than mmap_min_addr
This patch fixes the problem that get_unmapped_area() can return illegal
address and result in failing mmap(2) etc.
In case that the address higher than PAGE_SIZE is set to
/proc/sys/vm/mmap_min_addr, the address lower than mmap_min_addr can be
returned by get_unmapped_area(), even if you do not pass any virtual
address hint (i.e. the second argument).
This is because the current get_unmapped_area() code does not take into
account mmap_min_addr.
This leads to two actual problems as follows:
1. mmap(2) can fail with EPERM on the process without CAP_SYS_RAWIO,
although any illegal parameter is not passed.
2. The bottom-up search path after the top-down search might not work in
arch_get_unmapped_area_topdown().
Note: The first and third chunk of my patch, which changes "len" check,
are for more precise check using mmap_min_addr, and not for solving the
above problem.
[How to reproduce]
--- test.c -------------------------------------------------
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/errno.h>
int main(int argc, char *argv[])
{
void *ret = NULL, *last_map;
size_t pagesize = sysconf(_SC_PAGESIZE);
do {
last_map = ret;
ret = mmap(0, pagesize, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
// printf("ret=%p\n", ret);
} while (ret != MAP_FAILED);
if (errno != ENOMEM) {
printf("ERR: unexpected errno: %d (last map=%p)\n",
errno, last_map);
}
return 0;
}
---------------------------------------------------------------
$ gcc -m32 -o test test.c
$ sudo sysctl -w vm.mmap_min_addr=65536
vm.mmap_min_addr = 65536
$ ./test (run as non-priviledge user)
ERR: unexpected errno: 1 (last map=0x10000)
Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:21 +00:00
|
|
|
if (len > TASK_SIZE - mmap_min_addr)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2007-05-06 21:50:13 +00:00
|
|
|
if (flags & MAP_FIXED)
|
|
|
|
|
return addr;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* requesting a specific address */
|
|
|
|
|
if (addr) {
|
|
|
|
|
addr = PAGE_ALIGN(addr);
|
|
|
|
|
vma = find_vma(mm, addr);
|
mm: ensure get_unmapped_area() returns higher address than mmap_min_addr
This patch fixes the problem that get_unmapped_area() can return illegal
address and result in failing mmap(2) etc.
In case that the address higher than PAGE_SIZE is set to
/proc/sys/vm/mmap_min_addr, the address lower than mmap_min_addr can be
returned by get_unmapped_area(), even if you do not pass any virtual
address hint (i.e. the second argument).
This is because the current get_unmapped_area() code does not take into
account mmap_min_addr.
This leads to two actual problems as follows:
1. mmap(2) can fail with EPERM on the process without CAP_SYS_RAWIO,
although any illegal parameter is not passed.
2. The bottom-up search path after the top-down search might not work in
arch_get_unmapped_area_topdown().
Note: The first and third chunk of my patch, which changes "len" check,
are for more precise check using mmap_min_addr, and not for solving the
above problem.
[How to reproduce]
--- test.c -------------------------------------------------
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/errno.h>
int main(int argc, char *argv[])
{
void *ret = NULL, *last_map;
size_t pagesize = sysconf(_SC_PAGESIZE);
do {
last_map = ret;
ret = mmap(0, pagesize, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
// printf("ret=%p\n", ret);
} while (ret != MAP_FAILED);
if (errno != ENOMEM) {
printf("ERR: unexpected errno: %d (last map=%p)\n",
errno, last_map);
}
return 0;
}
---------------------------------------------------------------
$ gcc -m32 -o test test.c
$ sudo sysctl -w vm.mmap_min_addr=65536
vm.mmap_min_addr = 65536
$ ./test (run as non-priviledge user)
ERR: unexpected errno: 1 (last map=0x10000)
Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:21 +00:00
|
|
|
if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
|
2005-04-16 22:20:36 +00:00
|
|
|
(!vma || addr + len <= vma->vm_start))
|
|
|
|
|
return addr;
|
|
|
|
|
}
|
|
|
|
|
|
2012-12-12 00:01:49 +00:00
|
|
|
info.flags = VM_UNMAPPED_AREA_TOPDOWN;
|
|
|
|
|
info.length = len;
|
mm: ensure get_unmapped_area() returns higher address than mmap_min_addr
This patch fixes the problem that get_unmapped_area() can return illegal
address and result in failing mmap(2) etc.
In case that the address higher than PAGE_SIZE is set to
/proc/sys/vm/mmap_min_addr, the address lower than mmap_min_addr can be
returned by get_unmapped_area(), even if you do not pass any virtual
address hint (i.e. the second argument).
This is because the current get_unmapped_area() code does not take into
account mmap_min_addr.
This leads to two actual problems as follows:
1. mmap(2) can fail with EPERM on the process without CAP_SYS_RAWIO,
although any illegal parameter is not passed.
2. The bottom-up search path after the top-down search might not work in
arch_get_unmapped_area_topdown().
Note: The first and third chunk of my patch, which changes "len" check,
are for more precise check using mmap_min_addr, and not for solving the
above problem.
[How to reproduce]
--- test.c -------------------------------------------------
#include <stdio.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/errno.h>
int main(int argc, char *argv[])
{
void *ret = NULL, *last_map;
size_t pagesize = sysconf(_SC_PAGESIZE);
do {
last_map = ret;
ret = mmap(0, pagesize, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
// printf("ret=%p\n", ret);
} while (ret != MAP_FAILED);
if (errno != ENOMEM) {
printf("ERR: unexpected errno: %d (last map=%p)\n",
errno, last_map);
}
return 0;
}
---------------------------------------------------------------
$ gcc -m32 -o test test.c
$ sudo sysctl -w vm.mmap_min_addr=65536
vm.mmap_min_addr = 65536
$ ./test (run as non-priviledge user)
ERR: unexpected errno: 1 (last map=0x10000)
Signed-off-by: Akira Takeuchi <takeuchi.akr@jp.panasonic.com>
Signed-off-by: Kiyoshi Owada <owada.kiyoshi@jp.panasonic.com>
Reviewed-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Cc: <stable@vger.kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-12 23:08:21 +00:00
|
|
|
info.low_limit = max(PAGE_SIZE, mmap_min_addr);
|
2012-12-12 00:01:49 +00:00
|
|
|
info.high_limit = mm->mmap_base;
|
|
|
|
|
info.align_mask = 0;
|
|
|
|
|
addr = vm_unmapped_area(&info);
|
2012-03-21 23:33:56 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* A failed mmap() very likely causes application failure,
|
|
|
|
|
* so fall back to the bottom-up function here. This scenario
|
|
|
|
|
* can happen with large stack limits and large mmap()
|
|
|
|
|
* allocations.
|
|
|
|
|
*/
|
2015-11-06 02:46:54 +00:00
|
|
|
if (offset_in_page(addr)) {
|
2012-12-12 00:01:49 +00:00
|
|
|
VM_BUG_ON(addr != -ENOMEM);
|
|
|
|
|
info.flags = 0;
|
|
|
|
|
info.low_limit = TASK_UNMAPPED_BASE;
|
|
|
|
|
info.high_limit = TASK_SIZE;
|
|
|
|
|
addr = vm_unmapped_area(&info);
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
return addr;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
unsigned long
|
|
|
|
|
get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
|
|
|
|
|
unsigned long pgoff, unsigned long flags)
|
|
|
|
|
{
|
2007-05-06 21:50:13 +00:00
|
|
|
unsigned long (*get_area)(struct file *, unsigned long,
|
|
|
|
|
unsigned long, unsigned long, unsigned long);
|
|
|
|
|
|
2009-12-03 20:23:11 +00:00
|
|
|
unsigned long error = arch_mmap_check(addr, len, flags);
|
|
|
|
|
if (error)
|
|
|
|
|
return error;
|
|
|
|
|
|
|
|
|
|
/* Careful about overflows.. */
|
|
|
|
|
if (len > TASK_SIZE)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2007-05-06 21:50:13 +00:00
|
|
|
get_area = current->mm->get_unmapped_area;
|
2016-07-26 22:26:15 +00:00
|
|
|
if (file) {
|
|
|
|
|
if (file->f_op->get_unmapped_area)
|
|
|
|
|
get_area = file->f_op->get_unmapped_area;
|
|
|
|
|
} else if (flags & MAP_SHARED) {
|
|
|
|
|
/*
|
|
|
|
|
* mmap_region() will call shmem_zero_setup() to create a file,
|
|
|
|
|
* so use shmem's get_unmapped_area in case it can be huge.
|
|
|
|
|
* do_mmap_pgoff() will clear pgoff, so match alignment.
|
|
|
|
|
*/
|
|
|
|
|
pgoff = 0;
|
|
|
|
|
get_area = shmem_get_unmapped_area;
|
|
|
|
|
}
|
|
|
|
|
|
2007-05-06 21:50:13 +00:00
|
|
|
addr = get_area(file, addr, len, pgoff, flags);
|
|
|
|
|
if (IS_ERR_VALUE(addr))
|
|
|
|
|
return addr;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2005-05-20 05:43:37 +00:00
|
|
|
if (addr > TASK_SIZE - len)
|
|
|
|
|
return -ENOMEM;
|
2015-11-06 02:46:54 +00:00
|
|
|
if (offset_in_page(addr))
|
2005-05-20 05:43:37 +00:00
|
|
|
return -EINVAL;
|
2007-05-06 21:50:13 +00:00
|
|
|
|
2012-05-30 21:13:15 +00:00
|
|
|
error = security_mmap_addr(addr);
|
|
|
|
|
return error ? error : addr;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(get_unmapped_area);
|
|
|
|
|
|
|
|
|
|
/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
|
2009-01-06 22:40:21 +00:00
|
|
|
struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
struct rb_node *rb_node;
|
|
|
|
|
struct vm_area_struct *vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-05-29 22:06:21 +00:00
|
|
|
/* Check the cache first. */
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
vma = vmacache_find(mm, addr);
|
|
|
|
|
if (likely(vma))
|
|
|
|
|
return vma;
|
2012-05-29 22:06:21 +00:00
|
|
|
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
rb_node = mm->mm_rb.rb_node;
|
2012-05-29 22:06:21 +00:00
|
|
|
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
while (rb_node) {
|
|
|
|
|
struct vm_area_struct *tmp;
|
|
|
|
|
|
|
|
|
|
tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
|
|
|
|
|
|
|
|
|
|
if (tmp->vm_end > addr) {
|
|
|
|
|
vma = tmp;
|
|
|
|
|
if (tmp->vm_start <= addr)
|
|
|
|
|
break;
|
|
|
|
|
rb_node = rb_node->rb_left;
|
|
|
|
|
} else
|
|
|
|
|
rb_node = rb_node->rb_right;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
|
|
|
|
|
if (vma)
|
|
|
|
|
vmacache_update(addr, vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
return vma;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
EXPORT_SYMBOL(find_vma);
|
|
|
|
|
|
2012-01-10 23:08:07 +00:00
|
|
|
/*
|
|
|
|
|
* Same as find_vma, but also return a pointer to the previous VMA in *pprev.
|
|
|
|
|
*/
|
2005-04-16 22:20:36 +00:00
|
|
|
struct vm_area_struct *
|
|
|
|
|
find_vma_prev(struct mm_struct *mm, unsigned long addr,
|
|
|
|
|
struct vm_area_struct **pprev)
|
|
|
|
|
{
|
2012-01-10 23:08:07 +00:00
|
|
|
struct vm_area_struct *vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-01-10 23:08:07 +00:00
|
|
|
vma = find_vma(mm, addr);
|
2012-03-05 00:52:03 +00:00
|
|
|
if (vma) {
|
|
|
|
|
*pprev = vma->vm_prev;
|
|
|
|
|
} else {
|
|
|
|
|
struct rb_node *rb_node = mm->mm_rb.rb_node;
|
|
|
|
|
*pprev = NULL;
|
|
|
|
|
while (rb_node) {
|
|
|
|
|
*pprev = rb_entry(rb_node, struct vm_area_struct, vm_rb);
|
|
|
|
|
rb_node = rb_node->rb_right;
|
|
|
|
|
}
|
|
|
|
|
}
|
2012-01-10 23:08:07 +00:00
|
|
|
return vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Verify that the stack growth is acceptable and
|
|
|
|
|
* update accounting. This is shared with both the
|
|
|
|
|
* grow-up and grow-down cases.
|
|
|
|
|
*/
|
2009-01-06 22:40:21 +00:00
|
|
|
static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
|
|
|
struct rlimit *rlim = current->signal->rlim;
|
2015-01-11 19:33:57 +00:00
|
|
|
unsigned long new_start, actual_size;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* address space limit tests */
|
2016-01-14 23:22:07 +00:00
|
|
|
if (!may_expand_vm(mm, vma->vm_flags, grow))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* Stack limit test */
|
2015-01-11 19:33:57 +00:00
|
|
|
actual_size = size;
|
|
|
|
|
if (size && (vma->vm_flags & (VM_GROWSUP | VM_GROWSDOWN)))
|
|
|
|
|
actual_size -= PAGE_SIZE;
|
2015-04-15 23:14:08 +00:00
|
|
|
if (actual_size > READ_ONCE(rlim[RLIMIT_STACK].rlim_cur))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* mlock limit tests */
|
|
|
|
|
if (vma->vm_flags & VM_LOCKED) {
|
|
|
|
|
unsigned long locked;
|
|
|
|
|
unsigned long limit;
|
|
|
|
|
locked = mm->locked_vm + grow;
|
2015-04-15 23:14:08 +00:00
|
|
|
limit = READ_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
|
2010-03-05 21:41:44 +00:00
|
|
|
limit >>= PAGE_SHIFT;
|
2005-04-16 22:20:36 +00:00
|
|
|
if (locked > limit && !capable(CAP_IPC_LOCK))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
|
2007-01-30 22:35:39 +00:00
|
|
|
/* Check to ensure the stack will not grow into a hugetlb-only region */
|
|
|
|
|
new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
|
|
|
|
|
vma->vm_end - size;
|
|
|
|
|
if (is_hugepage_only_range(vma->vm_mm, new_start, size))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* Overcommit.. This must be the final test, as it will
|
|
|
|
|
* update security statistics.
|
|
|
|
|
*/
|
2009-04-16 20:58:12 +00:00
|
|
|
if (security_vm_enough_memory_mm(mm, grow))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2005-10-30 01:16:20 +00:00
|
|
|
#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2005-10-30 01:16:20 +00:00
|
|
|
* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
|
|
|
|
|
* vma is the last one with address > vma->vm_end. Have to extend vma.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2005-10-30 01:16:20 +00:00
|
|
|
int expand_upwards(struct vm_area_struct *vma, unsigned long address)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-11-06 02:48:17 +00:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2016-02-05 23:36:50 +00:00
|
|
|
int error = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
if (!(vma->vm_flags & VM_GROWSUP))
|
|
|
|
|
return -EFAULT;
|
|
|
|
|
|
2016-02-05 23:36:50 +00:00
|
|
|
/* Guard against wrapping around to address 0. */
|
|
|
|
|
if (address < PAGE_ALIGN(address+4))
|
|
|
|
|
address = PAGE_ALIGN(address+4);
|
|
|
|
|
else
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* We must make sure the anon_vma is allocated. */
|
2005-04-16 22:20:36 +00:00
|
|
|
if (unlikely(anon_vma_prepare(vma)))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* vma->vm_start/vm_end cannot change under us because the caller
|
|
|
|
|
* is required to hold the mmap_sem in read mode. We need the
|
|
|
|
|
* anon_vma lock to serialize against concurrent expand_stacks.
|
|
|
|
|
*/
|
2016-02-05 23:36:50 +00:00
|
|
|
anon_vma_lock_write(vma->anon_vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* Somebody else might have raced and expanded it already */
|
|
|
|
|
if (address > vma->vm_end) {
|
|
|
|
|
unsigned long size, grow;
|
|
|
|
|
|
|
|
|
|
size = address - vma->vm_start;
|
|
|
|
|
grow = (address - vma->vm_end) >> PAGE_SHIFT;
|
|
|
|
|
|
2011-05-10 00:44:42 +00:00
|
|
|
error = -ENOMEM;
|
|
|
|
|
if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
|
|
|
|
|
error = acct_stack_growth(vma, size, grow);
|
|
|
|
|
if (!error) {
|
2012-12-12 21:52:25 +00:00
|
|
|
/*
|
|
|
|
|
* vma_gap_update() doesn't support concurrent
|
|
|
|
|
* updates, but we only hold a shared mmap_sem
|
|
|
|
|
* lock here, so we need to protect against
|
|
|
|
|
* concurrent vma expansions.
|
2016-02-05 23:36:50 +00:00
|
|
|
* anon_vma_lock_write() doesn't help here, as
|
2012-12-12 21:52:25 +00:00
|
|
|
* we don't guarantee that all growable vmas
|
|
|
|
|
* in a mm share the same root anon vma.
|
|
|
|
|
* So, we reuse mm->page_table_lock to guard
|
|
|
|
|
* against concurrent vma expansions.
|
|
|
|
|
*/
|
2015-11-06 02:48:17 +00:00
|
|
|
spin_lock(&mm->page_table_lock);
|
2015-11-06 02:48:14 +00:00
|
|
|
if (vma->vm_flags & VM_LOCKED)
|
2015-11-06 02:48:17 +00:00
|
|
|
mm->locked_vm += grow;
|
2016-01-14 23:22:07 +00:00
|
|
|
vm_stat_account(mm, vma->vm_flags, grow);
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma_interval_tree_pre_update_vma(vma);
|
2011-05-10 00:44:42 +00:00
|
|
|
vma->vm_end = address;
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma_interval_tree_post_update_vma(vma);
|
2012-12-12 00:01:38 +00:00
|
|
|
if (vma->vm_next)
|
|
|
|
|
vma_gap_update(vma->vm_next);
|
|
|
|
|
else
|
2015-11-06 02:48:17 +00:00
|
|
|
mm->highest_vm_end = address;
|
|
|
|
|
spin_unlock(&mm->page_table_lock);
|
2012-12-12 21:52:25 +00:00
|
|
|
|
2011-05-10 00:44:42 +00:00
|
|
|
perf_event_mmap(vma);
|
|
|
|
|
}
|
2010-05-18 14:30:49 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-02-05 23:36:50 +00:00
|
|
|
anon_vma_unlock_write(vma->anon_vma);
|
2014-10-29 21:50:31 +00:00
|
|
|
khugepaged_enter_vma_merge(vma, vma->vm_flags);
|
2015-11-06 02:48:17 +00:00
|
|
|
validate_mm(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
return error;
|
|
|
|
|
}
|
2005-10-30 01:16:20 +00:00
|
|
|
#endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* vma is the first one with address < vma->vm_start. Have to extend vma.
|
|
|
|
|
*/
|
2011-05-25 00:11:44 +00:00
|
|
|
int expand_downwards(struct vm_area_struct *vma,
|
2007-07-19 08:48:16 +00:00
|
|
|
unsigned long address)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2015-11-06 02:48:17 +00:00
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
2005-04-16 22:20:36 +00:00
|
|
|
int error;
|
|
|
|
|
|
2007-11-26 23:47:26 +00:00
|
|
|
address &= PAGE_MASK;
|
2012-05-30 17:30:51 +00:00
|
|
|
error = security_mmap_addr(address);
|
2007-11-26 23:47:26 +00:00
|
|
|
if (error)
|
|
|
|
|
return error;
|
|
|
|
|
|
2016-02-05 23:36:50 +00:00
|
|
|
/* We must make sure the anon_vma is allocated. */
|
|
|
|
|
if (unlikely(anon_vma_prepare(vma)))
|
|
|
|
|
return -ENOMEM;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* vma->vm_start/vm_end cannot change under us because the caller
|
|
|
|
|
* is required to hold the mmap_sem in read mode. We need the
|
|
|
|
|
* anon_vma lock to serialize against concurrent expand_stacks.
|
|
|
|
|
*/
|
2016-02-05 23:36:50 +00:00
|
|
|
anon_vma_lock_write(vma->anon_vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* Somebody else might have raced and expanded it already */
|
|
|
|
|
if (address < vma->vm_start) {
|
|
|
|
|
unsigned long size, grow;
|
|
|
|
|
|
|
|
|
|
size = vma->vm_end - address;
|
|
|
|
|
grow = (vma->vm_start - address) >> PAGE_SHIFT;
|
|
|
|
|
|
2011-04-13 15:07:28 +00:00
|
|
|
error = -ENOMEM;
|
|
|
|
|
if (grow <= vma->vm_pgoff) {
|
|
|
|
|
error = acct_stack_growth(vma, size, grow);
|
|
|
|
|
if (!error) {
|
2012-12-12 21:52:25 +00:00
|
|
|
/*
|
|
|
|
|
* vma_gap_update() doesn't support concurrent
|
|
|
|
|
* updates, but we only hold a shared mmap_sem
|
|
|
|
|
* lock here, so we need to protect against
|
|
|
|
|
* concurrent vma expansions.
|
2016-02-05 23:36:50 +00:00
|
|
|
* anon_vma_lock_write() doesn't help here, as
|
2012-12-12 21:52:25 +00:00
|
|
|
* we don't guarantee that all growable vmas
|
|
|
|
|
* in a mm share the same root anon vma.
|
|
|
|
|
* So, we reuse mm->page_table_lock to guard
|
|
|
|
|
* against concurrent vma expansions.
|
|
|
|
|
*/
|
2015-11-06 02:48:17 +00:00
|
|
|
spin_lock(&mm->page_table_lock);
|
2015-11-06 02:48:14 +00:00
|
|
|
if (vma->vm_flags & VM_LOCKED)
|
2015-11-06 02:48:17 +00:00
|
|
|
mm->locked_vm += grow;
|
2016-01-14 23:22:07 +00:00
|
|
|
vm_stat_account(mm, vma->vm_flags, grow);
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma_interval_tree_pre_update_vma(vma);
|
2011-04-13 15:07:28 +00:00
|
|
|
vma->vm_start = address;
|
|
|
|
|
vma->vm_pgoff -= grow;
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
anon_vma_interval_tree_post_update_vma(vma);
|
2012-12-12 00:01:38 +00:00
|
|
|
vma_gap_update(vma);
|
2015-11-06 02:48:17 +00:00
|
|
|
spin_unlock(&mm->page_table_lock);
|
2012-12-12 21:52:25 +00:00
|
|
|
|
2011-04-13 15:07:28 +00:00
|
|
|
perf_event_mmap(vma);
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
}
|
2016-02-05 23:36:50 +00:00
|
|
|
anon_vma_unlock_write(vma->anon_vma);
|
2014-10-29 21:50:31 +00:00
|
|
|
khugepaged_enter_vma_merge(vma, vma->vm_flags);
|
2015-11-06 02:48:17 +00:00
|
|
|
validate_mm(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
return error;
|
|
|
|
|
}
|
|
|
|
|
|
2013-02-27 16:36:04 +00:00
|
|
|
/*
|
|
|
|
|
* Note how expand_stack() refuses to expand the stack all the way to
|
|
|
|
|
* abut the next virtual mapping, *unless* that mapping itself is also
|
|
|
|
|
* a stack mapping. We want to leave room for a guard page, after all
|
|
|
|
|
* (the guard page itself is not added here, that is done by the
|
|
|
|
|
* actual page faulting logic)
|
|
|
|
|
*
|
|
|
|
|
* This matches the behavior of the guard page logic (see mm/memory.c:
|
|
|
|
|
* check_stack_guard_page()), which only allows the guard page to be
|
|
|
|
|
* removed under these circumstances.
|
|
|
|
|
*/
|
2007-07-19 08:48:16 +00:00
|
|
|
#ifdef CONFIG_STACK_GROWSUP
|
|
|
|
|
int expand_stack(struct vm_area_struct *vma, unsigned long address)
|
|
|
|
|
{
|
2013-02-27 16:36:04 +00:00
|
|
|
struct vm_area_struct *next;
|
|
|
|
|
|
|
|
|
|
address &= PAGE_MASK;
|
|
|
|
|
next = vma->vm_next;
|
|
|
|
|
if (next && next->vm_start == address + PAGE_SIZE) {
|
|
|
|
|
if (!(next->vm_flags & VM_GROWSUP))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
2007-07-19 08:48:16 +00:00
|
|
|
return expand_upwards(vma, address);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct vm_area_struct *
|
|
|
|
|
find_extend_vma(struct mm_struct *mm, unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *vma, *prev;
|
|
|
|
|
|
|
|
|
|
addr &= PAGE_MASK;
|
|
|
|
|
vma = find_vma_prev(mm, addr, &prev);
|
|
|
|
|
if (vma && (vma->vm_start <= addr))
|
|
|
|
|
return vma;
|
2008-11-12 00:24:41 +00:00
|
|
|
if (!prev || expand_stack(prev, addr))
|
2007-07-19 08:48:16 +00:00
|
|
|
return NULL;
|
2013-02-23 00:32:44 +00:00
|
|
|
if (prev->vm_flags & VM_LOCKED)
|
2015-04-14 22:44:39 +00:00
|
|
|
populate_vma_page_range(prev, addr, prev->vm_end, NULL);
|
2007-07-19 08:48:16 +00:00
|
|
|
return prev;
|
|
|
|
|
}
|
|
|
|
|
#else
|
|
|
|
|
int expand_stack(struct vm_area_struct *vma, unsigned long address)
|
|
|
|
|
{
|
2013-02-27 16:36:04 +00:00
|
|
|
struct vm_area_struct *prev;
|
|
|
|
|
|
|
|
|
|
address &= PAGE_MASK;
|
|
|
|
|
prev = vma->vm_prev;
|
|
|
|
|
if (prev && prev->vm_end == address) {
|
|
|
|
|
if (!(prev->vm_flags & VM_GROWSDOWN))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
2007-07-19 08:48:16 +00:00
|
|
|
return expand_downwards(vma, address);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
struct vm_area_struct *
|
2014-10-09 22:26:29 +00:00
|
|
|
find_extend_vma(struct mm_struct *mm, unsigned long addr)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:29 +00:00
|
|
|
struct vm_area_struct *vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long start;
|
|
|
|
|
|
|
|
|
|
addr &= PAGE_MASK;
|
2014-10-09 22:26:29 +00:00
|
|
|
vma = find_vma(mm, addr);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!vma)
|
|
|
|
|
return NULL;
|
|
|
|
|
if (vma->vm_start <= addr)
|
|
|
|
|
return vma;
|
|
|
|
|
if (!(vma->vm_flags & VM_GROWSDOWN))
|
|
|
|
|
return NULL;
|
|
|
|
|
start = vma->vm_start;
|
|
|
|
|
if (expand_stack(vma, addr))
|
|
|
|
|
return NULL;
|
2013-02-23 00:32:44 +00:00
|
|
|
if (vma->vm_flags & VM_LOCKED)
|
2015-04-14 22:44:39 +00:00
|
|
|
populate_vma_page_range(vma, addr, start, NULL);
|
2005-04-16 22:20:36 +00:00
|
|
|
return vma;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2014-12-13 00:55:27 +00:00
|
|
|
EXPORT_SYMBOL_GPL(find_extend_vma);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
2005-10-30 01:15:56 +00:00
|
|
|
* Ok - we have the memory areas we should free on the vma list,
|
2005-04-16 22:20:36 +00:00
|
|
|
* so release them, and do the vma updates.
|
2005-10-30 01:15:56 +00:00
|
|
|
*
|
|
|
|
|
* Called with the mm semaphore held.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2005-10-30 01:15:56 +00:00
|
|
|
static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-05-06 20:54:06 +00:00
|
|
|
unsigned long nr_accounted = 0;
|
|
|
|
|
|
[PATCH] mm: update_hiwaters just in time
update_mem_hiwater has attracted various criticisms, in particular from those
concerned with mm scalability. Originally it was called whenever rss or
total_vm got raised. Then many of those callsites were replaced by a timer
tick call from account_system_time. Now Frank van Maarseveen reports that to
be found inadequate. How about this? Works for Frank.
Replace update_mem_hiwater, a poor combination of two unrelated ops, by macros
update_hiwater_rss and update_hiwater_vm. Don't attempt to keep
mm->hiwater_rss up to date at timer tick, nor every time we raise rss (usually
by 1): those are hot paths. Do the opposite, update only when about to lower
rss (usually by many), or just before final accounting in do_exit. Handle
mm->hiwater_vm in the same way, though it's much less of an issue. Demand
that whoever collects these hiwater statistics do the work of taking the
maximum with rss or total_vm.
And there has been no collector of these hiwater statistics in the tree. The
new convention needs an example, so match Frank's usage by adding a VmPeak
line above VmSize to /proc/<pid>/status, and also a VmHWM line above VmRSS
(High-Water-Mark or High-Water-Memory).
There was a particular anomaly during mremap move, that hiwater_vm might be
captured too high. A fleeting such anomaly remains, but it's quickly
corrected now, whereas before it would stick.
What locking? None: if the app is racy then these statistics will be racy,
it's not worth any overhead to make them exact. But whenever it suits,
hiwater_vm is updated under exclusive mmap_sem, and hiwater_rss under
page_table_lock (for now) or with preemption disabled (later on): without
going to any trouble, minimize the time between reading current values and
updating, to minimize those occasions when a racing thread bumps a count up
and back down in between.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:18 +00:00
|
|
|
/* Update high watermark before we lower total_vm */
|
|
|
|
|
update_hiwater_vm(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
do {
|
2005-10-30 01:15:56 +00:00
|
|
|
long nrpages = vma_pages(vma);
|
|
|
|
|
|
2012-05-06 20:54:06 +00:00
|
|
|
if (vma->vm_flags & VM_ACCOUNT)
|
|
|
|
|
nr_accounted += nrpages;
|
2016-01-14 23:22:07 +00:00
|
|
|
vm_stat_account(mm, vma->vm_flags, -nrpages);
|
2005-10-30 01:15:57 +00:00
|
|
|
vma = remove_vma(vma);
|
2005-04-19 20:29:18 +00:00
|
|
|
} while (vma);
|
2012-05-06 20:54:06 +00:00
|
|
|
vm_unacct_memory(nr_accounted);
|
2005-04-16 22:20:36 +00:00
|
|
|
validate_mm(mm);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Get rid of page table information in the indicated region.
|
|
|
|
|
*
|
2005-09-21 16:55:37 +00:00
|
|
|
* Called with the mm semaphore held.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
static void unmap_region(struct mm_struct *mm,
|
[PATCH] freepgt: free_pgtables use vma list
Recent woes with some arches needing their own pgd_addr_end macro; and 4-level
clear_page_range regression since 2.6.10's clear_page_tables; and its
long-standing well-known inefficiency in searching throughout the higher-level
page tables for those few entries to clear and free: all can be blamed on
ignoring the list of vmas when we free page tables.
Replace exit_mmap's clear_page_range of the total user address space by
free_pgtables operating on the mm's vma list; unmap_region use it in the same
way, giving floor and ceiling beyond which it may not free tables. This
brings lmbench fork/exec/sh numbers back to 2.6.10 (unless preempt is enabled,
in which case latency fixes spoil unmap_vmas throughput).
Beware: the do_mmap_pgoff driver failure case must now use unmap_region
instead of zap_page_range, since a page table might have been allocated, and
can only be freed while it is touched by some vma.
Move free_pgtables from mmap.c to memory.c, where its lower levels are adapted
from the clear_page_range levels. (Most of free_pgtables' old code was
actually for a non-existent case, prev not properly set up, dating from before
hch gave us split_vma.) Pass mmu_gather** in the public interfaces, since we
might want to add latency lockdrops later; but no attempt to do so yet, going
by vma should itself reduce latency.
But what if is_hugepage_only_range? Those ia64 and ppc64 cases need careful
examination: put that off until a later patch of the series.
What of x86_64's 32bit vdso page __map_syscall32 maps outside any vma?
And the range to sparc64's flush_tlb_pgtables? It's less clear to me now that
we need to do more than is done here - every PMD_SIZE ever occupied will be
flushed, do we really have to flush every PGDIR_SIZE ever partially occupied?
A shame to complicate it unnecessarily.
Special thanks to David Miller for time spent repairing my ceilings.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-04-19 20:29:15 +00:00
|
|
|
struct vm_area_struct *vma, struct vm_area_struct *prev,
|
|
|
|
|
unsigned long start, unsigned long end)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:29 +00:00
|
|
|
struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
|
2011-05-25 00:11:45 +00:00
|
|
|
struct mmu_gather tlb;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
lru_add_drain();
|
Fix TLB gather virtual address range invalidation corner cases
Ben Tebulin reported:
"Since v3.7.2 on two independent machines a very specific Git
repository fails in 9/10 cases on git-fsck due to an SHA1/memory
failures. This only occurs on a very specific repository and can be
reproduced stably on two independent laptops. Git mailing list ran
out of ideas and for me this looks like some very exotic kernel issue"
and bisected the failure to the backport of commit 53a59fc67f97 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").
That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.
The real bug is that the TLB gather virtual memory range setup is subtly
buggered. It was introduced in commit 597e1c3580b7 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range handling
was already fixed at least once in commit e6c495a96ce0 ("mm: fix the TLB
range flushed when __tlb_remove_page() runs out of slots"), but that fix
was not complete.
The problem with the TLB gather virtual address range is that it isn't
set up by the initial tlb_gather_mmu() initialization (which didn't get
the TLB range information), but it is set up ad-hoc later by the
functions that actually flush the TLB. And so any such case that forgot
to update the TLB range entries would potentially miss TLB invalidates.
Rather than try to figure out exactly which particular ad-hoc range
setup was missing (I personally suspect it's the hugetlb case in
zap_huge_pmd(), which didn't have the same logic as zap_pte_range()
did), this patch just gets rid of the problem at the source: make the
TLB range information available to tlb_gather_mmu(), and initialize it
when initializing all the other tlb gather fields.
This makes the patch larger, but conceptually much simpler. And the end
result is much more understandable; even if you want to play games with
partial ranges when invalidating the TLB contents in chunks, now the
range information is always there, and anybody who doesn't want to
bother with it won't introduce subtle bugs.
Ben verified that this fixes his problem.
Reported-bisected-and-tested-by: Ben Tebulin <tebulin@googlemail.com>
Build-testing-by: Stephen Rothwell <sfr@canb.auug.org.au>
Build-testing-by: Richard Weinberger <richard.weinberger@gmail.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-08-15 18:42:25 +00:00
|
|
|
tlb_gather_mmu(&tlb, mm, start, end);
|
[PATCH] mm: update_hiwaters just in time
update_mem_hiwater has attracted various criticisms, in particular from those
concerned with mm scalability. Originally it was called whenever rss or
total_vm got raised. Then many of those callsites were replaced by a timer
tick call from account_system_time. Now Frank van Maarseveen reports that to
be found inadequate. How about this? Works for Frank.
Replace update_mem_hiwater, a poor combination of two unrelated ops, by macros
update_hiwater_rss and update_hiwater_vm. Don't attempt to keep
mm->hiwater_rss up to date at timer tick, nor every time we raise rss (usually
by 1): those are hot paths. Do the opposite, update only when about to lower
rss (usually by many), or just before final accounting in do_exit. Handle
mm->hiwater_vm in the same way, though it's much less of an issue. Demand
that whoever collects these hiwater statistics do the work of taking the
maximum with rss or total_vm.
And there has been no collector of these hiwater statistics in the tree. The
new convention needs an example, so match Frank's usage by adding a VmPeak
line above VmSize to /proc/<pid>/status, and also a VmHWM line above VmRSS
(High-Water-Mark or High-Water-Memory).
There was a particular anomaly during mremap move, that hiwater_vm might be
captured too high. A fleeting such anomaly remains, but it's quickly
corrected now, whereas before it would stick.
What locking? None: if the app is racy then these statistics will be racy,
it's not worth any overhead to make them exact. But whenever it suits,
hiwater_vm is updated under exclusive mmap_sem, and hiwater_rss under
page_table_lock (for now) or with preemption disabled (later on): without
going to any trouble, minimize the time between reading current values and
updating, to minimize those occasions when a racing thread bumps a count up
and back down in between.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:18 +00:00
|
|
|
update_hiwater_rss(mm);
|
2012-05-06 20:54:06 +00:00
|
|
|
unmap_vmas(&tlb, vma, start, end);
|
2011-05-25 00:11:45 +00:00
|
|
|
free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
|
2013-04-29 22:07:44 +00:00
|
|
|
next ? next->vm_start : USER_PGTABLES_CEILING);
|
2011-05-25 00:11:45 +00:00
|
|
|
tlb_finish_mmu(&tlb, start, end);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Create a list of vma's touched by the unmap, removing them from the mm's
|
|
|
|
|
* vma list as we go..
|
|
|
|
|
*/
|
|
|
|
|
static void
|
|
|
|
|
detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
struct vm_area_struct *prev, unsigned long end)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct **insertion_point;
|
|
|
|
|
struct vm_area_struct *tail_vma = NULL;
|
|
|
|
|
|
|
|
|
|
insertion_point = (prev ? &prev->vm_next : &mm->mmap);
|
2010-08-20 23:24:55 +00:00
|
|
|
vma->vm_prev = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
do {
|
2012-12-12 00:01:38 +00:00
|
|
|
vma_rb_erase(vma, &mm->mm_rb);
|
2005-04-16 22:20:36 +00:00
|
|
|
mm->map_count--;
|
|
|
|
|
tail_vma = vma;
|
|
|
|
|
vma = vma->vm_next;
|
|
|
|
|
} while (vma && vma->vm_start < end);
|
|
|
|
|
*insertion_point = vma;
|
2012-12-12 00:01:38 +00:00
|
|
|
if (vma) {
|
2010-08-20 23:24:55 +00:00
|
|
|
vma->vm_prev = prev;
|
2012-12-12 00:01:38 +00:00
|
|
|
vma_gap_update(vma);
|
|
|
|
|
} else
|
|
|
|
|
mm->highest_vm_end = prev ? prev->vm_end : 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
tail_vma->vm_next = NULL;
|
mm: per-thread vma caching
This patch is a continuation of efforts trying to optimize find_vma(),
avoiding potentially expensive rbtree walks to locate a vma upon faults.
The original approach (https://lkml.org/lkml/2013/11/1/410), where the
largest vma was also cached, ended up being too specific and random,
thus further comparison with other approaches were needed. There are
two things to consider when dealing with this, the cache hit rate and
the latency of find_vma(). Improving the hit-rate does not necessarily
translate in finding the vma any faster, as the overhead of any fancy
caching schemes can be too high to consider.
We currently cache the last used vma for the whole address space, which
provides a nice optimization, reducing the total cycles in find_vma() by
up to 250%, for workloads with good locality. On the other hand, this
simple scheme is pretty much useless for workloads with poor locality.
Analyzing ebizzy runs shows that, no matter how many threads are
running, the mmap_cache hit rate is less than 2%, and in many situations
below 1%.
The proposed approach is to replace this scheme with a small per-thread
cache, maximizing hit rates at a very low maintenance cost.
Invalidations are performed by simply bumping up a 32-bit sequence
number. The only expensive operation is in the rare case of a seq
number overflow, where all caches that share the same address space are
flushed. Upon a miss, the proposed replacement policy is based on the
page number that contains the virtual address in question. Concretely,
the following results are seen on an 80 core, 8 socket x86-64 box:
1) System bootup: Most programs are single threaded, so the per-thread
scheme does improve ~50% hit rate by just adding a few more slots to
the cache.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 50.61% | 19.90 |
| patched | 73.45% | 13.58 |
+----------------+----------+------------------+
2) Kernel build: This one is already pretty good with the current
approach as we're dealing with good locality.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 75.28% | 11.03 |
| patched | 88.09% | 9.31 |
+----------------+----------+------------------+
3) Oracle 11g Data Mining (4k pages): Similar to the kernel build workload.
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 70.66% | 17.14 |
| patched | 91.15% | 12.57 |
+----------------+----------+------------------+
4) Ebizzy: There's a fair amount of variation from run to run, but this
approach always shows nearly perfect hit rates, while baseline is just
about non-existent. The amounts of cycles can fluctuate between
anywhere from ~60 to ~116 for the baseline scheme, but this approach
reduces it considerably. For instance, with 80 threads:
+----------------+----------+------------------+
| caching scheme | hit-rate | cycles (billion) |
+----------------+----------+------------------+
| baseline | 1.06% | 91.54 |
| patched | 99.97% | 14.18 |
+----------------+----------+------------------+
[akpm@linux-foundation.org: fix nommu build, per Davidlohr]
[akpm@linux-foundation.org: document vmacache_valid() logic]
[akpm@linux-foundation.org: attempt to untangle header files]
[akpm@linux-foundation.org: add vmacache_find() BUG_ON]
[hughd@google.com: add vmacache_valid_mm() (from Oleg)]
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: adjust and enhance comments]
Signed-off-by: Davidlohr Bueso <davidlohr@hp.com>
Reviewed-by: Rik van Riel <riel@redhat.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Reviewed-by: Michel Lespinasse <walken@google.com>
Cc: Oleg Nesterov <oleg@redhat.com>
Tested-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2014-04-07 22:37:25 +00:00
|
|
|
|
|
|
|
|
/* Kill the cache */
|
|
|
|
|
vmacache_invalidate(mm);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
mmap: don't return ENOMEM when mapcount is temporarily exceeded in munmap()
On ia64, the following test program exit abnormally, because glibc thread
library called abort().
========================================================
(gdb) bt
#0 0xa000000000010620 in __kernel_syscall_via_break ()
#1 0x20000000003208e0 in raise () from /lib/libc.so.6.1
#2 0x2000000000324090 in abort () from /lib/libc.so.6.1
#3 0x200000000027c3e0 in __deallocate_stack () from /lib/libpthread.so.0
#4 0x200000000027f7c0 in start_thread () from /lib/libpthread.so.0
#5 0x200000000047ef60 in __clone2 () from /lib/libc.so.6.1
========================================================
The fact is, glibc call munmap() when thread exitng time for freeing
stack, and it assume munlock() never fail. However, munmap() often make
vma splitting and it with many mapcount make -ENOMEM.
Oh well, that's crazy, because stack unmapping never increase mapcount.
The maxcount exceeding is only temporary. internal temporary exceeding
shouldn't make ENOMEM.
This patch does it.
test_max_mapcount.c
==================================================================
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<pthread.h>
#include<errno.h>
#include<unistd.h>
#define THREAD_NUM 30000
#define MAL_SIZE (8*1024*1024)
void *wait_thread(void *args)
{
void *addr;
addr = malloc(MAL_SIZE);
sleep(10);
return NULL;
}
void *wait_thread2(void *args)
{
sleep(60);
return NULL;
}
int main(int argc, char *argv[])
{
int i;
pthread_t thread[THREAD_NUM], th;
int ret, count = 0;
pthread_attr_t attr;
ret = pthread_attr_init(&attr);
if(ret) {
perror("pthread_attr_init");
}
ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(ret) {
perror("pthread_attr_setdetachstate");
}
for (i = 0; i < THREAD_NUM; i++) {
ret = pthread_create(&th, &attr, wait_thread, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
ret = pthread_create(&thread[i], &attr, wait_thread2, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
}
sleep(3600);
return 0;
}
==================================================================
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:57:56 +00:00
|
|
|
* __split_vma() bypasses sysctl_max_map_count checking. We use this on the
|
|
|
|
|
* munmap path where it doesn't make sense to fail.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2014-10-09 22:26:29 +00:00
|
|
|
static int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long addr, int new_below)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *new;
|
2015-09-08 22:03:38 +00:00
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-07-24 04:27:41 +00:00
|
|
|
if (is_vm_hugetlb_page(vma) && (addr &
|
|
|
|
|
~(huge_page_mask(hstate_vma(vma)))))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
2006-12-07 04:33:17 +00:00
|
|
|
new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!new)
|
2015-09-08 22:03:38 +00:00
|
|
|
return -ENOMEM;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* most fields are the same, copy all, and then fixup */
|
|
|
|
|
*new = *vma;
|
|
|
|
|
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
INIT_LIST_HEAD(&new->anon_vma_chain);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (new_below)
|
|
|
|
|
new->vm_end = addr;
|
|
|
|
|
else {
|
|
|
|
|
new->vm_start = addr;
|
|
|
|
|
new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-11 21:20:14 +00:00
|
|
|
err = vma_dup_policy(vma, new);
|
|
|
|
|
if (err)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
goto out_free_vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2014-12-02 23:59:42 +00:00
|
|
|
err = anon_vma_clone(new, vma);
|
|
|
|
|
if (err)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
goto out_free_mpol;
|
|
|
|
|
|
2012-10-08 23:28:54 +00:00
|
|
|
if (new->vm_file)
|
2005-04-16 22:20:36 +00:00
|
|
|
get_file(new->vm_file);
|
|
|
|
|
|
|
|
|
|
if (new->vm_ops && new->vm_ops->open)
|
|
|
|
|
new->vm_ops->open(new);
|
|
|
|
|
|
|
|
|
|
if (new_below)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
|
2005-04-16 22:20:36 +00:00
|
|
|
((addr - new->vm_start) >> PAGE_SHIFT), new);
|
|
|
|
|
else
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
/* Success. */
|
|
|
|
|
if (!err)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/* Clean everything up if vma_adjust failed. */
|
2010-04-26 16:33:03 +00:00
|
|
|
if (new->vm_ops && new->vm_ops->close)
|
|
|
|
|
new->vm_ops->close(new);
|
2012-10-08 23:28:54 +00:00
|
|
|
if (new->vm_file)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
fput(new->vm_file);
|
2010-09-22 20:05:12 +00:00
|
|
|
unlink_anon_vmas(new);
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
out_free_mpol:
|
2013-09-11 21:20:14 +00:00
|
|
|
mpol_put(vma_policy(new));
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
out_free_vma:
|
|
|
|
|
kmem_cache_free(vm_area_cachep, new);
|
|
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
mmap: don't return ENOMEM when mapcount is temporarily exceeded in munmap()
On ia64, the following test program exit abnormally, because glibc thread
library called abort().
========================================================
(gdb) bt
#0 0xa000000000010620 in __kernel_syscall_via_break ()
#1 0x20000000003208e0 in raise () from /lib/libc.so.6.1
#2 0x2000000000324090 in abort () from /lib/libc.so.6.1
#3 0x200000000027c3e0 in __deallocate_stack () from /lib/libpthread.so.0
#4 0x200000000027f7c0 in start_thread () from /lib/libpthread.so.0
#5 0x200000000047ef60 in __clone2 () from /lib/libc.so.6.1
========================================================
The fact is, glibc call munmap() when thread exitng time for freeing
stack, and it assume munlock() never fail. However, munmap() often make
vma splitting and it with many mapcount make -ENOMEM.
Oh well, that's crazy, because stack unmapping never increase mapcount.
The maxcount exceeding is only temporary. internal temporary exceeding
shouldn't make ENOMEM.
This patch does it.
test_max_mapcount.c
==================================================================
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<pthread.h>
#include<errno.h>
#include<unistd.h>
#define THREAD_NUM 30000
#define MAL_SIZE (8*1024*1024)
void *wait_thread(void *args)
{
void *addr;
addr = malloc(MAL_SIZE);
sleep(10);
return NULL;
}
void *wait_thread2(void *args)
{
sleep(60);
return NULL;
}
int main(int argc, char *argv[])
{
int i;
pthread_t thread[THREAD_NUM], th;
int ret, count = 0;
pthread_attr_t attr;
ret = pthread_attr_init(&attr);
if(ret) {
perror("pthread_attr_init");
}
ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(ret) {
perror("pthread_attr_setdetachstate");
}
for (i = 0; i < THREAD_NUM; i++) {
ret = pthread_create(&th, &attr, wait_thread, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
ret = pthread_create(&thread[i], &attr, wait_thread2, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
}
sleep(3600);
return 0;
}
==================================================================
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:57:56 +00:00
|
|
|
/*
|
|
|
|
|
* Split a vma into two pieces at address 'addr', a new vma is allocated
|
|
|
|
|
* either for the first part or the tail.
|
|
|
|
|
*/
|
|
|
|
|
int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
|
|
|
|
|
unsigned long addr, int new_below)
|
|
|
|
|
{
|
|
|
|
|
if (mm->map_count >= sysctl_max_map_count)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
return __split_vma(mm, vma, addr, new_below);
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Munmap is split into 2 main parts -- this part which finds
|
|
|
|
|
* what needs doing, and the areas themselves, which do the
|
|
|
|
|
* work. This now handles partial unmappings.
|
|
|
|
|
* Jeremy Fitzhardinge <jeremy@goop.org>
|
|
|
|
|
*/
|
|
|
|
|
int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
|
|
|
|
|
{
|
|
|
|
|
unsigned long end;
|
2005-04-19 20:29:18 +00:00
|
|
|
struct vm_area_struct *vma, *prev, *last;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-11-06 02:46:54 +00:00
|
|
|
if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
2014-10-09 22:26:29 +00:00
|
|
|
len = PAGE_ALIGN(len);
|
|
|
|
|
if (len == 0)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
|
|
/* Find the first overlapping VMA */
|
2011-06-16 07:35:09 +00:00
|
|
|
vma = find_vma(mm, start);
|
2005-04-19 20:29:18 +00:00
|
|
|
if (!vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
2011-06-16 07:35:09 +00:00
|
|
|
prev = vma->vm_prev;
|
2005-04-19 20:29:18 +00:00
|
|
|
/* we have start < vma->vm_end */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* if it doesn't overlap, we have nothing.. */
|
|
|
|
|
end = start + len;
|
2005-04-19 20:29:18 +00:00
|
|
|
if (vma->vm_start >= end)
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If we need to split any vma, do it now to save pain later.
|
|
|
|
|
*
|
|
|
|
|
* Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
|
|
|
|
|
* unmapped vm_area_struct will remain in use: so lower split_vma
|
|
|
|
|
* places tmp vma above, and higher split_vma places tmp vma below.
|
|
|
|
|
*/
|
2005-04-19 20:29:18 +00:00
|
|
|
if (start > vma->vm_start) {
|
mmap: don't return ENOMEM when mapcount is temporarily exceeded in munmap()
On ia64, the following test program exit abnormally, because glibc thread
library called abort().
========================================================
(gdb) bt
#0 0xa000000000010620 in __kernel_syscall_via_break ()
#1 0x20000000003208e0 in raise () from /lib/libc.so.6.1
#2 0x2000000000324090 in abort () from /lib/libc.so.6.1
#3 0x200000000027c3e0 in __deallocate_stack () from /lib/libpthread.so.0
#4 0x200000000027f7c0 in start_thread () from /lib/libpthread.so.0
#5 0x200000000047ef60 in __clone2 () from /lib/libc.so.6.1
========================================================
The fact is, glibc call munmap() when thread exitng time for freeing
stack, and it assume munlock() never fail. However, munmap() often make
vma splitting and it with many mapcount make -ENOMEM.
Oh well, that's crazy, because stack unmapping never increase mapcount.
The maxcount exceeding is only temporary. internal temporary exceeding
shouldn't make ENOMEM.
This patch does it.
test_max_mapcount.c
==================================================================
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<pthread.h>
#include<errno.h>
#include<unistd.h>
#define THREAD_NUM 30000
#define MAL_SIZE (8*1024*1024)
void *wait_thread(void *args)
{
void *addr;
addr = malloc(MAL_SIZE);
sleep(10);
return NULL;
}
void *wait_thread2(void *args)
{
sleep(60);
return NULL;
}
int main(int argc, char *argv[])
{
int i;
pthread_t thread[THREAD_NUM], th;
int ret, count = 0;
pthread_attr_t attr;
ret = pthread_attr_init(&attr);
if(ret) {
perror("pthread_attr_init");
}
ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(ret) {
perror("pthread_attr_setdetachstate");
}
for (i = 0; i < THREAD_NUM; i++) {
ret = pthread_create(&th, &attr, wait_thread, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
ret = pthread_create(&thread[i], &attr, wait_thread2, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
}
sleep(3600);
return 0;
}
==================================================================
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:57:56 +00:00
|
|
|
int error;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Make sure that map_count on return from munmap() will
|
|
|
|
|
* not exceed its limit; but let map_count go just above
|
|
|
|
|
* its limit temporarily, to help free resources as expected.
|
|
|
|
|
*/
|
|
|
|
|
if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
error = __split_vma(mm, vma, start, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (error)
|
|
|
|
|
return error;
|
2005-04-19 20:29:18 +00:00
|
|
|
prev = vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Does it split the last one? */
|
|
|
|
|
last = find_vma(mm, end);
|
|
|
|
|
if (last && end > last->vm_start) {
|
mmap: don't return ENOMEM when mapcount is temporarily exceeded in munmap()
On ia64, the following test program exit abnormally, because glibc thread
library called abort().
========================================================
(gdb) bt
#0 0xa000000000010620 in __kernel_syscall_via_break ()
#1 0x20000000003208e0 in raise () from /lib/libc.so.6.1
#2 0x2000000000324090 in abort () from /lib/libc.so.6.1
#3 0x200000000027c3e0 in __deallocate_stack () from /lib/libpthread.so.0
#4 0x200000000027f7c0 in start_thread () from /lib/libpthread.so.0
#5 0x200000000047ef60 in __clone2 () from /lib/libc.so.6.1
========================================================
The fact is, glibc call munmap() when thread exitng time for freeing
stack, and it assume munlock() never fail. However, munmap() often make
vma splitting and it with many mapcount make -ENOMEM.
Oh well, that's crazy, because stack unmapping never increase mapcount.
The maxcount exceeding is only temporary. internal temporary exceeding
shouldn't make ENOMEM.
This patch does it.
test_max_mapcount.c
==================================================================
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<pthread.h>
#include<errno.h>
#include<unistd.h>
#define THREAD_NUM 30000
#define MAL_SIZE (8*1024*1024)
void *wait_thread(void *args)
{
void *addr;
addr = malloc(MAL_SIZE);
sleep(10);
return NULL;
}
void *wait_thread2(void *args)
{
sleep(60);
return NULL;
}
int main(int argc, char *argv[])
{
int i;
pthread_t thread[THREAD_NUM], th;
int ret, count = 0;
pthread_attr_t attr;
ret = pthread_attr_init(&attr);
if(ret) {
perror("pthread_attr_init");
}
ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
if(ret) {
perror("pthread_attr_setdetachstate");
}
for (i = 0; i < THREAD_NUM; i++) {
ret = pthread_create(&th, &attr, wait_thread, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
ret = pthread_create(&thread[i], &attr, wait_thread2, NULL);
if(ret) {
fprintf(stderr, "[%d] ", count);
perror("pthread_create");
} else {
printf("[%d] create OK.\n", count);
}
count++;
}
sleep(3600);
return 0;
}
==================================================================
[akpm@linux-foundation.org: coding-style fixes]
Signed-off-by: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 01:57:56 +00:00
|
|
|
int error = __split_vma(mm, last, end, 1);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (error)
|
|
|
|
|
return error;
|
|
|
|
|
}
|
2014-10-09 22:26:29 +00:00
|
|
|
vma = prev ? prev->vm_next : mm->mmap;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-10-19 03:26:50 +00:00
|
|
|
/*
|
|
|
|
|
* unlock any mlock()ed ranges before detaching vmas
|
|
|
|
|
*/
|
|
|
|
|
if (mm->locked_vm) {
|
|
|
|
|
struct vm_area_struct *tmp = vma;
|
|
|
|
|
while (tmp && tmp->vm_start < end) {
|
|
|
|
|
if (tmp->vm_flags & VM_LOCKED) {
|
|
|
|
|
mm->locked_vm -= vma_pages(tmp);
|
|
|
|
|
munlock_vma_pages_all(tmp);
|
|
|
|
|
}
|
|
|
|
|
tmp = tmp->vm_next;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* Remove the vma's, and unmap the actual pages
|
|
|
|
|
*/
|
2005-04-19 20:29:18 +00:00
|
|
|
detach_vmas_to_be_unmapped(mm, vma, prev, end);
|
|
|
|
|
unmap_region(mm, vma, prev, start, end);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
x86, mpx: Cleanup unused bound tables
The previous patch allocates bounds tables on-demand. As noted in
an earlier description, these can add up to *HUGE* amounts of
memory. This has caused OOMs in practice when running tests.
This patch adds support for freeing bounds tables when they are no
longer in use.
There are two types of mappings in play when unmapping tables:
1. The mapping with the actual data, which userspace is
munmap()ing or brk()ing away, etc...
2. The mapping for the bounds table *backing* the data
(is tagged with VM_MPX, see the patch "add MPX specific
mmap interface").
If userspace use the prctl() indroduced earlier in this patchset
to enable the management of bounds tables in kernel, when it
unmaps the first type of mapping with the actual data, the kernel
needs to free the mapping for the bounds table backing the data.
This patch hooks in at the very end of do_unmap() to do so.
We look at the addresses being unmapped and find the bounds
directory entries and tables which cover those addresses. If
an entire table is unused, we clear associated directory entry
and free the table.
Once we unmap the bounds table, we would have a bounds directory
entry pointing at empty address space. That address space might
now be allocated for some other (random) use, and the MPX
hardware might now try to walk it as if it were a bounds table.
That would be bad. So any unmapping of an enture bounds table
has to be accompanied by a corresponding write to the bounds
directory entry to invalidate it. That write to the bounds
directory can fault, which causes the following problem:
Since we are doing the freeing from munmap() (and other paths
like it), we hold mmap_sem for write. If we fault, the page
fault handler will attempt to acquire mmap_sem for read and
we will deadlock. To avoid the deadlock, we pagefault_disable()
when touching the bounds directory entry and use a
get_user_pages() to resolve the fault.
The unmapping of bounds tables happends under vm_munmap(). We
also (indirectly) call vm_munmap() to _do_ the unmapping of the
bounds tables. We avoid unbounded recursion by disallowing
freeing of bounds tables *for* bounds tables. This would not
occur normally, so should not have any practical impact. Being
strict about it here helps ensure that we do not have an
exploitable stack overflow.
Based-on-patch-by: Qiaowei Ren <qiaowei.ren@intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Cc: linux-mm@kvack.org
Cc: linux-mips@linux-mips.org
Cc: Dave Hansen <dave@sr71.net>
Link: http://lkml.kernel.org/r/20141114151831.E4531C4A@viggo.jf.intel.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2014-11-14 15:18:31 +00:00
|
|
|
arch_unmap(mm, vma, start, end);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Fix up all other VM information */
|
2005-10-30 01:15:56 +00:00
|
|
|
remove_vma_list(mm, vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2012-04-21 01:57:04 +00:00
|
|
|
int vm_munmap(unsigned long start, size_t len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
int ret;
|
2012-04-21 01:57:04 +00:00
|
|
|
struct mm_struct *mm = current->mm;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-05-23 23:25:33 +00:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
|
return -EINTR;
|
|
|
|
|
|
2012-04-20 23:20:01 +00:00
|
|
|
ret = do_munmap(mm, start, len);
|
2005-04-16 22:20:36 +00:00
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2012-04-20 23:20:01 +00:00
|
|
|
EXPORT_SYMBOL(vm_munmap);
|
|
|
|
|
|
|
|
|
|
SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
|
|
|
|
|
{
|
2016-05-23 23:25:27 +00:00
|
|
|
int ret;
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
|
2012-04-20 23:20:01 +00:00
|
|
|
profile_munmap(addr);
|
2016-05-23 23:25:27 +00:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
|
return -EINTR;
|
|
|
|
|
ret = do_munmap(mm, addr, len);
|
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
return ret;
|
2012-04-20 23:20:01 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Emulation of deprecated remap_file_pages() syscall.
|
|
|
|
|
*/
|
|
|
|
|
SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
|
|
|
|
|
unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
unsigned long populate = 0;
|
|
|
|
|
unsigned long ret = -EINVAL;
|
|
|
|
|
struct file *file;
|
|
|
|
|
|
2016-03-17 21:19:47 +00:00
|
|
|
pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n",
|
|
|
|
|
current->comm, current->pid);
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
|
|
|
|
|
if (prot)
|
|
|
|
|
return ret;
|
|
|
|
|
start = start & PAGE_MASK;
|
|
|
|
|
size = size & PAGE_MASK;
|
|
|
|
|
|
|
|
|
|
if (start + size <= start)
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
|
|
/* Does pgoff wrap? */
|
|
|
|
|
if (pgoff + (size >> PAGE_SHIFT) < pgoff)
|
|
|
|
|
return ret;
|
|
|
|
|
|
2016-05-23 23:25:27 +00:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
|
return -EINTR;
|
|
|
|
|
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
vma = find_vma(mm, start);
|
|
|
|
|
|
|
|
|
|
if (!vma || !(vma->vm_flags & VM_SHARED))
|
|
|
|
|
goto out;
|
|
|
|
|
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
if (start < vma->vm_start)
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
goto out;
|
|
|
|
|
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
if (start + size > vma->vm_end) {
|
|
|
|
|
struct vm_area_struct *next;
|
|
|
|
|
|
|
|
|
|
for (next = vma->vm_next; next; next = next->vm_next) {
|
|
|
|
|
/* hole between vmas ? */
|
|
|
|
|
if (next->vm_start != next->vm_prev->vm_end)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
if (next->vm_file != vma->vm_file)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
if (next->vm_flags != vma->vm_flags)
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
if (start + size <= next->vm_end)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!next)
|
|
|
|
|
goto out;
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
|
|
|
|
|
prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
|
|
|
|
|
prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
|
|
|
|
|
|
|
|
|
|
flags &= MAP_NONBLOCK;
|
|
|
|
|
flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
|
|
|
|
|
if (vma->vm_flags & VM_LOCKED) {
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
struct vm_area_struct *tmp;
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
flags |= MAP_LOCKED;
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
/* drop PG_Mlocked flag for over-mapped range */
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
for (tmp = vma; tmp->vm_start >= start + size;
|
|
|
|
|
tmp = tmp->vm_next) {
|
2016-07-26 22:25:53 +00:00
|
|
|
/*
|
|
|
|
|
* Split pmd and munlock page on the border
|
|
|
|
|
* of the range.
|
|
|
|
|
*/
|
|
|
|
|
vma_adjust_trans_huge(tmp, start, start + size, 0);
|
|
|
|
|
|
mm: fix regression in remap_file_pages() emulation
Grazvydas Ignotas has reported a regression in remap_file_pages()
emulation.
Testcase:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define SIZE (4096 * 3)
int main(int argc, char **argv)
{
unsigned long *p;
long i;
p = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
if (remap_file_pages(p, 4096, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
if (remap_file_pages(p, 4096 * 2, 0, 1, 0)) {
perror("remap_file_pages");
return -1;
}
assert(p[0] == 1);
munmap(p, SIZE);
return 0;
}
The second remap_file_pages() fails with -EINVAL.
The reason is that remap_file_pages() emulation assumes that the target
vma covers whole area we want to over map. That assumption is broken by
first remap_file_pages() call: it split the area into two vma.
The solution is to check next adjacent vmas, if they map the same file
with the same flags.
Fixes: c8d78c1823f4 ("mm: replace remap_file_pages() syscall with emulation")
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Reported-by: Grazvydas Ignotas <notasas@gmail.com>
Tested-by: Grazvydas Ignotas <notasas@gmail.com>
Cc: <stable@vger.kernel.org> [4.0+]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2016-02-17 21:11:15 +00:00
|
|
|
munlock_vma_pages_range(tmp,
|
|
|
|
|
max(tmp->vm_start, start),
|
|
|
|
|
min(tmp->vm_end, start + size));
|
|
|
|
|
}
|
mm: replace remap_file_pages() syscall with emulation
remap_file_pages(2) was invented to be able efficiently map parts of
huge file into limited 32-bit virtual address space such as in database
workloads.
Nonlinear mappings are pain to support and it seems there's no
legitimate use-cases nowadays since 64-bit systems are widely available.
Let's drop it and get rid of all these special-cased code.
The patch replaces the syscall with emulation which creates new VMA on
each remap_file_pages(), unless they it can be merged with an adjacent
one.
I didn't find *any* real code that uses remap_file_pages(2) to test
emulation impact on. I've checked Debian code search and source of all
packages in ALT Linux. No real users: libc wrappers, mentions in
strace, gdb, valgrind and this kind of stuff.
There are few basic tests in LTP for the syscall. They work just fine
with emulation.
To test performance impact, I've written small test case which
demonstrate pretty much worst case scenario: map 4G shmfs file, write to
begin of every page pgoff of the page, remap pages in reverse order,
read every page.
The test creates 1 million of VMAs if emulation is in use, so I had to
set vm.max_map_count to 1100000 to avoid -ENOMEM.
Before: 23.3 ( +- 4.31% ) seconds
After: 43.9 ( +- 0.85% ) seconds
Slowdown: 1.88x
I believe we can live with that.
Test case:
#define _GNU_SOURCE
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/mman.h>
#define MB (1024UL * 1024)
#define SIZE (4096 * MB)
int main(int argc, char **argv)
{
unsigned long *p;
long i, pass;
for (pass = 0; pass < 10; pass++) {
p = mmap(NULL, SIZE, PROT_READ|PROT_WRITE,
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
perror("mmap");
return -1;
}
for (i = 0; i < SIZE / 4096; i++)
p[i * 4096 / sizeof(*p)] = i;
for (i = 0; i < SIZE / 4096; i++) {
if (remap_file_pages(p + i * 4096 / sizeof(*p), 4096,
0, (SIZE - 4096 * (i + 1)) >> 12, 0)) {
perror("remap_file_pages");
return -1;
}
}
for (i = SIZE / 4096 - 1; i >= 0; i--)
assert(p[i * 4096 / sizeof(*p)] == SIZE / 4096 - i - 1);
munmap(p, SIZE);
}
return 0;
}
[akpm@linux-foundation.org: fix spello]
[sasha.levin@oracle.com: initialize populate before usage]
[sasha.levin@oracle.com: grab file ref to prevent race while mmaping]
Signed-off-by: "Kirill A. Shutemov" <kirill@shutemov.name>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Dave Jones <davej@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Armin Rigo <arigo@tunes.org>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-10 22:09:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
file = get_file(vma->vm_file);
|
|
|
|
|
ret = do_mmap_pgoff(vma->vm_file, start, size,
|
|
|
|
|
prot, flags, pgoff, &populate);
|
|
|
|
|
fput(file);
|
|
|
|
|
out:
|
|
|
|
|
up_write(&mm->mmap_sem);
|
|
|
|
|
if (populate)
|
|
|
|
|
mm_populate(ret, populate);
|
|
|
|
|
if (!IS_ERR_VALUE(ret))
|
|
|
|
|
ret = 0;
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static inline void verify_mm_writelocked(struct mm_struct *mm)
|
|
|
|
|
{
|
2005-10-30 23:03:12 +00:00
|
|
|
#ifdef CONFIG_DEBUG_VM
|
2005-04-16 22:20:36 +00:00
|
|
|
if (unlikely(down_read_trylock(&mm->mmap_sem))) {
|
|
|
|
|
WARN_ON(1);
|
|
|
|
|
up_read(&mm->mmap_sem);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* this is really a simplified "do_mmap". it only handles
|
|
|
|
|
* anonymous maps. eventually we may be able to do some
|
|
|
|
|
* brk-specific accounting here.
|
|
|
|
|
*/
|
2016-05-27 22:57:31 +00:00
|
|
|
static int do_brk(unsigned long addr, unsigned long len)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2014-10-09 22:26:29 +00:00
|
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
|
struct vm_area_struct *vma, *prev;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long flags;
|
2014-10-09 22:26:29 +00:00
|
|
|
struct rb_node **rb_link, *rb_parent;
|
2005-04-16 22:20:36 +00:00
|
|
|
pgoff_t pgoff = addr >> PAGE_SHIFT;
|
2006-09-07 10:17:04 +00:00
|
|
|
int error;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
len = PAGE_ALIGN(len);
|
|
|
|
|
if (!len)
|
2016-05-27 22:57:31 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-09-07 10:17:04 +00:00
|
|
|
flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
|
|
|
|
|
|
2009-12-04 00:40:46 +00:00
|
|
|
error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
|
2015-11-06 02:46:54 +00:00
|
|
|
if (offset_in_page(error))
|
2006-09-07 10:17:04 +00:00
|
|
|
return error;
|
|
|
|
|
|
2014-01-21 23:49:15 +00:00
|
|
|
error = mlock_future_check(mm, mm->def_flags, len);
|
|
|
|
|
if (error)
|
|
|
|
|
return error;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* mm->mmap_sem is required to protect against another thread
|
|
|
|
|
* changing the mappings in case we sleep.
|
|
|
|
|
*/
|
|
|
|
|
verify_mm_writelocked(mm);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Clear old maps. this also does some error checking for us
|
|
|
|
|
*/
|
2015-04-15 23:14:32 +00:00
|
|
|
while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
|
|
|
|
|
&rb_parent)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (do_munmap(mm, addr, len))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Check against address space limits *after* clearing old maps... */
|
2016-01-14 23:22:07 +00:00
|
|
|
if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
if (mm->map_count > sysctl_max_map_count)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2012-02-13 03:58:52 +00:00
|
|
|
if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
|
2005-04-16 22:20:36 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
/* Can we just expand an old private anonymous mapping? */
|
2008-10-19 03:26:50 +00:00
|
|
|
vma = vma_merge(mm, prev, addr, addr + len, flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
|
2008-10-19 03:26:50 +00:00
|
|
|
if (vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* create a vma struct for an anonymous mapping
|
|
|
|
|
*/
|
2006-03-25 11:06:43 +00:00
|
|
|
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!vma) {
|
|
|
|
|
vm_unacct_memory(len >> PAGE_SHIFT);
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
|
|
|
|
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
INIT_LIST_HEAD(&vma->anon_vma_chain);
|
2005-04-16 22:20:36 +00:00
|
|
|
vma->vm_mm = mm;
|
|
|
|
|
vma->vm_start = addr;
|
|
|
|
|
vma->vm_end = addr + len;
|
|
|
|
|
vma->vm_pgoff = pgoff;
|
|
|
|
|
vma->vm_flags = flags;
|
2007-10-19 06:39:15 +00:00
|
|
|
vma->vm_page_prot = vm_get_page_prot(flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
vma_link(mm, vma, prev, rb_link, rb_parent);
|
|
|
|
|
out:
|
2010-05-18 14:30:49 +00:00
|
|
|
perf_event_mmap(vma);
|
2005-04-16 22:20:36 +00:00
|
|
|
mm->total_vm += len >> PAGE_SHIFT;
|
2016-01-14 23:22:07 +00:00
|
|
|
mm->data_vm += len >> PAGE_SHIFT;
|
2013-02-23 00:32:40 +00:00
|
|
|
if (flags & VM_LOCKED)
|
|
|
|
|
mm->locked_vm += (len >> PAGE_SHIFT);
|
2013-09-11 21:22:24 +00:00
|
|
|
vma->vm_flags |= VM_SOFTDIRTY;
|
2016-05-27 22:57:31 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2016-05-27 22:57:31 +00:00
|
|
|
int vm_brk(unsigned long addr, unsigned long len)
|
2012-04-20 22:35:40 +00:00
|
|
|
{
|
|
|
|
|
struct mm_struct *mm = current->mm;
|
2016-05-27 22:57:31 +00:00
|
|
|
int ret;
|
2013-02-23 00:32:40 +00:00
|
|
|
bool populate;
|
2012-04-20 22:35:40 +00:00
|
|
|
|
2016-05-23 23:25:42 +00:00
|
|
|
if (down_write_killable(&mm->mmap_sem))
|
|
|
|
|
return -EINTR;
|
|
|
|
|
|
2012-04-20 22:35:40 +00:00
|
|
|
ret = do_brk(addr, len);
|
2013-02-23 00:32:40 +00:00
|
|
|
populate = ((mm->def_flags & VM_LOCKED) != 0);
|
2012-04-20 22:35:40 +00:00
|
|
|
up_write(&mm->mmap_sem);
|
2016-05-27 22:57:31 +00:00
|
|
|
if (populate && !ret)
|
2013-02-23 00:32:40 +00:00
|
|
|
mm_populate(addr, len);
|
2012-04-20 22:35:40 +00:00
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(vm_brk);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/* Release all mmaps. */
|
|
|
|
|
void exit_mmap(struct mm_struct *mm)
|
|
|
|
|
{
|
2011-05-25 00:11:45 +00:00
|
|
|
struct mmu_gather tlb;
|
2008-10-19 03:26:50 +00:00
|
|
|
struct vm_area_struct *vma;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long nr_accounted = 0;
|
|
|
|
|
|
2007-05-02 17:27:14 +00:00
|
|
|
/* mm's last user has gone, and its about to be pulled down */
|
mmu-notifiers: core
With KVM/GFP/XPMEM there isn't just the primary CPU MMU pointing to pages.
There are secondary MMUs (with secondary sptes and secondary tlbs) too.
sptes in the kvm case are shadow pagetables, but when I say spte in
mmu-notifier context, I mean "secondary pte". In GRU case there's no
actual secondary pte and there's only a secondary tlb because the GRU
secondary MMU has no knowledge about sptes and every secondary tlb miss
event in the MMU always generates a page fault that has to be resolved by
the CPU (this is not the case of KVM where the a secondary tlb miss will
walk sptes in hardware and it will refill the secondary tlb transparently
to software if the corresponding spte is present). The same way
zap_page_range has to invalidate the pte before freeing the page, the spte
(and secondary tlb) must also be invalidated before any page is freed and
reused.
Currently we take a page_count pin on every page mapped by sptes, but that
means the pages can't be swapped whenever they're mapped by any spte
because they're part of the guest working set. Furthermore a spte unmap
event can immediately lead to a page to be freed when the pin is released
(so requiring the same complex and relatively slow tlb_gather smp safe
logic we have in zap_page_range and that can be avoided completely if the
spte unmap event doesn't require an unpin of the page previously mapped in
the secondary MMU).
The mmu notifiers allow kvm/GRU/XPMEM to attach to the tsk->mm and know
when the VM is swapping or freeing or doing anything on the primary MMU so
that the secondary MMU code can drop sptes before the pages are freed,
avoiding all page pinning and allowing 100% reliable swapping of guest
physical address space. Furthermore it avoids the code that teardown the
mappings of the secondary MMU, to implement a logic like tlb_gather in
zap_page_range that would require many IPI to flush other cpu tlbs, for
each fixed number of spte unmapped.
To make an example: if what happens on the primary MMU is a protection
downgrade (from writeable to wrprotect) the secondary MMU mappings will be
invalidated, and the next secondary-mmu-page-fault will call
get_user_pages and trigger a do_wp_page through get_user_pages if it
called get_user_pages with write=1, and it'll re-establishing an updated
spte or secondary-tlb-mapping on the copied page. Or it will setup a
readonly spte or readonly tlb mapping if it's a guest-read, if it calls
get_user_pages with write=0. This is just an example.
This allows to map any page pointed by any pte (and in turn visible in the
primary CPU MMU), into a secondary MMU (be it a pure tlb like GRU, or an
full MMU with both sptes and secondary-tlb like the shadow-pagetable layer
with kvm), or a remote DMA in software like XPMEM (hence needing of
schedule in XPMEM code to send the invalidate to the remote node, while no
need to schedule in kvm/gru as it's an immediate event like invalidating
primary-mmu pte).
At least for KVM without this patch it's impossible to swap guests
reliably. And having this feature and removing the page pin allows
several other optimizations that simplify life considerably.
Dependencies:
1) mm_take_all_locks() to register the mmu notifier when the whole VM
isn't doing anything with "mm". This allows mmu notifier users to keep
track if the VM is in the middle of the invalidate_range_begin/end
critical section with an atomic counter incraese in range_begin and
decreased in range_end. No secondary MMU page fault is allowed to map
any spte or secondary tlb reference, while the VM is in the middle of
range_begin/end as any page returned by get_user_pages in that critical
section could later immediately be freed without any further
->invalidate_page notification (invalidate_range_begin/end works on
ranges and ->invalidate_page isn't called immediately before freeing
the page). To stop all page freeing and pagetable overwrites the
mmap_sem must be taken in write mode and all other anon_vma/i_mmap
locks must be taken too.
2) It'd be a waste to add branches in the VM if nobody could possibly
run KVM/GRU/XPMEM on the kernel, so mmu notifiers will only enabled if
CONFIG_KVM=m/y. In the current kernel kvm won't yet take advantage of
mmu notifiers, but this already allows to compile a KVM external module
against a kernel with mmu notifiers enabled and from the next pull from
kvm.git we'll start using them. And GRU/XPMEM will also be able to
continue the development by enabling KVM=m in their config, until they
submit all GRU/XPMEM GPLv2 code to the mainline kernel. Then they can
also enable MMU_NOTIFIERS in the same way KVM does it (even if KVM=n).
This guarantees nobody selects MMU_NOTIFIER=y if KVM and GRU and XPMEM
are all =n.
The mmu_notifier_register call can fail because mm_take_all_locks may be
interrupted by a signal and return -EINTR. Because mmu_notifier_reigster
is used when a driver startup, a failure can be gracefully handled. Here
an example of the change applied to kvm to register the mmu notifiers.
Usually when a driver startups other allocations are required anyway and
-ENOMEM failure paths exists already.
struct kvm *kvm_arch_create_vm(void)
{
struct kvm *kvm = kzalloc(sizeof(struct kvm), GFP_KERNEL);
+ int err;
if (!kvm)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
+ kvm->arch.mmu_notifier.ops = &kvm_mmu_notifier_ops;
+ err = mmu_notifier_register(&kvm->arch.mmu_notifier, current->mm);
+ if (err) {
+ kfree(kvm);
+ return ERR_PTR(err);
+ }
+
return kvm;
}
mmu_notifier_unregister returns void and it's reliable.
The patch also adds a few needed but missing includes that would prevent
kernel to compile after these changes on non-x86 archs (x86 didn't need
them by luck).
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: fix mm/filemap_xip.c build]
[akpm@linux-foundation.org: fix mm/mmu_notifier.c build]
Signed-off-by: Andrea Arcangeli <andrea@qumranet.com>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Jack Steiner <steiner@sgi.com>
Cc: Robin Holt <holt@sgi.com>
Cc: Nick Piggin <npiggin@suse.de>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Kanoj Sarcar <kanojsarcar@yahoo.com>
Cc: Roland Dreier <rdreier@cisco.com>
Cc: Steve Wise <swise@opengridcomputing.com>
Cc: Avi Kivity <avi@qumranet.com>
Cc: Hugh Dickins <hugh@veritas.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
Cc: Anthony Liguori <aliguori@us.ibm.com>
Cc: Chris Wright <chrisw@redhat.com>
Cc: Marcelo Tosatti <marcelo@kvack.org>
Cc: Eric Dumazet <dada1@cosmosbay.com>
Cc: "Paul E. McKenney" <paulmck@us.ibm.com>
Cc: Izik Eidus <izike@qumranet.com>
Cc: Anthony Liguori <aliguori@us.ibm.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-28 22:46:29 +00:00
|
|
|
mmu_notifier_release(mm);
|
2007-05-02 17:27:14 +00:00
|
|
|
|
2008-10-19 03:26:50 +00:00
|
|
|
if (mm->locked_vm) {
|
|
|
|
|
vma = mm->mmap;
|
|
|
|
|
while (vma) {
|
|
|
|
|
if (vma->vm_flags & VM_LOCKED)
|
|
|
|
|
munlock_vma_pages_all(vma);
|
|
|
|
|
vma = vma->vm_next;
|
|
|
|
|
}
|
|
|
|
|
}
|
mm: rearrange exit_mmap() to unlock before arch_exit_mmap
Christophe Saout reported [in precursor to:
http://marc.info/?l=linux-kernel&m=123209902707347&w=4]:
> Note that I also some a different issue with CONFIG_UNEVICTABLE_LRU.
> Seems like Xen tears down current->mm early on process termination, so
> that __get_user_pages in exit_mmap causes nasty messages when the
> process had any mlocked pages. (in fact, it somehow manages to get into
> the swapping code and produces a null pointer dereference trying to get
> a swap token)
Jeremy explained:
Yes. In the normal case under Xen, an in-use pagetable is "pinned",
meaning that it is RO to the kernel, and all updates must go via hypercall
(or writes are trapped and emulated, which is much the same thing). An
unpinned pagetable is not currently in use by any process, and can be
directly accessed as normal RW pages.
As an optimisation at process exit time, we unpin the pagetable as early
as possible (switching the process to init_mm), so that all the normal
pagetable teardown can happen with direct memory accesses.
This happens in exit_mmap() -> arch_exit_mmap(). The munlocking happens
a few lines below. The obvious thing to do would be to move
arch_exit_mmap() to below the munlock code, but I think we'd want to
call it even if mm->mmap is NULL, just to be on the safe side.
Thus, this patch:
exit_mmap() needs to unlock any locked vmas before calling arch_exit_mmap,
as the latter may switch the current mm to init_mm, which would cause the
former to fail.
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christophe Saout <christophe@saout.de>
Cc: Keir Fraser <keir.fraser@eu.citrix.com>
Cc: Christophe Saout <christophe@saout.de>
Cc: Alex Williamson <alex.williamson@hp.com>
Cc: <stable@kernel.org> [2.6.28.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-02-11 21:04:41 +00:00
|
|
|
|
|
|
|
|
arch_exit_mmap(mm);
|
|
|
|
|
|
2008-10-19 03:26:50 +00:00
|
|
|
vma = mm->mmap;
|
mm: rearrange exit_mmap() to unlock before arch_exit_mmap
Christophe Saout reported [in precursor to:
http://marc.info/?l=linux-kernel&m=123209902707347&w=4]:
> Note that I also some a different issue with CONFIG_UNEVICTABLE_LRU.
> Seems like Xen tears down current->mm early on process termination, so
> that __get_user_pages in exit_mmap causes nasty messages when the
> process had any mlocked pages. (in fact, it somehow manages to get into
> the swapping code and produces a null pointer dereference trying to get
> a swap token)
Jeremy explained:
Yes. In the normal case under Xen, an in-use pagetable is "pinned",
meaning that it is RO to the kernel, and all updates must go via hypercall
(or writes are trapped and emulated, which is much the same thing). An
unpinned pagetable is not currently in use by any process, and can be
directly accessed as normal RW pages.
As an optimisation at process exit time, we unpin the pagetable as early
as possible (switching the process to init_mm), so that all the normal
pagetable teardown can happen with direct memory accesses.
This happens in exit_mmap() -> arch_exit_mmap(). The munlocking happens
a few lines below. The obvious thing to do would be to move
arch_exit_mmap() to below the munlock code, but I think we'd want to
call it even if mm->mmap is NULL, just to be on the safe side.
Thus, this patch:
exit_mmap() needs to unlock any locked vmas before calling arch_exit_mmap,
as the latter may switch the current mm to init_mm, which would cause the
former to fail.
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Signed-off-by: Lee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christophe Saout <christophe@saout.de>
Cc: Keir Fraser <keir.fraser@eu.citrix.com>
Cc: Christophe Saout <christophe@saout.de>
Cc: Alex Williamson <alex.williamson@hp.com>
Cc: <stable@kernel.org> [2.6.28.x]
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-02-11 21:04:41 +00:00
|
|
|
if (!vma) /* Can happen if dup_mmap() received an OOM */
|
|
|
|
|
return;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
lru_add_drain();
|
|
|
|
|
flush_cache_mm(mm);
|
Fix TLB gather virtual address range invalidation corner cases
Ben Tebulin reported:
"Since v3.7.2 on two independent machines a very specific Git
repository fails in 9/10 cases on git-fsck due to an SHA1/memory
failures. This only occurs on a very specific repository and can be
reproduced stably on two independent laptops. Git mailing list ran
out of ideas and for me this looks like some very exotic kernel issue"
and bisected the failure to the backport of commit 53a59fc67f97 ("mm:
limit mmu_gather batching to fix soft lockups on !CONFIG_PREEMPT").
That commit itself is not actually buggy, but what it does is to make it
much more likely to hit the partial TLB invalidation case, since it
introduces a new case in tlb_next_batch() that previously only ever
happened when running out of memory.
The real bug is that the TLB gather virtual memory range setup is subtly
buggered. It was introduced in commit 597e1c3580b7 ("mm/mmu_gather:
enable tlb flush range in generic mmu_gather"), and the range handling
was already fixed at least once in commit e6c495a96ce0 ("mm: fix the TLB
range flushed when __tlb_remove_page() runs out of slots"), but that fix
was not complete.
The problem with the TLB gather virtual address range is that it isn't
set up by the initial tlb_gather_mmu() initialization (which didn't get
the TLB range information), but it is set up ad-hoc later by the
functions that actually flush the TLB. And so any such case that forgot
to update the TLB range entries would potentially miss TLB invalidates.
Rather than try to figure out exactly which particular ad-hoc range
setup was missing (I personally suspect it's the hugetlb case in
zap_huge_pmd(), which didn't have the same logic as zap_pte_range()
did), this patch just gets rid of the problem at the source: make the
TLB range information available to tlb_gather_mmu(), and initialize it
when initializing all the other tlb gather fields.
This makes the patch larger, but conceptually much simpler. And the end
result is much more understandable; even if you want to play games with
partial ranges when invalidating the TLB contents in chunks, now the
range information is always there, and anybody who doesn't want to
bother with it won't introduce subtle bugs.
Ben verified that this fixes his problem.
Reported-bisected-and-tested-by: Ben Tebulin <tebulin@googlemail.com>
Build-testing-by: Stephen Rothwell <sfr@canb.auug.org.au>
Build-testing-by: Richard Weinberger <richard.weinberger@gmail.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Cc: stable@vger.kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-08-15 18:42:25 +00:00
|
|
|
tlb_gather_mmu(&tlb, mm, 0, -1);
|
2009-01-06 22:40:29 +00:00
|
|
|
/* update_hiwater_rss(mm) here? but nobody should be looking */
|
[PATCH] freepgt: free_pgtables use vma list
Recent woes with some arches needing their own pgd_addr_end macro; and 4-level
clear_page_range regression since 2.6.10's clear_page_tables; and its
long-standing well-known inefficiency in searching throughout the higher-level
page tables for those few entries to clear and free: all can be blamed on
ignoring the list of vmas when we free page tables.
Replace exit_mmap's clear_page_range of the total user address space by
free_pgtables operating on the mm's vma list; unmap_region use it in the same
way, giving floor and ceiling beyond which it may not free tables. This
brings lmbench fork/exec/sh numbers back to 2.6.10 (unless preempt is enabled,
in which case latency fixes spoil unmap_vmas throughput).
Beware: the do_mmap_pgoff driver failure case must now use unmap_region
instead of zap_page_range, since a page table might have been allocated, and
can only be freed while it is touched by some vma.
Move free_pgtables from mmap.c to memory.c, where its lower levels are adapted
from the clear_page_range levels. (Most of free_pgtables' old code was
actually for a non-existent case, prev not properly set up, dating from before
hch gave us split_vma.) Pass mmu_gather** in the public interfaces, since we
might want to add latency lockdrops later; but no attempt to do so yet, going
by vma should itself reduce latency.
But what if is_hugepage_only_range? Those ia64 and ppc64 cases need careful
examination: put that off until a later patch of the series.
What of x86_64's 32bit vdso page __map_syscall32 maps outside any vma?
And the range to sparc64's flush_tlb_pgtables? It's less clear to me now that
we need to do more than is done here - every PMD_SIZE ever occupied will be
flushed, do we really have to flush every PGDIR_SIZE ever partially occupied?
A shame to complicate it unnecessarily.
Special thanks to David Miller for time spent repairing my ceilings.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-04-19 20:29:15 +00:00
|
|
|
/* Use -1 here to ensure all VMAs in the mm are unmapped */
|
2012-05-06 20:54:06 +00:00
|
|
|
unmap_vmas(&tlb, vma, 0, -1);
|
ksm: fix oom deadlock
There's a now-obvious deadlock in KSM's out-of-memory handling:
imagine ksmd or KSM_RUN_UNMERGE handling, holding ksm_thread_mutex,
trying to allocate a page to break KSM in an mm which becomes the
OOM victim (quite likely in the unmerge case): it's killed and goes
to exit, and hangs there waiting to acquire ksm_thread_mutex.
Clearly we must not require ksm_thread_mutex in __ksm_exit, simple
though that made everything else: perhaps use mmap_sem somehow?
And part of the answer lies in the comments on unmerge_ksm_pages:
__ksm_exit should also leave all the rmap_item removal to ksmd.
But there's a fundamental problem, that KSM relies upon mmap_sem to
guarantee the consistency of the mm it's dealing with, yet exit_mmap
tears down an mm without taking mmap_sem. And bumping mm_users won't
help at all, that just ensures that the pages the OOM killer assumes
are on their way to being freed will not be freed.
The best answer seems to be, to move the ksm_exit callout from just
before exit_mmap, to the middle of exit_mmap: after the mm's pages
have been freed (if the mmu_gather is flushed), but before its page
tables and vma structures have been freed; and down_write,up_write
mmap_sem there to serialize with KSM's own reliance on mmap_sem.
But KSM then needs to be careful, whenever it downs mmap_sem, to
check that the mm is not already exiting: there's a danger of using
find_vma on a layout that's being torn apart, or writing into page
tables which have been freed for reuse; and even do_anonymous_page
and __do_fault need to check they're not being called by break_ksm
to reinstate a pte after zap_pte_range has zapped that page table.
Though it might be clearer to add an exiting flag, set while holding
mmap_sem in __ksm_exit, that wouldn't cover the issue of reinstating
a zapped pte. All we need is to check whether mm_users is 0 - but
must remember that ksmd may detect that before __ksm_exit is reached.
So, ksm_test_exit(mm) added to comment such checks on mm->mm_users.
__ksm_exit now has to leave clearing up the rmap_items to ksmd,
that needs ksm_thread_mutex; but shift the exiting mm just after the
ksm_scan cursor so that it will soon be dealt with. __ksm_enter raise
mm_count to hold the mm_struct, ksmd's exit processing (exactly like
its processing when it finds all VM_MERGEABLEs unmapped) mmdrop it,
similar procedure for KSM_RUN_UNMERGE (which has stopped ksmd).
But also give __ksm_exit a fast path: when there's no complication
(no rmap_items attached to mm and it's not at the ksm_scan cursor),
it can safely do all the exiting work itself. This is not just an
optimization: when ksmd is not running, the raised mm_count would
otherwise leak mm_structs.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Acked-by: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-09-22 00:02:20 +00:00
|
|
|
|
2013-04-29 22:07:44 +00:00
|
|
|
free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
|
2012-03-05 19:03:47 +00:00
|
|
|
tlb_finish_mmu(&tlb, 0, -1);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
[PATCH] mm: unlink vma before pagetables
In most places the descent from pgd to pud to pmd to pte holds mmap_sem
(exclusively or not), which ensures that free_pgtables cannot be freeing page
tables from any level at the same time. But truncation and reverse mapping
descend without mmap_sem.
No problem: just make sure that a vma is unlinked from its prio_tree (or
nonlinear list) and from its anon_vma list, after zapping the vma, but before
freeing its page tables. Then neither vmtruncate nor rmap can reach that vma
whose page tables are now volatile (nor do they need to reach it, since all
its page entries have been zapped by this stage).
The i_mmap_lock and anon_vma->lock already serialize this correctly; but the
locking hierarchy is such that we cannot take them while holding
page_table_lock. Well, we're trying to push that down anyway. So in this
patch, move anon_vma_unlink and unlink_file_vma into free_pgtables, at the
same time as moving page_table_lock around calls to unmap_vmas.
tlb_gather_mmu and tlb_finish_mmu then fall outside the page_table_lock, but
we made them preempt_disable and preempt_enable earlier; and a long source
audit of all the architectures has shown no problem with removing
page_table_lock from them. free_pgtables doesn't need page_table_lock for
itself, nor for what it calls; tlb->mm->nr_ptes is usually protected by
page_table_lock, but partly by non-exclusive mmap_sem - here it's decremented
with exclusive mmap_sem, or mm_users 0. update_hiwater_rss and
vm_unacct_memory don't need page_table_lock either.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2005-10-30 01:16:29 +00:00
|
|
|
* Walk the list again, actually closing and freeing it,
|
|
|
|
|
* with preemption enabled, without holding any MM locks.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2012-05-06 20:54:06 +00:00
|
|
|
while (vma) {
|
|
|
|
|
if (vma->vm_flags & VM_ACCOUNT)
|
|
|
|
|
nr_accounted += vma_pages(vma);
|
2005-10-30 01:15:57 +00:00
|
|
|
vma = remove_vma(vma);
|
2012-05-06 20:54:06 +00:00
|
|
|
}
|
|
|
|
|
vm_unacct_memory(nr_accounted);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Insert vm structure into process list sorted by address
|
|
|
|
|
* and into the inode's i_mmap tree. If vm_file is non-NULL
|
2014-12-13 00:54:24 +00:00
|
|
|
* then i_mmap_rwsem is taken here.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2012-10-08 23:29:07 +00:00
|
|
|
int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-10-08 23:29:07 +00:00
|
|
|
struct vm_area_struct *prev;
|
|
|
|
|
struct rb_node **rb_link, *rb_parent;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-09-08 22:04:08 +00:00
|
|
|
if (find_vma_links(mm, vma->vm_start, vma->vm_end,
|
|
|
|
|
&prev, &rb_link, &rb_parent))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
if ((vma->vm_flags & VM_ACCOUNT) &&
|
|
|
|
|
security_vm_enough_memory_mm(mm, vma_pages(vma)))
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* The vm_pgoff of a purely anonymous vma should be irrelevant
|
|
|
|
|
* until its first write fault, when page's anon_vma and index
|
|
|
|
|
* are set. But now set the vm_pgoff it will almost certainly
|
|
|
|
|
* end up with (unless mremap moves it elsewhere before that
|
|
|
|
|
* first wfault), so /proc/pid/maps tells a consistent story.
|
|
|
|
|
*
|
|
|
|
|
* By setting it to reflect the virtual start address of the
|
|
|
|
|
* vma, merges and splits can happen in a seamless way, just
|
|
|
|
|
* using the existing file pgoff checks and manipulations.
|
|
|
|
|
* Similarly in do_mmap_pgoff and in do_brk.
|
|
|
|
|
*/
|
mmap: fix the usage of ->vm_pgoff in special_mapping paths
Test-case:
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <assert.h>
void *find_vdso_vaddr(void)
{
FILE *perl;
char buf[32] = {};
perl = popen("perl -e 'open STDIN,qq|/proc/@{[getppid]}/maps|;"
"/^(.*?)-.*vdso/ && print hex $1 while <>'", "r");
fread(buf, sizeof(buf), 1, perl);
fclose(perl);
return (void *)atol(buf);
}
#define PAGE_SIZE 4096
int main(void)
{
void *vdso = find_vdso_vaddr();
assert(vdso);
// of course they should differ, and they do so far
printf("vdso pages differ: %d\n",
!!memcmp(vdso, vdso + PAGE_SIZE, PAGE_SIZE));
// split into 2 vma's
assert(mprotect(vdso, PAGE_SIZE, PROT_READ) == 0);
// force another fault on the next check
assert(madvise(vdso, 2 * PAGE_SIZE, MADV_DONTNEED) == 0);
// now they no longer differ, the 2nd vm_pgoff is wrong
printf("vdso pages differ: %d\n",
!!memcmp(vdso, vdso + PAGE_SIZE, PAGE_SIZE));
return 0;
}
Output:
vdso pages differ: 1
vdso pages differ: 0
This is because split_vma() correctly updates ->vm_pgoff, but the logic
in insert_vm_struct() and special_mapping_fault() is absolutely broken,
so the fault at vdso + PAGE_SIZE return the 1st page. The same happens
if you simply unmap the 1st page.
special_mapping_fault() does:
pgoff = vmf->pgoff - vma->vm_pgoff;
and this is _only_ correct if vma->vm_start mmaps the first page from
->vm_private_data array.
vdso or any other user of install_special_mapping() is not anonymous,
it has the "backing storage" even if it is just the array of pages.
So we actually need to make vm_pgoff work as an offset in this array.
Note: this also allows to fix another problem: currently gdb can't access
"[vvar]" memory because in this case special_mapping_fault() doesn't work.
Now that we can use ->vm_pgoff we can implement ->access() and fix this.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pavel Emelyanov <xemul@parallels.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 21:58:31 +00:00
|
|
|
if (vma_is_anonymous(vma)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
BUG_ON(vma->anon_vma);
|
|
|
|
|
vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
|
|
|
|
|
}
|
uprobes, mm, x86: Add the ability to install and remove uprobes breakpoints
Add uprobes support to the core kernel, with x86 support.
This commit adds the kernel facilities, the actual uprobes
user-space ABI and perf probe support comes in later commits.
General design:
Uprobes are maintained in an rb-tree indexed by inode and offset
(the offset here is from the start of the mapping). For a unique
(inode, offset) tuple, there can be at most one uprobe in the
rb-tree.
Since the (inode, offset) tuple identifies a unique uprobe, more
than one user may be interested in the same uprobe. This provides
the ability to connect multiple 'consumers' to the same uprobe.
Each consumer defines a handler and a filter (optional). The
'handler' is run every time the uprobe is hit, if it matches the
'filter' criteria.
The first consumer of a uprobe causes the breakpoint to be
inserted at the specified address and subsequent consumers are
appended to this list. On subsequent probes, the consumer gets
appended to the existing list of consumers. The breakpoint is
removed when the last consumer unregisters. For all other
unregisterations, the consumer is removed from the list of
consumers.
Given a inode, we get a list of the mms that have mapped the
inode. Do the actual registration if mm maps the page where a
probe needs to be inserted/removed.
We use a temporary list to walk through the vmas that map the
inode.
- The number of maps that map the inode, is not known before we
walk the rmap and keeps changing.
- extending vm_area_struct wasn't recommended, it's a
size-critical data structure.
- There can be more than one maps of the inode in the same mm.
We add callbacks to the mmap methods to keep an eye on text vmas
that are of interest to uprobes. When a vma of interest is mapped,
we insert the breakpoint at the right address.
Uprobe works by replacing the instruction at the address defined
by (inode, offset) with the arch specific breakpoint
instruction. We save a copy of the original instruction at the
uprobed address.
This is needed for:
a. executing the instruction out-of-line (xol).
b. instruction analysis for any subsequent fixups.
c. restoring the instruction back when the uprobe is unregistered.
We insert or delete a breakpoint instruction, and this
breakpoint instruction is assumed to be the smallest instruction
available on the platform. For fixed size instruction platforms
this is trivially true, for variable size instruction platforms
the breakpoint instruction is typically the smallest (often a
single byte).
Writing the instruction is done by COWing the page and changing
the instruction during the copy, this even though most platforms
allow atomic writes of the breakpoint instruction. This also
mirrors the behaviour of a ptrace() memory write to a PRIVATE
file map.
The core worker is derived from KSM's replace_page() logic.
In essence, similar to KSM:
a. allocate a new page and copy over contents of the page that
has the uprobed vaddr
b. modify the copy and insert the breakpoint at the required
address
c. switch the original page with the copy containing the
breakpoint
d. flush page tables.
replace_page() is being replicated here because of some minor
changes in the type of pages and also because Hugh Dickins had
plans to improve replace_page() for KSM specific work.
Instruction analysis on x86 is based on instruction decoder and
determines if an instruction can be probed and determines the
necessary fixups after singlestep. Instruction analysis is done
at probe insertion time so that we avoid having to repeat the
same analysis every time a probe is hit.
A lot of code here is due to the improvement/suggestions/inputs
from Peter Zijlstra.
Changelog:
(v10):
- Add code to clear REX.B prefix as suggested by Denys Vlasenko
and Masami Hiramatsu.
(v9):
- Use insn_offset_modrm as suggested by Masami Hiramatsu.
(v7):
Handle comments from Peter Zijlstra:
- Dont take reference to inode. (expect inode to uprobe_register to be sane).
- Use PTR_ERR to set the return value.
- No need to take reference to inode.
- use PTR_ERR to return error value.
- register and uprobe_unregister share code.
(v5):
- Modified del_consumer as per comments from Peter.
- Drop reference to inode before dropping reference to uprobe.
- Use i_size_read(inode) instead of inode->i_size.
- Ensure uprobe->consumers is NULL, before __uprobe_unregister() is called.
- Includes errno.h as recommended by Stephen Rothwell to fix a build issue
on sparc defconfig
- Remove restrictions while unregistering.
- Earlier code leaked inode references under some conditions while
registering/unregistering.
- Continue the vma-rmap walk even if the intermediate vma doesnt
meet the requirements.
- Validate the vma found by find_vma before inserting/removing the
breakpoint
- Call del_consumer under mutex_lock.
- Use hash locks.
- Handle mremap.
- Introduce find_least_offset_node() instead of close match logic in
find_uprobe
- Uprobes no more depends on MM_OWNER; No reference to task_structs
while inserting/removing a probe.
- Uses read_mapping_page instead of grab_cache_page so that the pages
have valid content.
- pass NULL to get_user_pages for the task parameter.
- call SetPageUptodate on the new page allocated in write_opcode.
- fix leaking a reference to the new page under certain conditions.
- Include Instruction Decoder if Uprobes gets defined.
- Remove const attributes for instruction prefix arrays.
- Uses mm_context to know if the application is 32 bit.
Signed-off-by: Srikar Dronamraju <srikar@linux.vnet.ibm.com>
Also-written-by: Jim Keniston <jkenisto@us.ibm.com>
Reviewed-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Andi Kleen <andi@firstfloor.org>
Cc: Christoph Hellwig <hch@infradead.org>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Roland McGrath <roland@hack.frob.com>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Arnaldo Carvalho de Melo <acme@infradead.org>
Cc: Anton Arapov <anton@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Stephen Rothwell <sfr@canb.auug.org.au>
Cc: Denys Vlasenko <vda.linux@googlemail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linux-mm <linux-mm@kvack.org>
Link: http://lkml.kernel.org/r/20120209092642.GE16600@linux.vnet.ibm.com
[ Made various small edits to the commit log ]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2012-02-09 09:26:42 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
vma_link(mm, vma, prev, rb_link, rb_parent);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Copy the vma structure to a new location in the same mm,
|
|
|
|
|
* prior to moving page table entries, to effect an mremap move.
|
|
|
|
|
*/
|
|
|
|
|
struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
|
2012-10-08 23:31:50 +00:00
|
|
|
unsigned long addr, unsigned long len, pgoff_t pgoff,
|
|
|
|
|
bool *need_rmap_locks)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
struct vm_area_struct *vma = *vmap;
|
|
|
|
|
unsigned long vma_start = vma->vm_start;
|
|
|
|
|
struct mm_struct *mm = vma->vm_mm;
|
|
|
|
|
struct vm_area_struct *new_vma, *prev;
|
|
|
|
|
struct rb_node **rb_link, *rb_parent;
|
mremap: enforce rmap src/dst vma ordering in case of vma_merge() succeeding in copy_vma()
migrate was doing an rmap_walk with speculative lock-less access on
pagetables. That could lead it to not serializing properly against mremap
PT locks. But a second problem remains in the order of vmas in the
same_anon_vma list used by the rmap_walk.
If vma_merge succeeds in copy_vma, the src vma could be placed after the
dst vma in the same_anon_vma list. That could still lead to migrate
missing some pte.
This patch adds an anon_vma_moveto_tail() function to force the dst vma at
the end of the list before mremap starts to solve the problem.
If the mremap is very large and there are a lots of parents or childs
sharing the anon_vma root lock, this should still scale better than taking
the anon_vma root lock around every pte copy practically for the whole
duration of mremap.
Update: Hugh noticed special care is needed in the error path where
move_page_tables goes in the reverse direction, a second
anon_vma_moveto_tail() call is needed in the error path.
This program exercises the anon_vma_moveto_tail:
===
int main()
{
static struct timeval oldstamp, newstamp;
long diffsec;
char *p, *p2, *p3, *p4;
if (posix_memalign((void **)&p, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p2, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p3, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
memset(p, 0xff, SIZE);
printf("%p\n", p);
memset(p2, 0xff, SIZE);
memset(p3, 0x77, 4096);
if (memcmp(p, p2, SIZE))
printf("error\n");
p4 = mremap(p+SIZE/2, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p3);
if (p4 != p3)
perror("mremap"), exit(1);
p4 = mremap(p4, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p+SIZE/2);
if (p4 != p+SIZE/2)
perror("mremap"), exit(1);
if (memcmp(p, p2, SIZE))
printf("error\n");
printf("ok\n");
return 0;
}
===
$ perf probe -a anon_vma_moveto_tail
Add new event:
probe:anon_vma_moveto_tail (on anon_vma_moveto_tail)
You can now use it on all perf tools, such as:
perf record -e probe:anon_vma_moveto_tail -aR sleep 1
$ perf record -e probe:anon_vma_moveto_tail -aR ./anon_vma_moveto_tail
0x7f2ca2800000
ok
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.043 MB perf.data (~1860 samples) ]
$ perf report --stdio
100.00% anon_vma_moveto [kernel.kallsyms] [k] anon_vma_moveto_tail
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Reported-by: Nai Xia <nai.xia@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pawel Sikora <pluto@agmk.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 23:08:05 +00:00
|
|
|
bool faulted_in_anon_vma = true;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If anonymous vma has not yet been faulted, update new pgoff
|
|
|
|
|
* to match new location, to increase its chance of merging.
|
|
|
|
|
*/
|
mremap: fix the wrong !vma->vm_file check in copy_vma()
Test-case:
#define _GNU_SOURCE
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <assert.h>
void *find_vdso_vaddr(void)
{
FILE *perl;
char buf[32] = {};
perl = popen("perl -e 'open STDIN,qq|/proc/@{[getppid]}/maps|;"
"/^(.*?)-.*vdso/ && print hex $1 while <>'", "r");
fread(buf, sizeof(buf), 1, perl);
fclose(perl);
return (void *)atol(buf);
}
#define PAGE_SIZE 4096
void *get_unmapped_area(void)
{
void *p = mmap(0, PAGE_SIZE, PROT_NONE,
MAP_PRIVATE|MAP_ANONYMOUS, -1,0);
assert(p != MAP_FAILED);
munmap(p, PAGE_SIZE);
return p;
}
char save[2][PAGE_SIZE];
int main(void)
{
void *vdso = find_vdso_vaddr();
void *page[2];
assert(vdso);
memcpy(save, vdso, sizeof (save));
// force another fault on the next check
assert(madvise(vdso, 2 * PAGE_SIZE, MADV_DONTNEED) == 0);
page[0] = mremap(vdso,
PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE,
get_unmapped_area());
page[1] = mremap(vdso + PAGE_SIZE,
PAGE_SIZE, PAGE_SIZE, MREMAP_FIXED | MREMAP_MAYMOVE,
get_unmapped_area());
assert(page[0] != MAP_FAILED && page[1] != MAP_FAILED);
printf("match: %d %d\n",
!memcmp(save[0], page[0], PAGE_SIZE),
!memcmp(save[1], page[1], PAGE_SIZE));
return 0;
}
fails without this patch. Before the previous commit it gets the wrong
page, now it segfaults (which is imho better).
This is because copy_vma() wrongly assumes that if vma->vm_file == NULL
is irrelevant until the first fault which will use do_anonymous_page().
This is obviously wrong for the special mapping.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pavel Emelyanov <xemul@parallels.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 21:58:34 +00:00
|
|
|
if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
pgoff = addr >> PAGE_SHIFT;
|
mremap: enforce rmap src/dst vma ordering in case of vma_merge() succeeding in copy_vma()
migrate was doing an rmap_walk with speculative lock-less access on
pagetables. That could lead it to not serializing properly against mremap
PT locks. But a second problem remains in the order of vmas in the
same_anon_vma list used by the rmap_walk.
If vma_merge succeeds in copy_vma, the src vma could be placed after the
dst vma in the same_anon_vma list. That could still lead to migrate
missing some pte.
This patch adds an anon_vma_moveto_tail() function to force the dst vma at
the end of the list before mremap starts to solve the problem.
If the mremap is very large and there are a lots of parents or childs
sharing the anon_vma root lock, this should still scale better than taking
the anon_vma root lock around every pte copy practically for the whole
duration of mremap.
Update: Hugh noticed special care is needed in the error path where
move_page_tables goes in the reverse direction, a second
anon_vma_moveto_tail() call is needed in the error path.
This program exercises the anon_vma_moveto_tail:
===
int main()
{
static struct timeval oldstamp, newstamp;
long diffsec;
char *p, *p2, *p3, *p4;
if (posix_memalign((void **)&p, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p2, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p3, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
memset(p, 0xff, SIZE);
printf("%p\n", p);
memset(p2, 0xff, SIZE);
memset(p3, 0x77, 4096);
if (memcmp(p, p2, SIZE))
printf("error\n");
p4 = mremap(p+SIZE/2, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p3);
if (p4 != p3)
perror("mremap"), exit(1);
p4 = mremap(p4, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p+SIZE/2);
if (p4 != p+SIZE/2)
perror("mremap"), exit(1);
if (memcmp(p, p2, SIZE))
printf("error\n");
printf("ok\n");
return 0;
}
===
$ perf probe -a anon_vma_moveto_tail
Add new event:
probe:anon_vma_moveto_tail (on anon_vma_moveto_tail)
You can now use it on all perf tools, such as:
perf record -e probe:anon_vma_moveto_tail -aR sleep 1
$ perf record -e probe:anon_vma_moveto_tail -aR ./anon_vma_moveto_tail
0x7f2ca2800000
ok
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.043 MB perf.data (~1860 samples) ]
$ perf report --stdio
100.00% anon_vma_moveto [kernel.kallsyms] [k] anon_vma_moveto_tail
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Reported-by: Nai Xia <nai.xia@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pawel Sikora <pluto@agmk.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 23:08:05 +00:00
|
|
|
faulted_in_anon_vma = false;
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-10-08 23:29:07 +00:00
|
|
|
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
|
|
|
|
|
return NULL; /* should never get here */
|
2005-04-16 22:20:36 +00:00
|
|
|
new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
|
2015-09-04 22:46:24 +00:00
|
|
|
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
|
|
|
|
|
vma->vm_userfaultfd_ctx);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (new_vma) {
|
|
|
|
|
/*
|
|
|
|
|
* Source vma may have been merged into new_vma
|
|
|
|
|
*/
|
mremap: enforce rmap src/dst vma ordering in case of vma_merge() succeeding in copy_vma()
migrate was doing an rmap_walk with speculative lock-less access on
pagetables. That could lead it to not serializing properly against mremap
PT locks. But a second problem remains in the order of vmas in the
same_anon_vma list used by the rmap_walk.
If vma_merge succeeds in copy_vma, the src vma could be placed after the
dst vma in the same_anon_vma list. That could still lead to migrate
missing some pte.
This patch adds an anon_vma_moveto_tail() function to force the dst vma at
the end of the list before mremap starts to solve the problem.
If the mremap is very large and there are a lots of parents or childs
sharing the anon_vma root lock, this should still scale better than taking
the anon_vma root lock around every pte copy practically for the whole
duration of mremap.
Update: Hugh noticed special care is needed in the error path where
move_page_tables goes in the reverse direction, a second
anon_vma_moveto_tail() call is needed in the error path.
This program exercises the anon_vma_moveto_tail:
===
int main()
{
static struct timeval oldstamp, newstamp;
long diffsec;
char *p, *p2, *p3, *p4;
if (posix_memalign((void **)&p, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p2, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
if (posix_memalign((void **)&p3, 2*1024*1024, SIZE))
perror("memalign"), exit(1);
memset(p, 0xff, SIZE);
printf("%p\n", p);
memset(p2, 0xff, SIZE);
memset(p3, 0x77, 4096);
if (memcmp(p, p2, SIZE))
printf("error\n");
p4 = mremap(p+SIZE/2, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p3);
if (p4 != p3)
perror("mremap"), exit(1);
p4 = mremap(p4, SIZE/2, SIZE/2, MREMAP_FIXED|MREMAP_MAYMOVE, p+SIZE/2);
if (p4 != p+SIZE/2)
perror("mremap"), exit(1);
if (memcmp(p, p2, SIZE))
printf("error\n");
printf("ok\n");
return 0;
}
===
$ perf probe -a anon_vma_moveto_tail
Add new event:
probe:anon_vma_moveto_tail (on anon_vma_moveto_tail)
You can now use it on all perf tools, such as:
perf record -e probe:anon_vma_moveto_tail -aR sleep 1
$ perf record -e probe:anon_vma_moveto_tail -aR ./anon_vma_moveto_tail
0x7f2ca2800000
ok
[ perf record: Woken up 1 times to write data ]
[ perf record: Captured and wrote 0.043 MB perf.data (~1860 samples) ]
$ perf report --stdio
100.00% anon_vma_moveto [kernel.kallsyms] [k] anon_vma_moveto_tail
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Reported-by: Nai Xia <nai.xia@gmail.com>
Acked-by: Mel Gorman <mgorman@suse.de>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pawel Sikora <pluto@agmk.net>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-01-10 23:08:05 +00:00
|
|
|
if (unlikely(vma_start >= new_vma->vm_start &&
|
|
|
|
|
vma_start < new_vma->vm_end)) {
|
|
|
|
|
/*
|
|
|
|
|
* The only way we can get a vma_merge with
|
|
|
|
|
* self during an mremap is if the vma hasn't
|
|
|
|
|
* been faulted in yet and we were allowed to
|
|
|
|
|
* reset the dst vma->vm_pgoff to the
|
|
|
|
|
* destination address of the mremap to allow
|
|
|
|
|
* the merge to happen. mremap must change the
|
|
|
|
|
* vm_pgoff linearity between src and dst vmas
|
|
|
|
|
* (in turn preventing a vma_merge) to be
|
|
|
|
|
* safe. It is only safe to keep the vm_pgoff
|
|
|
|
|
* linear if there are no pages mapped yet.
|
|
|
|
|
*/
|
2014-10-09 22:28:10 +00:00
|
|
|
VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
|
2012-10-08 23:31:50 +00:00
|
|
|
*vmap = vma = new_vma;
|
2012-10-08 23:31:36 +00:00
|
|
|
}
|
2012-10-08 23:31:50 +00:00
|
|
|
*need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2006-12-07 04:33:17 +00:00
|
|
|
new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
|
2015-09-08 22:03:38 +00:00
|
|
|
if (!new_vma)
|
|
|
|
|
goto out;
|
|
|
|
|
*new_vma = *vma;
|
|
|
|
|
new_vma->vm_start = addr;
|
|
|
|
|
new_vma->vm_end = addr + len;
|
|
|
|
|
new_vma->vm_pgoff = pgoff;
|
|
|
|
|
if (vma_dup_policy(vma, new_vma))
|
|
|
|
|
goto out_free_vma;
|
|
|
|
|
INIT_LIST_HEAD(&new_vma->anon_vma_chain);
|
|
|
|
|
if (anon_vma_clone(new_vma, vma))
|
|
|
|
|
goto out_free_mempol;
|
|
|
|
|
if (new_vma->vm_file)
|
|
|
|
|
get_file(new_vma->vm_file);
|
|
|
|
|
if (new_vma->vm_ops && new_vma->vm_ops->open)
|
|
|
|
|
new_vma->vm_ops->open(new_vma);
|
|
|
|
|
vma_link(mm, new_vma, prev, rb_link, rb_parent);
|
|
|
|
|
*need_rmap_locks = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
return new_vma;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
|
2015-09-08 22:03:38 +00:00
|
|
|
out_free_mempol:
|
2013-09-11 21:20:14 +00:00
|
|
|
mpol_put(vma_policy(new_vma));
|
2015-09-08 22:03:38 +00:00
|
|
|
out_free_vma:
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
kmem_cache_free(vm_area_cachep, new_vma);
|
2015-09-08 22:03:38 +00:00
|
|
|
out:
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
return NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-05-01 15:58:35 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Return true if the calling process may expand its vm space by the passed
|
|
|
|
|
* number of pages
|
|
|
|
|
*/
|
2016-01-14 23:22:07 +00:00
|
|
|
bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
|
2005-05-01 15:58:35 +00:00
|
|
|
{
|
2016-01-14 23:22:07 +00:00
|
|
|
if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
|
|
|
|
|
return false;
|
2005-05-01 15:58:35 +00:00
|
|
|
|
2016-02-03 00:57:43 +00:00
|
|
|
if (is_data_mapping(flags) &&
|
|
|
|
|
mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
|
2016-05-20 23:57:45 +00:00
|
|
|
/* Workaround for Valgrind */
|
|
|
|
|
if (rlimit(RLIMIT_DATA) == 0 &&
|
|
|
|
|
mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
|
|
|
|
|
return true;
|
|
|
|
|
if (!ignore_rlimit_data) {
|
|
|
|
|
pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits or use boot option ignore_rlimit_data.\n",
|
2016-02-03 00:57:43 +00:00
|
|
|
current->comm, current->pid,
|
|
|
|
|
(mm->data_vm + npages) << PAGE_SHIFT,
|
|
|
|
|
rlimit(RLIMIT_DATA));
|
|
|
|
|
return false;
|
2016-05-20 23:57:45 +00:00
|
|
|
}
|
2016-02-03 00:57:43 +00:00
|
|
|
}
|
2005-05-01 15:58:35 +00:00
|
|
|
|
2016-01-14 23:22:07 +00:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
|
|
|
|
|
{
|
|
|
|
|
mm->total_vm += npages;
|
|
|
|
|
|
2016-02-03 00:57:43 +00:00
|
|
|
if (is_exec_mapping(flags))
|
2016-01-14 23:22:07 +00:00
|
|
|
mm->exec_vm += npages;
|
2016-02-03 00:57:43 +00:00
|
|
|
else if (is_stack_mapping(flags))
|
2016-01-14 23:22:07 +00:00
|
|
|
mm->stack_vm += npages;
|
2016-02-03 00:57:43 +00:00
|
|
|
else if (is_data_mapping(flags))
|
2016-01-14 23:22:07 +00:00
|
|
|
mm->data_vm += npages;
|
2005-05-01 15:58:35 +00:00
|
|
|
}
|
2007-02-08 22:20:41 +00:00
|
|
|
|
2014-05-19 22:58:33 +00:00
|
|
|
static int special_mapping_fault(struct vm_area_struct *vma,
|
|
|
|
|
struct vm_fault *vmf);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Having a close hook prevents vma merging regardless of flags.
|
|
|
|
|
*/
|
|
|
|
|
static void special_mapping_close(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static const char *special_mapping_name(struct vm_area_struct *vma)
|
|
|
|
|
{
|
|
|
|
|
return ((struct vm_special_mapping *)vma->vm_private_data)->name;
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-28 11:35:38 +00:00
|
|
|
static int special_mapping_mremap(struct vm_area_struct *new_vma)
|
|
|
|
|
{
|
|
|
|
|
struct vm_special_mapping *sm = new_vma->vm_private_data;
|
|
|
|
|
|
|
|
|
|
if (sm->mremap)
|
|
|
|
|
return sm->mremap(sm, new_vma);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-19 22:58:33 +00:00
|
|
|
static const struct vm_operations_struct special_mapping_vmops = {
|
|
|
|
|
.close = special_mapping_close,
|
|
|
|
|
.fault = special_mapping_fault,
|
2016-06-28 11:35:38 +00:00
|
|
|
.mremap = special_mapping_mremap,
|
2014-05-19 22:58:33 +00:00
|
|
|
.name = special_mapping_name,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct vm_operations_struct legacy_special_mapping_vmops = {
|
|
|
|
|
.close = special_mapping_close,
|
|
|
|
|
.fault = special_mapping_fault,
|
|
|
|
|
};
|
2007-02-08 22:20:41 +00:00
|
|
|
|
2008-02-09 00:15:19 +00:00
|
|
|
static int special_mapping_fault(struct vm_area_struct *vma,
|
|
|
|
|
struct vm_fault *vmf)
|
2007-02-08 22:20:41 +00:00
|
|
|
{
|
2008-02-09 00:15:19 +00:00
|
|
|
pgoff_t pgoff;
|
2007-02-08 22:20:41 +00:00
|
|
|
struct page **pages;
|
|
|
|
|
|
2015-12-30 04:12:19 +00:00
|
|
|
if (vma->vm_ops == &legacy_special_mapping_vmops) {
|
2014-05-19 22:58:33 +00:00
|
|
|
pages = vma->vm_private_data;
|
2015-12-30 04:12:19 +00:00
|
|
|
} else {
|
|
|
|
|
struct vm_special_mapping *sm = vma->vm_private_data;
|
|
|
|
|
|
|
|
|
|
if (sm->fault)
|
|
|
|
|
return sm->fault(sm, vma, vmf);
|
|
|
|
|
|
|
|
|
|
pages = sm->pages;
|
|
|
|
|
}
|
2014-05-19 22:58:33 +00:00
|
|
|
|
mmap: fix the usage of ->vm_pgoff in special_mapping paths
Test-case:
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <assert.h>
void *find_vdso_vaddr(void)
{
FILE *perl;
char buf[32] = {};
perl = popen("perl -e 'open STDIN,qq|/proc/@{[getppid]}/maps|;"
"/^(.*?)-.*vdso/ && print hex $1 while <>'", "r");
fread(buf, sizeof(buf), 1, perl);
fclose(perl);
return (void *)atol(buf);
}
#define PAGE_SIZE 4096
int main(void)
{
void *vdso = find_vdso_vaddr();
assert(vdso);
// of course they should differ, and they do so far
printf("vdso pages differ: %d\n",
!!memcmp(vdso, vdso + PAGE_SIZE, PAGE_SIZE));
// split into 2 vma's
assert(mprotect(vdso, PAGE_SIZE, PROT_READ) == 0);
// force another fault on the next check
assert(madvise(vdso, 2 * PAGE_SIZE, MADV_DONTNEED) == 0);
// now they no longer differ, the 2nd vm_pgoff is wrong
printf("vdso pages differ: %d\n",
!!memcmp(vdso, vdso + PAGE_SIZE, PAGE_SIZE));
return 0;
}
Output:
vdso pages differ: 1
vdso pages differ: 0
This is because split_vma() correctly updates ->vm_pgoff, but the logic
in insert_vm_struct() and special_mapping_fault() is absolutely broken,
so the fault at vdso + PAGE_SIZE return the 1st page. The same happens
if you simply unmap the 1st page.
special_mapping_fault() does:
pgoff = vmf->pgoff - vma->vm_pgoff;
and this is _only_ correct if vma->vm_start mmaps the first page from
->vm_private_data array.
vdso or any other user of install_special_mapping() is not anonymous,
it has the "backing storage" even if it is just the array of pages.
So we actually need to make vm_pgoff work as an offset in this array.
Note: this also allows to fix another problem: currently gdb can't access
"[vvar]" memory because in this case special_mapping_fault() doesn't work.
Now that we can use ->vm_pgoff we can implement ->access() and fix this.
Signed-off-by: Oleg Nesterov <oleg@redhat.com>
Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Hugh Dickins <hughd@google.com>
Cc: Pavel Emelyanov <xemul@parallels.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-09-08 21:58:31 +00:00
|
|
|
for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
|
2008-02-09 00:15:19 +00:00
|
|
|
pgoff--;
|
2007-02-08 22:20:41 +00:00
|
|
|
|
|
|
|
|
if (*pages) {
|
|
|
|
|
struct page *page = *pages;
|
|
|
|
|
get_page(page);
|
2008-02-09 00:15:19 +00:00
|
|
|
vmf->page = page;
|
|
|
|
|
return 0;
|
2007-02-08 22:20:41 +00:00
|
|
|
}
|
|
|
|
|
|
2008-02-09 00:15:19 +00:00
|
|
|
return VM_FAULT_SIGBUS;
|
2007-02-08 22:20:41 +00:00
|
|
|
}
|
|
|
|
|
|
2014-05-19 22:58:33 +00:00
|
|
|
static struct vm_area_struct *__install_special_mapping(
|
|
|
|
|
struct mm_struct *mm,
|
|
|
|
|
unsigned long addr, unsigned long len,
|
2015-11-06 02:48:41 +00:00
|
|
|
unsigned long vm_flags, void *priv,
|
|
|
|
|
const struct vm_operations_struct *ops)
|
2007-02-08 22:20:41 +00:00
|
|
|
{
|
2010-12-09 14:29:42 +00:00
|
|
|
int ret;
|
2007-02-08 22:20:41 +00:00
|
|
|
struct vm_area_struct *vma;
|
|
|
|
|
|
|
|
|
|
vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
|
|
|
|
|
if (unlikely(vma == NULL))
|
2014-03-17 22:22:02 +00:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2007-02-08 22:20:41 +00:00
|
|
|
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
INIT_LIST_HEAD(&vma->anon_vma_chain);
|
2007-02-08 22:20:41 +00:00
|
|
|
vma->vm_mm = mm;
|
|
|
|
|
vma->vm_start = addr;
|
|
|
|
|
vma->vm_end = addr + len;
|
|
|
|
|
|
2013-09-11 21:22:24 +00:00
|
|
|
vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
|
2007-10-19 06:39:15 +00:00
|
|
|
vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
|
2007-02-08 22:20:41 +00:00
|
|
|
|
2014-05-19 22:58:33 +00:00
|
|
|
vma->vm_ops = ops;
|
|
|
|
|
vma->vm_private_data = priv;
|
2007-02-08 22:20:41 +00:00
|
|
|
|
2010-12-09 14:29:42 +00:00
|
|
|
ret = insert_vm_struct(mm, vma);
|
|
|
|
|
if (ret)
|
|
|
|
|
goto out;
|
2007-02-08 22:20:41 +00:00
|
|
|
|
2016-01-14 23:22:07 +00:00
|
|
|
vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
|
2007-02-08 22:20:41 +00:00
|
|
|
|
perf: Do the big rename: Performance Counters -> Performance Events
Bye-bye Performance Counters, welcome Performance Events!
In the past few months the perfcounters subsystem has grown out its
initial role of counting hardware events, and has become (and is
becoming) a much broader generic event enumeration, reporting, logging,
monitoring, analysis facility.
Naming its core object 'perf_counter' and naming the subsystem
'perfcounters' has become more and more of a misnomer. With pending
code like hw-breakpoints support the 'counter' name is less and
less appropriate.
All in one, we've decided to rename the subsystem to 'performance
events' and to propagate this rename through all fields, variables
and API names. (in an ABI compatible fashion)
The word 'event' is also a bit shorter than 'counter' - which makes
it slightly more convenient to write/handle as well.
Thanks goes to Stephane Eranian who first observed this misnomer and
suggested a rename.
User-space tooling and ABI compatibility is not affected - this patch
should be function-invariant. (Also, defconfigs were not touched to
keep the size down.)
This patch has been generated via the following script:
FILES=$(find * -type f | grep -vE 'oprofile|[^K]config')
sed -i \
-e 's/PERF_EVENT_/PERF_RECORD_/g' \
-e 's/PERF_COUNTER/PERF_EVENT/g' \
-e 's/perf_counter/perf_event/g' \
-e 's/nb_counters/nb_events/g' \
-e 's/swcounter/swevent/g' \
-e 's/tpcounter_event/tp_event/g' \
$FILES
for N in $(find . -name perf_counter.[ch]); do
M=$(echo $N | sed 's/perf_counter/perf_event/g')
mv $N $M
done
FILES=$(find . -name perf_event.*)
sed -i \
-e 's/COUNTER_MASK/REG_MASK/g' \
-e 's/COUNTER/EVENT/g' \
-e 's/\<event\>/event_id/g' \
-e 's/counter/event/g' \
-e 's/Counter/Event/g' \
$FILES
... to keep it as correct as possible. This script can also be
used by anyone who has pending perfcounters patches - it converts
a Linux kernel tree over to the new naming. We tried to time this
change to the point in time where the amount of pending patches
is the smallest: the end of the merge window.
Namespace clashes were fixed up in a preparatory patch - and some
stylistic fallout will be fixed up in a subsequent patch.
( NOTE: 'counters' are still the proper terminology when we deal
with hardware registers - and these sed scripts are a bit
over-eager in renaming them. I've undone some of that, but
in case there's something left where 'counter' would be
better than 'event' we can undo that on an individual basis
instead of touching an otherwise nicely automated patch. )
Suggested-by: Stephane Eranian <eranian@google.com>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Paul Mackerras <paulus@samba.org>
Reviewed-by: Arjan van de Ven <arjan@linux.intel.com>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: David Howells <dhowells@redhat.com>
Cc: Kyle McMartin <kyle@mcmartin.ca>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: <linux-arch@vger.kernel.org>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-09-21 10:02:48 +00:00
|
|
|
perf_event_mmap(vma);
|
2009-06-05 12:04:55 +00:00
|
|
|
|
2014-03-17 22:22:02 +00:00
|
|
|
return vma;
|
2010-12-09 14:29:42 +00:00
|
|
|
|
|
|
|
|
out:
|
|
|
|
|
kmem_cache_free(vm_area_cachep, vma);
|
2014-03-17 22:22:02 +00:00
|
|
|
return ERR_PTR(ret);
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-19 22:58:33 +00:00
|
|
|
/*
|
|
|
|
|
* Called with mm->mmap_sem held for writing.
|
|
|
|
|
* Insert a new vma covering the given region, with the given flags.
|
|
|
|
|
* Its pages are supplied by the given array of struct page *.
|
|
|
|
|
* The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
|
|
|
|
|
* The region past the last page supplied will always produce SIGBUS.
|
|
|
|
|
* The array pointer and the pages it points to are assumed to stay alive
|
|
|
|
|
* for as long as this mapping might exist.
|
|
|
|
|
*/
|
|
|
|
|
struct vm_area_struct *_install_special_mapping(
|
|
|
|
|
struct mm_struct *mm,
|
|
|
|
|
unsigned long addr, unsigned long len,
|
|
|
|
|
unsigned long vm_flags, const struct vm_special_mapping *spec)
|
|
|
|
|
{
|
2015-11-06 02:48:41 +00:00
|
|
|
return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
|
|
|
|
|
&special_mapping_vmops);
|
2014-05-19 22:58:33 +00:00
|
|
|
}
|
|
|
|
|
|
2014-03-17 22:22:02 +00:00
|
|
|
int install_special_mapping(struct mm_struct *mm,
|
|
|
|
|
unsigned long addr, unsigned long len,
|
|
|
|
|
unsigned long vm_flags, struct page **pages)
|
|
|
|
|
{
|
2014-05-19 22:58:33 +00:00
|
|
|
struct vm_area_struct *vma = __install_special_mapping(
|
2015-11-06 02:48:41 +00:00
|
|
|
mm, addr, len, vm_flags, (void *)pages,
|
|
|
|
|
&legacy_special_mapping_vmops);
|
2014-03-17 22:22:02 +00:00
|
|
|
|
2014-06-04 23:07:05 +00:00
|
|
|
return PTR_ERR_OR_ZERO(vma);
|
2007-02-08 22:20:41 +00:00
|
|
|
}
|
2008-07-28 22:46:26 +00:00
|
|
|
|
|
|
|
|
static DEFINE_MUTEX(mm_all_locks_mutex);
|
|
|
|
|
|
2008-08-11 07:30:25 +00:00
|
|
|
static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
|
2008-07-28 22:46:26 +00:00
|
|
|
{
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
|
2008-07-28 22:46:26 +00:00
|
|
|
/*
|
|
|
|
|
* The LSB of head.next can't change from under us
|
|
|
|
|
* because we hold the mm_all_locks_mutex.
|
|
|
|
|
*/
|
mm: mmap: annotate vm_lock_anon_vma locking properly for lockdep
Commit 5a505085f043 ("mm/rmap: Convert the struct anon_vma::mutex to an
rwsem") turned anon_vma mutex to rwsem.
However, the properly annotated nested locking in mm_take_all_locks()
has been converted from
mutex_lock_nest_lock(&anon_vma->root->mutex, &mm->mmap_sem);
to
down_write(&anon_vma->root->rwsem);
which is incomplete, and causes the false positive report from lockdep
below.
Annotate the fact that mmap_sem is used as an outter lock to serialize
taking of all the anon_vma rwsems at once no matter the order, using the
down_write_nest_lock() primitive.
This patch fixes this lockdep report:
=============================================
[ INFO: possible recursive locking detected ]
3.8.0-rc2-00036-g5f73896 #171 Not tainted
---------------------------------------------
qemu-kvm/2315 is trying to acquire lock:
(&anon_vma->rwsem){+.+...}, at: mm_take_all_locks+0x149/0x1b0
but task is already holding lock:
(&anon_vma->rwsem){+.+...}, at: mm_take_all_locks+0x149/0x1b0
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&anon_vma->rwsem);
lock(&anon_vma->rwsem);
*** DEADLOCK ***
May be due to missing lock nesting notation
4 locks held by qemu-kvm/2315:
#0: (&mm->mmap_sem){++++++}, at: do_mmu_notifier_register+0xfc/0x170
#1: (mm_all_locks_mutex){+.+...}, at: mm_take_all_locks+0x36/0x1b0
#2: (&mapping->i_mmap_mutex){+.+...}, at: mm_take_all_locks+0xc9/0x1b0
#3: (&anon_vma->rwsem){+.+...}, at: mm_take_all_locks+0x149/0x1b0
stack backtrace:
Pid: 2315, comm: qemu-kvm Not tainted 3.8.0-rc2-00036-g5f73896 #171
Call Trace:
print_deadlock_bug+0xf2/0x100
validate_chain+0x4f6/0x720
__lock_acquire+0x359/0x580
lock_acquire+0x121/0x190
down_write+0x3f/0x70
mm_take_all_locks+0x149/0x1b0
do_mmu_notifier_register+0x68/0x170
mmu_notifier_register+0xe/0x10
kvm_create_vm+0x22b/0x330 [kvm]
kvm_dev_ioctl+0xf8/0x1a0 [kvm]
do_vfs_ioctl+0x9d/0x350
sys_ioctl+0x91/0xb0
system_call_fastpath+0x16/0x1b
Signed-off-by: Jiri Kosina <jkosina@suse.cz>
Cc: Rik van Riel <riel@redhat.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mel Gorman <mel@csn.ul.ie>
Tested-by: Sedat Dilek <sedat.dilek@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-01-11 22:31:59 +00:00
|
|
|
down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
|
2008-07-28 22:46:26 +00:00
|
|
|
/*
|
|
|
|
|
* We can safely modify head.next after taking the
|
2012-12-02 19:56:46 +00:00
|
|
|
* anon_vma->root->rwsem. If some other vma in this mm shares
|
2008-07-28 22:46:26 +00:00
|
|
|
* the same anon_vma we won't take it again.
|
|
|
|
|
*
|
|
|
|
|
* No need of atomic instructions here, head.next
|
|
|
|
|
* can't change from under us thanks to the
|
2012-12-02 19:56:46 +00:00
|
|
|
* anon_vma->root->rwsem.
|
2008-07-28 22:46:26 +00:00
|
|
|
*/
|
|
|
|
|
if (__test_and_set_bit(0, (unsigned long *)
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
&anon_vma->root->rb_root.rb_node))
|
2008-07-28 22:46:26 +00:00
|
|
|
BUG();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2008-08-11 07:30:25 +00:00
|
|
|
static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
|
2008-07-28 22:46:26 +00:00
|
|
|
{
|
|
|
|
|
if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
|
|
|
|
|
/*
|
|
|
|
|
* AS_MM_ALL_LOCKS can't change from under us because
|
|
|
|
|
* we hold the mm_all_locks_mutex.
|
|
|
|
|
*
|
|
|
|
|
* Operations on ->flags have to be atomic because
|
|
|
|
|
* even if AS_MM_ALL_LOCKS is stable thanks to the
|
|
|
|
|
* mm_all_locks_mutex, there may be other cpus
|
|
|
|
|
* changing other bitflags in parallel to us.
|
|
|
|
|
*/
|
|
|
|
|
if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
|
|
|
|
|
BUG();
|
2014-12-13 00:54:24 +00:00
|
|
|
down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
|
2008-07-28 22:46:26 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This operation locks against the VM for all pte/vma/mm related
|
|
|
|
|
* operations that could ever happen on a certain mm. This includes
|
|
|
|
|
* vmtruncate, try_to_unmap, and all page faults.
|
|
|
|
|
*
|
|
|
|
|
* The caller must take the mmap_sem in write mode before calling
|
|
|
|
|
* mm_take_all_locks(). The caller isn't allowed to release the
|
|
|
|
|
* mmap_sem until mm_drop_all_locks() returns.
|
|
|
|
|
*
|
|
|
|
|
* mmap_sem in write mode is required in order to block all operations
|
|
|
|
|
* that could modify pagetables and free pages without need of
|
2015-02-10 22:09:59 +00:00
|
|
|
* altering the vma layout. It's also needed in write mode to avoid new
|
2008-07-28 22:46:26 +00:00
|
|
|
* anon_vmas to be associated with existing vmas.
|
|
|
|
|
*
|
|
|
|
|
* A single task can't take more than one mm_take_all_locks() in a row
|
|
|
|
|
* or it would deadlock.
|
|
|
|
|
*
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
* The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
|
2008-07-28 22:46:26 +00:00
|
|
|
* mapping->flags avoid to take the same lock twice, if more than one
|
|
|
|
|
* vma in this mm is backed by the same anon_vma or address_space.
|
|
|
|
|
*
|
2016-01-16 00:57:31 +00:00
|
|
|
* We take locks in following order, accordingly to comment at beginning
|
|
|
|
|
* of mm/rmap.c:
|
|
|
|
|
* - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
|
|
|
|
|
* hugetlb mapping);
|
|
|
|
|
* - all i_mmap_rwsem locks;
|
|
|
|
|
* - all anon_vma->rwseml
|
|
|
|
|
*
|
|
|
|
|
* We can take all locks within these types randomly because the VM code
|
|
|
|
|
* doesn't nest them and we protected from parallel mm_take_all_locks() by
|
|
|
|
|
* mm_all_locks_mutex.
|
2008-07-28 22:46:26 +00:00
|
|
|
*
|
|
|
|
|
* mm_take_all_locks() and mm_drop_all_locks are expensive operations
|
|
|
|
|
* that may have to take thousand of locks.
|
|
|
|
|
*
|
|
|
|
|
* mm_take_all_locks() can fail if it's interrupted by signals.
|
|
|
|
|
*/
|
|
|
|
|
int mm_take_all_locks(struct mm_struct *mm)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *vma;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
struct anon_vma_chain *avc;
|
2008-07-28 22:46:26 +00:00
|
|
|
|
|
|
|
|
BUG_ON(down_read_trylock(&mm->mmap_sem));
|
|
|
|
|
|
|
|
|
|
mutex_lock(&mm_all_locks_mutex);
|
|
|
|
|
|
|
|
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
|
|
|
|
if (signal_pending(current))
|
|
|
|
|
goto out_unlock;
|
2016-01-16 00:57:31 +00:00
|
|
|
if (vma->vm_file && vma->vm_file->f_mapping &&
|
|
|
|
|
is_vm_hugetlb_page(vma))
|
|
|
|
|
vm_lock_mapping(mm, vma->vm_file->f_mapping);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
|
|
|
|
if (signal_pending(current))
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
if (vma->vm_file && vma->vm_file->f_mapping &&
|
|
|
|
|
!is_vm_hugetlb_page(vma))
|
2008-08-11 07:30:25 +00:00
|
|
|
vm_lock_mapping(mm, vma->vm_file->f_mapping);
|
2008-07-28 22:46:26 +00:00
|
|
|
}
|
2008-08-11 07:30:25 +00:00
|
|
|
|
|
|
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
|
|
|
|
if (signal_pending(current))
|
|
|
|
|
goto out_unlock;
|
|
|
|
|
if (vma->anon_vma)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
|
|
|
|
vm_lock_anon_vma(mm, avc->anon_vma);
|
2008-07-28 22:46:26 +00:00
|
|
|
}
|
2008-08-11 07:30:25 +00:00
|
|
|
|
2011-11-01 00:08:59 +00:00
|
|
|
return 0;
|
2008-07-28 22:46:26 +00:00
|
|
|
|
|
|
|
|
out_unlock:
|
2011-11-01 00:08:59 +00:00
|
|
|
mm_drop_all_locks(mm);
|
|
|
|
|
return -EINTR;
|
2008-07-28 22:46:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
|
|
|
|
|
{
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_node)) {
|
2008-07-28 22:46:26 +00:00
|
|
|
/*
|
|
|
|
|
* The LSB of head.next can't change to 0 from under
|
|
|
|
|
* us because we hold the mm_all_locks_mutex.
|
|
|
|
|
*
|
|
|
|
|
* We must however clear the bitflag before unlocking
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
* the vma so the users using the anon_vma->rb_root will
|
2008-07-28 22:46:26 +00:00
|
|
|
* never see our bitflag.
|
|
|
|
|
*
|
|
|
|
|
* No need of atomic instructions here, head.next
|
|
|
|
|
* can't change from under us until we release the
|
2012-12-02 19:56:46 +00:00
|
|
|
* anon_vma->root->rwsem.
|
2008-07-28 22:46:26 +00:00
|
|
|
*/
|
|
|
|
|
if (!__test_and_clear_bit(0, (unsigned long *)
|
mm anon rmap: replace same_anon_vma linked list with an interval tree.
When a large VMA (anon or private file mapping) is first touched, which
will populate its anon_vma field, and then split into many regions through
the use of mprotect(), the original anon_vma ends up linking all of the
vmas on a linked list. This can cause rmap to become inefficient, as we
have to walk potentially thousands of irrelevent vmas before finding the
one a given anon page might fall into.
By replacing the same_anon_vma linked list with an interval tree (where
each avc's interval is determined by its vma's start and last pgoffs), we
can make rmap efficient for this use case again.
While the change is large, all of its pieces are fairly simple.
Most places that were walking the same_anon_vma list were looking for a
known pgoff, so they can just use the anon_vma_interval_tree_foreach()
interval tree iterator instead. The exception here is ksm, where the
page's index is not known. It would probably be possible to rework ksm so
that the index would be known, but for now I have decided to keep things
simple and just walk the entirety of the interval tree there.
When updating vma's that already have an anon_vma assigned, we must take
care to re-index the corresponding avc's on their interval tree. This is
done through the use of anon_vma_interval_tree_pre_update_vma() and
anon_vma_interval_tree_post_update_vma(), which remove the avc's from
their interval tree before the update and re-insert them after the update.
The anon_vma stays locked during the update, so there is no chance that
rmap would miss the vmas that are being updated.
Signed-off-by: Michel Lespinasse <walken@google.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Daniel Santos <daniel.santos@pobox.com>
Cc: Hugh Dickins <hughd@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-10-08 23:31:39 +00:00
|
|
|
&anon_vma->root->rb_root.rb_node))
|
2008-07-28 22:46:26 +00:00
|
|
|
BUG();
|
2013-02-23 00:34:40 +00:00
|
|
|
anon_vma_unlock_write(anon_vma);
|
2008-07-28 22:46:26 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vm_unlock_mapping(struct address_space *mapping)
|
|
|
|
|
{
|
|
|
|
|
if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
|
|
|
|
|
/*
|
|
|
|
|
* AS_MM_ALL_LOCKS can't change to 0 from under us
|
|
|
|
|
* because we hold the mm_all_locks_mutex.
|
|
|
|
|
*/
|
2014-12-13 00:54:21 +00:00
|
|
|
i_mmap_unlock_write(mapping);
|
2008-07-28 22:46:26 +00:00
|
|
|
if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
|
|
|
|
|
&mapping->flags))
|
|
|
|
|
BUG();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The mmap_sem cannot be released by the caller until
|
|
|
|
|
* mm_drop_all_locks() returns.
|
|
|
|
|
*/
|
|
|
|
|
void mm_drop_all_locks(struct mm_struct *mm)
|
|
|
|
|
{
|
|
|
|
|
struct vm_area_struct *vma;
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
struct anon_vma_chain *avc;
|
2008-07-28 22:46:26 +00:00
|
|
|
|
|
|
|
|
BUG_ON(down_read_trylock(&mm->mmap_sem));
|
|
|
|
|
BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
|
|
|
|
|
|
|
|
|
|
for (vma = mm->mmap; vma; vma = vma->vm_next) {
|
|
|
|
|
if (vma->anon_vma)
|
mm: change anon_vma linking to fix multi-process server scalability issue
The old anon_vma code can lead to scalability issues with heavily forking
workloads. Specifically, each anon_vma will be shared between the parent
process and all its child processes.
In a workload with 1000 child processes and a VMA with 1000 anonymous
pages per process that get COWed, this leads to a system with a million
anonymous pages in the same anon_vma, each of which is mapped in just one
of the 1000 processes. However, the current rmap code needs to walk them
all, leading to O(N) scanning complexity for each page.
This can result in systems where one CPU is walking the page tables of
1000 processes in page_referenced_one, while all other CPUs are stuck on
the anon_vma lock. This leads to catastrophic failure for a benchmark
like AIM7, where the total number of processes can reach in the tens of
thousands. Real workloads are still a factor 10 less process intensive
than AIM7, but they are catching up.
This patch changes the way anon_vmas and VMAs are linked, which allows us
to associate multiple anon_vmas with a VMA. At fork time, each child
process gets its own anon_vmas, in which its COWed pages will be
instantiated. The parents' anon_vma is also linked to the VMA, because
non-COWed pages could be present in any of the children.
This reduces rmap scanning complexity to O(1) for the pages of the 1000
child processes, with O(N) complexity for at most 1/N pages in the system.
This reduces the average scanning cost in heavily forking workloads from
O(N) to 2.
The only real complexity in this patch stems from the fact that linking a
VMA to anon_vmas now involves memory allocations. This means vma_adjust
can fail, if it needs to attach a VMA to anon_vma structures. This in
turn means error handling needs to be added to the calling functions.
A second source of complexity is that, because there can be multiple
anon_vmas, the anon_vma linking in vma_adjust can no longer be done under
"the" anon_vma lock. To prevent the rmap code from walking up an
incomplete VMA, this patch introduces the VM_LOCK_RMAP VMA flag. This bit
flag uses the same slot as the NOMMU VM_MAPPED_COPY, with an ifdef in mm.h
to make sure it is impossible to compile a kernel that needs both symbolic
values for the same bitflag.
Some test results:
Without the anon_vma changes, when AIM7 hits around 9.7k users (on a test
box with 16GB RAM and not quite enough IO), the system ends up running
>99% in system time, with every CPU on the same anon_vma lock in the
pageout code.
With these changes, AIM7 hits the cross-over point around 29.7k users.
This happens with ~99% IO wait time, there never seems to be any spike in
system time. The anon_vma lock contention appears to be resolved.
[akpm@linux-foundation.org: cleanups]
Signed-off-by: Rik van Riel <riel@redhat.com>
Cc: KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>
Cc: Larry Woodman <lwoodman@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2010-03-05 21:42:07 +00:00
|
|
|
list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
|
|
|
|
|
vm_unlock_anon_vma(avc->anon_vma);
|
2008-07-28 22:46:26 +00:00
|
|
|
if (vma->vm_file && vma->vm_file->f_mapping)
|
|
|
|
|
vm_unlock_mapping(vma->vm_file->f_mapping);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mutex_unlock(&mm_all_locks_mutex);
|
|
|
|
|
}
|
2009-01-08 12:04:47 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* initialise the VMA slab
|
|
|
|
|
*/
|
|
|
|
|
void __init mmap_init(void)
|
|
|
|
|
{
|
2009-04-30 22:08:51 +00:00
|
|
|
int ret;
|
|
|
|
|
|
2014-09-08 00:51:29 +00:00
|
|
|
ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
|
2009-04-30 22:08:51 +00:00
|
|
|
VM_BUG_ON(ret);
|
2009-01-08 12:04:47 +00:00
|
|
|
}
|
mm: limit growth of 3% hardcoded other user reserve
Add user_reserve_kbytes knob.
Limit the growth of the memory reserved for other user processes to
min(3% current process size, user_reserve_pages). Only about 8MB is
necessary to enable recovery in the default mode, and only a few hundred
MB are required even when overcommit is disabled.
user_reserve_pages defaults to min(3% free pages, 128MB)
I arrived at 128MB by taking the max VSZ of sshd, login, bash, and top ...
then adding the RSS of each.
This only affects OVERCOMMIT_NEVER mode.
Background
1. user reserve
__vm_enough_memory reserves a hardcoded 3% of the current process size for
other applications when overcommit is disabled. This was done so that a
user could recover if they launched a memory hogging process. Without the
reserve, a user would easily run into a message such as:
bash: fork: Cannot allocate memory
2. admin reserve
Additionally, a hardcoded 3% of free memory is reserved for root in both
overcommit 'guess' and 'never' modes. This was intended to prevent a
scenario where root-cant-log-in and perform recovery operations.
Note that this reserve shrinks, and doesn't guarantee a useful reserve.
Motivation
The two hardcoded memory reserves should be updated to account for current
memory sizes.
Also, the admin reserve would be more useful if it didn't shrink too much.
When the current code was originally written, 1GB was considered
"enterprise". Now the 3% reserve can grow to multiple GB on large memory
systems, and it only needs to be a few hundred MB at most to enable a user
or admin to recover a system with an unwanted memory hogging process.
I've found that reducing these reserves is especially beneficial for a
specific type of application load:
* single application system
* one or few processes (e.g. one per core)
* allocating all available memory
* not initializing every page immediately
* long running
I've run scientific clusters with this sort of load. A long running job
sometimes failed many hours (weeks of CPU time) into a calculation. They
weren't initializing all of their memory immediately, and they weren't
using calloc, so I put systems into overcommit 'never' mode. These
clusters run diskless and have no swap.
However, with the current reserves, a user wishing to allocate as much
memory as possible to one process may be prevented from using, for
example, almost 2GB out of 32GB.
The effect is less, but still significant when a user starts a job with
one process per core. I have repeatedly seen a set of processes
requesting the same amount of memory fail because one of them could not
allocate the amount of memory a user would expect to be able to allocate.
For example, Message Passing Interfce (MPI) processes, one per core. And
it is similar for other parallel programming frameworks.
Changing this reserve code will make the overcommit never mode more useful
by allowing applications to allocate nearly all of the available memory.
Also, the new admin_reserve_kbytes will be safer than the current behavior
since the hardcoded 3% of available memory reserve can shrink to something
useless in the case where applications have grabbed all available memory.
Risks
* "bash: fork: Cannot allocate memory"
The downside of the first patch-- which creates a tunable user reserve
that is only used in overcommit 'never' mode--is that an admin can set
it so low that a user may not be able to kill their process, even if
they already have a shell prompt.
Of course, a user can get in the same predicament with the current 3%
reserve--they just have to launch processes until 3% becomes negligible.
* root-cant-log-in problem
The second patch, adding the tunable rootuser_reserve_pages, allows
the admin to shoot themselves in the foot by setting it too small. They
can easily get the system into a state where root-can't-log-in.
However, the new admin_reserve_kbytes will be safer than the current
behavior since the hardcoded 3% of available memory reserve can shrink
to something useless in the case where applications have grabbed all
available memory.
Alternatives
* Memory cgroups provide a more flexible way to limit application memory.
Not everyone wants to set up cgroups or deal with their overhead.
* We could create a fourth overcommit mode which provides smaller reserves.
The size of useful reserves may be drastically different depending
on the whether the system is embedded or enterprise.
* Force users to initialize all of their memory or use calloc.
Some users don't want/expect the system to overcommit when they malloc.
Overcommit 'never' mode is for this scenario, and it should work well.
The new user and admin reserve tunables are simple to use, with low
overhead compared to cgroups. The patches preserve current behavior where
3% of memory is less than 128MB, except that the admin reserve doesn't
shrink to an unusable size under pressure. The code allows admins to tune
for embedded and enterprise usage.
FAQ
* How is the root-cant-login problem addressed?
What happens if admin_reserve_pages is set to 0?
Root is free to shoot themselves in the foot by setting
admin_reserve_kbytes too low.
On x86_64, the minimum useful reserve is:
8MB for overcommit 'guess'
128MB for overcommit 'never'
admin_reserve_pages defaults to min(3% free memory, 8MB)
So, anyone switching to 'never' mode needs to adjust
admin_reserve_pages.
* How do you calculate a minimum useful reserve?
A user or the admin needs enough memory to login and perform
recovery operations, which includes, at a minimum:
sshd or login + bash (or some other shell) + top (or ps, kill, etc.)
For overcommit 'guess', we can sum resident set sizes (RSS)
because we only need enough memory to handle what the recovery
programs will typically use. On x86_64 this is about 8MB.
For overcommit 'never', we can take the max of their virtual sizes (VSZ)
and add the sum of their RSS. We use VSZ instead of RSS because mode
forces us to ensure we can fulfill all of the requested memory allocations--
even if the programs only use a fraction of what they ask for.
On x86_64 this is about 128MB.
When swap is enabled, reserves are useful even when they are as
small as 10MB, regardless of overcommit mode.
When both swap and overcommit are disabled, then the admin should
tune the reserves higher to be absolutley safe. Over 230MB each
was safest in my testing.
* What happens if user_reserve_pages is set to 0?
Note, this only affects overcomitt 'never' mode.
Then a user will be able to allocate all available memory minus
admin_reserve_kbytes.
However, they will easily see a message such as:
"bash: fork: Cannot allocate memory"
And they won't be able to recover/kill their application.
The admin should be able to recover the system if
admin_reserve_kbytes is set appropriately.
* What's the difference between overcommit 'guess' and 'never'?
"Guess" allows an allocation if there are enough free + reclaimable
pages. It has a hardcoded 3% of free pages reserved for root.
"Never" allows an allocation if there is enough swap + a configurable
percentage (default is 50) of physical RAM. It has a hardcoded 3% of
free pages reserved for root, like "Guess" mode. It also has a
hardcoded 3% of the current process size reserved for additional
applications.
* Why is overcommit 'guess' not suitable even when an app eventually
writes to every page? It takes free pages, file pages, available
swap pages, reclaimable slab pages into consideration. In other words,
these are all pages available, then why isn't overcommit suitable?
Because it only looks at the present state of the system. It
does not take into account the memory that other applications have
malloced, but haven't initialized yet. It overcommits the system.
Test Summary
There was little change in behavior in the default overcommit 'guess'
mode with swap enabled before and after the patch. This was expected.
Systems run most predictably (i.e. no oom kills) in overcommit 'never'
mode with swap enabled. This also allowed the most memory to be allocated
to a user application.
Overcommit 'guess' mode without swap is a bad idea. It is easy to
crash the system. None of the other tested combinations crashed.
This matches my experience on the Roadrunner supercomputer.
Without the tunable user reserve, a system in overcommit 'never' mode
and without swap does not allow the admin to recover, although the
admin can.
With the new tunable reserves, a system in overcommit 'never' mode
and without swap can be configured to:
1. maximize user-allocatable memory, running close to the edge of
recoverability
2. maximize recoverability, sacrificing allocatable memory to
ensure that a user cannot take down a system
Test Description
Fedora 18 VM - 4 x86_64 cores, 5725MB RAM, 4GB Swap
System is booted into multiuser console mode, with unnecessary services
turned off. Caches were dropped before each test.
Hogs are user memtester processes that attempt to allocate all free memory
as reported by /proc/meminfo
In overcommit 'never' mode, memory_ratio=100
Test Results
3.9.0-rc1-mm1
Overcommit | Swap | Hogs | MB Got/Wanted | OOMs | User Recovery | Admin Recovery
---------- ---- ---- ------------- ---- ------------- --------------
guess yes 1 5432/5432 no yes yes
guess yes 4 5444/5444 1 yes yes
guess no 1 5302/5449 no yes yes
guess no 4 - crash no no
never yes 1 5460/5460 1 yes yes
never yes 4 5460/5460 1 yes yes
never no 1 5218/5432 no no yes
never no 4 5203/5448 no no yes
3.9.0-rc1-mm1-tunablereserves
User and Admin Recovery show their respective reserves, if applicable.
Overcommit | Swap | Hogs | MB Got/Wanted | OOMs | User Recovery | Admin Recovery
---------- ---- ---- ------------- ---- ------------- --------------
guess yes 1 5419/5419 no - yes 8MB yes
guess yes 4 5436/5436 1 - yes 8MB yes
guess no 1 5440/5440 * - yes 8MB yes
guess no 4 - crash - no 8MB no
* process would successfully mlock, then the oom killer would pick it
never yes 1 5446/5446 no 10MB yes 20MB yes
never yes 4 5456/5456 no 10MB yes 20MB yes
never no 1 5387/5429 no 128MB no 8MB barely
never no 1 5323/5428 no 226MB barely 8MB barely
never no 1 5323/5428 no 226MB barely 8MB barely
never no 1 5359/5448 no 10MB no 10MB barely
never no 1 5323/5428 no 0MB no 10MB barely
never no 1 5332/5428 no 0MB no 50MB yes
never no 1 5293/5429 no 0MB no 90MB yes
never no 1 5001/5427 no 230MB yes 338MB yes
never no 4* 4998/5424 no 230MB yes 338MB yes
* more memtesters were launched, able to allocate approximately another 100MB
Future Work
- Test larger memory systems.
- Test an embedded image.
- Test other architectures.
- Time malloc microbenchmarks.
- Would it be useful to be able to set overcommit policy for
each memory cgroup?
- Some lines are slightly above 80 chars.
Perhaps define a macro to convert between pages and kb?
Other places in the kernel do this.
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: make init_user_reserve() static]
Signed-off-by: Andrew Shewmaker <agshew@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-04-29 22:08:10 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Initialise sysctl_user_reserve_kbytes.
|
|
|
|
|
*
|
|
|
|
|
* This is intended to prevent a user from starting a single memory hogging
|
|
|
|
|
* process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
|
|
|
|
|
* mode.
|
|
|
|
|
*
|
|
|
|
|
* The default value is min(3% of free memory, 128MB)
|
|
|
|
|
* 128MB is enough to recover with sshd/login, bash, and top/kill.
|
|
|
|
|
*/
|
2013-04-29 22:08:12 +00:00
|
|
|
static int init_user_reserve(void)
|
mm: limit growth of 3% hardcoded other user reserve
Add user_reserve_kbytes knob.
Limit the growth of the memory reserved for other user processes to
min(3% current process size, user_reserve_pages). Only about 8MB is
necessary to enable recovery in the default mode, and only a few hundred
MB are required even when overcommit is disabled.
user_reserve_pages defaults to min(3% free pages, 128MB)
I arrived at 128MB by taking the max VSZ of sshd, login, bash, and top ...
then adding the RSS of each.
This only affects OVERCOMMIT_NEVER mode.
Background
1. user reserve
__vm_enough_memory reserves a hardcoded 3% of the current process size for
other applications when overcommit is disabled. This was done so that a
user could recover if they launched a memory hogging process. Without the
reserve, a user would easily run into a message such as:
bash: fork: Cannot allocate memory
2. admin reserve
Additionally, a hardcoded 3% of free memory is reserved for root in both
overcommit 'guess' and 'never' modes. This was intended to prevent a
scenario where root-cant-log-in and perform recovery operations.
Note that this reserve shrinks, and doesn't guarantee a useful reserve.
Motivation
The two hardcoded memory reserves should be updated to account for current
memory sizes.
Also, the admin reserve would be more useful if it didn't shrink too much.
When the current code was originally written, 1GB was considered
"enterprise". Now the 3% reserve can grow to multiple GB on large memory
systems, and it only needs to be a few hundred MB at most to enable a user
or admin to recover a system with an unwanted memory hogging process.
I've found that reducing these reserves is especially beneficial for a
specific type of application load:
* single application system
* one or few processes (e.g. one per core)
* allocating all available memory
* not initializing every page immediately
* long running
I've run scientific clusters with this sort of load. A long running job
sometimes failed many hours (weeks of CPU time) into a calculation. They
weren't initializing all of their memory immediately, and they weren't
using calloc, so I put systems into overcommit 'never' mode. These
clusters run diskless and have no swap.
However, with the current reserves, a user wishing to allocate as much
memory as possible to one process may be prevented from using, for
example, almost 2GB out of 32GB.
The effect is less, but still significant when a user starts a job with
one process per core. I have repeatedly seen a set of processes
requesting the same amount of memory fail because one of them could not
allocate the amount of memory a user would expect to be able to allocate.
For example, Message Passing Interfce (MPI) processes, one per core. And
it is similar for other parallel programming frameworks.
Changing this reserve code will make the overcommit never mode more useful
by allowing applications to allocate nearly all of the available memory.
Also, the new admin_reserve_kbytes will be safer than the current behavior
since the hardcoded 3% of available memory reserve can shrink to something
useless in the case where applications have grabbed all available memory.
Risks
* "bash: fork: Cannot allocate memory"
The downside of the first patch-- which creates a tunable user reserve
that is only used in overcommit 'never' mode--is that an admin can set
it so low that a user may not be able to kill their process, even if
they already have a shell prompt.
Of course, a user can get in the same predicament with the current 3%
reserve--they just have to launch processes until 3% becomes negligible.
* root-cant-log-in problem
The second patch, adding the tunable rootuser_reserve_pages, allows
the admin to shoot themselves in the foot by setting it too small. They
can easily get the system into a state where root-can't-log-in.
However, the new admin_reserve_kbytes will be safer than the current
behavior since the hardcoded 3% of available memory reserve can shrink
to something useless in the case where applications have grabbed all
available memory.
Alternatives
* Memory cgroups provide a more flexible way to limit application memory.
Not everyone wants to set up cgroups or deal with their overhead.
* We could create a fourth overcommit mode which provides smaller reserves.
The size of useful reserves may be drastically different depending
on the whether the system is embedded or enterprise.
* Force users to initialize all of their memory or use calloc.
Some users don't want/expect the system to overcommit when they malloc.
Overcommit 'never' mode is for this scenario, and it should work well.
The new user and admin reserve tunables are simple to use, with low
overhead compared to cgroups. The patches preserve current behavior where
3% of memory is less than 128MB, except that the admin reserve doesn't
shrink to an unusable size under pressure. The code allows admins to tune
for embedded and enterprise usage.
FAQ
* How is the root-cant-login problem addressed?
What happens if admin_reserve_pages is set to 0?
Root is free to shoot themselves in the foot by setting
admin_reserve_kbytes too low.
On x86_64, the minimum useful reserve is:
8MB for overcommit 'guess'
128MB for overcommit 'never'
admin_reserve_pages defaults to min(3% free memory, 8MB)
So, anyone switching to 'never' mode needs to adjust
admin_reserve_pages.
* How do you calculate a minimum useful reserve?
A user or the admin needs enough memory to login and perform
recovery operations, which includes, at a minimum:
sshd or login + bash (or some other shell) + top (or ps, kill, etc.)
For overcommit 'guess', we can sum resident set sizes (RSS)
because we only need enough memory to handle what the recovery
programs will typically use. On x86_64 this is about 8MB.
For overcommit 'never', we can take the max of their virtual sizes (VSZ)
and add the sum of their RSS. We use VSZ instead of RSS because mode
forces us to ensure we can fulfill all of the requested memory allocations--
even if the programs only use a fraction of what they ask for.
On x86_64 this is about 128MB.
When swap is enabled, reserves are useful even when they are as
small as 10MB, regardless of overcommit mode.
When both swap and overcommit are disabled, then the admin should
tune the reserves higher to be absolutley safe. Over 230MB each
was safest in my testing.
* What happens if user_reserve_pages is set to 0?
Note, this only affects overcomitt 'never' mode.
Then a user will be able to allocate all available memory minus
admin_reserve_kbytes.
However, they will easily see a message such as:
"bash: fork: Cannot allocate memory"
And they won't be able to recover/kill their application.
The admin should be able to recover the system if
admin_reserve_kbytes is set appropriately.
* What's the difference between overcommit 'guess' and 'never'?
"Guess" allows an allocation if there are enough free + reclaimable
pages. It has a hardcoded 3% of free pages reserved for root.
"Never" allows an allocation if there is enough swap + a configurable
percentage (default is 50) of physical RAM. It has a hardcoded 3% of
free pages reserved for root, like "Guess" mode. It also has a
hardcoded 3% of the current process size reserved for additional
applications.
* Why is overcommit 'guess' not suitable even when an app eventually
writes to every page? It takes free pages, file pages, available
swap pages, reclaimable slab pages into consideration. In other words,
these are all pages available, then why isn't overcommit suitable?
Because it only looks at the present state of the system. It
does not take into account the memory that other applications have
malloced, but haven't initialized yet. It overcommits the system.
Test Summary
There was little change in behavior in the default overcommit 'guess'
mode with swap enabled before and after the patch. This was expected.
Systems run most predictably (i.e. no oom kills) in overcommit 'never'
mode with swap enabled. This also allowed the most memory to be allocated
to a user application.
Overcommit 'guess' mode without swap is a bad idea. It is easy to
crash the system. None of the other tested combinations crashed.
This matches my experience on the Roadrunner supercomputer.
Without the tunable user reserve, a system in overcommit 'never' mode
and without swap does not allow the admin to recover, although the
admin can.
With the new tunable reserves, a system in overcommit 'never' mode
and without swap can be configured to:
1. maximize user-allocatable memory, running close to the edge of
recoverability
2. maximize recoverability, sacrificing allocatable memory to
ensure that a user cannot take down a system
Test Description
Fedora 18 VM - 4 x86_64 cores, 5725MB RAM, 4GB Swap
System is booted into multiuser console mode, with unnecessary services
turned off. Caches were dropped before each test.
Hogs are user memtester processes that attempt to allocate all free memory
as reported by /proc/meminfo
In overcommit 'never' mode, memory_ratio=100
Test Results
3.9.0-rc1-mm1
Overcommit | Swap | Hogs | MB Got/Wanted | OOMs | User Recovery | Admin Recovery
---------- ---- ---- ------------- ---- ------------- --------------
guess yes 1 5432/5432 no yes yes
guess yes 4 5444/5444 1 yes yes
guess no 1 5302/5449 no yes yes
guess no 4 - crash no no
never yes 1 5460/5460 1 yes yes
never yes 4 5460/5460 1 yes yes
never no 1 5218/5432 no no yes
never no 4 5203/5448 no no yes
3.9.0-rc1-mm1-tunablereserves
User and Admin Recovery show their respective reserves, if applicable.
Overcommit | Swap | Hogs | MB Got/Wanted | OOMs | User Recovery | Admin Recovery
---------- ---- ---- ------------- ---- ------------- --------------
guess yes 1 5419/5419 no - yes 8MB yes
guess yes 4 5436/5436 1 - yes 8MB yes
guess no 1 5440/5440 * - yes 8MB yes
guess no 4 - crash - no 8MB no
* process would successfully mlock, then the oom killer would pick it
never yes 1 5446/5446 no 10MB yes 20MB yes
never yes 4 5456/5456 no 10MB yes 20MB yes
never no 1 5387/5429 no 128MB no 8MB barely
never no 1 5323/5428 no 226MB barely 8MB barely
never no 1 5323/5428 no 226MB barely 8MB barely
never no 1 5359/5448 no 10MB no 10MB barely
never no 1 5323/5428 no 0MB no 10MB barely
never no 1 5332/5428 no 0MB no 50MB yes
never no 1 5293/5429 no 0MB no 90MB yes
never no 1 5001/5427 no 230MB yes 338MB yes
never no 4* 4998/5424 no 230MB yes 338MB yes
* more memtesters were launched, able to allocate approximately another 100MB
Future Work
- Test larger memory systems.
- Test an embedded image.
- Test other architectures.
- Time malloc microbenchmarks.
- Would it be useful to be able to set overcommit policy for
each memory cgroup?
- Some lines are slightly above 80 chars.
Perhaps define a macro to convert between pages and kb?
Other places in the kernel do this.
[akpm@linux-foundation.org: coding-style fixes]
[akpm@linux-foundation.org: make init_user_reserve() static]
Signed-off-by: Andrew Shewmaker <agshew@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-04-29 22:08:10 +00:00
|
|
|
{
|
|
|
|
|
unsigned long free_kbytes;
|
|
|
|
|
|
|
|
|
|
free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
|
|
|
|
|
|
|
|
|
|
sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2014-01-23 23:53:30 +00:00
|
|
|
subsys_initcall(init_user_reserve);
|
2013-04-29 22:08:11 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Initialise sysctl_admin_reserve_kbytes.
|
|
|
|
|
*
|
|
|
|
|
* The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
|
|
|
|
|
* to log in and kill a memory hogging process.
|
|
|
|
|
*
|
|
|
|
|
* Systems with more than 256MB will reserve 8MB, enough to recover
|
|
|
|
|
* with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
|
|
|
|
|
* only reserve 3% of free pages by default.
|
|
|
|
|
*/
|
2013-04-29 22:08:12 +00:00
|
|
|
static int init_admin_reserve(void)
|
2013-04-29 22:08:11 +00:00
|
|
|
{
|
|
|
|
|
unsigned long free_kbytes;
|
|
|
|
|
|
|
|
|
|
free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
|
|
|
|
|
|
|
|
|
|
sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2014-01-23 23:53:30 +00:00
|
|
|
subsys_initcall(init_admin_reserve);
|
2013-04-29 22:08:12 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Reinititalise user and admin reserves if memory is added or removed.
|
|
|
|
|
*
|
|
|
|
|
* The default user reserve max is 128MB, and the default max for the
|
|
|
|
|
* admin reserve is 8MB. These are usually, but not always, enough to
|
|
|
|
|
* enable recovery from a memory hogging process using login/sshd, a shell,
|
|
|
|
|
* and tools like top. It may make sense to increase or even disable the
|
|
|
|
|
* reserve depending on the existence of swap or variations in the recovery
|
|
|
|
|
* tools. So, the admin may have changed them.
|
|
|
|
|
*
|
|
|
|
|
* If memory is added and the reserves have been eliminated or increased above
|
|
|
|
|
* the default max, then we'll trust the admin.
|
|
|
|
|
*
|
|
|
|
|
* If memory is removed and there isn't enough free memory, then we
|
|
|
|
|
* need to reset the reserves.
|
|
|
|
|
*
|
|
|
|
|
* Otherwise keep the reserve set by the admin.
|
|
|
|
|
*/
|
|
|
|
|
static int reserve_mem_notifier(struct notifier_block *nb,
|
|
|
|
|
unsigned long action, void *data)
|
|
|
|
|
{
|
|
|
|
|
unsigned long tmp, free_kbytes;
|
|
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
|
case MEM_ONLINE:
|
|
|
|
|
/* Default max is 128MB. Leave alone if modified by operator. */
|
|
|
|
|
tmp = sysctl_user_reserve_kbytes;
|
|
|
|
|
if (0 < tmp && tmp < (1UL << 17))
|
|
|
|
|
init_user_reserve();
|
|
|
|
|
|
|
|
|
|
/* Default max is 8MB. Leave alone if modified by operator. */
|
|
|
|
|
tmp = sysctl_admin_reserve_kbytes;
|
|
|
|
|
if (0 < tmp && tmp < (1UL << 13))
|
|
|
|
|
init_admin_reserve();
|
|
|
|
|
|
|
|
|
|
break;
|
|
|
|
|
case MEM_OFFLINE:
|
|
|
|
|
free_kbytes = global_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
|
|
|
|
|
|
|
|
|
|
if (sysctl_user_reserve_kbytes > free_kbytes) {
|
|
|
|
|
init_user_reserve();
|
|
|
|
|
pr_info("vm.user_reserve_kbytes reset to %lu\n",
|
|
|
|
|
sysctl_user_reserve_kbytes);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (sysctl_admin_reserve_kbytes > free_kbytes) {
|
|
|
|
|
init_admin_reserve();
|
|
|
|
|
pr_info("vm.admin_reserve_kbytes reset to %lu\n",
|
|
|
|
|
sysctl_admin_reserve_kbytes);
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
return NOTIFY_OK;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static struct notifier_block reserve_mem_nb = {
|
|
|
|
|
.notifier_call = reserve_mem_notifier,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int __meminit init_reserve_notifier(void)
|
|
|
|
|
{
|
|
|
|
|
if (register_hotmemory_notifier(&reserve_mem_nb))
|
2014-06-06 21:38:30 +00:00
|
|
|
pr_err("Failed registering memory add/remove notifier for admin reserve\n");
|
2013-04-29 22:08:12 +00:00
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2014-01-23 23:53:30 +00:00
|
|
|
subsys_initcall(init_reserve_notifier);
|
