Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-10 14:11:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

userfaultfd: use maple tree iterator to iterate VMAs

Browse Source 
Don't use the mm_struct linked list or the vma->vm_next in prep for
removal.

Link: https://lkml.kernel.org/r/20220906194824.2110408-45-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Tested-by: Yu Zhao <yuzhao@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: David Hildenbrand <david@redhat.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Davidlohr Bueso <dave@stgolabs.net>
Cc: "Matthew Wilcox (Oracle)" <willy@infradead.org>
Cc: SeongJae Park <sj@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Liam R. Howlett 2022-09-06 19:48:57 +00:00 committed by Andrew Morton
parent c4c84f0628
commit 69dbe6daf1
3 changed files with 46 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
fs/userfaultfd.c
View File

@ -611,14 +611,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
if (release_new_ctx) { if (release_new_ctx) {
struct vm_area_struct *vma; struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm; struct mm_struct *mm = release_new_ctx->mm;
VMA_ITERATOR(vmi, mm, 0);
/* the various vma->vm_userfaultfd_ctx still points to it */ /* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm); mmap_write_lock(mm);
for (vma = mm->mmap; vma; vma = vma->vm_next) for_each_vma(vmi, vma) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) { if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~__VM_UFFD_FLAGS; vma->vm_flags &= ~__VM_UFFD_FLAGS;
} }
}
mmap_write_unlock(mm); mmap_write_unlock(mm);
userfaultfd_ctx_put(release_new_ctx); userfaultfd_ctx_put(release_new_ctx);
@ -799,11 +801,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
return false; return false;
} }
int userfaultfd_unmap_prep(struct vm_area_struct *vma, int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
unsigned long start, unsigned long end, unsigned long end, struct list_head *unmaps)
struct list_head *unmaps)
{ {
for ( ; vma && vma->vm_start < end; vma = vma->vm_next) { VMA_ITERATOR(vmi, mm, start);
struct vm_area_struct *vma;
for_each_vma_range(vmi, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx; struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx; struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
@ -853,6 +857,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
/* len == 0 means wake all */ /* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, }; struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags; unsigned long new_flags;
MA_STATE(mas, &mm->mm_mt, 0, 0);
WRITE_ONCE(ctx->released, true); WRITE_ONCE(ctx->released, true);
@ -869,7 +874,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
*/ */
mmap_write_lock(mm); mmap_write_lock(mm);
prev = NULL; prev = NULL;
for (vma = mm->mmap; vma; vma = vma->vm_next) { mas_for_each(&mas, vma, ULONG_MAX) {
cond_resched(); cond_resched();
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^ BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & __VM_UFFD_FLAGS)); !!(vma->vm_flags & __VM_UFFD_FLAGS));
@ -883,10 +888,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
vma->vm_file, vma->vm_pgoff, vma->vm_file, vma->vm_pgoff,
vma_policy(vma), vma_policy(vma),
NULL_VM_UFFD_CTX, anon_vma_name(vma)); NULL_VM_UFFD_CTX, anon_vma_name(vma));
if (prev) if (prev) {
mas_pause(&mas);
vma = prev; vma = prev;
else } else {
prev = vma; prev = vma;
}
vma->vm_flags = new_flags; vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX; vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
} }
@ -1268,6 +1276,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
bool found; bool found;
bool basic_ioctls; bool basic_ioctls;
unsigned long start, end, vma_end; unsigned long start, end, vma_end;
MA_STATE(mas, &mm->mm_mt, 0, 0);
user_uffdio_register = (struct uffdio_register __user *) arg; user_uffdio_register = (struct uffdio_register __user *) arg;
@ -1310,7 +1319,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
goto out; goto out;
mmap_write_lock(mm); mmap_write_lock(mm);
vma = find_vma_prev(mm, start, &prev); mas_set(&mas, start);
vma = mas_find(&mas, ULONG_MAX);
if (!vma) if (!vma)
goto out_unlock; goto out_unlock;
@ -1335,7 +1345,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
*/ */
found = false; found = false;
basic_ioctls = false; basic_ioctls = false;
for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched(); cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
@ -1395,8 +1405,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
} }
BUG_ON(!found); BUG_ON(!found);
if (vma->vm_start < start) mas_set(&mas, start);
prev = vma; prev = mas_prev(&mas, 0);
if (prev != vma)
mas_next(&mas, ULONG_MAX);
ret = 0; ret = 0;
do { do {
@ -1426,6 +1438,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
((struct vm_userfaultfd_ctx){ ctx }), ((struct vm_userfaultfd_ctx){ ctx }),
anon_vma_name(vma)); anon_vma_name(vma));
if (prev) { if (prev) {
/* vma_merge() invalidated the mas */
mas_pause(&mas);
vma = prev; vma = prev;
goto next; goto next;
} }
@ -1433,11 +1447,15 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
ret = split_vma(mm, vma, start, 1); ret = split_vma(mm, vma, start, 1);
if (ret) if (ret)
break; break;
/* split_vma() invalidated the mas */
mas_pause(&mas);
} }
if (vma->vm_end > end) { if (vma->vm_end > end) {
ret = split_vma(mm, vma, end, 0); ret = split_vma(mm, vma, end, 0);
if (ret) if (ret)
break; break;
/* split_vma() invalidated the mas */
mas_pause(&mas);
} }
next: next:
/* /*
@ -1454,8 +1472,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
skip: skip:
prev = vma; prev = vma;
start = vma->vm_end; start = vma->vm_end;
vma = vma->vm_next; vma = mas_next(&mas, end - 1);
} while (vma && vma->vm_start < end); } while (vma);
out_unlock: out_unlock:
mmap_write_unlock(mm); mmap_write_unlock(mm);
mmput(mm); mmput(mm);
@ -1499,6 +1517,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
bool found; bool found;
unsigned long start, end, vma_end; unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg; const void __user *buf = (void __user *)arg;
MA_STATE(mas, &mm->mm_mt, 0, 0);
ret = -EFAULT; ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister))) if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
@ -1517,7 +1536,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
goto out; goto out;
mmap_write_lock(mm); mmap_write_lock(mm);
vma = find_vma_prev(mm, start, &prev); mas_set(&mas, start);
vma = mas_find(&mas, ULONG_MAX);
if (!vma) if (!vma)
goto out_unlock; goto out_unlock;
@ -1542,7 +1562,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
*/ */
found = false; found = false;
ret = -EINVAL; ret = -EINVAL;
for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) { for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched(); cond_resched();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^ BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
@ -1562,8 +1582,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
} }
BUG_ON(!found); BUG_ON(!found);
if (vma->vm_start < start) mas_set(&mas, start);
prev = vma; prev = mas_prev(&mas, 0);
if (prev != vma)
mas_next(&mas, ULONG_MAX);
ret = 0; ret = 0;
do { do {
@ -1632,8 +1654,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
skip: skip:
prev = vma; prev = vma;
start = vma->vm_end; start = vma->vm_end;
vma = vma->vm_next; vma = mas_next(&mas, end - 1);
} while (vma && vma->vm_start < end); } while (vma);
out_unlock: out_unlock:
mmap_write_unlock(mm); mmap_write_unlock(mm);
mmput(mm); mmput(mm);

7
include/linux/userfaultfd_k.h
View File

@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start, unsigned long start,
unsigned long end); unsigned long end);
extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
unsigned long start, unsigned long end, unsigned long end, struct list_head *uf);
struct list_head *uf);
extern void userfaultfd_unmap_complete(struct mm_struct *mm, extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf); struct list_head *uf);
@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma,
return true; return true;
} }
static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, static inline int userfaultfd_unmap_prep(struct mm_struct *mm,
unsigned long start, unsigned long end, unsigned long start, unsigned long end,
struct list_head *uf) struct list_head *uf)
{ {

2
mm/mmap.c
View File

@ -2545,7 +2545,7 @@ do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
* split, despite we could. This is unlikely enough * split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for. * failure that it's not worth optimizing it for.
*/ */
error = userfaultfd_unmap_prep(vma, start, end, uf); error = userfaultfd_unmap_prep(mm, start, end, uf);
if (error) if (error)
goto userfaultfd_error; goto userfaultfd_error;