Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-09-20 15:03:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

migrate_pages: move split folios processing out of migrate_pages_batch()

Browse Source 
To simplify the code logic and reduce the line number.

Link: https://lkml.kernel.org/r/20230303030155.160983-3-ying.huang@intel.com
Fixes: 5dfab109d5 ("migrate_pages: batch _unmap and _move")
Signed-off-by: "Huang, Ying" <ying.huang@intel.com>
Reviewed-by: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Xu, Pengfei" <pengfei.xu@intel.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Stefan Roesch <shr@devkernel.io>
Cc: Tejun Heo <tj@kernel.org>
Cc: Xin Hao <xhao@linux.alibaba.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Yang Shi <shy828301@gmail.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mike Kravetz <mike.kravetz@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Huang Ying 2023-03-03 11:01:54 +08:00 committed by Andrew Morton
parent fb3592c41a
commit a21d213321
1 changed files with 28 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
mm/migrate.c
View File

@ -1616,9 +1616,10 @@ static int migrate_hugetlbs(struct list_head *from, new_page_t get_new_page,
static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
free_page_t put_new_page, unsigned long private,
enum migrate_mode mode, int reason, struct list_head *ret_folios,
struct migrate_pages_stats *stats)
struct list_head *split_folios, struct migrate_pages_stats *stats,
int nr_pass)
{
int retry;
int retry = 1;
int large_retry = 1;
int thp_retry = 1;
int nr_failed = 0;
@ -1628,21 +1629,15 @@ static int migrate_pages_batch(struct list_head *from, new_page_t get_new_page,
bool is_large = false;
bool is_thp = false;
struct folio *folio, *folio2, *dst = NULL, *dst2;
int rc, rc_saved, nr_pages;
LIST_HEAD(split_folios);
int rc, rc_saved = 0, nr_pages;
LIST_HEAD(unmap_folios);
LIST_HEAD(dst_folios);
bool nosplit = (reason == MR_NUMA_MISPLACED);
bool no_split_folio_counting = false;
VM_WARN_ON_ONCE(mode != MIGRATE_ASYNC &&
!list_empty(from) && !list_is_singular(from));
retry:
rc_saved = 0;
retry = 1;
for (pass = 0;
pass < NR_MAX_MIGRATE_PAGES_RETRY && (retry || large_retry);
pass++) {
for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) {
retry = 0;
large_retry = 0;
thp_retry = 0;
@ -1673,7 +1668,7 @@ retry:
if (!thp_migration_supported() && is_thp) {
nr_large_failed++;
stats->nr_thp_failed++;
if (!try_split_folio(folio, &split_folios)) {
if (!try_split_folio(folio, split_folios)) {
stats->nr_thp_split++;
continue;
}
@ -1705,7 +1700,7 @@ retry:
stats->nr_thp_failed += is_thp;
/* Large folio NUMA faulting doesn't split to retry. */
if (!nosplit) {
int ret = try_split_folio(folio, &split_folios);
int ret = try_split_folio(folio, split_folios);
if (!ret) {
stats->nr_thp_split += is_thp;
@ -1722,18 +1717,11 @@ retry:
break;
}
}
} else if (!no_split_folio_counting) {
} else {
nr_failed++;
}
stats->nr_failed_pages += nr_pages + nr_retry_pages;
/*
* There might be some split folios of fail-to-migrate large
* folios left in split_folios list. Move them to ret_folios
* list so that they could be put back to the right list by
* the caller otherwise the folio refcnt will be leaked.
*/
list_splice_init(&split_folios, ret_folios);
/* nr_failed isn't updated for not used */
nr_large_failed += large_retry;
stats->nr_thp_failed += thp_retry;
@ -1746,7 +1734,7 @@ retry:
if (is_large) {
large_retry++;
thp_retry += is_thp;
} else if (!no_split_folio_counting) {
} else {
retry++;
}
nr_retry_pages += nr_pages;
@ -1769,7 +1757,7 @@ retry:
if (is_large) {
nr_large_failed++;
stats->nr_thp_failed += is_thp;
} else if (!no_split_folio_counting) {
} else {
nr_failed++;
}
@ -1787,9 +1775,7 @@ move:
try_to_unmap_flush();
retry = 1;
for (pass = 0;
pass < NR_MAX_MIGRATE_PAGES_RETRY && (retry || large_retry);
pass++) {
for (pass = 0; pass < nr_pass && (retry || large_retry); pass++) {
retry = 0;
large_retry = 0;
thp_retry = 0;
@ -1818,7 +1804,7 @@ move:
if (is_large) {
large_retry++;
thp_retry += is_thp;
} else if (!no_split_folio_counting) {
} else {
retry++;
}
nr_retry_pages += nr_pages;
@ -1831,7 +1817,7 @@ move:
if (is_large) {
nr_large_failed++;
stats->nr_thp_failed += is_thp;
} else if (!no_split_folio_counting) {
} else {
nr_failed++;
}
@ -1868,27 +1854,6 @@ out:
dst2 = list_next_entry(dst, lru);
}
/*
* Try to migrate split folios of fail-to-migrate large folios, no
* nr_failed counting in this round, since all split folios of a
* large folio is counted as 1 failure in the first round.
*/
if (rc >= 0 && !list_empty(&split_folios)) {
/*
* Move non-migrated folios (after NR_MAX_MIGRATE_PAGES_RETRY
* retries) to ret_folios to avoid migrating them again.
*/
list_splice_init(from, ret_folios);
list_splice_init(&split_folios, from);
/*
* Force async mode to avoid to wait lock or bit when we have
* locked more than one folios.
*/
mode = MIGRATE_ASYNC;
no_split_folio_counting = true;
goto retry;
}
return rc;
}
@ -1927,6 +1892,7 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
struct folio *folio, *folio2;
LIST_HEAD(folios);
LIST_HEAD(ret_folios);
LIST_HEAD(split_folios);
struct migrate_pages_stats stats;
trace_mm_migrate_pages_start(mode, reason);
@ -1960,12 +1926,24 @@ again:
else
list_splice_init(from, &folios);
rc = migrate_pages_batch(&folios, get_new_page, put_new_page, private,
mode, reason, &ret_folios, &stats);
mode, reason, &ret_folios, &split_folios, &stats,
NR_MAX_MIGRATE_PAGES_RETRY);
list_splice_tail_init(&folios, &ret_folios);
if (rc < 0) {
rc_gather = rc;
list_splice_tail(&split_folios, &ret_folios);
goto out;
}
if (!list_empty(&split_folios)) {
/*
* Failure isn't counted since all split folios of a large folio
* is counted as 1 failure already. And, we only try to migrate
* with minimal effort, force MIGRATE_ASYNC mode and retry once.
*/
migrate_pages_batch(&split_folios, get_new_page, put_new_page, private,
MIGRATE_ASYNC, reason, &ret_folios, NULL, &stats, 1);
list_splice_tail_init(&split_folios, &ret_folios);
}
rc_gather += rc;
if (!list_empty(from))
goto again;