2354ad252b
commit: d9c2340052 ("Do not depend on MAX_ORDER when grouping pages by mobility")
introduced pageblock_order which will be used to group pages better.
The kernel now groups pages based on the value of HPAGE_SHIFT. Hence HPAGE_SHIFT
should be set before we call set_pageblock_order.
set_pageblock_order happens early in the boot and default hugetlb page size
should be initialized before that to compute the right pageblock_order value.
Currently, default hugetlbe page size is set via arch_initcalls which happens
late in the boot as shown via the below callstack:
[c000000007383b10] [c000000001289328] hugetlbpage_init+0x2b8/0x2f8
[c000000007383bc0] [c0000000012749e4] do_one_initcall+0x14c/0x320
[c000000007383c90] [c00000000127505c] kernel_init_freeable+0x410/0x4e8
[c000000007383da0] [c000000000012664] kernel_init+0x30/0x15c
[c000000007383e10] [c00000000000cf14] ret_from_kernel_thread+0x5c/0x64
and the pageblock_order initialization is done early during the boot.
[c0000000018bfc80] [c0000000012ae120] set_pageblock_order+0x50/0x64
[c0000000018bfca0] [c0000000012b3d94] sparse_init+0x188/0x268
[c0000000018bfd60] [c000000001288bfc] initmem_init+0x28c/0x328
[c0000000018bfe50] [c00000000127b370] setup_arch+0x410/0x480
[c0000000018bfed0] [c00000000127401c] start_kernel+0xb8/0x934
[c0000000018bff90] [c00000000000d984] start_here_common+0x1c/0x98
delaying default hugetlb page size initialization implies the kernel will
initialize pageblock_order to (MAX_ORDER - 1) which is not an optimal
value for mobility grouping. IIUC we always had this issue. But it was not
a problem for hash translation mode because (MAX_ORDER - 1) is the same as
HUGETLB_PAGE_ORDER (8) in the case of hash (16MB). With radix,
HUGETLB_PAGE_ORDER will be 5 (2M size) and hence pageblock_order should be
5 instead of 8.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20220211065215.101767-1-aneesh.kumar@linux.ibm.com
85 lines
2.2 KiB
C
85 lines
2.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _ASM_POWERPC_HUGETLB_H
|
|
#define _ASM_POWERPC_HUGETLB_H
|
|
|
|
#ifdef CONFIG_HUGETLB_PAGE
|
|
#include <asm/page.h>
|
|
|
|
#ifdef CONFIG_PPC_BOOK3S_64
|
|
#include <asm/book3s/64/hugetlb.h>
|
|
#elif defined(CONFIG_PPC_FSL_BOOK3E)
|
|
#include <asm/nohash/hugetlb-book3e.h>
|
|
#elif defined(CONFIG_PPC_8xx)
|
|
#include <asm/nohash/32/hugetlb-8xx.h>
|
|
#endif /* CONFIG_PPC_BOOK3S_64 */
|
|
|
|
extern bool hugetlb_disabled;
|
|
|
|
void __init hugetlbpage_init_defaultsize(void);
|
|
|
|
int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
|
|
unsigned long len);
|
|
|
|
static inline int is_hugepage_only_range(struct mm_struct *mm,
|
|
unsigned long addr,
|
|
unsigned long len)
|
|
{
|
|
if (IS_ENABLED(CONFIG_PPC_MM_SLICES) && !radix_enabled())
|
|
return slice_is_hugepage_only_range(mm, addr, len);
|
|
return 0;
|
|
}
|
|
#define is_hugepage_only_range is_hugepage_only_range
|
|
|
|
#define __HAVE_ARCH_HUGETLB_FREE_PGD_RANGE
|
|
void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
|
|
unsigned long end, unsigned long floor,
|
|
unsigned long ceiling);
|
|
|
|
#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR
|
|
static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
|
|
unsigned long addr, pte_t *ptep)
|
|
{
|
|
return __pte(pte_update(mm, addr, ptep, ~0UL, 0, 1));
|
|
}
|
|
|
|
#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH
|
|
static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
|
|
unsigned long addr, pte_t *ptep)
|
|
{
|
|
huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
|
|
flush_hugetlb_page(vma, addr);
|
|
}
|
|
|
|
#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS
|
|
int huge_ptep_set_access_flags(struct vm_area_struct *vma,
|
|
unsigned long addr, pte_t *ptep,
|
|
pte_t pte, int dirty);
|
|
|
|
void gigantic_hugetlb_cma_reserve(void) __init;
|
|
#include <asm-generic/hugetlb.h>
|
|
|
|
#else /* ! CONFIG_HUGETLB_PAGE */
|
|
static inline void flush_hugetlb_page(struct vm_area_struct *vma,
|
|
unsigned long vmaddr)
|
|
{
|
|
}
|
|
|
|
#define hugepd_shift(x) 0
|
|
static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,
|
|
unsigned pdshift)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
|
|
static inline void __init gigantic_hugetlb_cma_reserve(void)
|
|
{
|
|
}
|
|
|
|
static inline void __init hugetlbpage_init_defaultsize(void)
|
|
{
|
|
}
|
|
#endif /* CONFIG_HUGETLB_PAGE */
|
|
|
|
#endif /* _ASM_POWERPC_HUGETLB_H */
|