x86/cpu_entry_area: Move it out of the fixmap
Put the cpu_entry_area into a separate P4D entry. The fixmap gets too big and 0-day already hit a case where the fixmap PTEs were cleared by cleanup_highmap(). Aside of that the fixmap API is a pain as it's all backwards. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Cc: Andy Lutomirski <luto@kernel.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Josh Poimboeuf <jpoimboe@redhat.com> Cc: Juergen Gross <jgross@suse.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: linux-kernel@vger.kernel.org Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
		
							parent
							
								
									ed1bbc40a0
								
							
						
					
					
						commit
						92a0f81d89
					
				| @ -12,6 +12,7 @@ ffffea0000000000 - ffffeaffffffffff (=40 bits) virtual memory map (1TB) | |||||||
| ... unused hole ... | ... unused hole ... | ||||||
| ffffec0000000000 - fffffbffffffffff (=44 bits) kasan shadow memory (16TB) | ffffec0000000000 - fffffbffffffffff (=44 bits) kasan shadow memory (16TB) | ||||||
| ... unused hole ... | ... unused hole ... | ||||||
|  | fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping | ||||||
| ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks | ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks | ||||||
| ... unused hole ... | ... unused hole ... | ||||||
| ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space | ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space | ||||||
| @ -35,6 +36,7 @@ ffd4000000000000 - ffd5ffffffffffff (=49 bits) virtual memory map (512TB) | |||||||
| ... unused hole ... | ... unused hole ... | ||||||
| ffdf000000000000 - fffffc0000000000 (=53 bits) kasan shadow memory (8PB) | ffdf000000000000 - fffffc0000000000 (=53 bits) kasan shadow memory (8PB) | ||||||
| ... unused hole ... | ... unused hole ... | ||||||
|  | fffffe8000000000 - fffffeffffffffff (=39 bits) cpu_entry_area mapping | ||||||
| ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks | ffffff0000000000 - ffffff7fffffffff (=39 bits) %esp fixup stacks | ||||||
| ... unused hole ... | ... unused hole ... | ||||||
| ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space | ffffffef00000000 - fffffffeffffffff (=64 GB) EFI region mapping space | ||||||
|  | |||||||
| @ -43,10 +43,26 @@ struct cpu_entry_area { | |||||||
| }; | }; | ||||||
| 
 | 
 | ||||||
| #define CPU_ENTRY_AREA_SIZE	(sizeof(struct cpu_entry_area)) | #define CPU_ENTRY_AREA_SIZE	(sizeof(struct cpu_entry_area)) | ||||||
| #define CPU_ENTRY_AREA_PAGES	(CPU_ENTRY_AREA_SIZE / PAGE_SIZE) | #define CPU_ENTRY_AREA_TOT_SIZE	(CPU_ENTRY_AREA_SIZE * NR_CPUS) | ||||||
| 
 | 
 | ||||||
| DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area); | DECLARE_PER_CPU(struct cpu_entry_area *, cpu_entry_area); | ||||||
| 
 | 
 | ||||||
| extern void setup_cpu_entry_areas(void); | extern void setup_cpu_entry_areas(void); | ||||||
|  | extern void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags); | ||||||
|  | 
 | ||||||
|  | #define	CPU_ENTRY_AREA_RO_IDT		CPU_ENTRY_AREA_BASE | ||||||
|  | #define CPU_ENTRY_AREA_PER_CPU		(CPU_ENTRY_AREA_RO_IDT + PAGE_SIZE) | ||||||
|  | 
 | ||||||
|  | #define CPU_ENTRY_AREA_RO_IDT_VADDR	((void *)CPU_ENTRY_AREA_RO_IDT) | ||||||
|  | 
 | ||||||
|  | #define CPU_ENTRY_AREA_MAP_SIZE			\ | ||||||
|  | 	(CPU_ENTRY_AREA_PER_CPU + CPU_ENTRY_AREA_TOT_SIZE - CPU_ENTRY_AREA_BASE) | ||||||
|  | 
 | ||||||
|  | extern struct cpu_entry_area *get_cpu_entry_area(int cpu); | ||||||
|  | 
 | ||||||
|  | static inline struct entry_stack *cpu_entry_stack(int cpu) | ||||||
|  | { | ||||||
|  | 	return &get_cpu_entry_area(cpu)->entry_stack_page.stack; | ||||||
|  | } | ||||||
| 
 | 
 | ||||||
| #endif | #endif | ||||||
|  | |||||||
| @ -7,6 +7,7 @@ | |||||||
| #include <asm/mmu.h> | #include <asm/mmu.h> | ||||||
| #include <asm/fixmap.h> | #include <asm/fixmap.h> | ||||||
| #include <asm/irq_vectors.h> | #include <asm/irq_vectors.h> | ||||||
|  | #include <asm/cpu_entry_area.h> | ||||||
| 
 | 
 | ||||||
| #include <linux/smp.h> | #include <linux/smp.h> | ||||||
| #include <linux/percpu.h> | #include <linux/percpu.h> | ||||||
|  | |||||||
| @ -25,7 +25,6 @@ | |||||||
| #else | #else | ||||||
| #include <uapi/asm/vsyscall.h> | #include <uapi/asm/vsyscall.h> | ||||||
| #endif | #endif | ||||||
| #include <asm/cpu_entry_area.h> |  | ||||||
| 
 | 
 | ||||||
| /*
 | /*
 | ||||||
|  * We can't declare FIXADDR_TOP as variable for x86_64 because vsyscall |  * We can't declare FIXADDR_TOP as variable for x86_64 because vsyscall | ||||||
| @ -84,7 +83,6 @@ enum fixed_addresses { | |||||||
| 	FIX_IO_APIC_BASE_0, | 	FIX_IO_APIC_BASE_0, | ||||||
| 	FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS - 1, | 	FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + MAX_IO_APICS - 1, | ||||||
| #endif | #endif | ||||||
| 	FIX_RO_IDT,	/* Virtual mapping for read-only IDT */ |  | ||||||
| #ifdef CONFIG_X86_32 | #ifdef CONFIG_X86_32 | ||||||
| 	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */ | 	FIX_KMAP_BEGIN,	/* reserved pte's for temporary kernel mappings */ | ||||||
| 	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, | 	FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, | ||||||
| @ -100,9 +98,6 @@ enum fixed_addresses { | |||||||
| #ifdef	CONFIG_X86_INTEL_MID | #ifdef	CONFIG_X86_INTEL_MID | ||||||
| 	FIX_LNW_VRTC, | 	FIX_LNW_VRTC, | ||||||
| #endif | #endif | ||||||
| 	/* Fixmap entries to remap the GDTs, one per processor. */ |  | ||||||
| 	FIX_CPU_ENTRY_AREA_TOP, |  | ||||||
| 	FIX_CPU_ENTRY_AREA_BOTTOM = FIX_CPU_ENTRY_AREA_TOP + (CPU_ENTRY_AREA_PAGES * NR_CPUS) - 1, |  | ||||||
| 
 | 
 | ||||||
| #ifdef CONFIG_ACPI_APEI_GHES | #ifdef CONFIG_ACPI_APEI_GHES | ||||||
| 	/* Used for GHES mapping from assorted contexts */ | 	/* Used for GHES mapping from assorted contexts */ | ||||||
| @ -191,30 +186,5 @@ void __init *early_memremap_decrypted_wp(resource_size_t phys_addr, | |||||||
| void __early_set_fixmap(enum fixed_addresses idx, | void __early_set_fixmap(enum fixed_addresses idx, | ||||||
| 			phys_addr_t phys, pgprot_t flags); | 			phys_addr_t phys, pgprot_t flags); | ||||||
| 
 | 
 | ||||||
| static inline unsigned int __get_cpu_entry_area_page_index(int cpu, int page) |  | ||||||
| { |  | ||||||
| 	BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0); |  | ||||||
| 
 |  | ||||||
| 	return FIX_CPU_ENTRY_AREA_BOTTOM - cpu*CPU_ENTRY_AREA_PAGES - page; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| #define __get_cpu_entry_area_offset_index(cpu, offset) ({		\ |  | ||||||
| 	BUILD_BUG_ON(offset % PAGE_SIZE != 0);				\ |  | ||||||
| 	__get_cpu_entry_area_page_index(cpu, offset / PAGE_SIZE);	\ |  | ||||||
| 	}) |  | ||||||
| 
 |  | ||||||
| #define get_cpu_entry_area_index(cpu, field)				\ |  | ||||||
| 	__get_cpu_entry_area_offset_index((cpu), offsetof(struct cpu_entry_area, field)) |  | ||||||
| 
 |  | ||||||
| static inline struct cpu_entry_area *get_cpu_entry_area(int cpu) |  | ||||||
| { |  | ||||||
| 	return (struct cpu_entry_area *)__fix_to_virt(__get_cpu_entry_area_page_index(cpu, 0)); |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| static inline struct entry_stack *cpu_entry_stack(int cpu) |  | ||||||
| { |  | ||||||
| 	return &get_cpu_entry_area(cpu)->entry_stack_page.stack; |  | ||||||
| } |  | ||||||
| 
 |  | ||||||
| #endif /* !__ASSEMBLY__ */ | #endif /* !__ASSEMBLY__ */ | ||||||
| #endif /* _ASM_X86_FIXMAP_H */ | #endif /* _ASM_X86_FIXMAP_H */ | ||||||
|  | |||||||
| @ -38,13 +38,22 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */ | |||||||
| #define LAST_PKMAP 1024 | #define LAST_PKMAP 1024 | ||||||
| #endif | #endif | ||||||
| 
 | 
 | ||||||
| #define PKMAP_BASE ((FIXADDR_START - PAGE_SIZE * (LAST_PKMAP + 1))	\ | /*
 | ||||||
| 		    & PMD_MASK) |  * Define this here and validate with BUILD_BUG_ON() in pgtable_32.c | ||||||
|  |  * to avoid include recursion hell | ||||||
|  |  */ | ||||||
|  | #define CPU_ENTRY_AREA_PAGES	(NR_CPUS * 40) | ||||||
|  | 
 | ||||||
|  | #define CPU_ENTRY_AREA_BASE				\ | ||||||
|  | 	((FIXADDR_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1)) & PMD_MASK) | ||||||
|  | 
 | ||||||
|  | #define PKMAP_BASE		\ | ||||||
|  | 	((CPU_ENTRY_AREA_BASE - PAGE_SIZE) & PMD_MASK) | ||||||
| 
 | 
 | ||||||
| #ifdef CONFIG_HIGHMEM | #ifdef CONFIG_HIGHMEM | ||||||
| # define VMALLOC_END	(PKMAP_BASE - 2 * PAGE_SIZE) | # define VMALLOC_END	(PKMAP_BASE - 2 * PAGE_SIZE) | ||||||
| #else | #else | ||||||
| # define VMALLOC_END	(FIXADDR_START - 2 * PAGE_SIZE) | # define VMALLOC_END	(CPU_ENTRY_AREA_BASE - 2 * PAGE_SIZE) | ||||||
| #endif | #endif | ||||||
| 
 | 
 | ||||||
| #define MODULES_VADDR	VMALLOC_START | #define MODULES_VADDR	VMALLOC_START | ||||||
|  | |||||||
| @ -77,6 +77,7 @@ typedef struct { pteval_t pte; } pte_t; | |||||||
| 
 | 
 | ||||||
| /* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */ | /* See Documentation/x86/x86_64/mm.txt for a description of the memory map. */ | ||||||
| #define MAXMEM			_AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) | #define MAXMEM			_AC(__AC(1, UL) << MAX_PHYSMEM_BITS, UL) | ||||||
|  | 
 | ||||||
| #ifdef CONFIG_X86_5LEVEL | #ifdef CONFIG_X86_5LEVEL | ||||||
| # define VMALLOC_SIZE_TB	_AC(16384, UL) | # define VMALLOC_SIZE_TB	_AC(16384, UL) | ||||||
| # define __VMALLOC_BASE		_AC(0xff92000000000000, UL) | # define __VMALLOC_BASE		_AC(0xff92000000000000, UL) | ||||||
| @ -86,6 +87,7 @@ typedef struct { pteval_t pte; } pte_t; | |||||||
| # define __VMALLOC_BASE		_AC(0xffffc90000000000, UL) | # define __VMALLOC_BASE		_AC(0xffffc90000000000, UL) | ||||||
| # define __VMEMMAP_BASE		_AC(0xffffea0000000000, UL) | # define __VMEMMAP_BASE		_AC(0xffffea0000000000, UL) | ||||||
| #endif | #endif | ||||||
|  | 
 | ||||||
| #ifdef CONFIG_RANDOMIZE_MEMORY | #ifdef CONFIG_RANDOMIZE_MEMORY | ||||||
| # define VMALLOC_START		vmalloc_base | # define VMALLOC_START		vmalloc_base | ||||||
| # define VMEMMAP_START		vmemmap_base | # define VMEMMAP_START		vmemmap_base | ||||||
| @ -93,13 +95,20 @@ typedef struct { pteval_t pte; } pte_t; | |||||||
| # define VMALLOC_START		__VMALLOC_BASE | # define VMALLOC_START		__VMALLOC_BASE | ||||||
| # define VMEMMAP_START		__VMEMMAP_BASE | # define VMEMMAP_START		__VMEMMAP_BASE | ||||||
| #endif /* CONFIG_RANDOMIZE_MEMORY */ | #endif /* CONFIG_RANDOMIZE_MEMORY */ | ||||||
|  | 
 | ||||||
| #define VMALLOC_END		(VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL)) | #define VMALLOC_END		(VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL)) | ||||||
|  | 
 | ||||||
| #define MODULES_VADDR		(__START_KERNEL_map + KERNEL_IMAGE_SIZE) | #define MODULES_VADDR		(__START_KERNEL_map + KERNEL_IMAGE_SIZE) | ||||||
| /* The module sections ends with the start of the fixmap */ | /* The module sections ends with the start of the fixmap */ | ||||||
| #define MODULES_END		__fix_to_virt(__end_of_fixed_addresses + 1) | #define MODULES_END		__fix_to_virt(__end_of_fixed_addresses + 1) | ||||||
| #define MODULES_LEN		(MODULES_END - MODULES_VADDR) | #define MODULES_LEN		(MODULES_END - MODULES_VADDR) | ||||||
|  | 
 | ||||||
| #define ESPFIX_PGD_ENTRY	_AC(-2, UL) | #define ESPFIX_PGD_ENTRY	_AC(-2, UL) | ||||||
| #define ESPFIX_BASE_ADDR	(ESPFIX_PGD_ENTRY << P4D_SHIFT) | #define ESPFIX_BASE_ADDR	(ESPFIX_PGD_ENTRY << P4D_SHIFT) | ||||||
|  | 
 | ||||||
|  | #define CPU_ENTRY_AREA_PGD	_AC(-3, UL) | ||||||
|  | #define CPU_ENTRY_AREA_BASE	(CPU_ENTRY_AREA_PGD << P4D_SHIFT) | ||||||
|  | 
 | ||||||
| #define EFI_VA_START		( -4 * (_AC(1, UL) << 30)) | #define EFI_VA_START		( -4 * (_AC(1, UL) << 30)) | ||||||
| #define EFI_VA_END		(-68 * (_AC(1, UL) << 30)) | #define EFI_VA_END		(-68 * (_AC(1, UL) << 30)) | ||||||
| 
 | 
 | ||||||
|  | |||||||
| @ -18,6 +18,7 @@ | |||||||
| #include <linux/nmi.h> | #include <linux/nmi.h> | ||||||
| #include <linux/sysfs.h> | #include <linux/sysfs.h> | ||||||
| 
 | 
 | ||||||
|  | #include <asm/cpu_entry_area.h> | ||||||
| #include <asm/stacktrace.h> | #include <asm/stacktrace.h> | ||||||
| #include <asm/unwind.h> | #include <asm/unwind.h> | ||||||
| 
 | 
 | ||||||
|  | |||||||
| @ -951,8 +951,9 @@ void __init trap_init(void) | |||||||
| 	 * "sidt" instruction will not leak the location of the kernel, and | 	 * "sidt" instruction will not leak the location of the kernel, and | ||||||
| 	 * to defend the IDT against arbitrary memory write vulnerabilities. | 	 * to defend the IDT against arbitrary memory write vulnerabilities. | ||||||
| 	 * It will be reloaded in cpu_init() */ | 	 * It will be reloaded in cpu_init() */ | ||||||
| 	__set_fixmap(FIX_RO_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO); | 	cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table), | ||||||
| 	idt_descr.address = fix_to_virt(FIX_RO_IDT); | 		    PAGE_KERNEL_RO); | ||||||
|  | 	idt_descr.address = CPU_ENTRY_AREA_RO_IDT; | ||||||
| 
 | 
 | ||||||
| 	/*
 | 	/*
 | ||||||
| 	 * Should be a barrier for any external CPU state: | 	 * Should be a barrier for any external CPU state: | ||||||
|  | |||||||
| @ -15,11 +15,27 @@ static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks | |||||||
| 	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); | 	[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); | ||||||
| #endif | #endif | ||||||
| 
 | 
 | ||||||
| static void __init | struct cpu_entry_area *get_cpu_entry_area(int cpu) | ||||||
| set_percpu_fixmap_pages(int idx, void *ptr, int pages, pgprot_t prot) |  | ||||||
| { | { | ||||||
| 	for ( ; pages; pages--, idx--, ptr += PAGE_SIZE) | 	unsigned long va = CPU_ENTRY_AREA_PER_CPU + cpu * CPU_ENTRY_AREA_SIZE; | ||||||
| 		__set_fixmap(idx, per_cpu_ptr_to_phys(ptr), prot); | 	BUILD_BUG_ON(sizeof(struct cpu_entry_area) % PAGE_SIZE != 0); | ||||||
|  | 
 | ||||||
|  | 	return (struct cpu_entry_area *) va; | ||||||
|  | } | ||||||
|  | EXPORT_SYMBOL(get_cpu_entry_area); | ||||||
|  | 
 | ||||||
|  | void cea_set_pte(void *cea_vaddr, phys_addr_t pa, pgprot_t flags) | ||||||
|  | { | ||||||
|  | 	unsigned long va = (unsigned long) cea_vaddr; | ||||||
|  | 
 | ||||||
|  | 	set_pte_vaddr(va, pfn_pte(pa >> PAGE_SHIFT, flags)); | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | static void __init | ||||||
|  | cea_map_percpu_pages(void *cea_vaddr, void *ptr, int pages, pgprot_t prot) | ||||||
|  | { | ||||||
|  | 	for ( ; pages; pages--, cea_vaddr+= PAGE_SIZE, ptr += PAGE_SIZE) | ||||||
|  | 		cea_set_pte(cea_vaddr, per_cpu_ptr_to_phys(ptr), prot); | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| /* Setup the fixmap mappings only once per-processor */ | /* Setup the fixmap mappings only once per-processor */ | ||||||
| @ -47,8 +63,10 @@ static void __init setup_cpu_entry_area(int cpu) | |||||||
| 	pgprot_t tss_prot = PAGE_KERNEL; | 	pgprot_t tss_prot = PAGE_KERNEL; | ||||||
| #endif | #endif | ||||||
| 
 | 
 | ||||||
| 	__set_fixmap(get_cpu_entry_area_index(cpu, gdt), get_cpu_gdt_paddr(cpu), gdt_prot); | 	cea_set_pte(&get_cpu_entry_area(cpu)->gdt, get_cpu_gdt_paddr(cpu), | ||||||
| 	set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, entry_stack_page), | 		    gdt_prot); | ||||||
|  | 
 | ||||||
|  | 	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->entry_stack_page, | ||||||
| 			     per_cpu_ptr(&entry_stack_storage, cpu), 1, | 			     per_cpu_ptr(&entry_stack_storage, cpu), 1, | ||||||
| 			     PAGE_KERNEL); | 			     PAGE_KERNEL); | ||||||
| 
 | 
 | ||||||
| @ -72,10 +90,9 @@ static void __init setup_cpu_entry_area(int cpu) | |||||||
| 	BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^ | 	BUILD_BUG_ON((offsetof(struct tss_struct, x86_tss) ^ | ||||||
| 		      offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK); | 		      offsetofend(struct tss_struct, x86_tss)) & PAGE_MASK); | ||||||
| 	BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0); | 	BUILD_BUG_ON(sizeof(struct tss_struct) % PAGE_SIZE != 0); | ||||||
| 	set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, tss), | 	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->tss, | ||||||
| 			     &per_cpu(cpu_tss_rw, cpu), | 			     &per_cpu(cpu_tss_rw, cpu), | ||||||
| 				sizeof(struct tss_struct) / PAGE_SIZE, | 			     sizeof(struct tss_struct) / PAGE_SIZE, tss_prot); | ||||||
| 				tss_prot); |  | ||||||
| 
 | 
 | ||||||
| #ifdef CONFIG_X86_32 | #ifdef CONFIG_X86_32 | ||||||
| 	per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); | 	per_cpu(cpu_entry_area, cpu) = get_cpu_entry_area(cpu); | ||||||
| @ -85,20 +102,37 @@ static void __init setup_cpu_entry_area(int cpu) | |||||||
| 	BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); | 	BUILD_BUG_ON(sizeof(exception_stacks) % PAGE_SIZE != 0); | ||||||
| 	BUILD_BUG_ON(sizeof(exception_stacks) != | 	BUILD_BUG_ON(sizeof(exception_stacks) != | ||||||
| 		     sizeof(((struct cpu_entry_area *)0)->exception_stacks)); | 		     sizeof(((struct cpu_entry_area *)0)->exception_stacks)); | ||||||
| 	set_percpu_fixmap_pages(get_cpu_entry_area_index(cpu, exception_stacks), | 	cea_map_percpu_pages(&get_cpu_entry_area(cpu)->exception_stacks, | ||||||
| 			     &per_cpu(exception_stacks, cpu), | 			     &per_cpu(exception_stacks, cpu), | ||||||
| 				sizeof(exception_stacks) / PAGE_SIZE, | 			     sizeof(exception_stacks) / PAGE_SIZE, PAGE_KERNEL); | ||||||
| 				PAGE_KERNEL); |  | ||||||
| 
 | 
 | ||||||
| 	__set_fixmap(get_cpu_entry_area_index(cpu, entry_trampoline), | 	cea_set_pte(&get_cpu_entry_area(cpu)->entry_trampoline, | ||||||
| 		     __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); | 		     __pa_symbol(_entry_trampoline), PAGE_KERNEL_RX); | ||||||
| #endif | #endif | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
|  | static __init void setup_cpu_entry_area_ptes(void) | ||||||
|  | { | ||||||
|  | #ifdef CONFIG_X86_32 | ||||||
|  | 	unsigned long start, end; | ||||||
|  | 
 | ||||||
|  | 	BUILD_BUG_ON(CPU_ENTRY_AREA_PAGES * PAGE_SIZE < CPU_ENTRY_AREA_MAP_SIZE); | ||||||
|  | 	BUG_ON(CPU_ENTRY_AREA_BASE & ~PMD_MASK); | ||||||
|  | 
 | ||||||
|  | 	start = CPU_ENTRY_AREA_BASE; | ||||||
|  | 	end = start + CPU_ENTRY_AREA_MAP_SIZE; | ||||||
|  | 
 | ||||||
|  | 	for (; start < end; start += PMD_SIZE) | ||||||
|  | 		populate_extra_pte(start); | ||||||
|  | #endif | ||||||
|  | } | ||||||
|  | 
 | ||||||
| void __init setup_cpu_entry_areas(void) | void __init setup_cpu_entry_areas(void) | ||||||
| { | { | ||||||
| 	unsigned int cpu; | 	unsigned int cpu; | ||||||
| 
 | 
 | ||||||
|  | 	setup_cpu_entry_area_ptes(); | ||||||
|  | 
 | ||||||
| 	for_each_possible_cpu(cpu) | 	for_each_possible_cpu(cpu) | ||||||
| 		setup_cpu_entry_area(cpu); | 		setup_cpu_entry_area(cpu); | ||||||
| } | } | ||||||
|  | |||||||
| @ -58,6 +58,7 @@ enum address_markers_idx { | |||||||
| 	KASAN_SHADOW_START_NR, | 	KASAN_SHADOW_START_NR, | ||||||
| 	KASAN_SHADOW_END_NR, | 	KASAN_SHADOW_END_NR, | ||||||
| #endif | #endif | ||||||
|  | 	CPU_ENTRY_AREA_NR, | ||||||
| #ifdef CONFIG_X86_ESPFIX64 | #ifdef CONFIG_X86_ESPFIX64 | ||||||
| 	ESPFIX_START_NR, | 	ESPFIX_START_NR, | ||||||
| #endif | #endif | ||||||
| @ -81,6 +82,7 @@ static struct addr_marker address_markers[] = { | |||||||
| 	[KASAN_SHADOW_START_NR]	= { KASAN_SHADOW_START,	"KASAN shadow" }, | 	[KASAN_SHADOW_START_NR]	= { KASAN_SHADOW_START,	"KASAN shadow" }, | ||||||
| 	[KASAN_SHADOW_END_NR]	= { KASAN_SHADOW_END,	"KASAN shadow end" }, | 	[KASAN_SHADOW_END_NR]	= { KASAN_SHADOW_END,	"KASAN shadow end" }, | ||||||
| #endif | #endif | ||||||
|  | 	[CPU_ENTRY_AREA_NR]	= { CPU_ENTRY_AREA_BASE,"CPU entry Area" }, | ||||||
| #ifdef CONFIG_X86_ESPFIX64 | #ifdef CONFIG_X86_ESPFIX64 | ||||||
| 	[ESPFIX_START_NR]	= { ESPFIX_BASE_ADDR,	"ESPfix Area", 16 }, | 	[ESPFIX_START_NR]	= { ESPFIX_BASE_ADDR,	"ESPfix Area", 16 }, | ||||||
| #endif | #endif | ||||||
| @ -104,6 +106,7 @@ enum address_markers_idx { | |||||||
| #ifdef CONFIG_HIGHMEM | #ifdef CONFIG_HIGHMEM | ||||||
| 	PKMAP_BASE_NR, | 	PKMAP_BASE_NR, | ||||||
| #endif | #endif | ||||||
|  | 	CPU_ENTRY_AREA_NR, | ||||||
| 	FIXADDR_START_NR, | 	FIXADDR_START_NR, | ||||||
| 	END_OF_SPACE_NR, | 	END_OF_SPACE_NR, | ||||||
| }; | }; | ||||||
| @ -116,6 +119,7 @@ static struct addr_marker address_markers[] = { | |||||||
| #ifdef CONFIG_HIGHMEM | #ifdef CONFIG_HIGHMEM | ||||||
| 	[PKMAP_BASE_NR]		= { 0UL,		"Persistent kmap() Area" }, | 	[PKMAP_BASE_NR]		= { 0UL,		"Persistent kmap() Area" }, | ||||||
| #endif | #endif | ||||||
|  | 	[CPU_ENTRY_AREA_NR]	= { 0UL,		"CPU entry area" }, | ||||||
| 	[FIXADDR_START_NR]	= { 0UL,		"Fixmap area" }, | 	[FIXADDR_START_NR]	= { 0UL,		"Fixmap area" }, | ||||||
| 	[END_OF_SPACE_NR]	= { -1,			NULL } | 	[END_OF_SPACE_NR]	= { -1,			NULL } | ||||||
| }; | }; | ||||||
| @ -541,8 +545,8 @@ static int __init pt_dump_init(void) | |||||||
| 	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE; | 	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE; | ||||||
| # endif | # endif | ||||||
| 	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; | 	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START; | ||||||
|  | 	address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE; | ||||||
| #endif | #endif | ||||||
| 
 |  | ||||||
| 	return 0; | 	return 0; | ||||||
| } | } | ||||||
| __initcall(pt_dump_init); | __initcall(pt_dump_init); | ||||||
|  | |||||||
| @ -50,6 +50,7 @@ | |||||||
| #include <asm/setup.h> | #include <asm/setup.h> | ||||||
| #include <asm/set_memory.h> | #include <asm/set_memory.h> | ||||||
| #include <asm/page_types.h> | #include <asm/page_types.h> | ||||||
|  | #include <asm/cpu_entry_area.h> | ||||||
| #include <asm/init.h> | #include <asm/init.h> | ||||||
| 
 | 
 | ||||||
| #include "mm_internal.h" | #include "mm_internal.h" | ||||||
| @ -766,6 +767,7 @@ void __init mem_init(void) | |||||||
| 	mem_init_print_info(NULL); | 	mem_init_print_info(NULL); | ||||||
| 	printk(KERN_INFO "virtual kernel memory layout:\n" | 	printk(KERN_INFO "virtual kernel memory layout:\n" | ||||||
| 		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n" | 		"    fixmap  : 0x%08lx - 0x%08lx   (%4ld kB)\n" | ||||||
|  | 		"  cpu_entry : 0x%08lx - 0x%08lx   (%4ld kB)\n" | ||||||
| #ifdef CONFIG_HIGHMEM | #ifdef CONFIG_HIGHMEM | ||||||
| 		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n" | 		"    pkmap   : 0x%08lx - 0x%08lx   (%4ld kB)\n" | ||||||
| #endif | #endif | ||||||
| @ -777,6 +779,10 @@ void __init mem_init(void) | |||||||
| 		FIXADDR_START, FIXADDR_TOP, | 		FIXADDR_START, FIXADDR_TOP, | ||||||
| 		(FIXADDR_TOP - FIXADDR_START) >> 10, | 		(FIXADDR_TOP - FIXADDR_START) >> 10, | ||||||
| 
 | 
 | ||||||
|  | 		CPU_ENTRY_AREA_BASE, | ||||||
|  | 		CPU_ENTRY_AREA_BASE + CPU_ENTRY_AREA_MAP_SIZE, | ||||||
|  | 		CPU_ENTRY_AREA_MAP_SIZE >> 10, | ||||||
|  | 
 | ||||||
| #ifdef CONFIG_HIGHMEM | #ifdef CONFIG_HIGHMEM | ||||||
| 		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, | 		PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, | ||||||
| 		(LAST_PKMAP*PAGE_SIZE) >> 10, | 		(LAST_PKMAP*PAGE_SIZE) >> 10, | ||||||
|  | |||||||
| @ -15,6 +15,7 @@ | |||||||
| #include <asm/tlbflush.h> | #include <asm/tlbflush.h> | ||||||
| #include <asm/sections.h> | #include <asm/sections.h> | ||||||
| #include <asm/pgtable.h> | #include <asm/pgtable.h> | ||||||
|  | #include <asm/cpu_entry_area.h> | ||||||
| 
 | 
 | ||||||
| extern struct range pfn_mapped[E820_MAX_ENTRIES]; | extern struct range pfn_mapped[E820_MAX_ENTRIES]; | ||||||
| 
 | 
 | ||||||
| @ -322,31 +323,33 @@ void __init kasan_init(void) | |||||||
| 		map_range(&pfn_mapped[i]); | 		map_range(&pfn_mapped[i]); | ||||||
| 	} | 	} | ||||||
| 
 | 
 | ||||||
|  | 	shadow_cpu_entry_begin = (void *)CPU_ENTRY_AREA_BASE; | ||||||
|  | 	shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); | ||||||
|  | 	shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin, | ||||||
|  | 						PAGE_SIZE); | ||||||
|  | 
 | ||||||
|  | 	shadow_cpu_entry_end = (void *)(CPU_ENTRY_AREA_BASE + | ||||||
|  | 					CPU_ENTRY_AREA_MAP_SIZE); | ||||||
|  | 	shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); | ||||||
|  | 	shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end, | ||||||
|  | 					PAGE_SIZE); | ||||||
|  | 
 | ||||||
| 	kasan_populate_zero_shadow( | 	kasan_populate_zero_shadow( | ||||||
| 		kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), | 		kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM), | ||||||
|  | 		shadow_cpu_entry_begin); | ||||||
|  | 
 | ||||||
|  | 	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, | ||||||
|  | 			      (unsigned long)shadow_cpu_entry_end, 0); | ||||||
|  | 
 | ||||||
|  | 	kasan_populate_zero_shadow(shadow_cpu_entry_end, | ||||||
| 				kasan_mem_to_shadow((void *)__START_KERNEL_map)); | 				kasan_mem_to_shadow((void *)__START_KERNEL_map)); | ||||||
| 
 | 
 | ||||||
| 	kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext), | 	kasan_populate_shadow((unsigned long)kasan_mem_to_shadow(_stext), | ||||||
| 			      (unsigned long)kasan_mem_to_shadow(_end), | 			      (unsigned long)kasan_mem_to_shadow(_end), | ||||||
| 			      early_pfn_to_nid(__pa(_stext))); | 			      early_pfn_to_nid(__pa(_stext))); | ||||||
| 
 | 
 | ||||||
| 	shadow_cpu_entry_begin = (void *)__fix_to_virt(FIX_CPU_ENTRY_AREA_BOTTOM); |  | ||||||
| 	shadow_cpu_entry_begin = kasan_mem_to_shadow(shadow_cpu_entry_begin); |  | ||||||
| 	shadow_cpu_entry_begin = (void *)round_down((unsigned long)shadow_cpu_entry_begin, |  | ||||||
| 						PAGE_SIZE); |  | ||||||
| 
 |  | ||||||
| 	shadow_cpu_entry_end = (void *)(__fix_to_virt(FIX_CPU_ENTRY_AREA_TOP) + PAGE_SIZE); |  | ||||||
| 	shadow_cpu_entry_end = kasan_mem_to_shadow(shadow_cpu_entry_end); |  | ||||||
| 	shadow_cpu_entry_end = (void *)round_up((unsigned long)shadow_cpu_entry_end, |  | ||||||
| 					PAGE_SIZE); |  | ||||||
| 
 |  | ||||||
| 	kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), | 	kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), | ||||||
| 				   shadow_cpu_entry_begin); | 				(void *)KASAN_SHADOW_END); | ||||||
| 
 |  | ||||||
| 	kasan_populate_shadow((unsigned long)shadow_cpu_entry_begin, |  | ||||||
| 			      (unsigned long)shadow_cpu_entry_end, 0); |  | ||||||
| 
 |  | ||||||
| 	kasan_populate_zero_shadow(shadow_cpu_entry_end, (void *)KASAN_SHADOW_END); |  | ||||||
| 
 | 
 | ||||||
| 	load_cr3(init_top_pgt); | 	load_cr3(init_top_pgt); | ||||||
| 	__flush_tlb_all(); | 	__flush_tlb_all(); | ||||||
|  | |||||||
| @ -10,6 +10,7 @@ | |||||||
| #include <linux/pagemap.h> | #include <linux/pagemap.h> | ||||||
| #include <linux/spinlock.h> | #include <linux/spinlock.h> | ||||||
| 
 | 
 | ||||||
|  | #include <asm/cpu_entry_area.h> | ||||||
| #include <asm/pgtable.h> | #include <asm/pgtable.h> | ||||||
| #include <asm/pgalloc.h> | #include <asm/pgalloc.h> | ||||||
| #include <asm/fixmap.h> | #include <asm/fixmap.h> | ||||||
|  | |||||||
| @ -2261,7 +2261,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) | |||||||
| 
 | 
 | ||||||
| 	switch (idx) { | 	switch (idx) { | ||||||
| 	case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: | 	case FIX_BTMAP_END ... FIX_BTMAP_BEGIN: | ||||||
| 	case FIX_RO_IDT: |  | ||||||
| #ifdef CONFIG_X86_32 | #ifdef CONFIG_X86_32 | ||||||
| 	case FIX_WP_TEST: | 	case FIX_WP_TEST: | ||||||
| # ifdef CONFIG_HIGHMEM | # ifdef CONFIG_HIGHMEM | ||||||
| @ -2272,7 +2271,6 @@ static void xen_set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t prot) | |||||||
| #endif | #endif | ||||||
| 	case FIX_TEXT_POKE0: | 	case FIX_TEXT_POKE0: | ||||||
| 	case FIX_TEXT_POKE1: | 	case FIX_TEXT_POKE1: | ||||||
| 	case FIX_CPU_ENTRY_AREA_TOP ... FIX_CPU_ENTRY_AREA_BOTTOM: |  | ||||||
| 		/* All local page mappings */ | 		/* All local page mappings */ | ||||||
| 		pte = pfn_pte(phys, prot); | 		pte = pfn_pte(phys, prot); | ||||||
| 		break; | 		break; | ||||||
|  | |||||||
		Loading…
	
		Reference in New Issue
	
	Block a user