Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-12-26 12:52:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

x86: support GB hugepages on 64-bit

Browse Source 
Acked-by: Adam Litke <agl@us.ibm.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Andi Kleen 2008-07-23 21:27:50 -07:00 committed by Linus Torvalds
parent ceb8687961
commit 39c11e6c05
1 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
arch/x86/mm/hugetlbpage.c
View File

@ -134,10 +134,15 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
pud = pud_alloc(mm, pgd, addr); pud = pud_alloc(mm, pgd, addr);
if (pud) { if (pud) {
if (sz == PUD_SIZE) {
pte = (pte_t *)pud;
} else {
BUG_ON(sz != PMD_SIZE);
if (pud_none(*pud)) if (pud_none(*pud))
huge_pmd_share(mm, addr, pud); huge_pmd_share(mm, addr, pud);
pte = (pte_t *) pmd_alloc(mm, pud, addr); pte = (pte_t *) pmd_alloc(mm, pud, addr);
} }
}
BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte)); BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
return pte; return pte;
@ -152,9 +157,12 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
pgd = pgd_offset(mm, addr); pgd = pgd_offset(mm, addr);
if (pgd_present(*pgd)) { if (pgd_present(*pgd)) {
pud = pud_offset(pgd, addr); pud = pud_offset(pgd, addr);
if (pud_present(*pud)) if (pud_present(*pud)) {
if (pud_large(*pud))
return (pte_t *)pud;
pmd = pmd_offset(pud, addr); pmd = pmd_offset(pud, addr);
} }
}
return (pte_t *) pmd; return (pte_t *) pmd;
} }
@ -216,7 +224,7 @@ int pmd_huge(pmd_t pmd)
int pud_huge(pud_t pud) int pud_huge(pud_t pud)
{ {
return 0; return !!(pud_val(pud) & _PAGE_PSE);
} }
struct page * struct page *
@ -252,6 +260,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
unsigned long addr, unsigned long len, unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags) unsigned long pgoff, unsigned long flags)
{ {
struct hstate *h = hstate_file(file);
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long start_addr; unsigned long start_addr;
@ -264,7 +273,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
} }
full_search: full_search:
addr = ALIGN(start_addr, HPAGE_SIZE); addr = ALIGN(start_addr, huge_page_size(h));
for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
/* At this point: (!vma || addr < vma->vm_end). */ /* At this point: (!vma || addr < vma->vm_end). */
@ -286,7 +295,7 @@ full_search:
} }
if (addr + mm->cached_hole_size < vma->vm_start) if (addr + mm->cached_hole_size < vma->vm_start)
mm->cached_hole_size = vma->vm_start - addr; mm->cached_hole_size = vma->vm_start - addr;
addr = ALIGN(vma->vm_end, HPAGE_SIZE); addr = ALIGN(vma->vm_end, huge_page_size(h));
} }
} }
@ -294,6 +303,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
unsigned long addr0, unsigned long len, unsigned long addr0, unsigned long len,
unsigned long pgoff, unsigned long flags) unsigned long pgoff, unsigned long flags)
{ {
struct hstate *h = hstate_file(file);
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma, *prev_vma; struct vm_area_struct *vma, *prev_vma;
unsigned long base = mm->mmap_base, addr = addr0; unsigned long base = mm->mmap_base, addr = addr0;
@ -314,7 +324,7 @@ try_again:
goto fail; goto fail;
/* either no address requested or cant fit in requested address hole */ /* either no address requested or cant fit in requested address hole */
addr = (mm->free_area_cache - len) & HPAGE_MASK; addr = (mm->free_area_cache - len) & huge_page_mask(h);
do { do {
/* /*
* Lookup failure means no vma is above this address, * Lookup failure means no vma is above this address,
@ -345,7 +355,7 @@ try_again:
largest_hole = vma->vm_start - addr; largest_hole = vma->vm_start - addr;
/* try just below the current vma->vm_start */ /* try just below the current vma->vm_start */
addr = (vma->vm_start - len) & HPAGE_MASK; addr = (vma->vm_start - len) & huge_page_mask(h);
} while (len <= vma->vm_start); } while (len <= vma->vm_start);
fail: fail:
@ -383,10 +393,11 @@ unsigned long
hugetlb_get_unmapped_area(struct file *file, unsigned long addr, hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags) unsigned long len, unsigned long pgoff, unsigned long flags)
{ {
struct hstate *h = hstate_file(file);
struct mm_struct *mm = current->mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
if (len & ~HPAGE_MASK) if (len & ~huge_page_mask(h))
return -EINVAL; return -EINVAL;
if (len > TASK_SIZE) if (len > TASK_SIZE)
return -ENOMEM; return -ENOMEM;
@ -398,7 +409,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
} }
if (addr) { if (addr) {
addr = ALIGN(addr, HPAGE_SIZE); addr = ALIGN(addr, huge_page_size(h));
vma = find_vma(mm, addr); vma = find_vma(mm, addr);
if (TASK_SIZE - len >= addr && if (TASK_SIZE - len >= addr &&
(!vma || addr + len <= vma->vm_start)) (!vma || addr + len <= vma->vm_start))