linux/arch/tile/lib/memcpy_tile64.c
Chris Metcalf 76c567fbba arch/tile: support 4KB page size as well as 64KB
The Tilera architecture traditionally supports 64KB page sizes
to improve TLB utilization and improve performance when the
hardware is being used primarily to run a single application.

For more generic server scenarios, it can be beneficial to run
with 4KB page sizes, so this commit allows that to be specified
(by modifying the arch/tile/include/hv/pagesize.h header).

As part of this change, we also re-worked the PTE management
slightly so that PTE writes all go through a __set_pte() function
where we can do some additional validation.  The set_pte_order()
function was eliminated since the "order" argument wasn't being used.

One bug uncovered was in the PCI DMA code, which wasn't properly
flushing the specified range.  This was benign with 64KB pages,
but with 4KB pages we were getting some larger flushes wrong.

The per-cpu memory reservation code also needed updating to
conform with the newer percpu stuff; before it always chose 64KB,
and that was always correct, but with 4KB granularity we now have
to pay closer attention and reserve the amount of memory that will
be requested when the percpu code starts allocating.

Signed-off-by: Chris Metcalf <cmetcalf@tilera.com>
2011-03-10 13:17:53 -05:00

277 lines
8.8 KiB
C

/*
* Copyright 2010 Tilera Corporation. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
* NON INFRINGEMENT. See the GNU General Public License for
* more details.
*/
#include <linux/string.h>
#include <linux/smp.h>
#include <linux/module.h>
#include <linux/uaccess.h>
#include <asm/fixmap.h>
#include <asm/kmap_types.h>
#include <asm/tlbflush.h>
#include <hv/hypervisor.h>
#include <arch/chip.h>
#if !CHIP_HAS_COHERENT_LOCAL_CACHE()
/* Defined in memcpy.S */
extern unsigned long __memcpy_asm(void *to, const void *from, unsigned long n);
extern unsigned long __copy_to_user_inatomic_asm(
void __user *to, const void *from, unsigned long n);
extern unsigned long __copy_from_user_inatomic_asm(
void *to, const void __user *from, unsigned long n);
extern unsigned long __copy_from_user_zeroing_asm(
void *to, const void __user *from, unsigned long n);
typedef unsigned long (*memcpy_t)(void *, const void *, unsigned long);
/* Size above which to consider TLB games for performance */
#define LARGE_COPY_CUTOFF 2048
/* Communicate to the simulator what we are trying to do. */
#define sim_allow_multiple_caching(b) \
__insn_mtspr(SPR_SIM_CONTROL, \
SIM_CONTROL_ALLOW_MULTIPLE_CACHING | ((b) << _SIM_CONTROL_OPERATOR_BITS))
/*
* Copy memory by briefly enabling incoherent cacheline-at-a-time mode.
*
* We set up our own source and destination PTEs that we fully control.
* This is the only way to guarantee that we don't race with another
* thread that is modifying the PTE; we can't afford to try the
* copy_{to,from}_user() technique of catching the interrupt, since
* we must run with interrupts disabled to avoid the risk of some
* other code seeing the incoherent data in our cache. (Recall that
* our cache is indexed by PA, so even if the other code doesn't use
* our kmap_atomic virtual addresses, they'll still hit in cache using
* the normal VAs that aren't supposed to hit in cache.)
*/
static void memcpy_multicache(void *dest, const void *source,
pte_t dst_pte, pte_t src_pte, int len)
{
int idx;
unsigned long flags, newsrc, newdst;
pmd_t *pmdp;
pte_t *ptep;
int type0, type1;
int cpu = get_cpu();
/*
* Disable interrupts so that we don't recurse into memcpy()
* in an interrupt handler, nor accidentally reference
* the PA of the source from an interrupt routine. Also
* notify the simulator that we're playing games so we don't
* generate spurious coherency warnings.
*/
local_irq_save(flags);
sim_allow_multiple_caching(1);
/* Set up the new dest mapping */
type0 = kmap_atomic_idx_push();
idx = FIX_KMAP_BEGIN + (KM_TYPE_NR * cpu) + type0;
newdst = __fix_to_virt(idx) + ((unsigned long)dest & (PAGE_SIZE-1));
pmdp = pmd_offset(pud_offset(pgd_offset_k(newdst), newdst), newdst);
ptep = pte_offset_kernel(pmdp, newdst);
if (pte_val(*ptep) != pte_val(dst_pte)) {
set_pte(ptep, dst_pte);
local_flush_tlb_page(NULL, newdst, PAGE_SIZE);
}
/* Set up the new source mapping */
type1 = kmap_atomic_idx_push();
idx += (type0 - type1);
src_pte = hv_pte_set_nc(src_pte);
src_pte = hv_pte_clear_writable(src_pte); /* be paranoid */
newsrc = __fix_to_virt(idx) + ((unsigned long)source & (PAGE_SIZE-1));
pmdp = pmd_offset(pud_offset(pgd_offset_k(newsrc), newsrc), newsrc);
ptep = pte_offset_kernel(pmdp, newsrc);
__set_pte(ptep, src_pte); /* set_pte() would be confused by this */
local_flush_tlb_page(NULL, newsrc, PAGE_SIZE);
/* Actually move the data. */
__memcpy_asm((void *)newdst, (const void *)newsrc, len);
/*
* Remap the source as locally-cached and not OLOC'ed so that
* we can inval without also invaling the remote cpu's cache.
* This also avoids known errata with inv'ing cacheable oloc data.
*/
src_pte = hv_pte_set_mode(src_pte, HV_PTE_MODE_CACHE_NO_L3);
src_pte = hv_pte_set_writable(src_pte); /* need write access for inv */
__set_pte(ptep, src_pte); /* set_pte() would be confused by this */
local_flush_tlb_page(NULL, newsrc, PAGE_SIZE);
/*
* Do the actual invalidation, covering the full L2 cache line
* at the end since __memcpy_asm() is somewhat aggressive.
*/
__inv_buffer((void *)newsrc, len);
/*
* We're done: notify the simulator that all is back to normal,
* and re-enable interrupts and pre-emption.
*/
kmap_atomic_idx_pop();
kmap_atomic_idx_pop();
sim_allow_multiple_caching(0);
local_irq_restore(flags);
put_cpu();
}
/*
* Identify large copies from remotely-cached memory, and copy them
* via memcpy_multicache() if they look good, otherwise fall back
* to the particular kind of copying passed as the memcpy_t function.
*/
static unsigned long fast_copy(void *dest, const void *source, int len,
memcpy_t func)
{
/*
* Check if it's big enough to bother with. We may end up doing a
* small copy via TLB manipulation if we're near a page boundary,
* but presumably we'll make it up when we hit the second page.
*/
while (len >= LARGE_COPY_CUTOFF) {
int copy_size, bytes_left_on_page;
pte_t *src_ptep, *dst_ptep;
pte_t src_pte, dst_pte;
struct page *src_page, *dst_page;
/* Is the source page oloc'ed to a remote cpu? */
retry_source:
src_ptep = virt_to_pte(current->mm, (unsigned long)source);
if (src_ptep == NULL)
break;
src_pte = *src_ptep;
if (!hv_pte_get_present(src_pte) ||
!hv_pte_get_readable(src_pte) ||
hv_pte_get_mode(src_pte) != HV_PTE_MODE_CACHE_TILE_L3)
break;
if (get_remote_cache_cpu(src_pte) == smp_processor_id())
break;
src_page = pfn_to_page(hv_pte_get_pfn(src_pte));
get_page(src_page);
if (pte_val(src_pte) != pte_val(*src_ptep)) {
put_page(src_page);
goto retry_source;
}
if (pte_huge(src_pte)) {
/* Adjust the PTE to correspond to a small page */
int pfn = hv_pte_get_pfn(src_pte);
pfn += (((unsigned long)source & (HPAGE_SIZE-1))
>> PAGE_SHIFT);
src_pte = pfn_pte(pfn, src_pte);
src_pte = pte_mksmall(src_pte);
}
/* Is the destination page writable? */
retry_dest:
dst_ptep = virt_to_pte(current->mm, (unsigned long)dest);
if (dst_ptep == NULL) {
put_page(src_page);
break;
}
dst_pte = *dst_ptep;
if (!hv_pte_get_present(dst_pte) ||
!hv_pte_get_writable(dst_pte)) {
put_page(src_page);
break;
}
dst_page = pfn_to_page(hv_pte_get_pfn(dst_pte));
if (dst_page == src_page) {
/*
* Source and dest are on the same page; this
* potentially exposes us to incoherence if any
* part of src and dest overlap on a cache line.
* Just give up rather than trying to be precise.
*/
put_page(src_page);
break;
}
get_page(dst_page);
if (pte_val(dst_pte) != pte_val(*dst_ptep)) {
put_page(dst_page);
goto retry_dest;
}
if (pte_huge(dst_pte)) {
/* Adjust the PTE to correspond to a small page */
int pfn = hv_pte_get_pfn(dst_pte);
pfn += (((unsigned long)dest & (HPAGE_SIZE-1))
>> PAGE_SHIFT);
dst_pte = pfn_pte(pfn, dst_pte);
dst_pte = pte_mksmall(dst_pte);
}
/* All looks good: create a cachable PTE and copy from it */
copy_size = len;
bytes_left_on_page =
PAGE_SIZE - (((int)source) & (PAGE_SIZE-1));
if (copy_size > bytes_left_on_page)
copy_size = bytes_left_on_page;
bytes_left_on_page =
PAGE_SIZE - (((int)dest) & (PAGE_SIZE-1));
if (copy_size > bytes_left_on_page)
copy_size = bytes_left_on_page;
memcpy_multicache(dest, source, dst_pte, src_pte, copy_size);
/* Release the pages */
put_page(dst_page);
put_page(src_page);
/* Continue on the next page */
dest += copy_size;
source += copy_size;
len -= copy_size;
}
return func(dest, source, len);
}
void *memcpy(void *to, const void *from, __kernel_size_t n)
{
if (n < LARGE_COPY_CUTOFF)
return (void *)__memcpy_asm(to, from, n);
else
return (void *)fast_copy(to, from, n, __memcpy_asm);
}
unsigned long __copy_to_user_inatomic(void __user *to, const void *from,
unsigned long n)
{
if (n < LARGE_COPY_CUTOFF)
return __copy_to_user_inatomic_asm(to, from, n);
else
return fast_copy(to, from, n, __copy_to_user_inatomic_asm);
}
unsigned long __copy_from_user_inatomic(void *to, const void __user *from,
unsigned long n)
{
if (n < LARGE_COPY_CUTOFF)
return __copy_from_user_inatomic_asm(to, from, n);
else
return fast_copy(to, from, n, __copy_from_user_inatomic_asm);
}
unsigned long __copy_from_user_zeroing(void *to, const void __user *from,
unsigned long n)
{
if (n < LARGE_COPY_CUTOFF)
return __copy_from_user_zeroing_asm(to, from, n);
else
return fast_copy(to, from, n, __copy_from_user_zeroing_asm);
}
#endif /* !CHIP_HAS_COHERENT_LOCAL_CACHE() */