diff options
Diffstat (limited to 'arch')
| -rw-r--r-- | arch/arc/Kconfig | 4 | ||||
| -rw-r--r-- | arch/arc/include/asm/cacheflush.h | 53 | ||||
| -rw-r--r-- | arch/arc/include/asm/page.h | 16 | ||||
| -rw-r--r-- | arch/arc/include/asm/tlb.h | 11 | ||||
| -rw-r--r-- | arch/arc/mm/cache_arc700.c | 134 | ||||
| -rw-r--r-- | arch/arc/mm/tlb.c | 27 |
6 files changed, 223 insertions, 22 deletions
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig index 491ae7923b1..5917099470e 100644 --- a/arch/arc/Kconfig +++ b/arch/arc/Kconfig @@ -182,6 +182,10 @@ config ARC_CACHE_PAGES Note that Global I/D ENABLE + Per Page DISABLE works but corollary Global DISABLE + Per Page ENABLE won't work +config ARC_CACHE_VIPT_ALIASING + bool "Support VIPT Aliasing D$" + default n + endif #ARC_CACHE config ARC_HAS_ICCM diff --git a/arch/arc/include/asm/cacheflush.h b/arch/arc/include/asm/cacheflush.h index ed820bcb745..d692fbb1725 100644 --- a/arch/arc/include/asm/cacheflush.h +++ b/arch/arc/include/asm/cacheflush.h @@ -50,18 +50,55 @@ void dma_cache_wback(unsigned long start, unsigned long sz); #define flush_cache_vmap(start, end) flush_cache_all() #define flush_cache_vunmap(start, end) flush_cache_all() -/* - * VM callbacks when entire/range of user-space V-P mappings are - * torn-down/get-invalidated - * - * Currently we don't support D$ aliasing configs for our VIPT caches - * NOPS for VIPT Cache with non-aliasing D$ configurations only - */ -#define flush_cache_dup_mm(mm) /* called on fork */ +#define flush_cache_dup_mm(mm) /* called on fork (VIVT only) */ + +#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING + #define flush_cache_mm(mm) /* called on munmap/exit */ #define flush_cache_range(mm, u_vstart, u_vend) #define flush_cache_page(vma, u_vaddr, pfn) /* PF handling/COW-break */ +#else /* VIPT aliasing dcache */ + +/* To clear out stale userspace mappings */ +void flush_cache_mm(struct mm_struct *mm); +void flush_cache_range(struct vm_area_struct *vma, + unsigned long start,unsigned long end); +void flush_cache_page(struct vm_area_struct *vma, + unsigned long user_addr, unsigned long page); + +/* + * To make sure that userspace mapping is flushed to memory before + * get_user_pages() uses a kernel mapping to access the page + */ +#define ARCH_HAS_FLUSH_ANON_PAGE +void flush_anon_page(struct vm_area_struct *vma, + struct page *page, unsigned long u_vaddr); + +#endif /* CONFIG_ARC_CACHE_VIPT_ALIASING */ + +/* + * Simple wrapper over config option + * Bootup code ensures that hardware matches kernel configuration + */ +static inline int cache_is_vipt_aliasing(void) +{ +#ifdef CONFIG_ARC_CACHE_VIPT_ALIASING + return 1; +#else + return 0; +#endif +} + +#define CACHE_COLOR(addr) (((unsigned long)(addr) >> (PAGE_SHIFT)) & 3) + +/* + * checks if two addresses (after page aligning) index into same cache set + */ +#define addr_not_cache_congruent(addr1, addr2) \ + cache_is_vipt_aliasing() ? \ + (CACHE_COLOR(addr1) != CACHE_COLOR(addr2)) : 0 \ + #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ memcpy(dst, src, len); \ diff --git a/arch/arc/include/asm/page.h b/arch/arc/include/asm/page.h index bdf54610455..374a3551411 100644 --- a/arch/arc/include/asm/page.h +++ b/arch/arc/include/asm/page.h @@ -16,13 +16,27 @@ #define get_user_page(vaddr) __get_free_page(GFP_KERNEL) #define free_user_page(page, addr) free_page(addr) -/* TBD: for now don't worry about VIPT D$ aliasing */ #define clear_page(paddr) memset((paddr), 0, PAGE_SIZE) #define copy_page(to, from) memcpy((to), (from), PAGE_SIZE) +#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING + #define clear_user_page(addr, vaddr, pg) clear_page(addr) #define copy_user_page(vto, vfrom, vaddr, pg) copy_page(vto, vfrom) +#else /* VIPT aliasing dcache */ + +struct vm_area_struct; +struct page; + +#define __HAVE_ARCH_COPY_USER_HIGHPAGE + +void copy_user_highpage(struct page *to, struct page *from, + unsigned long u_vaddr, struct vm_area_struct *vma); +void clear_user_page(void *to, unsigned long u_vaddr, struct page *page); + +#endif /* CONFIG_ARC_CACHE_VIPT_ALIASING */ + #undef STRICT_MM_TYPECHECKS #ifdef STRICT_MM_TYPECHECKS diff --git a/arch/arc/include/asm/tlb.h b/arch/arc/include/asm/tlb.h index fe91719866a..85b6df839bd 100644 --- a/arch/arc/include/asm/tlb.h +++ b/arch/arc/include/asm/tlb.h @@ -30,13 +30,20 @@ do { \ /* * This pair is called at time of munmap/exit to flush cache and TLB entries * for mappings being torn down. - * 1) cache-flush part -implemented via tlb_start_vma( ) can be NOP (for now) - * as we don't support aliasing configs in our VIPT D$. + * 1) cache-flush part -implemented via tlb_start_vma( ) for VIPT aliasing D$ * 2) tlb-flush part - implemted via tlb_end_vma( ) flushes the TLB range * * Note, read http://lkml.org/lkml/2004/1/15/6 */ +#ifndef CONFIG_ARC_CACHE_VIPT_ALIASING #define tlb_start_vma(tlb, vma) +#else +#define tlb_start_vma(tlb, vma) \ +do { \ + if (!tlb->fullmm) \ + flush_cache_range(vma, vma->vm_start, vma->vm_end); \ +} while(0) +#endif #define tlb_end_vma(tlb, vma) \ do { \ diff --git a/arch/arc/mm/cache_arc700.c b/arch/arc/mm/cache_arc700.c index a9a37089257..9887195379e 100644 --- a/arch/arc/mm/cache_arc700.c +++ b/arch/arc/mm/cache_arc700.c @@ -68,6 +68,7 @@ #include <linux/mmu_context.h> #include <linux/syscalls.h> #include <linux/uaccess.h> +#include <linux/pagemap.h> #include <asm/cacheflush.h> #include <asm/cachectl.h> #include <asm/setup.h> @@ -138,6 +139,7 @@ void __cpuinit arc_cache_init(void) struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; int way_pg_ratio = way_pg_ratio; + int dcache_does_alias; char str[256]; printk(arc_cache_mumbojumbo(0, str, sizeof(str))); @@ -184,9 +186,13 @@ chk_dc: panic("Cache H/W doesn't match kernel Config"); } + dcache_does_alias = (dc->sz / ARC_DCACHE_WAYS) > PAGE_SIZE; + /* check for D-Cache aliasing */ - if ((dc->sz / ARC_DCACHE_WAYS) > PAGE_SIZE) - panic("D$ aliasing not handled right now\n"); + if (dcache_does_alias && !cache_is_vipt_aliasing()) + panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); + else if (!dcache_does_alias && cache_is_vipt_aliasing()) + panic("Don't need CONFIG_ARC_CACHE_VIPT_ALIASING\n"); #endif /* Set the default Invalidate Mode to "simpy discard dirty lines" @@ -312,7 +318,7 @@ static inline void __dc_line_loop(unsigned long paddr, unsigned long vaddr, } } -/* For kernel mappings cache op index is same as paddr */ +/* For kernel mappings cache operation: index is same as paddr */ #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) /* @@ -464,10 +470,47 @@ static void __ic_line_inv_vaddr(unsigned long paddr, unsigned long vaddr, * Exported APIs */ +/* + * Handle cache congruency of kernel and userspace mappings of page when kernel + * writes-to/reads-from + * + * The idea is to defer flushing of kernel mapping after a WRITE, possible if: + * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent + * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) + * -In SMP, if hardware caches are coherent + * + * There's a corollary case, where kernel READs from a userspace mapped page. + * If the U-mapping is not congruent to to K-mapping, former needs flushing. + */ void flush_dcache_page(struct page *page) { - /* Make a note that dcache is not yet flushed for this page */ - set_bit(PG_arch_1, &page->flags); + struct address_space *mapping; + + if (!cache_is_vipt_aliasing()) { + set_bit(PG_arch_1, &page->flags); + return; + } + + /* don't handle anon pages here */ + mapping = page_mapping(page); + if (!mapping) + return; + + /* + * pagecache page, file not yet mapped to userspace + * Make a note that K-mapping is dirty + */ + if (!mapping_mapped(mapping)) { + set_bit(PG_arch_1, &page->flags); + } else if (page_mapped(page)) { + + /* kernel reading from page with U-mapping */ + void *paddr = page_address(page); + unsigned long vaddr = page->index << PAGE_CACHE_SHIFT; + + if (addr_not_cache_congruent(paddr, vaddr)) + __flush_dcache_page(paddr, vaddr); + } } EXPORT_SYMBOL(flush_dcache_page); @@ -612,6 +655,87 @@ noinline void flush_cache_all(void) } +#ifdef CONFIG_ARC_CACHE_VIPT_ALIASING + +void flush_cache_mm(struct mm_struct *mm) +{ + flush_cache_all(); +} + +void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, + unsigned long pfn) +{ + unsigned int paddr = pfn << PAGE_SHIFT; + + __sync_icache_dcache(paddr, u_vaddr, PAGE_SIZE); +} + +void flush_cache_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end) +{ + flush_cache_all(); +} + +void copy_user_highpage(struct page *to, struct page *from, + unsigned long u_vaddr, struct vm_area_struct *vma) +{ + void *kfrom = page_address(from); + void *kto = page_address(to); + int clean_src_k_mappings = 0; + + /* + * If SRC page was already mapped in userspace AND it's U-mapping is + * not congruent with K-mapping, sync former to physical page so that + * K-mapping in memcpy below, sees the right data + * + * Note that while @u_vaddr refers to DST page's userspace vaddr, it is + * equally valid for SRC page as well + */ + if (page_mapped(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { + __flush_dcache_page(kfrom, u_vaddr); + clean_src_k_mappings = 1; + } + + copy_page(kto, kfrom); + + /* + * Mark DST page K-mapping as dirty for a later finalization by + * update_mmu_cache(). Although the finalization could have been done + * here as well (given that both vaddr/paddr are available). + * But update_mmu_cache() already has code to do that for other + * non copied user pages (e.g. read faults which wire in pagecache page + * directly). + */ + set_bit(PG_arch_1, &to->flags); + + /* + * if SRC was already usermapped and non-congruent to kernel mapping + * sync the kernel mapping back to physical page + */ + if (clean_src_k_mappings) { + __flush_dcache_page(kfrom, kfrom); + } else { + set_bit(PG_arch_1, &from->flags); + } +} + +void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) +{ + clear_page(to); + set_bit(PG_arch_1, &page->flags); +} + +void flush_anon_page(struct vm_area_struct *vma, struct page *page, + unsigned long u_vaddr) +{ + /* TBD: do we really need to clear the kernel mapping */ + __flush_dcache_page(page_address(page), u_vaddr); + __flush_dcache_page(page_address(page), page_address(page)); + +} + +#endif + /********************************************************************** * Explicit Cache flush request from user space via syscall * Needed for JITs which generate code on the fly diff --git a/arch/arc/mm/tlb.c b/arch/arc/mm/tlb.c index 5810c7a92b7..066145b5f34 100644 --- a/arch/arc/mm/tlb.c +++ b/arch/arc/mm/tlb.c @@ -421,25 +421,40 @@ void create_tlb(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) /* * Called at the end of pagefault, for a userspace mapped page * -pre-install the corresponding TLB entry into MMU - * -Finalize the delayed D-cache flush (wback+inv kernel mapping) + * -Finalize the delayed D-cache flush of kernel mapping of page due to + * flush_dcache_page(), copy_user_page() + * + * Note that flush (when done) involves both WBACK - so physical page is + * in sync as well as INV - so any non-congruent aliases don't remain */ void update_mmu_cache(struct vm_area_struct *vma, unsigned long vaddr_unaligned, pte_t *ptep) { unsigned long vaddr = vaddr_unaligned & PAGE_MASK; + unsigned long paddr = pte_val(*ptep) & PAGE_MASK; create_tlb(vma, vaddr, ptep); - /* icache doesn't snoop dcache, thus needs to be made coherent here */ - if (vma->vm_flags & VM_EXEC) { + /* + * Exec page : Independent of aliasing/page-color considerations, + * since icache doesn't snoop dcache on ARC, any dirty + * K-mapping of a code page needs to be wback+inv so that + * icache fetch by userspace sees code correctly. + * !EXEC page: If K-mapping is NOT congruent to U-mapping, flush it + * so userspace sees the right data. + * (Avoids the flush for Non-exec + congruent mapping case) + */ + if (vma->vm_flags & VM_EXEC || addr_not_cache_congruent(paddr, vaddr)) { struct page *page = pfn_to_page(pte_pfn(*ptep)); - /* if page was dcache dirty, flush now */ int dirty = test_and_clear_bit(PG_arch_1, &page->flags); if (dirty) { - unsigned long paddr = pte_val(*ptep) & PAGE_MASK; + /* wback + inv dcache lines */ __flush_dcache_page(paddr, paddr); - __inv_icache_page(paddr, vaddr); + + /* invalidate any existing icache lines */ + if (vma->vm_flags & VM_EXEC) + __inv_icache_page(paddr, vaddr); } } } |