diff options
Diffstat (limited to 'arch/sparc/mm')
28 files changed, 2390 insertions, 5121 deletions
diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile index 79836a7dd00..30c3eccfdf5 100644 --- a/arch/sparc/mm/Makefile +++ b/arch/sparc/mm/Makefile @@ -4,23 +4,16 @@ asflags-y := -ansi ccflags-y := -Werror -obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o +obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o obj-y += fault_$(BITS).o obj-y += init_$(BITS).o -obj-$(CONFIG_SPARC32) += loadmmu.o -obj-y += generic_$(BITS).o -obj-$(CONFIG_SPARC32) += extable.o btfixup.o srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += extable.o srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += srmmu_access.o obj-$(CONFIG_SPARC32) += hypersparc.o viking.o tsunami.o swift.o -obj-$(CONFIG_SPARC_LEON)+= leon_mm.o +obj-$(CONFIG_SPARC32) += leon_mm.o # Only used by sparc64 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o # Only used by sparc32 obj-$(CONFIG_HIGHMEM) += highmem.o - -ifdef CONFIG_SMP -obj-$(CONFIG_SPARC32) += nosun4c.o -else -obj-$(CONFIG_SPARC32) += sun4c.o -endif diff --git a/arch/sparc/mm/btfixup.c b/arch/sparc/mm/btfixup.c deleted file mode 100644 index 5175ac2f482..00000000000 --- a/arch/sparc/mm/btfixup.c +++ /dev/null @@ -1,330 +0,0 @@ -/* btfixup.c: Boot time code fixup and relocator, so that - * we can get rid of most indirect calls to achieve single - * image sun4c and srmmu kernel. - * - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/init.h> -#include <asm/btfixup.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/oplib.h> -#include <asm/system.h> -#include <asm/cacheflush.h> - -#define BTFIXUP_OPTIMIZE_NOP -#define BTFIXUP_OPTIMIZE_OTHER - -extern char *srmmu_name; -static char version[] __initdata = "Boot time fixup v1.6. 4/Mar/98 Jakub Jelinek (jj@ultra.linux.cz). Patching kernel for "; -static char str_sun4c[] __initdata = "sun4c\n"; -static char str_srmmu[] __initdata = "srmmu[%s]/"; -static char str_iommu[] __initdata = "iommu\n"; -static char str_iounit[] __initdata = "io-unit\n"; - -static int visited __initdata = 0; -extern unsigned int ___btfixup_start[], ___btfixup_end[], __init_begin[], __init_end[], __init_text_end[]; -extern unsigned int _stext[], _end[], __start___ksymtab[], __stop___ksymtab[]; -static char wrong_f[] __initdata = "Trying to set f fixup %p to invalid function %08x\n"; -static char wrong_b[] __initdata = "Trying to set b fixup %p to invalid function %08x\n"; -static char wrong_s[] __initdata = "Trying to set s fixup %p to invalid value %08x\n"; -static char wrong_h[] __initdata = "Trying to set h fixup %p to invalid value %08x\n"; -static char wrong_a[] __initdata = "Trying to set a fixup %p to invalid value %08x\n"; -static char wrong[] __initdata = "Wrong address for %c fixup %p\n"; -static char insn_f[] __initdata = "Fixup f %p refers to weird instructions at %p[%08x,%08x]\n"; -static char insn_b[] __initdata = "Fixup b %p doesn't refer to a SETHI at %p[%08x]\n"; -static char insn_s[] __initdata = "Fixup s %p doesn't refer to an OR at %p[%08x]\n"; -static char insn_h[] __initdata = "Fixup h %p doesn't refer to a SETHI at %p[%08x]\n"; -static char insn_a[] __initdata = "Fixup a %p doesn't refer to a SETHI nor OR at %p[%08x]\n"; -static char insn_i[] __initdata = "Fixup i %p doesn't refer to a valid instruction at %p[%08x]\n"; -static char fca_und[] __initdata = "flush_cache_all undefined in btfixup()\n"; -static char wrong_setaddr[] __initdata = "Garbled CALL/INT patch at %p[%08x,%08x,%08x]=%08x\n"; - -#ifdef BTFIXUP_OPTIMIZE_OTHER -static void __init set_addr(unsigned int *addr, unsigned int q1, int fmangled, unsigned int value) -{ - if (!fmangled) - *addr = value; - else { - unsigned int *q = (unsigned int *)q1; - if (*addr == 0x01000000) { - /* Noped */ - *q = value; - } else if (addr[-1] == *q) { - /* Moved */ - addr[-1] = value; - *q = value; - } else { - prom_printf(wrong_setaddr, addr-1, addr[-1], *addr, *q, value); - prom_halt(); - } - } -} -#else -static inline void set_addr(unsigned int *addr, unsigned int q1, int fmangled, unsigned int value) -{ - *addr = value; -} -#endif - -void __init btfixup(void) -{ - unsigned int *p, *q; - int type, count; - unsigned insn; - unsigned *addr; - int fmangled = 0; - void (*flush_cacheall)(void); - - if (!visited) { - visited++; - printk(version); - if (ARCH_SUN4C) - printk(str_sun4c); - else { - printk(str_srmmu, srmmu_name); - if (sparc_cpu_model == sun4d) - printk(str_iounit); - else - printk(str_iommu); - } - } - for (p = ___btfixup_start; p < ___btfixup_end; ) { - count = p[2]; - q = p + 3; - switch (type = *(unsigned char *)p) { - case 'f': - count = p[3]; - q = p + 4; - if (((p[0] & 1) || p[1]) - && ((p[1] & 3) || (unsigned *)(p[1]) < _stext || (unsigned *)(p[1]) >= _end)) { - prom_printf(wrong_f, p, p[1]); - prom_halt(); - } - break; - case 'b': - if (p[1] < (unsigned long)__init_begin || p[1] >= (unsigned long)__init_text_end || (p[1] & 3)) { - prom_printf(wrong_b, p, p[1]); - prom_halt(); - } - break; - case 's': - if (p[1] + 0x1000 >= 0x2000) { - prom_printf(wrong_s, p, p[1]); - prom_halt(); - } - break; - case 'h': - if (p[1] & 0x3ff) { - prom_printf(wrong_h, p, p[1]); - prom_halt(); - } - break; - case 'a': - if (p[1] + 0x1000 >= 0x2000 && (p[1] & 0x3ff)) { - prom_printf(wrong_a, p, p[1]); - prom_halt(); - } - break; - } - if (p[0] & 1) { - p[0] &= ~1; - while (count) { - fmangled = 0; - addr = (unsigned *)*q; - if (addr < _stext || addr >= _end) { - prom_printf(wrong, type, p); - prom_halt(); - } - insn = *addr; -#ifdef BTFIXUP_OPTIMIZE_OTHER - if (type != 'f' && q[1]) { - insn = *(unsigned int *)q[1]; - if (!insn || insn == 1) - insn = *addr; - else - fmangled = 1; - } -#endif - switch (type) { - case 'f': /* CALL */ - if (addr >= __start___ksymtab && addr < __stop___ksymtab) { - *addr = p[1]; - break; - } else if (!q[1]) { - if ((insn & 0xc1c00000) == 0x01000000) { /* SETHI */ - *addr = (insn & 0xffc00000) | (p[1] >> 10); break; - } else if ((insn & 0xc1f82000) == 0x80102000) { /* OR X, %LO(i), Y */ - *addr = (insn & 0xffffe000) | (p[1] & 0x3ff); break; - } else if ((insn & 0xc0000000) != 0x40000000) { /* !CALL */ - bad_f: - prom_printf(insn_f, p, addr, insn, addr[1]); - prom_halt(); - } - } else if (q[1] != 1) - addr[1] = q[1]; - if (p[2] == BTFIXUPCALL_NORM) { - norm_f: - *addr = 0x40000000 | ((p[1] - (unsigned)addr) >> 2); - q[1] = 0; - break; - } -#ifndef BTFIXUP_OPTIMIZE_NOP - goto norm_f; -#else - if (!(addr[1] & 0x80000000)) { - if ((addr[1] & 0xc1c00000) != 0x01000000) /* !SETHI */ - goto bad_f; /* CALL, Bicc, FBfcc, CBccc are weird in delay slot, aren't they? */ - } else { - if ((addr[1] & 0x01800000) == 0x01800000) { - if ((addr[1] & 0x01f80000) == 0x01e80000) { - /* RESTORE */ - goto norm_f; /* It is dangerous to patch that */ - } - goto bad_f; - } - if ((addr[1] & 0xffffe003) == 0x9e03e000) { - /* ADD %O7, XX, %o7 */ - int displac = (addr[1] << 19); - - displac = (displac >> 21) + 2; - *addr = (0x10800000) + (displac & 0x3fffff); - q[1] = addr[1]; - addr[1] = p[2]; - break; - } - if ((addr[1] & 0x201f) == 0x200f || (addr[1] & 0x7c000) == 0x3c000) - goto norm_f; /* Someone is playing bad tricks with us: rs1 or rs2 is o7 */ - if ((addr[1] & 0x3e000000) == 0x1e000000) - goto norm_f; /* rd is %o7. We'd better take care. */ - } - if (p[2] == BTFIXUPCALL_NOP) { - *addr = 0x01000000; - q[1] = 1; - break; - } -#ifndef BTFIXUP_OPTIMIZE_OTHER - goto norm_f; -#else - if (addr[1] == 0x01000000) { /* NOP in the delay slot */ - q[1] = addr[1]; - *addr = p[2]; - break; - } - if ((addr[1] & 0xc0000000) != 0xc0000000) { - /* Not a memory operation */ - if ((addr[1] & 0x30000000) == 0x10000000) { - /* Ok, non-memory op with rd %oX */ - if ((addr[1] & 0x3e000000) == 0x1c000000) - goto bad_f; /* Aiee. Someone is playing strange %sp tricks */ - if ((addr[1] & 0x3e000000) > 0x12000000 || - ((addr[1] & 0x3e000000) == 0x12000000 && - p[2] != BTFIXUPCALL_STO1O0 && p[2] != BTFIXUPCALL_SWAPO0O1) || - ((p[2] & 0xffffe000) == BTFIXUPCALL_RETINT(0))) { - /* Nobody uses the result. We can nop it out. */ - *addr = p[2]; - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - if ((addr[1] & 0xf1ffffe0) == 0x90100000) { - /* MOV %reg, %Ox */ - if ((addr[1] & 0x3e000000) == 0x10000000 && - (p[2] & 0x7c000) == 0x20000) { - /* Ok, it is call xx; mov reg, %o0 and call optimizes - to doing something on %o0. Patch the patch. */ - *addr = (p[2] & ~0x7c000) | ((addr[1] & 0x1f) << 14); - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - if ((addr[1] & 0x3e000000) == 0x12000000 && - p[2] == BTFIXUPCALL_STO1O0) { - *addr = (p[2] & ~0x3e000000) | ((addr[1] & 0x1f) << 25); - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - } - } - } - *addr = addr[1]; - q[1] = addr[1]; - addr[1] = p[2]; - break; -#endif /* BTFIXUP_OPTIMIZE_OTHER */ -#endif /* BTFIXUP_OPTIMIZE_NOP */ - case 'b': /* BLACKBOX */ - /* Has to be sethi i, xx */ - if ((insn & 0xc1c00000) != 0x01000000) { - prom_printf(insn_b, p, addr, insn); - prom_halt(); - } else { - void (*do_fixup)(unsigned *); - - do_fixup = (void (*)(unsigned *))p[1]; - do_fixup(addr); - } - break; - case 's': /* SIMM13 */ - /* Has to be or %g0, i, xx */ - if ((insn & 0xc1ffe000) != 0x80102000) { - prom_printf(insn_s, p, addr, insn); - prom_halt(); - } - set_addr(addr, q[1], fmangled, (insn & 0xffffe000) | (p[1] & 0x1fff)); - break; - case 'h': /* SETHI */ - /* Has to be sethi i, xx */ - if ((insn & 0xc1c00000) != 0x01000000) { - prom_printf(insn_h, p, addr, insn); - prom_halt(); - } - set_addr(addr, q[1], fmangled, (insn & 0xffc00000) | (p[1] >> 10)); - break; - case 'a': /* HALF */ - /* Has to be sethi i, xx or or %g0, i, xx */ - if ((insn & 0xc1c00000) != 0x01000000 && - (insn & 0xc1ffe000) != 0x80102000) { - prom_printf(insn_a, p, addr, insn); - prom_halt(); - } - if (p[1] & 0x3ff) - set_addr(addr, q[1], fmangled, - (insn & 0x3e000000) | 0x80102000 | (p[1] & 0x1fff)); - else - set_addr(addr, q[1], fmangled, - (insn & 0x3e000000) | 0x01000000 | (p[1] >> 10)); - break; - case 'i': /* INT */ - if ((insn & 0xc1c00000) == 0x01000000) /* %HI */ - set_addr(addr, q[1], fmangled, (insn & 0xffc00000) | (p[1] >> 10)); - else if ((insn & 0x80002000) == 0x80002000 && - (insn & 0x01800000) != 0x01800000) /* %LO */ - set_addr(addr, q[1], fmangled, (insn & 0xffffe000) | (p[1] & 0x3ff)); - else { - prom_printf(insn_i, p, addr, insn); - prom_halt(); - } - break; - } - count -= 2; - q += 2; - } - } else - p = q + count; - } -#ifdef CONFIG_SMP - flush_cacheall = (void (*)(void))BTFIXUPVAL_CALL(local_flush_cache_all); -#else - flush_cacheall = (void (*)(void))BTFIXUPVAL_CALL(flush_cache_all); -#endif - if (!flush_cacheall) { - prom_printf(fca_und); - prom_halt(); - } - (*flush_cacheall)(); -} diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 5b836f5aea9..908e8c17c90 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -20,97 +20,48 @@ #include <linux/smp.h> #include <linux/perf_event.h> #include <linux/interrupt.h> -#include <linux/module.h> #include <linux/kdebug.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgtable.h> -#include <asm/memreg.h> #include <asm/openprom.h> #include <asm/oplib.h> +#include <asm/setup.h> #include <asm/smp.h> #include <asm/traps.h> #include <asm/uaccess.h> -extern int prom_node_root; +#include "mm_32.h" int show_unhandled_signals = 1; -/* At boot time we determine these two values necessary for setting - * up the segment maps and page table entries (pte's). - */ - -int num_segmaps, num_contexts; -int invalid_segment; - -/* various Virtual Address Cache parameters we find at boot time... */ - -int vac_size, vac_linesize, vac_do_hw_vac_flushes; -int vac_entries_per_context, vac_entries_per_segment; -int vac_entries_per_page; - -/* Return how much physical memory we have. */ -unsigned long probe_memory(void) -{ - unsigned long total = 0; - int i; - - for (i = 0; sp_banks[i].num_bytes; i++) - total += sp_banks[i].num_bytes; - - return total; -} - -extern void sun4c_complete_all_stores(void); - -/* Whee, a level 15 NMI interrupt memory error. Let's have fun... */ -asmlinkage void sparc_lvl15_nmi(struct pt_regs *regs, unsigned long serr, - unsigned long svaddr, unsigned long aerr, - unsigned long avaddr) -{ - sun4c_complete_all_stores(); - printk("FAULT: NMI received\n"); - printk("SREGS: Synchronous Error %08lx\n", serr); - printk(" Synchronous Vaddr %08lx\n", svaddr); - printk(" Asynchronous Error %08lx\n", aerr); - printk(" Asynchronous Vaddr %08lx\n", avaddr); - if (sun4c_memerr_reg) - printk(" Memory Parity Error %08lx\n", *sun4c_memerr_reg); - printk("REGISTER DUMP:\n"); - show_regs(regs); - prom_halt(); -} - -static void unhandled_fault(unsigned long, struct task_struct *, - struct pt_regs *) __attribute__ ((noreturn)); - -static void unhandled_fault(unsigned long address, struct task_struct *tsk, - struct pt_regs *regs) +static void __noreturn unhandled_fault(unsigned long address, + struct task_struct *tsk, + struct pt_regs *regs) { - if((unsigned long) address < PAGE_SIZE) { + if ((unsigned long) address < PAGE_SIZE) { printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference\n"); } else { - printk(KERN_ALERT "Unable to handle kernel paging request " - "at virtual address %08lx\n", address); + printk(KERN_ALERT "Unable to handle kernel paging request at virtual address %08lx\n", + address); } printk(KERN_ALERT "tsk->{mm,active_mm}->context = %08lx\n", (tsk->mm ? tsk->mm->context : tsk->active_mm->context)); printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %08lx\n", (tsk->mm ? (unsigned long) tsk->mm->pgd : - (unsigned long) tsk->active_mm->pgd)); + (unsigned long) tsk->active_mm->pgd)); die_if_kernel("Oops", regs); } -asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, +asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, unsigned long address) { struct pt_regs regs; unsigned long g2; unsigned int insn; int i; - + i = search_extables_range(ret_pc, &g2); switch (i) { case 3: @@ -130,14 +81,14 @@ asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, /* for _from_ macros */ insn = *((unsigned int *) pc); if (!((insn >> 21) & 1) || ((insn>>19)&0x3f) == 15) - return 2; - break; + return 2; + break; default: break; - }; + } - memset(®s, 0, sizeof (regs)); + memset(®s, 0, sizeof(regs)); regs.pc = pc; regs.npc = pc + 4; __asm__ __volatile__( @@ -190,9 +141,6 @@ static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs, force_sig_info (sig, &info, current); } -extern unsigned long safe_compute_effective_address(struct pt_regs *, - unsigned int); - static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) { unsigned int insn; @@ -200,11 +148,10 @@ static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) if (text_fault) return regs->pc; - if (regs->psr & PSR_PS) { + if (regs->psr & PSR_PS) insn = *(unsigned int *) regs->pc; - } else { + else __get_user(insn, (unsigned int *) regs->pc); - } return safe_compute_effective_address(regs, insn); } @@ -227,8 +174,9 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, unsigned long g2; int from_user = !(regs->psr & PSR_PS); int fault, code; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; - if(text_fault) + if (text_fault) address = regs->pc; /* @@ -240,37 +188,33 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, * only copy the information from the master page table, * nothing more. */ - if (!ARCH_SUN4C && address >= TASK_SIZE) - goto vmalloc_fault; - code = SEGV_MAPERR; + if (address >= TASK_SIZE) + goto vmalloc_fault; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm) - goto no_context; + if (in_atomic() || !mm) + goto no_context; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); +retry: down_read(&mm->mmap_sem); - /* - * The kernel referencing a bad kernel pointer can lock up - * a sun4c machine completely, so we must attempt recovery. - */ - if(!from_user && address >= PAGE_OFFSET) + if (!from_user && address >= PAGE_OFFSET) goto bad_area; vma = find_vma(mm, address); - if(!vma) + if (!vma) goto bad_area; - if(vma->vm_start <= address) + if (vma->vm_start <= address) goto good_area; - if(!(vma->vm_flags & VM_GROWSDOWN)) + if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; - if(expand_stack(vma, address)) + if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so @@ -278,21 +222,30 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, */ good_area: code = SEGV_ACCERR; - if(write) { - if(!(vma->vm_flags & VM_WRITE)) + if (write) { + if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { /* Allow reads even for write-only mappings */ - if(!(vma->vm_flags & (VM_READ | VM_EXEC))) + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } + if (from_user) + flags |= FAULT_FLAG_USER; + if (write) + flags |= FAULT_FLAG_WRITE; + /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0); + fault = handle_mm_fault(mm, vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -300,15 +253,30 @@ good_area: goto do_sigbus; BUG(); } - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) { + current->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, + 1, regs, address); + } else { + current->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, + 1, regs, address); + } + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } } + up_read(&mm->mmap_sem); return; @@ -331,14 +299,16 @@ no_context: g2 = regs->u_regs[UREG_G2]; if (!from_user) { fixup = search_extables_range(regs->pc, &g2); - if (fixup > 10) { /* Values below are reserved for other things */ + /* Values below 10 are reserved for other things */ + if (fixup > 10) { extern const unsigned __memset_start[]; extern const unsigned __memset_end[]; extern const unsigned __csum_partial_copy_start[]; extern const unsigned __csum_partial_copy_end[]; #ifdef DEBUG_EXCEPTIONS - printk("Exception: PC<%08lx> faddr<%08lx>\n", regs->pc, address); + printk("Exception: PC<%08lx> faddr<%08lx>\n", + regs->pc, address); printk("EX_TABLE: insn<%08lx> fixup<%08x> g2<%08lx>\n", regs->pc, fixup, g2); #endif @@ -346,7 +316,7 @@ no_context: regs->pc < (unsigned long)__memset_end) || (regs->pc >= (unsigned long)__csum_partial_copy_start && regs->pc < (unsigned long)__csum_partial_copy_end)) { - regs->u_regs[UREG_I4] = address; + regs->u_regs[UREG_I4] = address; regs->u_regs[UREG_I5] = regs->pc; } regs->u_regs[UREG_G2] = g2; @@ -355,8 +325,8 @@ no_context: return; } } - - unhandled_fault (address, tsk, regs); + + unhandled_fault(address, tsk, regs); do_exit(SIGKILL); /* @@ -402,127 +372,44 @@ vmalloc_fault: if (pmd_present(*pmd) || !pmd_present(*pmd_k)) goto bad_area_nosemaphore; + *pmd = *pmd_k; return; } } -asmlinkage void do_sun4c_fault(struct pt_regs *regs, int text_fault, int write, - unsigned long address) -{ - extern void sun4c_update_mmu_cache(struct vm_area_struct *, - unsigned long,pte_t *); - extern pte_t *sun4c_pte_offset_kernel(pmd_t *,unsigned long); - struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; - pgd_t *pgdp; - pte_t *ptep; - - if (text_fault) { - address = regs->pc; - } else if (!write && - !(regs->psr & PSR_PS)) { - unsigned int insn, __user *ip; - - ip = (unsigned int __user *)regs->pc; - if (!get_user(insn, ip)) { - if ((insn & 0xc1680000) == 0xc0680000) - write = 1; - } - } - - if (!mm) { - /* We are oopsing. */ - do_sparc_fault(regs, text_fault, write, address); - BUG(); /* P3 Oops already, you bitch */ - } - - pgdp = pgd_offset(mm, address); - ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, address); - - if (pgd_val(*pgdp)) { - if (write) { - if ((pte_val(*ptep) & (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_PRESENT)) - == (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_PRESENT)) { - unsigned long flags; - - *ptep = __pte(pte_val(*ptep) | _SUN4C_PAGE_ACCESSED | - _SUN4C_PAGE_MODIFIED | - _SUN4C_PAGE_VALID | - _SUN4C_PAGE_DIRTY); - - local_irq_save(flags); - if (sun4c_get_segmap(address) != invalid_segment) { - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); - return; - } - local_irq_restore(flags); - } - } else { - if ((pte_val(*ptep) & (_SUN4C_PAGE_READ|_SUN4C_PAGE_PRESENT)) - == (_SUN4C_PAGE_READ|_SUN4C_PAGE_PRESENT)) { - unsigned long flags; - - *ptep = __pte(pte_val(*ptep) | _SUN4C_PAGE_ACCESSED | - _SUN4C_PAGE_VALID); - - local_irq_save(flags); - if (sun4c_get_segmap(address) != invalid_segment) { - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); - return; - } - local_irq_restore(flags); - } - } - } - - /* This conditional is 'interesting'. */ - if (pgd_val(*pgdp) && !(write && !(pte_val(*ptep) & _SUN4C_PAGE_WRITE)) - && (pte_val(*ptep) & _SUN4C_PAGE_VALID)) - /* Note: It is safe to not grab the MMAP semaphore here because - * we know that update_mmu_cache() will not sleep for - * any reason (at least not in the current implementation) - * and therefore there is no danger of another thread getting - * on the CPU and doing a shrink_mmap() on this vma. - */ - sun4c_update_mmu_cache (find_vma(current->mm, address), address, - ptep); - else - do_sparc_fault(regs, text_fault, write, address); -} - /* This always deals with user addresses. */ static void force_user_fault(unsigned long address, int write) { struct vm_area_struct *vma; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; + unsigned int flags = FAULT_FLAG_USER; int code; code = SEGV_MAPERR; down_read(&mm->mmap_sem); vma = find_vma(mm, address); - if(!vma) + if (!vma) goto bad_area; - if(vma->vm_start <= address) + if (vma->vm_start <= address) goto good_area; - if(!(vma->vm_flags & VM_GROWSDOWN)) + if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; - if(expand_stack(vma, address)) + if (expand_stack(vma, address)) goto bad_area; good_area: code = SEGV_ACCERR; - if(write) { - if(!(vma->vm_flags & VM_WRITE)) + if (write) { + if (!(vma->vm_flags & VM_WRITE)) goto bad_area; + flags |= FAULT_FLAG_WRITE; } else { - if(!(vma->vm_flags & (VM_READ | VM_EXEC))) + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0)) { + switch (handle_mm_fault(mm, vma, address, flags)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; @@ -550,7 +437,7 @@ void window_overflow_fault(void) unsigned long sp; sp = current_thread_info()->rwbuf_stkptrs[0]; - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 1); force_user_fault(sp, 1); @@ -559,7 +446,7 @@ void window_overflow_fault(void) void window_underflow_fault(unsigned long sp) { - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 0); force_user_fault(sp, 0); @@ -571,7 +458,7 @@ void window_ret_fault(struct pt_regs *regs) unsigned long sp; sp = regs->u_regs[UREG_FP]; - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 0); force_user_fault(sp, 0); diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index f92ce56a8b2..587cd056512 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -21,6 +21,7 @@ #include <linux/kprobes.h> #include <linux/kdebug.h> #include <linux/percpu.h> +#include <linux/context_tracking.h> #include <asm/page.h> #include <asm/pgtable.h> @@ -31,6 +32,7 @@ #include <asm/lsu.h> #include <asm/sections.h> #include <asm/mmu_context.h> +#include <asm/setup.h> int show_unhandled_signals = 1; @@ -95,38 +97,51 @@ static unsigned int get_user_insn(unsigned long tpc) pte_t *ptep, pte; unsigned long pa; u32 insn = 0; - unsigned long pstate; - if (pgd_none(*pgdp)) - goto outret; + if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) + goto out; pudp = pud_offset(pgdp, tpc); - if (pud_none(*pudp)) - goto outret; - pmdp = pmd_offset(pudp, tpc); - if (pmd_none(*pmdp)) - goto outret; - - /* This disables preemption for us as well. */ - __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); - __asm__ __volatile__("wrpr %0, %1, %%pstate" - : : "r" (pstate), "i" (PSTATE_IE)); - ptep = pte_offset_map(pmdp, tpc); - pte = *ptep; - if (!pte_present(pte)) + if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) goto out; - pa = (pte_pfn(pte) << PAGE_SHIFT); - pa += (tpc & ~PAGE_MASK); - - /* Use phys bypass so we don't pollute dtlb/dcache. */ - __asm__ __volatile__("lduwa [%1] %2, %0" - : "=r" (insn) - : "r" (pa), "i" (ASI_PHYS_USE_EC)); + /* This disables preemption for us as well. */ + local_irq_disable(); + pmdp = pmd_offset(pudp, tpc); + if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) + goto out_irq_enable; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_trans_huge(*pmdp)) { + if (pmd_trans_splitting(*pmdp)) + goto out_irq_enable; + + pa = pmd_pfn(*pmdp) << PAGE_SHIFT; + pa += tpc & ~HPAGE_MASK; + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } else +#endif + { + ptep = pte_offset_map(pmdp, tpc); + pte = *ptep; + if (pte_present(pte)) { + pa = (pte_pfn(pte) << PAGE_SHIFT); + pa += (tpc & ~PAGE_MASK); + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } + pte_unmap(ptep); + } +out_irq_enable: + local_irq_enable(); out: - pte_unmap(ptep); - __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); -outret: return insn; } @@ -151,10 +166,9 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, printk(KERN_CONT "\n"); } -extern unsigned long compute_effective_address(struct pt_regs *, unsigned int, unsigned int); - static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, - unsigned int insn, int fault_code) + unsigned long fault_addr, unsigned int insn, + int fault_code) { unsigned long addr; siginfo_t info; @@ -162,10 +176,18 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, info.si_code = code; info.si_signo = sig; info.si_errno = 0; - if (fault_code & FAULT_CODE_ITLB) + if (fault_code & FAULT_CODE_ITLB) { addr = regs->tpc; - else - addr = compute_effective_address(regs, insn, 0); + } else { + /* If we were able to probe the faulting instruction, use it + * to compute a precise fault address. Otherwise use the fault + * time provided address which may only have page granularity. + */ + if (insn) + addr = compute_effective_address(regs, insn, 0); + else + addr = fault_addr; + } info.si_addr = (void __user *) addr; info.si_trapno = 0; @@ -175,9 +197,6 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, force_sig_info(sig, &info, current); } -extern int handle_ldf_stq(u32, struct pt_regs *); -extern int handle_ld_nf(u32, struct pt_regs *); - static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) { if (!insn) { @@ -240,7 +259,7 @@ static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code, /* The si_code was set to make clear whether * this was a SEGV_MAPERR or SEGV_ACCERR fault. */ - do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code); + do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code); return; } @@ -260,30 +279,20 @@ static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) show_regs(regs); } -static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs, - unsigned long addr) -{ - static int times; - - if (times++ < 10) - printk(KERN_ERR "FAULT[%s:%d]: 32-bit process " - "reports 64-bit fault address [%lx]\n", - current->comm, current->pid, addr); - show_regs(regs); -} - asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned int insn = 0; int si_code, fault_code, fault; unsigned long address, mm_rss; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; fault_code = get_thread_fault_code(); if (notify_page_fault(regs)) - return; + goto exit_exception; si_code = SEGV_MAPERR; address = current_thread_info()->fault_address; @@ -299,10 +308,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) goto intr_or_no_mm; } } - if (unlikely((address >> 32) != 0)) { - bogus_32bit_fault_address(regs, address); + if (unlikely((address >> 32) != 0)) goto intr_or_no_mm; - } } if (regs->tstate & TSTATE_PRIV) { @@ -314,9 +321,10 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) /* Valid, no problems... */ } else { bad_kernel_pc(regs, address); - return; + goto exit_exception; } - } + } else + flags |= FAULT_FLAG_USER; /* * If we're in an interrupt or have no user @@ -325,7 +333,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) if (in_atomic() || !mm) goto intr_or_no_mm; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); if (!down_read_trylock(&mm->mmap_sem)) { if ((regs->tstate & TSTATE_PRIV) && @@ -333,6 +341,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) insn = get_fault_insn(regs, insn); goto handle_kernel_fault; } + +retry: down_read(&mm->mmap_sem); } @@ -417,13 +427,19 @@ good_area: vma->vm_file != NULL) set_thread_fault_code(fault_code | FAULT_CODE_BLKCOMMIT); + + flags |= FAULT_FLAG_WRITE; } else { /* Allow reads even for write-only mappings */ if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - fault = handle_mm_fault(mm, vma, address, (fault_code & FAULT_CODE_WRITE) ? FAULT_FLAG_WRITE : 0); + fault = handle_mm_fault(mm, vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + goto exit_exception; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -431,30 +447,51 @@ good_area: goto do_sigbus; BUG(); } - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) { + current->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, + 1, regs, address); + } else { + current->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, + 1, regs, address); + } + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } } up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); #endif if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mm_rss = mm->context.huge_pte_count; if (unlikely(mm_rss > - mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) - tsb_grow(mm, MM_TSB_HUGE, mm_rss); + mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) + tsb_grow(mm, MM_TSB_HUGE, mm_rss); + else + hugetlb_setup(regs); + + } #endif +exit_exception: + exception_exit(prev_state); return; /* @@ -467,7 +504,7 @@ bad_area: handle_kernel_fault: do_kernel_fault(regs, si_code, fault_code, insn, address); - return; + goto exit_exception; /* * We ran out of memory, or some other thing happened to us that made @@ -478,7 +515,7 @@ out_of_memory: up_read(&mm->mmap_sem); if (!(regs->tstate & TSTATE_PRIV)) { pagefault_out_of_memory(); - return; + goto exit_exception; } goto handle_kernel_fault; @@ -494,7 +531,7 @@ do_sigbus: * Send a sigbus, regardless of whether we were in kernel * or user mode. */ - do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code); + do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code); /* Kernel mode? Handle exceptions or die */ if (regs->tstate & TSTATE_PRIV) diff --git a/arch/sparc/mm/generic_32.c b/arch/sparc/mm/generic_32.c deleted file mode 100644 index 5edcac184ea..00000000000 --- a/arch/sparc/mm/generic_32.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * generic.c: Generic Sparc mm routines that are not dependent upon - * MMU type but are Sparc specific. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/swap.h> -#include <linux/pagemap.h> - -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/page.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> - -/* Remap IO memory, the same way as remap_pfn_range(), but use - * the obio memory space. - * - * They use a pgprot that sets PAGE_IO and does not check the - * mem_map table as this is independent of normal memory. - */ -static inline void io_remap_pte_range(struct mm_struct *mm, pte_t * pte, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; - do { - set_pte_at(mm, address, pte, mk_pte_io(offset, prot, space)); - address += PAGE_SIZE; - offset += PAGE_SIZE; - pte++; - } while (address < end); -} - -static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - offset -= address; - do { - pte_t * pte = pte_alloc_map(mm, pmd, address); - if (!pte) - return -ENOMEM; - io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address < end); - return 0; -} - -int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, - unsigned long pfn, unsigned long size, pgprot_t prot) -{ - int error = 0; - pgd_t * dir; - unsigned long beg = from; - unsigned long end = from + size; - struct mm_struct *mm = vma->vm_mm; - int space = GET_IOSPACE(pfn); - unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; - - /* See comment in mm/memory.c remap_pfn_range */ - vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; - vma->vm_pgoff = (offset >> PAGE_SHIFT) | - ((unsigned long)space << 28UL); - - offset -= from; - dir = pgd_offset(mm, from); - flush_cache_range(vma, beg, end); - - while (from < end) { - pmd_t *pmd = pmd_alloc(mm, dir, from); - error = -ENOMEM; - if (!pmd) - break; - error = io_remap_pmd_range(mm, pmd, from, end - from, offset + from, prot, space); - if (error) - break; - from = (from + PGDIR_SIZE) & PGDIR_MASK; - dir++; - } - - flush_tlb_range(vma, beg, end); - return error; -} -EXPORT_SYMBOL(io_remap_pfn_range); diff --git a/arch/sparc/mm/generic_64.c b/arch/sparc/mm/generic_64.c deleted file mode 100644 index 04f2bf4cd57..00000000000 --- a/arch/sparc/mm/generic_64.c +++ /dev/null @@ -1,164 +0,0 @@ -/* - * generic.c: Generic Sparc mm routines that are not dependent upon - * MMU type but are Sparc specific. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/swap.h> -#include <linux/pagemap.h> - -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/page.h> -#include <asm/tlbflush.h> - -/* Remap IO memory, the same way as remap_pfn_range(), but use - * the obio memory space. - * - * They use a pgprot that sets PAGE_IO and does not check the - * mem_map table as this is independent of normal memory. - */ -static inline void io_remap_pte_range(struct mm_struct *mm, pte_t * pte, - unsigned long address, - unsigned long size, - unsigned long offset, pgprot_t prot, - int space) -{ - unsigned long end; - - /* clear hack bit that was used as a write_combine side-effect flag */ - offset &= ~0x1UL; - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; - do { - pte_t entry; - unsigned long curend = address + PAGE_SIZE; - - entry = mk_pte_io(offset, prot, space, PAGE_SIZE); - if (!(address & 0xffff)) { - if (PAGE_SIZE < (4 * 1024 * 1024) && - !(address & 0x3fffff) && - !(offset & 0x3ffffe) && - end >= address + 0x400000) { - entry = mk_pte_io(offset, prot, space, - 4 * 1024 * 1024); - curend = address + 0x400000; - offset += 0x400000; - } else if (PAGE_SIZE < (512 * 1024) && - !(address & 0x7ffff) && - !(offset & 0x7fffe) && - end >= address + 0x80000) { - entry = mk_pte_io(offset, prot, space, - 512 * 1024 * 1024); - curend = address + 0x80000; - offset += 0x80000; - } else if (PAGE_SIZE < (64 * 1024) && - !(offset & 0xfffe) && - end >= address + 0x10000) { - entry = mk_pte_io(offset, prot, space, - 64 * 1024); - curend = address + 0x10000; - offset += 0x10000; - } else - offset += PAGE_SIZE; - } else - offset += PAGE_SIZE; - - if (pte_write(entry)) - entry = pte_mkdirty(entry); - do { - BUG_ON(!pte_none(*pte)); - set_pte_at(mm, address, pte, entry); - address += PAGE_SIZE; - pte_val(entry) += PAGE_SIZE; - pte++; - } while (address < curend); - } while (address < end); -} - -static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - offset -= address; - do { - pte_t * pte = pte_alloc_map(mm, pmd, address); - if (!pte) - return -ENOMEM; - io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space); - pte_unmap(pte); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address < end); - return 0; -} - -static inline int io_remap_pud_range(struct mm_struct *mm, pud_t * pud, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PUD_MASK; - end = address + size; - if (end > PUD_SIZE) - end = PUD_SIZE; - offset -= address; - do { - pmd_t *pmd = pmd_alloc(mm, pud, address); - if (!pud) - return -ENOMEM; - io_remap_pmd_range(mm, pmd, address, end - address, address + offset, prot, space); - address = (address + PUD_SIZE) & PUD_MASK; - pud++; - } while (address < end); - return 0; -} - -int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, - unsigned long pfn, unsigned long size, pgprot_t prot) -{ - int error = 0; - pgd_t * dir; - unsigned long beg = from; - unsigned long end = from + size; - struct mm_struct *mm = vma->vm_mm; - int space = GET_IOSPACE(pfn); - unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; - unsigned long phys_base; - - phys_base = offset | (((unsigned long) space) << 32UL); - - /* See comment in mm/memory.c remap_pfn_range */ - vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; - vma->vm_pgoff = phys_base >> PAGE_SHIFT; - - offset -= from; - dir = pgd_offset(mm, from); - flush_cache_range(vma, beg, end); - - while (from < end) { - pud_t *pud = pud_alloc(mm, dir, from); - error = -ENOMEM; - if (!pud) - break; - error = io_remap_pud_range(mm, pud, from, end - from, offset + from, prot, space); - if (error) - break; - from = (from + PGDIR_SIZE) & PGDIR_MASK; - dir++; - } - - flush_tlb_range(vma, beg, end); - return error; -} -EXPORT_SYMBOL(io_remap_pfn_range); diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c new file mode 100644 index 00000000000..1aed0432c64 --- /dev/null +++ b/arch/sparc/mm/gup.c @@ -0,0 +1,237 @@ +/* + * Lockless get_user_pages_fast for sparc, cribbed from powerpc + * + * Copyright (C) 2008 Nick Piggin + * Copyright (C) 2008 Novell Inc. + */ + +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/vmstat.h> +#include <linux/pagemap.h> +#include <linux/rwsem.h> +#include <asm/pgtable.h> + +/* + * The performance critical leaf functions are made noinline otherwise gcc + * inlines everything into a single function which results in too much + * register pressure. + */ +static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, + unsigned long end, int write, struct page **pages, int *nr) +{ + unsigned long mask, result; + pte_t *ptep; + + if (tlb_type == hypervisor) { + result = _PAGE_PRESENT_4V|_PAGE_P_4V; + if (write) + result |= _PAGE_WRITE_4V; + } else { + result = _PAGE_PRESENT_4U|_PAGE_P_4U; + if (write) + result |= _PAGE_WRITE_4U; + } + mask = result | _PAGE_SPECIAL; + + ptep = pte_offset_kernel(&pmd, addr); + do { + struct page *page, *head; + pte_t pte = *ptep; + + if ((pte_val(pte) & mask) != result) + return 0; + VM_BUG_ON(!pfn_valid(pte_pfn(pte))); + + /* The hugepage case is simplified on sparc64 because + * we encode the sub-page pfn offsets into the + * hugepage PTEs. We could optimize this in the future + * use page_cache_add_speculative() for the hugepage case. + */ + page = pte_page(pte); + head = compound_head(page); + if (!page_cache_get_speculative(head)) + return 0; + if (unlikely(pte_val(pte) != pte_val(*ptep))) { + put_page(head); + return 0; + } + if (head != page) + get_huge_page_tail(page); + + pages[*nr] = page; + (*nr)++; + } while (ptep++, addr += PAGE_SIZE, addr != end); + + return 1; +} + +static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, + unsigned long end, int write, struct page **pages, + int *nr) +{ + struct page *head, *page, *tail; + int refs; + + if (!(pmd_val(pmd) & _PAGE_VALID)) + return 0; + + if (write && !pmd_write(pmd)) + return 0; + + refs = 0; + head = pmd_page(pmd); + page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + tail = page; + do { + VM_BUG_ON(compound_head(page) != head); + pages[*nr] = page; + (*nr)++; + page++; + refs++; + } while (addr += PAGE_SIZE, addr != end); + + if (!page_cache_add_speculative(head, refs)) { + *nr -= refs; + return 0; + } + + if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { + *nr -= refs; + while (refs--) + put_page(head); + return 0; + } + + /* Any tail page need their mapcount reference taken before we + * return. + */ + while (refs--) { + if (PageTail(tail)) + get_huge_page_tail(tail); + tail++; + } + + return 1; +} + +static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + unsigned long next; + pmd_t *pmdp; + + pmdp = pmd_offset(&pud, addr); + do { + pmd_t pmd = *pmdp; + + next = pmd_addr_end(addr, end); + if (pmd_none(pmd) || pmd_trans_splitting(pmd)) + return 0; + if (unlikely(pmd_large(pmd))) { + if (!gup_huge_pmd(pmdp, pmd, addr, next, + write, pages, nr)) + return 0; + } else if (!gup_pte_range(pmd, addr, next, write, + pages, nr)) + return 0; + } while (pmdp++, addr = next, addr != end); + + return 1; +} + +static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + unsigned long next; + pud_t *pudp; + + pudp = pud_offset(&pgd, addr); + do { + pud_t pud = *pudp; + + next = pud_addr_end(addr, end); + if (pud_none(pud)) + return 0; + if (!gup_pmd_range(pud, addr, next, write, pages, nr)) + return 0; + } while (pudp++, addr = next, addr != end); + + return 1; +} + +int get_user_pages_fast(unsigned long start, int nr_pages, int write, + struct page **pages) +{ + struct mm_struct *mm = current->mm; + unsigned long addr, len, end; + unsigned long next; + pgd_t *pgdp; + int nr = 0; + + start &= PAGE_MASK; + addr = start; + len = (unsigned long) nr_pages << PAGE_SHIFT; + end = start + len; + + /* + * XXX: batch / limit 'nr', to avoid large irq off latency + * needs some instrumenting to determine the common sizes used by + * important workloads (eg. DB2), and whether limiting the batch size + * will decrease performance. + * + * It seems like we're in the clear for the moment. Direct-IO is + * the main guy that batches up lots of get_user_pages, and even + * they are limited to 64-at-a-time which is not so many. + */ + /* + * This doesn't prevent pagetable teardown, but does prevent + * the pagetables from being freed on sparc. + * + * So long as we atomically load page table pointers versus teardown, + * we can follow the address down to the the page and take a ref on it. + */ + local_irq_disable(); + + pgdp = pgd_offset(mm, addr); + do { + pgd_t pgd = *pgdp; + + next = pgd_addr_end(addr, end); + if (pgd_none(pgd)) + goto slow; + if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) + goto slow; + } while (pgdp++, addr = next, addr != end); + + local_irq_enable(); + + VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT); + return nr; + + { + int ret; + +slow: + local_irq_enable(); + + /* Try to get the remaining pages with get_user_pages */ + start += nr << PAGE_SHIFT; + pages += nr; + + down_read(&mm->mmap_sem); + ret = get_user_pages(current, mm, start, + (end - start) >> PAGE_SHIFT, write, 0, pages, NULL); + up_read(&mm->mmap_sem); + + /* Have to be a bit careful with return values */ + if (nr > 0) { + if (ret < 0) + ret = nr; + else + ret += nr; + } + + return ret; + } +} diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 4730eac0747..449f864f0ce 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -22,14 +22,33 @@ * shared by CPUs, and so precious, and establishing them requires IPI. * Atomic kmaps are lightweight and we may have NCPUS more of them. */ -#include <linux/mm.h> #include <linux/highmem.h> -#include <asm/pgalloc.h> +#include <linux/export.h> +#include <linux/mm.h> + #include <asm/cacheflush.h> #include <asm/tlbflush.h> -#include <asm/fixmap.h> +#include <asm/pgalloc.h> +#include <asm/vaddrs.h> + +pgprot_t kmap_prot; -void *__kmap_atomic(struct page *page) +static pte_t *kmap_pte; + +void __init kmap_init(void) +{ + unsigned long address; + pmd_t *dir; + + address = __fix_to_virt(FIX_KMAP_BEGIN); + dir = pmd_offset(pgd_offset_k(address), address); + + /* cache the first kmap pte */ + kmap_pte = pte_offset_kernel(dir, address); + kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); +} + +void *kmap_atomic(struct page *page) { unsigned long vaddr; long idx, type; @@ -63,7 +82,7 @@ void *__kmap_atomic(struct page *page) return (void*) vaddr; } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { @@ -109,21 +128,3 @@ void __kunmap_atomic(void *kvaddr) pagefault_enable(); } EXPORT_SYMBOL(__kunmap_atomic); - -/* We may be fed a pagetable here by ptep_to_xxx and others. */ -struct page *kmap_atomic_to_page(void *ptr) -{ - unsigned long idx, vaddr = (unsigned long)ptr; - pte_t *pte; - - if (vaddr < SRMMU_NOCACHE_VADDR) - return virt_to_page(ptr); - if (vaddr < PKMAP_BASE) - return pfn_to_page(__nocache_pa(vaddr) >> PAGE_SHIFT); - BUG_ON(vaddr < FIXADDR_START); - BUG_ON(vaddr > FIXADDR_TOP); - - idx = virt_to_fix(vaddr); - pte = kmap_pte - (idx - FIX_KMAP_BEGIN); - return pte_page(*pte); -} diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index 5fdddf134ca..d329537739c 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -4,8 +4,6 @@ * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) */ -#include <linux/init.h> -#include <linux/module.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/hugetlb.h> @@ -22,8 +20,6 @@ /* Slightly simplified from the non-hugepage variant because by * definition we don't have to worry about any page coloring stuff */ -#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) -#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long addr, @@ -31,55 +27,28 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long pgoff, unsigned long flags) { - struct mm_struct *mm = current->mm; - struct vm_area_struct * vma; unsigned long task_size = TASK_SIZE; - unsigned long start_addr; + struct vm_unmapped_area_info info; if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; - if (unlikely(len >= VA_EXCLUDE_START)) - return -ENOMEM; - if (len > mm->cached_hole_size) { - start_addr = addr = mm->free_area_cache; - } else { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; + info.flags = 0; + info.length = len; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = min(task_size, VA_EXCLUDE_START); + info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { + VM_BUG_ON(addr != -ENOMEM); + info.low_limit = VA_EXCLUDE_END; + info.high_limit = task_size; + addr = vm_unmapped_area(&info); } - task_size -= len; - -full_search: - addr = ALIGN(addr, HPAGE_SIZE); - - for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { - /* At this point: (!vma || addr < vma->vm_end). */ - if (addr < VA_EXCLUDE_START && - (addr + len) >= VA_EXCLUDE_START) { - addr = VA_EXCLUDE_END; - vma = find_vma(mm, VA_EXCLUDE_END); - } - if (unlikely(task_size < addr)) { - if (start_addr != TASK_UNMAPPED_BASE) { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; - goto full_search; - } - return -ENOMEM; - } - if (likely(!vma || addr + len <= vma->vm_start)) { - /* - * Remember the place where we stopped the search: - */ - mm->free_area_cache = addr + len; - return addr; - } - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - addr = ALIGN(vma->vm_end, HPAGE_SIZE); - } + return addr; } static unsigned long @@ -88,71 +57,34 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long pgoff, const unsigned long flags) { - struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; + struct vm_unmapped_area_info info; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); - /* check if free_area_cache is useful for us */ - if (len <= mm->cached_hole_size) { - mm->cached_hole_size = 0; - mm->free_area_cache = mm->mmap_base; - } - - /* either no address requested or can't fit in requested address hole */ - addr = mm->free_area_cache & HPAGE_MASK; - - /* make sure it can fit in the remaining address space */ - if (likely(addr > len)) { - vma = find_vma(mm, addr-len); - if (!vma || addr <= vma->vm_start) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr-len); - } - } - - if (unlikely(mm->mmap_base < len)) - goto bottomup; - - addr = (mm->mmap_base-len) & HPAGE_MASK; - - do { - /* - * Lookup failure means no vma is above this address, - * else if new region fits below vma->vm_start, - * return with success: - */ - vma = find_vma(mm, addr); - if (likely(!vma || addr+len <= vma->vm_start)) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr); - } + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = mm->mmap_base; + info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_offset = 0; + addr = vm_unmapped_area(&info); - /* remember the largest hole we saw so far */ - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - /* try just below the current vma->vm_start */ - addr = (vma->vm_start-len) & HPAGE_MASK; - } while (likely(len < vma->vm_start)); - -bottomup: /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ - mm->cached_hole_size = ~0UL; - mm->free_area_cache = TASK_UNMAPPED_BASE; - addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags); - /* - * Restore the topdown base: - */ - mm->free_area_cache = mm->mmap_base; - mm->cached_hole_size = ~0UL; + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = STACK_TOP32; + addr = vm_unmapped_area(&info); + } return addr; } @@ -214,7 +146,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, if (pud) { pmd = pmd_alloc(mm, pud, addr); if (pmd) - pte = pte_alloc_map(mm, pmd, addr); + pte = pte_alloc_map(mm, NULL, pmd, addr); } return pte; } @@ -304,53 +236,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, { return NULL; } - -static void context_reload(void *__data) -{ - struct mm_struct *mm = __data; - - if (mm == current->mm) - load_secondary_context(mm); -} - -void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ - struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE]; - - if (likely(tp->tsb != NULL)) - return; - - tsb_grow(mm, MM_TSB_HUGE, 0); - tsb_context_switch(mm); - smp_tsb_sync(mm); - - /* On UltraSPARC-III+ and later, configure the second half of - * the Data-TLB for huge pages. - */ - if (tlb_type == cheetah_plus) { - unsigned long ctx; - - spin_lock(&ctx_alloc_lock); - ctx = mm->context.sparc64_ctx_val; - ctx &= ~CTX_PGSZ_MASK; - ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; - ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; - - if (ctx != mm->context.sparc64_ctx_val) { - /* When changing the page size fields, we - * must perform a context flush so that no - * stale entries match. This flush must - * occur with the original context register - * settings. - */ - do_flush_tlb_mm(mm); - - /* Reload the context register of all processors - * also executing in this address space. - */ - mm->context.sparc64_ctx_val = ctx; - on_each_cpu(context_reload, mm, 0); - } - spin_unlock(&ctx_alloc_lock); - } -} diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S index 44aad32eeb4..969f96450f6 100644 --- a/arch/sparc/mm/hypersparc.S +++ b/arch/sparc/mm/hypersparc.S @@ -74,7 +74,7 @@ hypersparc_flush_cache_mm_out: /* The things we do for performance... */ hypersparc_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 #ifndef CONFIG_SMP ld [%o0 + AOFF_mm_context], %g1 cmp %g1, -1 @@ -163,7 +163,7 @@ hypersparc_flush_cache_range_out: */ /* Verified, my ass... */ hypersparc_flush_cache_page: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %g2 #ifndef CONFIG_SMP cmp %g2, -1 @@ -284,7 +284,7 @@ hypersparc_flush_tlb_mm_out: sta %g5, [%g1] ASI_M_MMUREGS hypersparc_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -307,7 +307,7 @@ hypersparc_flush_tlb_range_out: sta %g5, [%g1] ASI_M_MMUREGS hypersparc_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index 6d0e02c4fe0..eb828715527 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -27,17 +27,16 @@ #include <linux/gfp.h> #include <asm/sections.h> -#include <asm/system.h> -#include <asm/vac-ops.h> #include <asm/page.h> #include <asm/pgtable.h> #include <asm/vaddrs.h> #include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */ +#include <asm/setup.h> #include <asm/tlb.h> #include <asm/prom.h> #include <asm/leon.h> -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); +#include "mm_32.h" unsigned long *sparc_valid_addr_bitmap; EXPORT_SYMBOL(sparc_valid_addr_bitmap); @@ -48,13 +47,7 @@ EXPORT_SYMBOL(phys_base); unsigned long pfn_base; EXPORT_SYMBOL(pfn_base); -unsigned long page_kernel; -EXPORT_SYMBOL(page_kernel); - struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; -unsigned long sparc_unmapped_base; - -struct pgtable_cache_struct pgt_quicklists; /* Initial ramdisk setup */ extern unsigned int sparc_ramdisk_image; @@ -62,56 +55,17 @@ extern unsigned int sparc_ramdisk_size; unsigned long highstart_pfn, highend_pfn; -pte_t *kmap_pte; -pgprot_t kmap_prot; - -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)) - -void __init kmap_init(void) -{ - /* cache the first kmap pte */ - kmap_pte = kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN)); - kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); -} - -void show_mem(void) +void show_mem(unsigned int filter) { printk("Mem-info:\n"); - show_free_areas(); + show_free_areas(filter); printk("Free swap: %6ldkB\n", - nr_swap_pages << (PAGE_SHIFT-10)); + get_nr_swap_pages() << (PAGE_SHIFT-10)); printk("%ld pages of RAM\n", totalram_pages); printk("%ld free pages\n", nr_free_pages()); -#if 0 /* undefined pgtable_cache_size, pgd_cache_size */ - printk("%ld pages in page table cache\n",pgtable_cache_size); -#ifndef CONFIG_SMP - if (sparc_cpu_model == sun4m || sparc_cpu_model == sun4d) - printk("%ld entries in page dir cache\n",pgd_cache_size); -#endif -#endif } -void __init sparc_context_init(int numctx) -{ - int ctx; - - ctx_list_pool = __alloc_bootmem(numctx * sizeof(struct ctx_list), SMP_CACHE_BYTES, 0UL); - - for(ctx = 0; ctx < numctx; ctx++) { - struct ctx_list *clist; - clist = (ctx_list_pool + ctx); - clist->ctx_number = ctx; - clist->ctx_mm = NULL; - } - ctx_free.next = ctx_free.prev = &ctx_free; - ctx_used.next = ctx_used.prev = &ctx_used; - for(ctx = 0; ctx < numctx; ctx++) - add_to_free_ctxlist(ctx_list_pool + ctx); -} - -extern unsigned long cmdline_memory_size; unsigned long last_valid_pfn; unsigned long calc_highpages(void) @@ -290,78 +244,13 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) } /* - * check_pgt_cache - * - * This is called at the end of unmapping of VMA (zap_page_range), - * to rescan the page cache for architecture specific things, - * presumably something like sun4/sun4c PMEGs. Most architectures - * define check_pgt_cache empty. - * - * We simply copy the 2.4 implementation for now. - */ -static int pgt_cache_water[2] = { 25, 50 }; - -void check_pgt_cache(void) -{ - do_check_pgt_cache(pgt_cache_water[0], pgt_cache_water[1]); -} - -/* * paging_init() sets up the page tables: We call the MMU specific * init routine based upon the Sun model type on the Sparc. * */ -extern void sun4c_paging_init(void); -extern void srmmu_paging_init(void); -extern void device_scan(void); - -pgprot_t PAGE_SHARED __read_mostly; -EXPORT_SYMBOL(PAGE_SHARED); - void __init paging_init(void) { - switch(sparc_cpu_model) { - case sun4c: - case sun4e: - case sun4: - sun4c_paging_init(); - sparc_unmapped_base = 0xe0000000; - BTFIXUPSET_SETHI(sparc_unmapped_base, 0xe0000000); - break; - case sparc_leon: - leon_init(); - /* fall through */ - case sun4m: - case sun4d: - srmmu_paging_init(); - sparc_unmapped_base = 0x50000000; - BTFIXUPSET_SETHI(sparc_unmapped_base, 0x50000000); - break; - default: - prom_printf("paging_init: Cannot init paging on this Sparc\n"); - prom_printf("paging_init: sparc_cpu_model = %d\n", sparc_cpu_model); - prom_printf("paging_init: Halting...\n"); - prom_halt(); - }; - - /* Initialize the protection map with non-constant, MMU dependent values. */ - protection_map[0] = PAGE_NONE; - protection_map[1] = PAGE_READONLY; - protection_map[2] = PAGE_COPY; - protection_map[3] = PAGE_COPY; - protection_map[4] = PAGE_READONLY; - protection_map[5] = PAGE_READONLY; - protection_map[6] = PAGE_COPY; - protection_map[7] = PAGE_COPY; - protection_map[8] = PAGE_NONE; - protection_map[9] = PAGE_READONLY; - protection_map[10] = PAGE_SHARED; - protection_map[11] = PAGE_SHARED; - protection_map[12] = PAGE_READONLY; - protection_map[13] = PAGE_READONLY; - protection_map[14] = PAGE_SHARED; - protection_map[15] = PAGE_SHARED; - btfixup(); + srmmu_paging_init(); prom_build_devicetree(); of_fill_in_cpu_data(); device_scan(); @@ -392,22 +281,12 @@ static void map_high_region(unsigned long start_pfn, unsigned long end_pfn) printk("mapping high region %08lx - %08lx\n", start_pfn, end_pfn); #endif - for (tmp = start_pfn; tmp < end_pfn; tmp++) { - struct page *page = pfn_to_page(tmp); - - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - totalhigh_pages++; - } + for (tmp = start_pfn; tmp < end_pfn; tmp++) + free_highmem_page(pfn_to_page(tmp)); } void __init mem_init(void) { - int codepages = 0; - int datapages = 0; - int initpages = 0; - int reservedpages = 0; int i; if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) { @@ -439,15 +318,12 @@ void __init mem_init(void) max_mapnr = last_valid_pfn - pfn_base; high_memory = __va(max_low_pfn << PAGE_SHIFT); - - totalram_pages = free_all_bootmem(); + free_all_bootmem(); for (i = 0; sp_banks[i].num_bytes != 0; i++) { unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; - num_physpages += sp_banks[i].num_bytes >> PAGE_SHIFT; - if (end_pfn <= highstart_pfn) continue; @@ -457,72 +333,19 @@ void __init mem_init(void) map_high_region(start_pfn, end_pfn); } - totalram_pages += totalhigh_pages; - - codepages = (((unsigned long) &_etext) - ((unsigned long)&_start)); - codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT; - datapages = (((unsigned long) &_edata) - ((unsigned long)&_etext)); - datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT; - initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin)); - initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT; - - /* Ignore memory holes for the purpose of counting reserved pages */ - for (i=0; i < max_low_pfn; i++) - if (test_bit(i >> (20 - PAGE_SHIFT), sparc_valid_addr_bitmap) - && PageReserved(pfn_to_page(i))) - reservedpages++; - - printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n", - nr_free_pages() << (PAGE_SHIFT-10), - num_physpages << (PAGE_SHIFT - 10), - codepages << (PAGE_SHIFT-10), - reservedpages << (PAGE_SHIFT - 10), - datapages << (PAGE_SHIFT-10), - initpages << (PAGE_SHIFT-10), - totalhigh_pages << (PAGE_SHIFT-10)); + mem_init_print_info(NULL); } void free_initmem (void) { - unsigned long addr; - unsigned long freed; - - addr = (unsigned long)(&__init_begin); - freed = (unsigned long)(&__init_end) - addr; - for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) { - struct page *p; - - memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); - p = virt_to_page(addr); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - totalram_pages++; - num_physpages++; - } - printk(KERN_INFO "Freeing unused kernel memory: %ldk freed\n", - freed >> 10); + free_initmem_default(POISON_FREE_INITMEM); } #ifdef CONFIG_BLK_DEV_INITRD void free_initrd_mem(unsigned long start, unsigned long end) { - if (start < end) - printk(KERN_INFO "Freeing initrd memory: %ldk freed\n", - (end - start) >> 10); - for (; start < end; start += PAGE_SIZE) { - struct page *p; - - memset((void *)start, POISON_FREE_INITMEM, PAGE_SIZE); - p = virt_to_page(start); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - totalram_pages++; - num_physpages++; - } + free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, + "initrd"); } #endif diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 2f6ae1d1fb6..16b58ff11e6 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -28,7 +28,6 @@ #include <linux/gfp.h> #include <asm/head.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> @@ -48,30 +47,49 @@ #include <asm/prom.h> #include <asm/mdesc.h> #include <asm/cpudata.h> +#include <asm/setup.h> #include <asm/irq.h> #include "init_64.h" -unsigned long kern_linear_pte_xor[2] __read_mostly; +unsigned long kern_linear_pte_xor[4] __read_mostly; -/* A bitmap, one bit for every 256MB of physical memory. If the bit - * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else - * if set we should use a 256MB page (via kern_linear_pte_xor[1]). +/* A bitmap, two bits for every 256MB of physical memory. These two + * bits determine what page size we use for kernel linear + * translations. They form an index into kern_linear_pte_xor[]. The + * value in the indexed slot is XOR'd with the TLB miss virtual + * address to form the resulting TTE. The mapping is: + * + * 0 ==> 4MB + * 1 ==> 256MB + * 2 ==> 2GB + * 3 ==> 16GB + * + * All sun4v chips support 256MB pages. Only SPARC-T4 and later + * support 2GB pages, and hopefully future cpus will support the 16GB + * pages as well. For slots 2 and 3, we encode a 256MB TTE xor there + * if these larger page sizes are not supported by the cpu. + * + * It would be nice to determine this from the machine description + * 'cpu' properties, but we need to have this table setup before the + * MDESC is initialized. */ unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; #ifndef CONFIG_DEBUG_PAGEALLOC -/* A special kernel TSB for 4MB and 256MB linear mappings. - * Space is allocated for this right after the trap table - * in arch/sparc64/kernel/head.S +/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. + * Space is allocated for this right after the trap table in + * arch/sparc64/kernel/head.S */ extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; #endif +static unsigned long cpu_pgsz_mask; + #define MAX_BANKS 32 -static struct linux_prom64_registers pavail[MAX_BANKS] __devinitdata; -static int pavail_ents __devinitdata; +static struct linux_prom64_registers pavail[MAX_BANKS]; +static int pavail_ents; static int cmp_p64(const void *a, const void *b) { @@ -102,7 +120,8 @@ static void __init read_obp_memory(const char *property, ret = prom_getproperty(node, property, (char *) regs, prop_size); if (ret == -1) { - prom_printf("Couldn't get %s property from /memory.\n"); + prom_printf("Couldn't get %s property from /memory.\n", + property); prom_halt(); } @@ -258,7 +277,6 @@ static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long } unsigned long _PAGE_ALL_SZ_BITS __read_mostly; -unsigned long _PAGE_SZBITS __read_mostly; static void flush_dcache(unsigned long pfn) { @@ -289,12 +307,39 @@ static void flush_dcache(unsigned long pfn) } } +/* mm->context.lock must be held */ +static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index, + unsigned long tsb_hash_shift, unsigned long address, + unsigned long tte) +{ + struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb; + unsigned long tag; + + if (unlikely(!tsb)) + return; + + tsb += ((address >> tsb_hash_shift) & + (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); + tag = (address >> 22UL); + tsb_insert(tsb, tag, tte); +} + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static inline bool is_hugetlb_pte(pte_t pte) +{ + if ((tlb_type == hypervisor && + (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || + (tlb_type != hypervisor && + (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) + return true; + return false; +} +#endif + void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { struct mm_struct *mm; - struct tsb *tsb; - unsigned long tag, flags; - unsigned long tsb_index, tsb_hash_shift; + unsigned long flags; pte_t pte = *ptep; if (tlb_type != hypervisor) { @@ -306,28 +351,16 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * mm = vma->vm_mm; - tsb_index = MM_TSB_BASE; - tsb_hash_shift = PAGE_SHIFT; - spin_lock_irqsave(&mm->context.lock, flags); -#ifdef CONFIG_HUGETLB_PAGE - if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) { - if ((tlb_type == hypervisor && - (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || - (tlb_type != hypervisor && - (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) { - tsb_index = MM_TSB_HUGE; - tsb_hash_shift = HPAGE_SHIFT; - } - } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, + address, pte_val(pte)); + else #endif - - tsb = mm->context.tsb_block[tsb_index].tsb; - tsb += ((address >> tsb_hash_shift) & - (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); - tag = (address >> 22UL); - tsb_insert(tsb, tag, pte_val(pte)); + __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, + address, pte_val(pte)); spin_unlock_irqrestore(&mm->context.lock, flags); } @@ -404,6 +437,12 @@ EXPORT_SYMBOL(flush_icache_range); void mmu_info(struct seq_file *m) { + static const char *pgsz_strings[] = { + "8K", "64K", "512K", "4MB", "32MB", + "256MB", "2GB", "16GB", + }; + int i, printed; + if (tlb_type == cheetah) seq_printf(m, "MMU Type\t: Cheetah\n"); else if (tlb_type == cheetah_plus) @@ -415,6 +454,17 @@ void mmu_info(struct seq_file *m) else seq_printf(m, "MMU Type\t: ???\n"); + seq_printf(m, "MMU PGSZs\t: "); + printed = 0; + for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) { + if (cpu_pgsz_mask & (1UL << i)) { + seq_printf(m, "%s%s", + printed ? "," : "", pgsz_strings[i]); + printed++; + } + } + seq_putc(m, '\n'); + #ifdef CONFIG_DEBUG_DCFLUSH seq_printf(m, "DCPageFlushes\t: %d\n", atomic_read(&dcpage_flushes)); @@ -463,7 +513,7 @@ static void __init read_obp_translations(void) prom_halt(); } if (unlikely(n > sizeof(prom_trans))) { - prom_printf("prom_mappings: Size %Zd is too big.\n", n); + prom_printf("prom_mappings: Size %d is too big.\n", n); prom_halt(); } @@ -511,6 +561,11 @@ static void __init read_obp_translations(void) for (i = 0; i < prom_trans_ents; i++) prom_trans[i].data &= ~0x0003fe0000000000UL; } + + /* Force execute bit on. */ + for (i = 0; i < prom_trans_ents; i++) + prom_trans[i].data |= (tlb_type == hypervisor ? + _PAGE_EXEC_4V : _PAGE_EXEC_4U); } static void __init hypervisor_tlb_lock(unsigned long vaddr, @@ -520,7 +575,7 @@ static void __init hypervisor_tlb_lock(unsigned long vaddr, unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu); if (ret != 0) { - prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: " + prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: " "errors with %lx\n", vaddr, 0, pte, mmu, ret); prom_halt(); } @@ -534,7 +589,7 @@ static void __init remap_kernel(void) int i, tlb_ent = sparc64_highest_locked_tlbent(); tte_vaddr = (unsigned long) KERNBASE; - phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; tte_data = kern_large_tte(phys_page); kern_locked_tte_data = tte_data; @@ -576,7 +631,7 @@ static void __init inherit_prom_mappings(void) void prom_world(int enter) { if (!enter) - set_fs((mm_segment_t) { get_thread_current_ds() }); + set_fs(get_fs()); __asm__ __volatile__("flushw"); } @@ -627,10 +682,9 @@ void get_new_mmu_context(struct mm_struct *mm) { unsigned long ctx, new_ctx; unsigned long orig_pgsz_bits; - unsigned long flags; int new_version; - spin_lock_irqsave(&ctx_alloc_lock, flags); + spin_lock(&ctx_alloc_lock); orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK); ctx = (tlb_context_cache + 1) & CTX_NR_MASK; new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx); @@ -666,7 +720,7 @@ void get_new_mmu_context(struct mm_struct *mm) out: tlb_context_cache = new_ctx; mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits; - spin_unlock_irqrestore(&ctx_alloc_lock, flags); + spin_unlock(&ctx_alloc_lock); if (unlikely(new_version)) smp_new_mmu_context_version(); @@ -737,16 +791,15 @@ static void __init find_ramdisk(unsigned long phys_base) struct node_mem_mask { unsigned long mask; unsigned long val; - unsigned long bootmem_paddr; }; static struct node_mem_mask node_masks[MAX_NUMNODES]; static int num_node_masks; +#ifdef CONFIG_NEED_MULTIPLE_NODES + int numa_cpu_lookup_table[NR_CPUS]; cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; -#ifdef CONFIG_NEED_MULTIPLE_NODES - struct mdesc_mblock { u64 base; u64 size; @@ -785,7 +838,7 @@ static int find_node(unsigned long addr) return -1; } -u64 memblock_nid_range(u64 start, u64 end, int *nid) +static u64 memblock_nid_range(u64 start, u64 end, int *nid) { *nid = find_node(start); start += PAGE_SIZE; @@ -802,24 +855,19 @@ u64 memblock_nid_range(u64 start, u64 end, int *nid) return start; } -#else -u64 memblock_nid_range(u64 start, u64 end, int *nid) -{ - *nid = 0; - return end; -} #endif /* This must be invoked after performing all of the necessary - * add_active_range() calls for 'nid'. We need to be able to get + * memblock_set_node() calls for 'nid'. We need to be able to get * correct data from get_pfn_range_for_nid(). */ static void __init allocate_node_data(int nid) { - unsigned long paddr, num_pages, start_pfn, end_pfn; struct pglist_data *p; - + unsigned long start_pfn, end_pfn; #ifdef CONFIG_NEED_MULTIPLE_NODES + unsigned long paddr; + paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid); if (!paddr) { prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid); @@ -828,7 +876,7 @@ static void __init allocate_node_data(int nid) NODE_DATA(nid) = __va(paddr); memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); - NODE_DATA(nid)->bdata = &bootmem_node_data[nid]; + NODE_DATA(nid)->node_id = nid; #endif p = NODE_DATA(nid); @@ -836,33 +884,25 @@ static void __init allocate_node_data(int nid) get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); p->node_start_pfn = start_pfn; p->node_spanned_pages = end_pfn - start_pfn; - - if (p->node_spanned_pages) { - num_pages = bootmem_bootmap_pages(p->node_spanned_pages); - - paddr = memblock_alloc_try_nid(num_pages << PAGE_SHIFT, PAGE_SIZE, nid); - if (!paddr) { - prom_printf("Cannot allocate bootmap for nid[%d]\n", - nid); - prom_halt(); - } - node_masks[nid].bootmem_paddr = paddr; - } } static void init_node_masks_nonnuma(void) { +#ifdef CONFIG_NEED_MULTIPLE_NODES int i; +#endif numadbg("Initializing tables for non-numa.\n"); node_masks[0].mask = node_masks[0].val = 0; num_node_masks = 1; +#ifdef CONFIG_NEED_MULTIPLE_NODES for (i = 0; i < NR_CPUS; i++) numa_cpu_lookup_table[i] = 0; - numa_cpumask_lookup_table[0] = CPU_MASK_ALL; + cpumask_setall(&numa_cpumask_lookup_table[0]); +#endif } #ifdef CONFIG_NEED_MULTIPLE_NODES @@ -982,14 +1022,12 @@ static void __init add_node_ranges(void) this_end = memblock_nid_range(start, end, &nid); - numadbg("Adding active range nid[%d] " + numadbg("Setting memblock NUMA node nid[%d] " "start[%lx] end[%lx]\n", nid, start, this_end); - add_active_range(nid, - start >> PAGE_SHIFT, - this_end >> PAGE_SHIFT); - + memblock_set_node(start, this_end - start, + &memblock.memory, nid); start = this_end; } } @@ -1066,7 +1104,14 @@ static int __init grab_mblocks(struct mdesc_handle *md) m->size = *val; val = mdesc_get_property(md, node, "address-congruence-offset", NULL); - m->offset = *val; + + /* The address-congruence-offset property is optional. + * Explicity zero it be identifty this. + */ + if (val) + m->offset = *val; + else + m->offset = 0UL; numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n", count - 1, m->base, m->size, m->offset); @@ -1080,7 +1125,7 @@ static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, { u64 arc; - cpus_clear(*mask); + cpumask_clear(mask); mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) { u64 target = mdesc_arc_target(md, arc); @@ -1091,7 +1136,7 @@ static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, continue; id = mdesc_get_property(md, target, "id", NULL); if (*id < nr_cpu_ids) - cpu_set(*id, *mask); + cpumask_set_cpu(*id, mask); } } @@ -1153,13 +1198,13 @@ static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp, numa_parse_mdesc_group_cpus(md, grp, &mask); - for_each_cpu_mask(cpu, mask) + for_each_cpu(cpu, &mask) numa_cpu_lookup_table[cpu] = index; - numa_cpumask_lookup_table[index] = mask; + cpumask_copy(&numa_cpumask_lookup_table[index], &mask); if (numa_debug) { printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index); - for_each_cpu_mask(cpu, mask) + for_each_cpu(cpu, &mask) printk("%d ", cpu); printk("]\n"); } @@ -1218,7 +1263,7 @@ static int __init numa_parse_jbus(void) index = 0; for_each_present_cpu(cpu) { numa_cpu_lookup_table[cpu] = index; - numa_cpumask_lookup_table[index] = cpumask_of_cpu(cpu); + cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu)); node_masks[index].mask = ~((1UL << 36UL) - 1UL); node_masks[index].val = cpu << 36UL; @@ -1277,7 +1322,6 @@ static void __init bootmem_init_nonnuma(void) { unsigned long top_of_ram = memblock_end_of_DRAM(); unsigned long total_ram = memblock_phys_mem_size(); - struct memblock_region *reg; numadbg("bootmem_init_nonnuma()\n"); @@ -1287,92 +1331,14 @@ static void __init bootmem_init_nonnuma(void) (top_of_ram - total_ram) >> 20); init_node_masks_nonnuma(); - - for_each_memblock(memory, reg) { - unsigned long start_pfn, end_pfn; - - if (!reg->size) - continue; - - start_pfn = memblock_region_memory_base_pfn(reg); - end_pfn = memblock_region_memory_end_pfn(reg); - add_active_range(0, start_pfn, end_pfn); - } - + memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0); allocate_node_data(0); - node_set_online(0); } -static void __init reserve_range_in_node(int nid, unsigned long start, - unsigned long end) -{ - numadbg(" reserve_range_in_node(nid[%d],start[%lx],end[%lx]\n", - nid, start, end); - while (start < end) { - unsigned long this_end; - int n; - - this_end = memblock_nid_range(start, end, &n); - if (n == nid) { - numadbg(" MATCH reserving range [%lx:%lx]\n", - start, this_end); - reserve_bootmem_node(NODE_DATA(nid), start, - (this_end - start), BOOTMEM_DEFAULT); - } else - numadbg(" NO MATCH, advancing start to %lx\n", - this_end); - - start = this_end; - } -} - -static void __init trim_reserved_in_node(int nid) -{ - struct memblock_region *reg; - - numadbg(" trim_reserved_in_node(%d)\n", nid); - - for_each_memblock(reserved, reg) - reserve_range_in_node(nid, reg->base, reg->base + reg->size); -} - -static void __init bootmem_init_one_node(int nid) -{ - struct pglist_data *p; - - numadbg("bootmem_init_one_node(%d)\n", nid); - - p = NODE_DATA(nid); - - if (p->node_spanned_pages) { - unsigned long paddr = node_masks[nid].bootmem_paddr; - unsigned long end_pfn; - - end_pfn = p->node_start_pfn + p->node_spanned_pages; - - numadbg(" init_bootmem_node(%d, %lx, %lx, %lx)\n", - nid, paddr >> PAGE_SHIFT, p->node_start_pfn, end_pfn); - - init_bootmem_node(p, paddr >> PAGE_SHIFT, - p->node_start_pfn, end_pfn); - - numadbg(" free_bootmem_with_active_regions(%d, %lx)\n", - nid, end_pfn); - free_bootmem_with_active_regions(nid, end_pfn); - - trim_reserved_in_node(nid); - - numadbg(" sparse_memory_present_with_active_regions(%d)\n", - nid); - sparse_memory_present_with_active_regions(nid); - } -} - static unsigned long __init bootmem_init(unsigned long phys_base) { unsigned long end_pfn; - int nid; end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; max_pfn = max_low_pfn = end_pfn; @@ -1381,11 +1347,12 @@ static unsigned long __init bootmem_init(unsigned long phys_base) if (bootmem_init_numa() < 0) bootmem_init_nonnuma(); - /* XXX cpu notifier XXX */ + /* Dump memblock with node info. */ + memblock_dump_all(); - for_each_online_node(nid) - bootmem_init_one_node(nid); + /* XXX cpu notifier XXX */ + sparse_memory_present_with_active_regions(MAX_NUMNODES); sparse_init(); return end_pfn; @@ -1453,32 +1420,75 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, extern unsigned int kvmap_linear_patch[1]; #endif /* CONFIG_DEBUG_PAGEALLOC */ -static void __init mark_kpte_bitmap(unsigned long start, unsigned long end) +static void __init kpte_set_val(unsigned long index, unsigned long val) { - const unsigned long shift_256MB = 28; - const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL); - const unsigned long size_256MB = (1UL << shift_256MB); + unsigned long *ptr = kpte_linear_bitmap; - while (start < end) { - long remains; + val <<= ((index % (BITS_PER_LONG / 2)) * 2); + ptr += (index / (BITS_PER_LONG / 2)); - remains = end - start; - if (remains < size_256MB) - break; + *ptr |= val; +} - if (start & mask_256MB) { - start = (start + size_256MB) & ~mask_256MB; - continue; - } +static const unsigned long kpte_shift_min = 28; /* 256MB */ +static const unsigned long kpte_shift_max = 34; /* 16GB */ +static const unsigned long kpte_shift_incr = 3; + +static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end, + unsigned long shift) +{ + unsigned long size = (1UL << shift); + unsigned long mask = (size - 1UL); + unsigned long remains = end - start; + unsigned long val; - while (remains >= size_256MB) { - unsigned long index = start >> shift_256MB; + if (remains < size || (start & mask)) + return start; - __set_bit(index, kpte_linear_bitmap); + /* VAL maps: + * + * shift 28 --> kern_linear_pte_xor index 1 + * shift 31 --> kern_linear_pte_xor index 2 + * shift 34 --> kern_linear_pte_xor index 3 + */ + val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1; + + remains &= ~mask; + if (shift != kpte_shift_max) + remains = size; - start += size_256MB; - remains -= size_256MB; + while (remains) { + unsigned long index = start >> kpte_shift_min; + + kpte_set_val(index, val); + + start += 1UL << kpte_shift_min; + remains -= 1UL << kpte_shift_min; + } + + return start; +} + +static void __init mark_kpte_bitmap(unsigned long start, unsigned long end) +{ + unsigned long smallest_size, smallest_mask; + unsigned long s; + + smallest_size = (1UL << kpte_shift_min); + smallest_mask = (smallest_size - 1UL); + + while (start < end) { + unsigned long orig_start = start; + + for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) { + start = kpte_mark_using_shift(start, end, s); + + if (start != orig_start) + break; } + + if (start == orig_start) + start = (start + smallest_size) & ~smallest_mask; } } @@ -1553,6 +1563,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size) return ~0UL; } +unsigned long PAGE_OFFSET; +EXPORT_SYMBOL(PAGE_OFFSET); + +static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits) +{ + unsigned long final_shift; + unsigned int val = *insn; + unsigned int cnt; + + /* We are patching in ilog2(max_supported_phys_address), and + * we are doing so in a manner similar to a relocation addend. + * That is, we are adding the shift value to whatever value + * is in the shift instruction count field already. + */ + cnt = (val & 0x3f); + val &= ~0x3f; + + /* If we are trying to shift >= 64 bits, clear the destination + * register. This can happen when phys_bits ends up being equal + * to MAX_PHYS_ADDRESS_BITS. + */ + final_shift = (cnt + (64 - phys_bits)); + if (final_shift >= 64) { + unsigned int rd = (val >> 25) & 0x1f; + + val = 0x80100000 | (rd << 25); + } else { + val |= final_shift; + } + *insn = val; + + __asm__ __volatile__("flush %0" + : /* no outputs */ + : "r" (insn)); +} + +static void __init page_offset_shift_patch(unsigned long phys_bits) +{ + extern unsigned int __page_offset_shift_patch; + extern unsigned int __page_offset_shift_patch_end; + unsigned int *p; + + p = &__page_offset_shift_patch; + while (p < &__page_offset_shift_patch_end) { + unsigned int *insn = (unsigned int *)(unsigned long)*p; + + page_offset_shift_patch_one(insn, phys_bits); + + p++; + } +} + +static void __init setup_page_offset(void) +{ + unsigned long max_phys_bits = 40; + + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + max_phys_bits = 42; + } else if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + max_phys_bits = 39; + break; + case SUN4V_CHIP_NIAGARA3: + max_phys_bits = 43; + break; + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC64X: + default: + max_phys_bits = 47; + break; + } + } + + if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) { + prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n", + max_phys_bits); + prom_halt(); + } + + PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits); + + pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", + PAGE_OFFSET, max_phys_bits); + + page_offset_shift_patch(max_phys_bits); +} + static void __init tsb_phys_patch(void) { struct tsb_ldquad_phys_patch_entry *pquad; @@ -1597,6 +1697,44 @@ static void __init tsb_phys_patch(void) static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; +static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) +{ + pa >>= KTSB_PHYS_SHIFT; + + while (start < end) { + unsigned int *ia = (unsigned int *)(unsigned long)*start; + + ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); + __asm__ __volatile__("flush %0" : : "r" (ia)); + + ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 1)); + + start++; + } +} + +static void ktsb_phys_patch(void) +{ + extern unsigned int __swapper_tsb_phys_patch; + extern unsigned int __swapper_tsb_phys_patch_end; + unsigned long ktsb_pa; + + ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE); + patch_one_ktsb_phys(&__swapper_tsb_phys_patch, + &__swapper_tsb_phys_patch_end, ktsb_pa); +#ifndef CONFIG_DEBUG_PAGEALLOC + { + extern unsigned int __swapper_4m_tsb_phys_patch; + extern unsigned int __swapper_4m_tsb_phys_patch_end; + ktsb_pa = (kern_base + + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); + patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch, + &__swapper_4m_tsb_phys_patch_end, ktsb_pa); + } +#endif +} + static void __init sun4v_ktsb_init(void) { unsigned long ktsb_pa; @@ -1625,7 +1763,7 @@ static void __init sun4v_ktsb_init(void) ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB; ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB; break; - }; + } ktsb_descr[0].assoc = 1; ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES; @@ -1634,13 +1772,16 @@ static void __init sun4v_ktsb_init(void) ktsb_descr[0].resv = 0; #ifndef CONFIG_DEBUG_PAGEALLOC - /* Second KTSB for 4MB/256MB mappings. */ + /* Second KTSB for 4MB/256MB/2GB/16GB mappings. */ ktsb_pa = (kern_base + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB; - ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB | - HV_PGSZ_MASK_256MB); + ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB | + HV_PGSZ_MASK_256MB | + HV_PGSZ_MASK_2GB | + HV_PGSZ_MASK_16GB) & + cpu_pgsz_mask); ktsb_descr[1].assoc = 1; ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES; ktsb_descr[1].ctx_idx = 0; @@ -1649,7 +1790,7 @@ static void __init sun4v_ktsb_init(void) #endif } -void __cpuinit sun4v_ktsb_register(void) +void sun4v_ktsb_register(void) { unsigned long pa, ret; @@ -1663,10 +1804,51 @@ void __cpuinit sun4v_ktsb_register(void) } } +static void __init sun4u_linear_pte_xor_finalize(void) +{ +#ifndef CONFIG_DEBUG_PAGEALLOC + /* This is where we would add Panther support for + * 32MB and 256MB pages. + */ +#endif +} + +static void __init sun4v_linear_pte_xor_finalize(void) +{ +#ifndef CONFIG_DEBUG_PAGEALLOC + if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { + kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { + kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[2] = kern_linear_pte_xor[1]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { + kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[3] = kern_linear_pte_xor[2]; + } +#endif +} + /* paging_init() sets up the page tables */ static unsigned long last_valid_pfn; -pgd_t swapper_pg_dir[2048]; +pgd_t swapper_pg_dir[PTRS_PER_PGD]; static void sun4u_pgprot_init(void); static void sun4v_pgprot_init(void); @@ -1675,6 +1857,9 @@ void __init paging_init(void) { unsigned long end_pfn, shift, phys_base; unsigned long real_end, i; + int node; + + setup_page_offset(); /* These build time checkes make sure that the dcache_dirty_cpu() * page->flags usage will work. @@ -1701,7 +1886,7 @@ void __init paging_init(void) BUILD_BUG_ON(NR_CPUS > 4096); - kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; kern_size = (unsigned long)&_end - (unsigned long)KERNBASE; /* Invalidate both kernel TSBs. */ @@ -1716,15 +1901,13 @@ void __init paging_init(void) sun4u_pgprot_init(); if (tlb_type == cheetah_plus || - tlb_type == hypervisor) + tlb_type == hypervisor) { tsb_phys_patch(); - - if (tlb_type == hypervisor) { - sun4v_patch_tlb_handlers(); - sun4v_ktsb_init(); + ktsb_phys_patch(); } - memblock_init(); + if (tlb_type == hypervisor) + sun4v_patch_tlb_handlers(); /* Find available physical memory... * @@ -1751,7 +1934,7 @@ void __init paging_init(void) memblock_enforce_memory_limit(cmdline_memory_size); - memblock_analyze(); + memblock_allow_resize(); memblock_dump_all(); set_bit(0, mmu_context_bmap); @@ -1759,7 +1942,7 @@ void __init paging_init(void) shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE); real_end = (unsigned long)_end; - num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22); + num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB); printk("Kernel: Using %d locked TLB entries for main kernel image.\n", num_kernel_image_mappings); @@ -1783,9 +1966,6 @@ void __init paging_init(void) __flush_tlb_all(); - if (tlb_type == hypervisor) - sun4v_ktsb_register(); - prom_build_devicetree(); of_populate_present_mask(); #ifndef CONFIG_SMP @@ -1798,24 +1978,54 @@ void __init paging_init(void) #ifndef CONFIG_SMP mdesc_fill_in_cpu_data(cpu_all_mask); #endif + mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask); + + sun4v_linear_pte_xor_finalize(); + + sun4v_ktsb_init(); + sun4v_ktsb_register(); + } else { + unsigned long impl, ver; + + cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K | + HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB); + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + impl = ((ver >> 32) & 0xffff); + if (impl == PANTHER_IMPL) + cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB | + HV_PGSZ_MASK_256MB); + + sun4u_linear_pte_xor_finalize(); } + /* Flush the TLBs and the 4M TSB so that the updated linear + * pte XOR settings are realized for all mappings. + */ + __flush_tlb_all(); +#ifndef CONFIG_DEBUG_PAGEALLOC + memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb)); +#endif + __flush_tlb_all(); + + /* Setup bootmem... */ + last_valid_pfn = end_pfn = bootmem_init(phys_base); + /* Once the OF device tree and MDESC have been setup, we know * the list of possible cpus. Therefore we can allocate the * IRQ stacks. */ for_each_possible_cpu(i) { - /* XXX Use node local allocations... XXX */ - softirq_stack[i] = __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE)); - hardirq_stack[i] = __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE)); - } + node = cpu_to_node(i); - /* Setup bootmem... */ - last_valid_pfn = end_pfn = bootmem_init(phys_base); + softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + } -#ifndef CONFIG_NEED_MULTIPLE_NODES - max_mapnr = last_valid_pfn; -#endif kernel_physical_mapping_init(); { @@ -1831,7 +2041,7 @@ void __init paging_init(void) printk("Booting Linux...\n"); } -int __devinit page_in_phys_avail(unsigned long paddr) +int page_in_phys_avail(unsigned long paddr) { int i; @@ -1889,7 +2099,7 @@ static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap) if (new_start <= old_start && new_end >= (old_start + PAGE_SIZE)) { - set_bit(old_start >> 22, bitmap); + set_bit(old_start >> ILOG2_4MB, bitmap); goto do_next_page; } } @@ -1921,15 +2131,24 @@ static void __init patch_tlb_miss_handler_bitmap(void) flushi(&valid_addr_bitmap_insn[0]); } +static void __init register_page_bootmem_info(void) +{ +#ifdef CONFIG_NEED_MULTIPLE_NODES + int i; + + for_each_online_node(i) + if (NODE_DATA(i)->node_spanned_pages) + register_page_bootmem_info_node(NODE_DATA(i)); +#endif +} void __init mem_init(void) { - unsigned long codepages, datapages, initpages; unsigned long addr, last; addr = PAGE_OFFSET + kern_base; last = PAGE_ALIGN(kern_size) + addr; while (addr < last) { - set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap); + set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap); addr += PAGE_SIZE; } @@ -1938,25 +2157,8 @@ void __init mem_init(void) high_memory = __va(last_valid_pfn << PAGE_SHIFT); -#ifdef CONFIG_NEED_MULTIPLE_NODES - { - int i; - for_each_online_node(i) { - if (NODE_DATA(i)->node_spanned_pages != 0) { - totalram_pages += - free_all_bootmem_node(NODE_DATA(i)); - } - } - } -#else - totalram_pages = free_all_bootmem(); -#endif - - /* We subtract one to account for the mem_map_zero page - * allocated below. - */ - totalram_pages -= 1; - num_physpages = totalram_pages; + register_page_bootmem_info(); + free_all_bootmem(); /* * Set up the zero page, mark it reserved, so that page count @@ -1967,21 +2169,9 @@ void __init mem_init(void) prom_printf("paging_init: Cannot alloc zero page.\n"); prom_halt(); } - SetPageReserved(mem_map_zero); + mark_page_reserved(mem_map_zero); - codepages = (((unsigned long) _etext) - ((unsigned long) _start)); - codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT; - datapages = (((unsigned long) _edata) - ((unsigned long) _etext)); - datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT; - initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin)); - initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT; - - printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n", - nr_free_pages() << (PAGE_SHIFT-10), - codepages << (PAGE_SHIFT-10), - datapages << (PAGE_SHIFT-10), - initpages << (PAGE_SHIFT-10), - PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT)); + mem_init_print_info(NULL); if (tlb_type == cheetah || tlb_type == cheetah_plus) cheetah_ecache_flush_init(); @@ -2007,39 +2197,22 @@ void free_initmem(void) initend = (unsigned long)(__init_end) & PAGE_MASK; for (; addr < initend; addr += PAGE_SIZE) { unsigned long page; - struct page *p; page = (addr + ((unsigned long) __va(kern_base)) - ((unsigned long) KERNBASE)); memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); - if (do_free) { - p = virt_to_page(page); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - num_physpages++; - totalram_pages++; - } + if (do_free) + free_reserved_page(virt_to_page(page)); } } #ifdef CONFIG_BLK_DEV_INITRD void free_initrd_mem(unsigned long start, unsigned long end) { - if (start < end) - printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10); - for (; start < end; start += PAGE_SIZE) { - struct page *p = virt_to_page(start); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - num_physpages++; - totalram_pages++; - } + free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, + "initrd"); } #endif @@ -2073,10 +2246,12 @@ EXPORT_SYMBOL(_PAGE_CACHE); #ifdef CONFIG_SPARSEMEM_VMEMMAP unsigned long vmemmap_table[VMEMMAP_SIZE]; -int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) +static long __meminitdata addr_start, addr_end; +static int __meminitdata node_start; + +int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, + int node) { - unsigned long vstart = (unsigned long) start; - unsigned long vend = (unsigned long) (start + nr); unsigned long phys_start = (vstart - VMEMMAP_BASE); unsigned long phys_end = (vend - VMEMMAP_BASE); unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; @@ -2097,21 +2272,41 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) void *block; if (!(*vmem_pp & _PAGE_VALID)) { - block = vmemmap_alloc_block(1UL << 22, node); + block = vmemmap_alloc_block(1UL << ILOG2_4MB, node); if (!block) return -ENOMEM; *vmem_pp = pte_base | __pa(block); - printk(KERN_INFO "[%p-%p] page_structs=%lu " - "node=%d entry=%lu/%lu\n", start, block, nr, - node, - addr >> VMEMMAP_CHUNK_SHIFT, - VMEMMAP_SIZE); + /* check to see if we have contiguous blocks */ + if (addr_end != addr || node_start != node) { + if (addr_start) + printk(KERN_DEBUG " [%lx-%lx] on node %d\n", + addr_start, addr_end-1, node_start); + addr_start = addr; + node_start = node; + } + addr_end = addr + VMEMMAP_CHUNK; } } return 0; } + +void __meminit vmemmap_populate_print_last(void) +{ + if (addr_start) { + printk(KERN_DEBUG " [%lx-%lx] on node %d\n", + addr_start, addr_end-1, node_start); + addr_start = 0; + addr_end = 0; + node_start = 0; + } +} + +void vmemmap_free(unsigned long start, unsigned long end) +{ +} + #endif /* CONFIG_SPARSEMEM_VMEMMAP */ static void prot_init_common(unsigned long page_none, @@ -2145,6 +2340,7 @@ static void __init sun4u_pgprot_init(void) { unsigned long page_none, page_shared, page_copy, page_readonly; unsigned long page_exec_bit; + int i; PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID | _PAGE_CACHE_4U | _PAGE_P_4U | @@ -2163,19 +2359,17 @@ static void __init sun4u_pgprot_init(void) __ACCESS_BITS_4U | _PAGE_E_4U); #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^ - 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U | _PAGE_W_4U); - /* XXX Should use 256MB on Panther. XXX */ - kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; - _PAGE_SZBITS = _PAGE_SZBITS_4U; _PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U | _PAGE_SZ64K_4U | _PAGE_SZ8K_4U | _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U); @@ -2199,6 +2393,7 @@ static void __init sun4v_pgprot_init(void) { unsigned long page_none, page_shared, page_copy, page_readonly; unsigned long page_exec_bit; + int i; PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID | _PAGE_CACHE_4V | _PAGE_P_4V | @@ -2211,29 +2406,20 @@ static void __init sun4v_pgprot_init(void) _PAGE_CACHE = _PAGE_CACHE_4V; #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^ - 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V | _PAGE_W_4V); -#ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^ - 0xfffff80000000000UL; -#else - kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ - 0xfffff80000000000UL; -#endif - kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | - _PAGE_P_4V | _PAGE_W_4V); + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V | __ACCESS_BITS_4V | _PAGE_E_4V); - _PAGE_SZBITS = _PAGE_SZBITS_4V; _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V | _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V | _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V | @@ -2266,7 +2452,7 @@ unsigned long pte_sz_bits(unsigned long sz) return _PAGE_SZ512K_4V; case 4 * 1024 * 1024: return _PAGE_SZ4MB_4V; - }; + } } else { switch (sz) { case 8 * 1024: @@ -2278,7 +2464,7 @@ unsigned long pte_sz_bits(unsigned long sz) return _PAGE_SZ512K_4U; case 4 * 1024 * 1024: return _PAGE_SZ4MB_4U; - }; + } } } @@ -2366,3 +2552,150 @@ void __flush_tlb_all(void) __asm__ __volatile__("wrpr %0, 0, %%pstate" : : "r" (pstate)); } + +pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long address) +{ + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | + __GFP_REPEAT | __GFP_ZERO); + pte_t *pte = NULL; + + if (page) + pte = (pte_t *) page_address(page); + + return pte; +} + +pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) +{ + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | + __GFP_REPEAT | __GFP_ZERO); + if (!page) + return NULL; + if (!pgtable_page_ctor(page)) { + free_hot_cold_page(page, 0); + return NULL; + } + return (pte_t *) page_address(page); +} + +void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + free_page((unsigned long)pte); +} + +static void __pte_free(pgtable_t pte) +{ + struct page *page = virt_to_page(pte); + + pgtable_page_dtor(page); + __free_page(page); +} + +void pte_free(struct mm_struct *mm, pgtable_t pte) +{ + __pte_free(pte); +} + +void pgtable_free(void *table, bool is_page) +{ + if (is_page) + __pte_free(table); + else + kmem_cache_free(pgtable_cache, table); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd) +{ + unsigned long pte, flags; + struct mm_struct *mm; + pmd_t entry = *pmd; + + if (!pmd_large(entry) || !pmd_young(entry)) + return; + + pte = pmd_val(entry); + + /* We are fabricating 8MB pages using 4MB real hw pages. */ + pte |= (addr & (1UL << REAL_HPAGE_SHIFT)); + + mm = vma->vm_mm; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, + addr, pte); + + spin_unlock_irqrestore(&mm->context.lock, flags); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void context_reload(void *__data) +{ + struct mm_struct *mm = __data; + + if (mm == current->mm) + load_secondary_context(mm); +} + +void hugetlb_setup(struct pt_regs *regs) +{ + struct mm_struct *mm = current->mm; + struct tsb_config *tp; + + if (in_atomic() || !mm) { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + pr_alert("Unexpected HugeTLB setup in atomic context.\n"); + die_if_kernel("HugeTSB in atomic", regs); + } + + tp = &mm->context.tsb_block[MM_TSB_HUGE]; + if (likely(tp->tsb == NULL)) + tsb_grow(mm, MM_TSB_HUGE, 0); + + tsb_context_switch(mm); + smp_tsb_sync(mm); + + /* On UltraSPARC-III+ and later, configure the second half of + * the Data-TLB for huge pages. + */ + if (tlb_type == cheetah_plus) { + unsigned long ctx; + + spin_lock(&ctx_alloc_lock); + ctx = mm->context.sparc64_ctx_val; + ctx &= ~CTX_PGSZ_MASK; + ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; + ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; + + if (ctx != mm->context.sparc64_ctx_val) { + /* When changing the page size fields, we + * must perform a context flush so that no + * stale entries match. This flush must + * occur with the original context register + * settings. + */ + do_flush_tlb_mm(mm); + + /* Reload the context register of all processors + * also executing in this address space. + */ + mm->context.sparc64_ctx_val = ctx; + on_each_cpu(context_reload, mm, 0); + } + spin_unlock(&ctx_alloc_lock); + } +} +#endif diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h index 77d1b313e34..0668b364f44 100644 --- a/arch/sparc/mm/init_64.h +++ b/arch/sparc/mm/init_64.h @@ -1,25 +1,27 @@ #ifndef _SPARC64_MM_INIT_H #define _SPARC64_MM_INIT_H +#include <asm/page.h> + /* Most of the symbols in this file are defined in init.c and * marked non-static so that assembler code can get at them. */ -#define MAX_PHYS_ADDRESS (1UL << 41UL) +#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) #define KPTE_BITMAP_BYTES \ - ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8) + ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) #define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL) #define VALID_ADDR_BITMAP_BYTES \ ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8) -extern unsigned long kern_linear_pte_xor[2]; +extern unsigned long kern_linear_pte_xor[4]; extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; extern unsigned int sparc64_highest_unlocked_tlb_ent; extern unsigned long sparc64_kern_pri_context; extern unsigned long sparc64_kern_pri_nuc_bits; extern unsigned long sparc64_kern_sec_context; -extern void mmu_info(struct seq_file *m); +void mmu_info(struct seq_file *m); struct linux_prom_translation { unsigned long virt; @@ -34,9 +36,7 @@ extern unsigned int prom_trans_ents; /* Exported for SMP bootup purposes. */ extern unsigned long kern_locked_tte_data; -extern void prom_world(int enter); - -extern void free_initmem(void); +void prom_world(int enter); #ifdef CONFIG_SPARSEMEM_VMEMMAP #define VMEMMAP_CHUNK_SHIFT 22 diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index fc58c3e917d..f311bf21901 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -25,6 +25,8 @@ #include <asm/dma.h> #include <asm/oplib.h> +#include "mm_32.h" + /* #define IOUNIT_DEBUG */ #ifdef IOUNIT_DEBUG #define IOD(x) printk(x) @@ -38,7 +40,8 @@ static void __init iounit_iommu_init(struct platform_device *op) { struct iounit_struct *iounit; - iopte_t *xpt, *xptend; + iopte_t __iomem *xpt; + iopte_t __iomem *xptend; iounit = kzalloc(sizeof(struct iounit_struct), GFP_ATOMIC); if (!iounit) { @@ -62,10 +65,10 @@ static void __init iounit_iommu_init(struct platform_device *op) op->dev.archdata.iommu = iounit; iounit->page_table = xpt; spin_lock_init(&iounit->lock); - - for (xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); - xpt < xptend;) - iopte_val(*xpt++) = 0; + + xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); + for (; xpt < xptend; xpt++) + sbus_writel(0, xpt); } static int __init iounit_init(void) @@ -130,7 +133,7 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); vaddr = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + (vaddr & ~PAGE_MASK); for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) { set_bit(scan, iounit->bmap); - iounit->page_table[scan] = iopte; + sbus_writel(iopte, &iounit->page_table[scan]); } IOD(("%08lx\n", vaddr)); return vaddr; @@ -197,12 +200,12 @@ static void iounit_release_scsi_sgl(struct device *dev, struct scatterlist *sg, } #ifdef CONFIG_SBUS -static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, __u32 addr, int len) +static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, unsigned long addr, int len) { struct iounit_struct *iounit = dev->archdata.iommu; unsigned long page, end; pgprot_t dvma_prot; - iopte_t *iopte; + iopte_t __iomem *iopte; *pba = addr; @@ -224,8 +227,8 @@ static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned lon i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); - iopte = (iopte_t *)(iounit->page_table + i); - *iopte = MKIOPTE(__pa(page)); + iopte = iounit->page_table + i; + sbus_writel(MKIOPTE(__pa(page)), iopte); } addr += PAGE_SIZE; va += PAGE_SIZE; @@ -242,29 +245,18 @@ static void iounit_unmap_dma_area(struct device *dev, unsigned long addr, int le } #endif -static char *iounit_lockarea(char *vaddr, unsigned long len) -{ -/* FIXME: Write this */ - return vaddr; -} - -static void iounit_unlockarea(char *vaddr, unsigned long len) -{ -/* FIXME: Write this */ -} +static const struct sparc32_dma_ops iounit_dma_ops = { + .get_scsi_one = iounit_get_scsi_one, + .get_scsi_sgl = iounit_get_scsi_sgl, + .release_scsi_one = iounit_release_scsi_one, + .release_scsi_sgl = iounit_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iounit_map_dma_area, + .unmap_dma_area = iounit_unmap_dma_area, +#endif +}; void __init ld_mmu_iounit(void) { - BTFIXUPSET_CALL(mmu_lockarea, iounit_lockarea, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_unlockarea, iounit_unlockarea, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(mmu_get_scsi_one, iounit_get_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iounit_get_scsi_sgl, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_one, iounit_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, iounit_release_scsi_sgl, BTFIXUPCALL_NORM); - -#ifdef CONFIG_SBUS - BTFIXUPSET_CALL(mmu_map_dma_area, iounit_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, iounit_unmap_dma_area, BTFIXUPCALL_NORM); -#endif + sparc32_dma_ops = &iounit_dma_ops; } diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index 07fc6a65d9b..491511d37e3 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -27,6 +27,8 @@ #include <asm/iommu.h> #include <asm/dma.h> +#include "mm_32.h" + /* * This can be sized dynamically, but we will do this * only when we have a guidance about actual I/O pressures. @@ -34,14 +36,9 @@ #define IOMMU_RNGE IOMMU_RNGE_256MB #define IOMMU_START 0xF0000000 #define IOMMU_WINSIZE (256*1024*1024U) -#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */ +#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ -/* srmmu.c */ -extern int viking_mxcc_present; -BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long) -#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page) -extern int flush_page_for_dma_global; static int viking_flush; /* viking.S */ extern void viking_flush_page(unsigned long page); @@ -61,6 +58,8 @@ static void __init sbus_iommu_init(struct platform_device *op) struct iommu_struct *iommu; unsigned int impl, vers; unsigned long *bitmap; + unsigned long control; + unsigned long base; unsigned long tmp; iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); @@ -75,12 +74,14 @@ static void __init sbus_iommu_init(struct platform_device *op) prom_printf("Cannot map IOMMU registers\n"); prom_halt(); } - impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28; - vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24; - tmp = iommu->regs->control; - tmp &= ~(IOMMU_CTRL_RNGE); - tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); - iommu->regs->control = tmp; + + control = sbus_readl(&iommu->regs->control); + impl = (control & IOMMU_CTRL_IMPL) >> 28; + vers = (control & IOMMU_CTRL_VERS) >> 24; + control &= ~(IOMMU_CTRL_RNGE); + control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); + sbus_writel(control, &iommu->regs->control); + iommu_invalidate(iommu->regs); iommu->start = IOMMU_START; iommu->end = 0xffffffff; @@ -92,8 +93,8 @@ static void __init sbus_iommu_init(struct platform_device *op) it to us. */ tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); if (!tmp) { - prom_printf("Unable to allocate iommu table [0x%08x]\n", - IOMMU_NPTES*sizeof(iopte_t)); + prom_printf("Unable to allocate iommu table [0x%lx]\n", + IOMMU_NPTES * sizeof(iopte_t)); prom_halt(); } iommu->page_table = (iopte_t *)tmp; @@ -102,7 +103,9 @@ static void __init sbus_iommu_init(struct platform_device *op) memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); flush_cache_all(); flush_tlb_all(); - iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4; + + base = __pa((unsigned long)iommu->page_table) >> 4; + sbus_writel(base, &iommu->regs->base); iommu_invalidate(iommu->regs); bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); @@ -143,7 +146,6 @@ static int __init iommu_init(void) subsys_initcall(iommu_init); -/* This begs to be btfixup-ed by srmmu. */ /* Flush the iotlb entries to ram. */ /* This could be better if we didn't have to flush whole pages. */ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) @@ -216,11 +218,6 @@ static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len) return busa + off; } -static __u32 iommu_get_scsi_one_noflush(struct device *dev, char *vaddr, unsigned long len) -{ - return iommu_get_scsi_one(dev, vaddr, len); -} - static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len) { flush_page_for_dma(0); @@ -238,19 +235,6 @@ static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned return iommu_get_scsi_one(dev, vaddr, len); } -static void iommu_get_scsi_sgl_noflush(struct device *dev, struct scatterlist *sg, int sz) -{ - int n; - - while (sz != 0) { - --sz; - n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; - sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; - sg->dma_length = sg->length; - sg = sg_next(sg); - } -} - static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz) { int n; @@ -426,40 +410,36 @@ static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len } #endif -static char *iommu_lockarea(char *vaddr, unsigned long len) -{ - return vaddr; -} +static const struct sparc32_dma_ops iommu_dma_gflush_ops = { + .get_scsi_one = iommu_get_scsi_one_gflush, + .get_scsi_sgl = iommu_get_scsi_sgl_gflush, + .release_scsi_one = iommu_release_scsi_one, + .release_scsi_sgl = iommu_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iommu_map_dma_area, + .unmap_dma_area = iommu_unmap_dma_area, +#endif +}; -static void iommu_unlockarea(char *vaddr, unsigned long len) -{ -} +static const struct sparc32_dma_ops iommu_dma_pflush_ops = { + .get_scsi_one = iommu_get_scsi_one_pflush, + .get_scsi_sgl = iommu_get_scsi_sgl_pflush, + .release_scsi_one = iommu_release_scsi_one, + .release_scsi_sgl = iommu_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iommu_map_dma_area, + .unmap_dma_area = iommu_unmap_dma_area, +#endif +}; void __init ld_mmu_iommu(void) { - viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page); - BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP); - - if (!BTFIXUPVAL_CALL(flush_page_for_dma)) { - /* IO coherent chip */ - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM); - } else if (flush_page_for_dma_global) { + if (flush_page_for_dma_global) { /* flush_page_for_dma flushes everything, no matter of what page is it */ - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM); + sparc32_dma_ops = &iommu_dma_gflush_ops; } else { - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM); + sparc32_dma_ops = &iommu_dma_pflush_ops; } - BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM); - -#ifdef CONFIG_SBUS - BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM); -#endif if (viking_mxcc_present || srmmu_modtype == HyperSparc) { dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c index c0e01297e64..3b17b6f7895 100644 --- a/arch/sparc/mm/leon_mm.c +++ b/arch/sparc/mm/leon_mm.c @@ -15,10 +15,24 @@ #include <asm/leon.h> #include <asm/tlbflush.h> +#include "mm_32.h" + int leon_flush_during_switch = 1; -int srmmu_swprobe_trace; +static int srmmu_swprobe_trace; + +static inline unsigned long leon_get_ctable_ptr(void) +{ + unsigned int retval; + + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (SRMMU_CTXTBL_PTR), + "i" (ASI_LEON_MMUREGS)); + return (retval & SRMMU_CTX_PMASK) << 4; +} + -unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr) +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) { unsigned int ctxtbl; @@ -33,10 +47,10 @@ unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr) if (srmmu_swprobe_trace) printk(KERN_INFO "swprobe: trace on\n"); - ctxtbl = srmmu_get_ctable_ptr(); + ctxtbl = leon_get_ctable_ptr(); if (!(ctxtbl)) { if (srmmu_swprobe_trace) - printk(KERN_INFO "swprobe: srmmu_get_ctable_ptr returned 0=>0\n"); + printk(KERN_INFO "swprobe: leon_get_ctable_ptr returned 0=>0\n"); return 0; } if (!_pfn_valid(PFN(ctxtbl))) { @@ -162,7 +176,7 @@ ready: printk(KERN_INFO "swprobe: padde %x\n", paddr_calc); if (paddr) *paddr = paddr_calc; - return paddrbase; + return pte; } void leon_flush_icache_all(void) @@ -226,7 +240,7 @@ void leon3_getCacheRegs(struct leon3_cacheregs *regs) * Leon2 and Leon3 differ in their way of telling cache information * */ -int leon_flush_needed(void) +int __init leon_flush_needed(void) { int flush_needed = -1; unsigned int ssize, sets; @@ -258,3 +272,80 @@ void leon_switch_mm(void) if (leon_flush_during_switch) leon_flush_cache_all(); } + +static void leon_flush_cache_mm(struct mm_struct *mm) +{ + leon_flush_cache_all(); +} + +static void leon_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + leon_flush_pcache_all(vma, page); +} + +static void leon_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_cache_all(); +} + +static void leon_flush_tlb_mm(struct mm_struct *mm) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_page(struct vm_area_struct *vma, + unsigned long page) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_page_to_ram(unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_sig_insns(struct mm_struct *mm, unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_page_for_dma(unsigned long page) +{ + leon_flush_dcache_all(); +} + +void __init poke_leonsparc(void) +{ +} + +static const struct sparc32_cachetlb_ops leon_ops = { + .cache_all = leon_flush_cache_all, + .cache_mm = leon_flush_cache_mm, + .cache_page = leon_flush_cache_page, + .cache_range = leon_flush_cache_range, + .tlb_all = leon_flush_tlb_all, + .tlb_mm = leon_flush_tlb_mm, + .tlb_page = leon_flush_tlb_page, + .tlb_range = leon_flush_tlb_range, + .page_to_ram = leon_flush_page_to_ram, + .sig_insns = leon_flush_sig_insns, + .page_for_dma = leon_flush_page_for_dma, +}; + +void __init init_leon(void) +{ + srmmu_name = "LEON"; + sparc32_cachetlb_ops = &leon_ops; + poke_srmmu = poke_leonsparc; + + leon_flush_during_switch = leon_flush_needed(); +} diff --git a/arch/sparc/mm/loadmmu.c b/arch/sparc/mm/loadmmu.c deleted file mode 100644 index 82ec8f66603..00000000000 --- a/arch/sparc/mm/loadmmu.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * loadmmu.c: This code loads up all the mm function pointers once the - * machine type has been determined. It also sets the static - * mmu values such as PAGE_NONE, etc. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> - -#include <asm/system.h> -#include <asm/page.h> -#include <asm/pgtable.h> -#include <asm/mmu_context.h> -#include <asm/oplib.h> - -struct ctx_list *ctx_list_pool; -struct ctx_list ctx_free; -struct ctx_list ctx_used; - -extern void ld_mmu_sun4c(void); -extern void ld_mmu_srmmu(void); - -void __init load_mmu(void) -{ - switch(sparc_cpu_model) { - case sun4c: - case sun4: - ld_mmu_sun4c(); - break; - case sun4m: - case sun4d: - case sparc_leon: - ld_mmu_srmmu(); - break; - default: - prom_printf("load_mmu: %d unsupported\n", (int)sparc_cpu_model); - prom_halt(); - } - btfixup(); -} diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h new file mode 100644 index 00000000000..a6c27ca9a72 --- /dev/null +++ b/arch/sparc/mm/mm_32.h @@ -0,0 +1,24 @@ +/* fault_32.c - visible as they are called from assembler */ +asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, + unsigned long address); +asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, + unsigned long address); + +void window_overflow_fault(void); +void window_underflow_fault(unsigned long sp); +void window_ret_fault(struct pt_regs *regs); + +/* srmmu.c */ +extern char *srmmu_name; +extern int viking_mxcc_present; +extern int flush_page_for_dma_global; + +extern void (*poke_srmmu)(void); + +void __init srmmu_paging_init(void); + +/* iommu.c */ +void ld_mmu_iommu(void); + +/* io-unit.c */ +void ld_mmu_iounit(void); diff --git a/arch/sparc/mm/nosun4c.c b/arch/sparc/mm/nosun4c.c deleted file mode 100644 index 4e62c27147c..00000000000 --- a/arch/sparc/mm/nosun4c.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * nosun4c.c: This file is a bunch of dummies for SMP compiles, - * so that it does not need sun4c and avoid ifdefs. - * - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <asm/pgtable.h> - -static char shouldnothappen[] __initdata = "32bit SMP kernel only supports sun4m and sun4d\n"; - -/* Dummies */ -struct sun4c_mmu_ring { - unsigned long xxx1[3]; - unsigned char xxx2[2]; - int xxx3; -}; -struct sun4c_mmu_ring sun4c_kernel_ring; -struct sun4c_mmu_ring sun4c_kfree_ring; -unsigned long sun4c_kernel_faults; -unsigned long *sun4c_memerr_reg; - -static void __init should_not_happen(void) -{ - prom_printf(shouldnothappen); - prom_halt(); -} - -unsigned long __init sun4c_paging_init(unsigned long start_mem, unsigned long end_mem) -{ - should_not_happen(); - return 0; -} - -void __init ld_mmu_sun4c(void) -{ - should_not_happen(); -} - -void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr, int bus_type, int rdonly) -{ -} - -void sun4c_unmapioaddr(unsigned long virt_addr) -{ -} - -void sun4c_complete_all_stores(void) -{ -} - -pte_t *sun4c_pte_offset(pmd_t * dir, unsigned long address) -{ - return NULL; -} - -pte_t *sun4c_pte_offset_kernel(pmd_t *dir, unsigned long address) -{ - return NULL; -} - -void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) -{ -} - -void __init sun4c_probe_vac(void) -{ - should_not_happen(); -} - -void __init sun4c_probe_memerr_reg(void) -{ - should_not_happen(); -} diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 92319aa8b66..be65f035d18 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -8,47 +8,48 @@ * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) */ -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <linux/pagemap.h> -#include <linux/init.h> +#include <linux/seq_file.h> #include <linux/spinlock.h> #include <linux/bootmem.h> -#include <linux/fs.h> -#include <linux/seq_file.h> +#include <linux/pagemap.h> +#include <linux/vmalloc.h> #include <linux/kdebug.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/init.h> #include <linux/log2.h> #include <linux/gfp.h> +#include <linux/fs.h> +#include <linux/mm.h> -#include <asm/bitext.h> -#include <asm/page.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/io-unit.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> -#include <asm/io.h> +#include <asm/bitext.h> #include <asm/vaddrs.h> -#include <asm/traps.h> -#include <asm/smp.h> -#include <asm/mbus.h> #include <asm/cache.h> +#include <asm/traps.h> #include <asm/oplib.h> +#include <asm/mbus.h> +#include <asm/page.h> #include <asm/asi.h> #include <asm/msi.h> -#include <asm/mmu_context.h> -#include <asm/io-unit.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> +#include <asm/smp.h> +#include <asm/io.h> /* Now the cpu specific definitions. */ -#include <asm/viking.h> -#include <asm/mxcc.h> -#include <asm/ross.h> +#include <asm/turbosparc.h> #include <asm/tsunami.h> +#include <asm/viking.h> #include <asm/swift.h> -#include <asm/turbosparc.h> #include <asm/leon.h> +#include <asm/mxcc.h> +#include <asm/ross.h> -#include <asm/btfixup.h> +#include "mm_32.h" enum mbus_module srmmu_modtype; static unsigned int hwbug_bitmask; @@ -59,28 +60,23 @@ extern struct resource sparc_iomap; extern unsigned long last_valid_pfn; -extern unsigned long page_kernel; - static pgd_t *srmmu_swapper_pg_dir; +const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +EXPORT_SYMBOL(sparc32_cachetlb_ops); + #ifdef CONFIG_SMP +const struct sparc32_cachetlb_ops *local_ops; + #define FLUSH_BEGIN(mm) #define FLUSH_END #else -#define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) { +#define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) { #define FLUSH_END } #endif -BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long) -#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page) - int flush_page_for_dma_global = 1; -#ifdef CONFIG_SMP -BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long) -#define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page) -#endif - char *srmmu_name; ctxd_t *srmmu_ctx_table_phys; @@ -91,28 +87,6 @@ static DEFINE_SPINLOCK(srmmu_context_spinlock); static int is_hypersparc; -/* - * In general all page table modifications should use the V8 atomic - * swap instruction. This insures the mmu and the cpu are in sync - * with respect to ref/mod bits in the page tables. - */ -static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value) -{ - __asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr)); - return value; -} - -static inline void srmmu_set_pte(pte_t *ptep, pte_t pteval) -{ - srmmu_swap((unsigned long *)ptep, pte_val(pteval)); -} - -/* The very generic SRMMU page table operations. */ -static inline int srmmu_device_memory(unsigned long x) -{ - return ((x & 0xF0000000) != 0); -} - static int srmmu_cache_pagetables; /* these will be initialized in srmmu_nocache_calcsize() */ @@ -126,148 +100,41 @@ static unsigned long srmmu_nocache_end; #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS) void *srmmu_nocache_pool; -void *srmmu_nocache_bitmap; static struct bit_map srmmu_nocache_map; -static unsigned long srmmu_pte_pfn(pte_t pte) -{ - if (srmmu_device_memory(pte_val(pte))) { - /* Just return something that will cause - * pfn_valid() to return false. This makes - * copy_one_pte() to just directly copy to - * PTE over. - */ - return ~0UL; - } - return (pte_val(pte) & SRMMU_PTE_PMASK) >> (PAGE_SHIFT-4); -} - -static struct page *srmmu_pmd_page(pmd_t pmd) -{ - - if (srmmu_device_memory(pmd_val(pmd))) - BUG(); - return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); -} - -static inline unsigned long srmmu_pgd_page(pgd_t pgd) -{ return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); } - - -static inline int srmmu_pte_none(pte_t pte) -{ return !(pte_val(pte) & 0xFFFFFFF); } - -static inline int srmmu_pte_present(pte_t pte) -{ return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); } - -static inline void srmmu_pte_clear(pte_t *ptep) -{ srmmu_set_pte(ptep, __pte(0)); } - static inline int srmmu_pmd_none(pmd_t pmd) { return !(pmd_val(pmd) & 0xFFFFFFF); } -static inline int srmmu_pmd_bad(pmd_t pmd) -{ return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } - -static inline int srmmu_pmd_present(pmd_t pmd) -{ return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } - -static inline void srmmu_pmd_clear(pmd_t *pmdp) { - int i; - for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); -} - -static inline int srmmu_pgd_none(pgd_t pgd) -{ return !(pgd_val(pgd) & 0xFFFFFFF); } - -static inline int srmmu_pgd_bad(pgd_t pgd) -{ return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } - -static inline int srmmu_pgd_present(pgd_t pgd) -{ return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } - -static inline void srmmu_pgd_clear(pgd_t * pgdp) -{ srmmu_set_pte((pte_t *)pgdp, __pte(0)); } - -static inline pte_t srmmu_pte_wrprotect(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_WRITE);} - -static inline pte_t srmmu_pte_mkclean(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_DIRTY);} - -static inline pte_t srmmu_pte_mkold(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_REF);} - -static inline pte_t srmmu_pte_mkwrite(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_WRITE);} - -static inline pte_t srmmu_pte_mkdirty(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_DIRTY);} - -static inline pte_t srmmu_pte_mkyoung(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_REF);} - -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -static pte_t srmmu_mk_pte(struct page *page, pgprot_t pgprot) -{ return __pte((page_to_pfn(page) << (PAGE_SHIFT-4)) | pgprot_val(pgprot)); } - -static pte_t srmmu_mk_pte_phys(unsigned long page, pgprot_t pgprot) -{ return __pte(((page) >> 4) | pgprot_val(pgprot)); } - -static pte_t srmmu_mk_pte_io(unsigned long page, pgprot_t pgprot, int space) -{ return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); } - /* XXX should we hyper_flush_whole_icache here - Anton */ static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) -{ srmmu_set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } - -static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp) -{ srmmu_set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); } +{ set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } -static void srmmu_pmd_set(pmd_t *pmdp, pte_t *ptep) +void pmd_set(pmd_t *pmdp, pte_t *ptep) { unsigned long ptp; /* Physical address, shifted right by 4 */ int i; ptp = __nocache_pa((unsigned long) ptep) >> 4; for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); + set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); } } -static void srmmu_pmd_populate(pmd_t *pmdp, struct page *ptep) +void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) { unsigned long ptp; /* Physical address, shifted right by 4 */ int i; ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */ for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); + set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); } } -static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot) -{ return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); } - -/* to find an entry in a top-level page table... */ -static inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address) -{ return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); } - -/* Find an entry in the second-level page table.. */ -static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address) -{ - return (pmd_t *) srmmu_pgd_page(*dir) + - ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); -} - -/* Find an entry in the third-level page table.. */ -static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address) +/* Find an entry in the third-level page table.. */ +pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address) { void *pte; @@ -276,77 +143,66 @@ static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); } -static unsigned long srmmu_swp_type(swp_entry_t entry) -{ - return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK; -} - -static unsigned long srmmu_swp_offset(swp_entry_t entry) -{ - return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK; -} - -static swp_entry_t srmmu_swp_entry(unsigned long type, unsigned long offset) -{ - return (swp_entry_t) { - (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT - | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT }; -} - /* * size: bytes to allocate in the nocache area. * align: bytes, number to align at. * Returns the virtual address of the allocated area. */ -static unsigned long __srmmu_get_nocache(int size, int align) +static void *__srmmu_get_nocache(int size, int align) { int offset; + unsigned long addr; if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { - printk("Size 0x%x too small for nocache request\n", size); + printk(KERN_ERR "Size 0x%x too small for nocache request\n", + size); size = SRMMU_NOCACHE_BITMAP_SHIFT; } - if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) { - printk("Size 0x%x unaligned int nocache request\n", size); - size += SRMMU_NOCACHE_BITMAP_SHIFT-1; + if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) { + printk(KERN_ERR "Size 0x%x unaligned int nocache request\n", + size); + size += SRMMU_NOCACHE_BITMAP_SHIFT - 1; } BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); offset = bit_map_string_get(&srmmu_nocache_map, - size >> SRMMU_NOCACHE_BITMAP_SHIFT, - align >> SRMMU_NOCACHE_BITMAP_SHIFT); + size >> SRMMU_NOCACHE_BITMAP_SHIFT, + align >> SRMMU_NOCACHE_BITMAP_SHIFT); if (offset == -1) { - printk("srmmu: out of nocache %d: %d/%d\n", - size, (int) srmmu_nocache_size, - srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); - return 0; + printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n", + size, (int) srmmu_nocache_size, + srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); + return NULL; } - return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT)); + addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT); + return (void *)addr; } -static unsigned long srmmu_get_nocache(int size, int align) +void *srmmu_get_nocache(int size, int align) { - unsigned long tmp; + void *tmp; tmp = __srmmu_get_nocache(size, align); if (tmp) - memset((void *)tmp, 0, size); + memset(tmp, 0, size); return tmp; } -static void srmmu_free_nocache(unsigned long vaddr, int size) +void srmmu_free_nocache(void *addr, int size) { + unsigned long vaddr; int offset; + vaddr = (unsigned long)addr; if (vaddr < SRMMU_NOCACHE_VADDR) { printk("Vaddr %lx is smaller than nocache base 0x%lx\n", vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); BUG(); } - if (vaddr+size > srmmu_nocache_end) { + if (vaddr + size > srmmu_nocache_end) { printk("Vaddr %lx is bigger than nocache end 0x%lx\n", vaddr, srmmu_nocache_end); BUG(); @@ -359,7 +215,7 @@ static void srmmu_free_nocache(unsigned long vaddr, int size) printk("Size 0x%x is too small\n", size); BUG(); } - if (vaddr & (size-1)) { + if (vaddr & (size - 1)) { printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); BUG(); } @@ -373,13 +229,23 @@ static void srmmu_free_nocache(unsigned long vaddr, int size) static void srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end); -extern unsigned long probe_memory(void); /* in fault.c */ +/* Return how much physical memory we have. */ +static unsigned long __init probe_memory(void) +{ + unsigned long total = 0; + int i; + + for (i = 0; sp_banks[i].num_bytes; i++) + total += sp_banks[i].num_bytes; + + return total; +} /* * Reserve nocache dynamically proportionally to the amount of * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002 */ -static void srmmu_nocache_calcsize(void) +static void __init srmmu_nocache_calcsize(void) { unsigned long sysmemavail = probe_memory() / 1024; int srmmu_nocache_npages; @@ -402,6 +268,7 @@ static void srmmu_nocache_calcsize(void) static void __init srmmu_nocache_init(void) { + void *srmmu_nocache_bitmap; unsigned int bitmap_bits; pgd_t *pgd; pmd_t *pmd; @@ -415,10 +282,12 @@ static void __init srmmu_nocache_init(void) SRMMU_NOCACHE_ALIGN_MAX, 0UL); memset(srmmu_nocache_pool, 0, srmmu_nocache_size); - srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); + srmmu_nocache_bitmap = + __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long), + SMP_CACHE_BYTES, 0UL); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); - srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); init_mm.pgd = srmmu_swapper_pg_dir; @@ -429,15 +298,15 @@ static void __init srmmu_nocache_init(void) while (vaddr < srmmu_nocache_end) { pgd = pgd_offset_k(vaddr); - pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr); - pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr); + pmd = pmd_offset(__nocache_fix(pgd), vaddr); + pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV); if (srmmu_cache_pagetables) pteval |= SRMMU_CACHE; - srmmu_set_pte(__nocache_fix(pte), __pte(pteval)); + set_pte(__nocache_fix(pte), __pte(pteval)); vaddr += PAGE_SIZE; paddr += PAGE_SIZE; @@ -447,11 +316,11 @@ static void __init srmmu_nocache_init(void) flush_tlb_all(); } -static inline pgd_t *srmmu_get_pgd_fast(void) +pgd_t *get_pgd_fast(void) { pgd_t *pgd = NULL; - pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); if (pgd) { pgd_t *init = pgd_offset_k(0); memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); @@ -462,21 +331,6 @@ static inline pgd_t *srmmu_get_pgd_fast(void) return pgd; } -static void srmmu_free_pgd_fast(pgd_t *pgd) -{ - srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE); -} - -static pmd_t *srmmu_pmd_alloc_one(struct mm_struct *mm, unsigned long address) -{ - return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); -} - -static void srmmu_pmd_free(pmd_t * pmd) -{ - srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE); -} - /* * Hardware needs alignment to 256 only, but we align to whole page size * to reduce fragmentation problems due to the buddy principle. @@ -485,31 +339,22 @@ static void srmmu_pmd_free(pmd_t * pmd) * Alignments up to the page size are the same for physical and virtual * addresses of the nocache area. */ -static pte_t * -srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) -{ - return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE); -} - -static pgtable_t -srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { unsigned long pte; struct page *page; - if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0) + if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) return NULL; - page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); - pgtable_page_ctor(page); + page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT); + if (!pgtable_page_ctor(page)) { + __free_page(page); + return NULL; + } return page; } -static void srmmu_free_pte_fast(pte_t *pte) -{ - srmmu_free_nocache((unsigned long)pte, PTE_SIZE); -} - -static void srmmu_pte_free(pgtable_t pte) +void pte_free(struct mm_struct *mm, pgtable_t pte) { unsigned long p; @@ -518,18 +363,50 @@ static void srmmu_pte_free(pgtable_t pte) if (p == 0) BUG(); p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ - p = (unsigned long) __nocache_va(p); /* Nocached virtual */ - srmmu_free_nocache(p, PTE_SIZE); + + /* free non cached virtual address*/ + srmmu_free_nocache(__nocache_va(p), PTE_SIZE); } -/* - */ +/* context handling - a dynamically sized pool is used */ +#define NO_CONTEXT -1 + +struct ctx_list { + struct ctx_list *next; + struct ctx_list *prev; + unsigned int ctx_number; + struct mm_struct *ctx_mm; +}; + +static struct ctx_list *ctx_list_pool; +static struct ctx_list ctx_free; +static struct ctx_list ctx_used; + +/* At boot time we determine the number of contexts */ +static int num_contexts; + +static inline void remove_from_ctx_list(struct ctx_list *entry) +{ + entry->next->prev = entry->prev; + entry->prev->next = entry->next; +} + +static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) +{ + entry->next = head; + (entry->prev = head->prev)->next = entry; + head->prev = entry; +} +#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) +#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) + + static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) { struct ctx_list *ctxp; ctxp = ctx_free.next; - if(ctxp != &ctx_free) { + if (ctxp != &ctx_free) { remove_from_ctx_list(ctxp); add_to_used_ctxlist(ctxp); mm->context = ctxp->ctx_number; @@ -537,9 +414,9 @@ static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) return; } ctxp = ctx_used.next; - if(ctxp->ctx_mm == old_mm) + if (ctxp->ctx_mm == old_mm) ctxp = ctxp->next; - if(ctxp == &ctx_used) + if (ctxp == &ctx_used) panic("out of mmu contexts"); flush_cache_mm(ctxp->ctx_mm); flush_tlb_mm(ctxp->ctx_mm); @@ -559,11 +436,31 @@ static inline void free_context(int context) add_to_free_ctxlist(ctx_old); } +static void __init sparc_context_init(int numctx) +{ + int ctx; + unsigned long size; -static void srmmu_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, - struct task_struct *tsk, int cpu) + size = numctx * sizeof(struct ctx_list); + ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); + + for (ctx = 0; ctx < numctx; ctx++) { + struct ctx_list *clist; + + clist = (ctx_list_pool + ctx); + clist->ctx_number = ctx; + clist->ctx_mm = NULL; + } + ctx_free.next = ctx_free.prev = &ctx_free; + ctx_used.next = ctx_used.prev = &ctx_used; + for (ctx = 0; ctx < numctx; ctx++) + add_to_free_ctxlist(ctx_list_pool + ctx); +} + +void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, + struct task_struct *tsk) { - if(mm->context == NO_CONTEXT) { + if (mm->context == NO_CONTEXT) { spin_lock(&srmmu_context_spinlock); alloc_context(old_mm, mm); spin_unlock(&srmmu_context_spinlock); @@ -581,7 +478,7 @@ static void srmmu_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, /* Low level IO area allocation on the SRMMU. */ static inline void srmmu_mapioaddr(unsigned long physaddr, - unsigned long virt_addr, int bus_type) + unsigned long virt_addr, int bus_type) { pgd_t *pgdp; pmd_t *pmdp; @@ -590,23 +487,22 @@ static inline void srmmu_mapioaddr(unsigned long physaddr, physaddr &= PAGE_MASK; pgdp = pgd_offset_k(virt_addr); - pmdp = srmmu_pmd_offset(pgdp, virt_addr); - ptep = srmmu_pte_offset(pmdp, virt_addr); + pmdp = pmd_offset(pgdp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); tmp = (physaddr >> 4) | SRMMU_ET_PTE; - /* - * I need to test whether this is consistent over all + /* I need to test whether this is consistent over all * sun4m's. The bus_type represents the upper 4 bits of * 36-bit physical address on the I/O space lines... */ tmp |= (bus_type << 28); tmp |= SRMMU_PRIV; __flush_page_to_ram(virt_addr); - srmmu_set_pte(ptep, __pte(tmp)); + set_pte(ptep, __pte(tmp)); } -static void srmmu_mapiorange(unsigned int bus, unsigned long xpa, - unsigned long xva, unsigned int len) +void srmmu_mapiorange(unsigned int bus, unsigned long xpa, + unsigned long xva, unsigned int len) { while (len != 0) { len -= PAGE_SIZE; @@ -624,14 +520,14 @@ static inline void srmmu_unmapioaddr(unsigned long virt_addr) pte_t *ptep; pgdp = pgd_offset_k(virt_addr); - pmdp = srmmu_pmd_offset(pgdp, virt_addr); - ptep = srmmu_pte_offset(pmdp, virt_addr); + pmdp = pmd_offset(pgdp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); /* No need to flush uncacheable page. */ - srmmu_pte_clear(ptep); + __pte_clear(ptep); } -static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) +void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) { while (len != 0) { len -= PAGE_SIZE; @@ -641,34 +537,6 @@ static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) flush_tlb_all(); } -/* - * On the SRMMU we do not have the problems with limited tlb entries - * for mapping kernel pages, so we just take things from the free page - * pool. As a side effect we are putting a little too much pressure - * on the gfp() subsystem. This setup also makes the logic of the - * iommu mapping code a lot easier as we can transparently handle - * mappings on the kernel stack without any special code as we did - * need on the sun4c. - */ -static struct thread_info *srmmu_alloc_thread_info(void) -{ - struct thread_info *ret; - - ret = (struct thread_info *)__get_free_pages(GFP_KERNEL, - THREAD_INFO_ORDER); -#ifdef CONFIG_DEBUG_STACK_USAGE - if (ret) - memset(ret, 0, PAGE_SIZE << THREAD_INFO_ORDER); -#endif /* DEBUG_STACK_USAGE */ - - return ret; -} - -static void srmmu_free_thread_info(struct thread_info *ti) -{ - free_pages((unsigned long)ti, THREAD_INFO_ORDER); -} - /* tsunami.S */ extern void tsunami_flush_cache_all(void); extern void tsunami_flush_cache_mm(struct mm_struct *mm); @@ -683,38 +551,6 @@ extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long st extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); extern void tsunami_setup_blockops(void); -/* - * Workaround, until we find what's going on with Swift. When low on memory, - * it sometimes loops in fault/handle_mm_fault incl. flush_tlb_page to find - * out it is already in page tables/ fault again on the same instruction. - * I really don't understand it, have checked it and contexts - * are right, flush_tlb_all is done as well, and it faults again... - * Strange. -jj - * - * The following code is a deadwood that may be necessary when - * we start to make precise page flushes again. --zaitcev - */ -static void swift_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t *ptep) -{ -#if 0 - static unsigned long last; - unsigned int val; - /* unsigned int n; */ - - if (address == last) { - val = srmmu_hwprobe(address); - if (val != 0 && pte_val(*ptep) != val) { - printk("swift_update_mmu_cache: " - "addr %lx put %08x probed %08x from %p\n", - address, pte_val(*ptep), val, - __builtin_return_address(0)); - srmmu_flush_whole_tlb(); - } - } - last = address; -#endif -} - /* swift.S */ extern void swift_flush_cache_all(void); extern void swift_flush_cache_mm(struct mm_struct *mm); @@ -767,244 +603,6 @@ void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) * with respect to cache coherency. */ -/* Cypress flushes. */ -static void cypress_flush_cache_all(void) -{ - volatile unsigned long cypress_sucks; - unsigned long faddr, tagval; - - flush_user_windows(); - for(faddr = 0; faddr < 0x10000; faddr += 0x20) { - __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" : - "=r" (tagval) : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - - /* If modified and valid, kick it. */ - if((tagval & 0x60) == 0x60) - cypress_sucks = *(unsigned long *)(0xf0020000 + faddr); - } -} - -static void cypress_flush_cache_mm(struct mm_struct *mm) -{ - register unsigned long a, b, c, d, e, f, g; - unsigned long flags, faddr; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - faddr = (0x10000 - 0x100); - goto inside; - do { - faddr -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (faddr), "i" (ASI_M_FLUSH_CTX), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(faddr); - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -static void cypress_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - register unsigned long a, b, c, d, e, f, g; - unsigned long flags, faddr; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - start &= SRMMU_REAL_PMD_MASK; - while(start < end) { - faddr = (start + (0x10000 - 0x100)); - goto inside; - do { - faddr -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (faddr), - "i" (ASI_M_FLUSH_SEG), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while (faddr != start); - start += SRMMU_REAL_PMD_SIZE; - } - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -static void cypress_flush_cache_page(struct vm_area_struct *vma, unsigned long page) -{ - register unsigned long a, b, c, d, e, f, g; - struct mm_struct *mm = vma->vm_mm; - unsigned long flags, line; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - page &= PAGE_MASK; - line = (page + PAGE_SIZE) - 0x100; - goto inside; - do { - line -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (line), - "i" (ASI_M_FLUSH_PAGE), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(line != page); - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -/* Cypress is copy-back, at least that is how we configure it. */ -static void cypress_flush_page_to_ram(unsigned long page) -{ - register unsigned long a, b, c, d, e, f, g; - unsigned long line; - - a = 0x20; b = 0x40; c = 0x60; d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - page &= PAGE_MASK; - line = (page + PAGE_SIZE) - 0x100; - goto inside; - do { - line -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (line), - "i" (ASI_M_FLUSH_PAGE), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(line != page); -} - -/* Cypress is also IO cache coherent. */ -static void cypress_flush_page_for_dma(unsigned long page) -{ -} - -/* Cypress has unified L2 VIPT, from which both instructions and data - * are stored. It does not have an onboard icache of any sort, therefore - * no flush is necessary. - */ -static void cypress_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) -{ -} - -static void cypress_flush_tlb_all(void) -{ - srmmu_flush_whole_tlb(); -} - -static void cypress_flush_tlb_mm(struct mm_struct *mm) -{ - FLUSH_BEGIN(mm) - __asm__ __volatile__( - "lda [%0] %3, %%g5\n\t" - "sta %2, [%0] %3\n\t" - "sta %%g0, [%1] %4\n\t" - "sta %%g5, [%0] %3\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (0x300), "r" (mm->context), - "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE) - : "g5"); - FLUSH_END -} - -static void cypress_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - unsigned long size; - - FLUSH_BEGIN(mm) - start &= SRMMU_PGDIR_MASK; - size = SRMMU_PGDIR_ALIGN(end) - start; - __asm__ __volatile__( - "lda [%0] %5, %%g5\n\t" - "sta %1, [%0] %5\n" - "1:\n\t" - "subcc %3, %4, %3\n\t" - "bne 1b\n\t" - " sta %%g0, [%2 + %3] %6\n\t" - "sta %%g5, [%0] %5\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (start | 0x200), - "r" (size), "r" (SRMMU_PGDIR_SIZE), "i" (ASI_M_MMUREGS), - "i" (ASI_M_FLUSH_PROBE) - : "g5", "cc"); - FLUSH_END -} - -static void cypress_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - - FLUSH_BEGIN(mm) - __asm__ __volatile__( - "lda [%0] %3, %%g5\n\t" - "sta %1, [%0] %3\n\t" - "sta %%g0, [%2] %4\n\t" - "sta %%g5, [%0] %3\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (page & PAGE_MASK), - "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE) - : "g5"); - FLUSH_END -} - /* viking.S */ extern void viking_flush_cache_all(void); extern void viking_flush_cache_mm(struct mm_struct *mm); @@ -1063,23 +661,23 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); - if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *) __srmmu_get_nocache( + if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(__nocache_fix(pgdp), pmdp); + pgd_set(__nocache_fix(pgdp), pmdp); } - pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + pmdp = pmd_offset(__nocache_fix(pgdp), start); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); - srmmu_pmd_set(__nocache_fix(pmdp), ptep); + pmd_set(__nocache_fix(pmdp), ptep); } if (start > (0xffffffffUL - PMD_SIZE)) break; @@ -1094,23 +692,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); - if(srmmu_pgd_none(*pgdp)) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pgd_none(*pgdp)) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(pgdp, pmdp); + pgd_set(pgdp, pmdp); } - pmdp = srmmu_pmd_offset(pgdp, start); - if(srmmu_pmd_none(*pmdp)) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, + pmdp = pmd_offset(pgdp, start); + if (srmmu_pmd_none(*pmdp)) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(ptep, 0, PTE_SIZE); - srmmu_pmd_set(pmdp, ptep); + pmd_set(pmdp, ptep); } if (start > (0xffffffffUL - PMD_SIZE)) break; @@ -1118,6 +716,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, } } +/* These flush types are not available on all chips... */ +static inline unsigned long srmmu_probe(unsigned long vaddr) +{ + unsigned long retval; + + if (sparc_cpu_model != sparc_leon) { + + vaddr &= PAGE_MASK; + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); + } else { + retval = leon_swprobe(vaddr, NULL); + } + return retval; +} + /* * This is much cleaner than poking around physical address space * looking at the prom's page table directly which is what most @@ -1126,72 +741,76 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, static void __init srmmu_inherit_prom_mappings(unsigned long start, unsigned long end) { + unsigned long probed; + unsigned long addr; pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - unsigned long prompte; + int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - while(start <= end) { + while (start <= end) { if (start == 0) break; /* probably wrap around */ - if(start == 0xfef00000) + if (start == 0xfef00000) start = KADB_DEBUGGER_BEGVM; - if(!(prompte = srmmu_hwprobe(start))) { + probed = srmmu_probe(start); + if (!probed) { + /* continue probing until we find an entry */ start += PAGE_SIZE; continue; } - + /* A red snapper, see what it really is. */ what = 0; - - if(!(start & ~(SRMMU_REAL_PMD_MASK))) { - if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte) + addr = start - PAGE_SIZE; + + if (!(start & ~(SRMMU_REAL_PMD_MASK))) { + if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed) what = 1; } - - if(!(start & ~(SRMMU_PGDIR_MASK))) { - if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) == - prompte) + + if (!(start & ~(SRMMU_PGDIR_MASK))) { + if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed) what = 2; } - + pgdp = pgd_offset_k(start); - if(what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte); + if (what == 2) { + *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); start += SRMMU_PGDIR_SIZE; continue; } - if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(__nocache_fix(pgdp), pmdp); + pgd_set(__nocache_fix(pgdp), pmdp); } - pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, - PTE_SIZE); + pmdp = pmd_offset(__nocache_fix(pgdp), start); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); - srmmu_pmd_set(__nocache_fix(pmdp), ptep); + pmd_set(__nocache_fix(pmdp), ptep); } - if(what == 1) { - /* - * We bend the rule where all 16 PTPs in a pmd_t point + if (what == 1) { + /* We bend the rule where all 16 PTPs in a pmd_t point * inside the same PTE page, and we leak a perfectly * good hardware PTE piece. Alternatives seem worse. */ unsigned int x; /* Index of HW PMD in soft cluster */ + unsigned long *val; x = (start >> PMD_SHIFT) & 15; - *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte; + val = &pmdp->pmdv[x]; + *(unsigned long *)__nocache_fix(val) = probed; start += SRMMU_REAL_PMD_SIZE; continue; } - ptep = srmmu_pte_offset(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(prompte); + ptep = pte_offset_kernel(__nocache_fix(pmdp), start); + *(pte_t *)__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -1220,25 +839,18 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) if (vstart < min_vaddr || vstart >= max_vaddr) return vstart; - + if (vend > max_vaddr || vend < min_vaddr) vend = max_vaddr; - while(vstart < vend) { + while (vstart < vend) { do_large_mapping(vstart, pstart); vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; } return vstart; } -static inline void memprobe_error(char *msg) -{ - prom_printf(msg); - prom_printf("Halting now...\n"); - prom_halt(); -} - -static inline void map_kernel(void) +static void __init map_kernel(void) { int i; @@ -1249,16 +861,9 @@ static inline void map_kernel(void) for (i = 0; sp_banks[i].num_bytes != 0; i++) { map_spbank((unsigned long)__va(sp_banks[i].base_addr), i); } - - BTFIXUPSET_SIMM13(user_ptrs_per_pgd, PAGE_OFFSET / SRMMU_PGDIR_SIZE); } -/* Paging initialization on the Sparc Reference MMU. */ -extern void sparc_context_init(int); - -void (*poke_srmmu)(void) __cpuinitdata = NULL; - -extern unsigned long bootmem_init(unsigned long *pages_avail); +void (*poke_srmmu)(void) = NULL; void __init srmmu_paging_init(void) { @@ -1270,6 +875,7 @@ void __init srmmu_paging_init(void) pte_t *pte; unsigned long pages_avail; + init_mm.context = (unsigned long) NO_CONTEXT; sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ if (sparc_cpu_model == sun4d) @@ -1278,9 +884,9 @@ void __init srmmu_paging_init(void) /* Find the number of contexts on the srmmu. */ cpunode = prom_getchild(prom_root_node); num_contexts = 0; - while(cpunode != 0) { + while (cpunode != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); break; } @@ -1288,7 +894,7 @@ void __init srmmu_paging_init(void) } } - if(!num_contexts) { + if (!num_contexts) { prom_printf("Something wrong, can't find cpu node in paging_init.\n"); prom_halt(); } @@ -1298,21 +904,21 @@ void __init srmmu_paging_init(void) srmmu_nocache_calcsize(); srmmu_nocache_init(); - srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE)); + srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE)); map_kernel(); /* ctx table has to be physically aligned to its size */ - srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t)); + srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t)); srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); - for(i = 0; i < num_contexts; i++) + for (i = 0; i < num_contexts; i++) srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); #ifdef CONFIG_SMP /* Stop from hanging here... */ - local_flush_tlb_all(); + local_ops->tlb_all(); #else flush_tlb_all(); #endif @@ -1326,8 +932,8 @@ void __init srmmu_paging_init(void) srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); pgd = pgd_offset_k(PKMAP_BASE); - pmd = srmmu_pmd_offset(pgd, PKMAP_BASE); - pte = srmmu_pte_offset(pmd, PKMAP_BASE); + pmd = pmd_offset(pgd, PKMAP_BASE); + pte = pte_offset_kernel(pmd, PKMAP_BASE); pkmap_page_table = pte; flush_cache_all(); @@ -1359,9 +965,9 @@ void __init srmmu_paging_init(void) } } -static void srmmu_mmu_info(struct seq_file *m) +void mmu_info(struct seq_file *m) { - seq_printf(m, + seq_printf(m, "MMU type\t: %s\n" "contexts\t: %d\n" "nocache total\t: %ld\n" @@ -1372,14 +978,16 @@ static void srmmu_mmu_info(struct seq_file *m) srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); } -static void srmmu_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte) +int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { + mm->context = NO_CONTEXT; + return 0; } -static void srmmu_destroy_context(struct mm_struct *mm) +void destroy_context(struct mm_struct *mm) { - if(mm->context != NO_CONTEXT) { + if (mm->context != NO_CONTEXT) { flush_cache_mm(mm); srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); flush_tlb_mm(mm); @@ -1409,13 +1017,12 @@ static void __init init_vac_layout(void) #endif nd = prom_getchild(prom_root_node); - while((nd = prom_getsibling(nd)) != 0) { + while ((nd = prom_getsibling(nd)) != 0) { prom_getstring(nd, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { vac_line_size = prom_getint(nd, "cache-line-size"); if (vac_line_size == -1) { - prom_printf("can't determine cache-line-size, " - "halting.\n"); + prom_printf("can't determine cache-line-size, halting.\n"); prom_halt(); } cache_lines = prom_getint(nd, "cache-nlines"); @@ -1426,9 +1033,9 @@ static void __init init_vac_layout(void) vac_cache_size = cache_lines * vac_line_size; #ifdef CONFIG_SMP - if(vac_cache_size > max_size) + if (vac_cache_size > max_size) max_size = vac_cache_size; - if(vac_line_size < min_line_size) + if (vac_line_size < min_line_size) min_line_size = vac_line_size; //FIXME: cpus not contiguous!! cpu++; @@ -1439,7 +1046,7 @@ static void __init init_vac_layout(void) #endif } } - if(nd == 0) { + if (nd == 0) { prom_printf("No CPU nodes found, halting.\n"); prom_halt(); } @@ -1451,7 +1058,7 @@ static void __init init_vac_layout(void) (int)vac_cache_size, (int)vac_line_size); } -static void __cpuinit poke_hypersparc(void) +static void poke_hypersparc(void) { volatile unsigned long clear; unsigned long mreg = srmmu_get_mmureg(); @@ -1474,6 +1081,20 @@ static void __cpuinit poke_hypersparc(void) clear = srmmu_get_fstatus(); } +static const struct sparc32_cachetlb_ops hypersparc_ops = { + .cache_all = hypersparc_flush_cache_all, + .cache_mm = hypersparc_flush_cache_mm, + .cache_page = hypersparc_flush_cache_page, + .cache_range = hypersparc_flush_cache_range, + .tlb_all = hypersparc_flush_tlb_all, + .tlb_mm = hypersparc_flush_tlb_mm, + .tlb_page = hypersparc_flush_tlb_page, + .tlb_range = hypersparc_flush_tlb_range, + .page_to_ram = hypersparc_flush_page_to_ram, + .sig_insns = hypersparc_flush_sig_insns, + .page_for_dma = hypersparc_flush_page_for_dma, +}; + static void __init init_hypersparc(void) { srmmu_name = "ROSS HyperSparc"; @@ -1482,119 +1103,14 @@ static void __init init_hypersparc(void) init_vac_layout(); is_hypersparc = 1; - - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_all, hypersparc_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, hypersparc_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, hypersparc_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, hypersparc_flush_cache_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, hypersparc_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, hypersparc_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, hypersparc_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, hypersparc_flush_tlb_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, hypersparc_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, hypersparc_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, hypersparc_flush_page_for_dma, BTFIXUPCALL_NOP); - + sparc32_cachetlb_ops = &hypersparc_ops; poke_srmmu = poke_hypersparc; hypersparc_setup_blockops(); } -static void __cpuinit poke_cypress(void) -{ - unsigned long mreg = srmmu_get_mmureg(); - unsigned long faddr, tagval; - volatile unsigned long cypress_sucks; - volatile unsigned long clear; - - clear = srmmu_get_faddr(); - clear = srmmu_get_fstatus(); - - if (!(mreg & CYPRESS_CENABLE)) { - for(faddr = 0x0; faddr < 0x10000; faddr += 20) { - __asm__ __volatile__("sta %%g0, [%0 + %1] %2\n\t" - "sta %%g0, [%0] %2\n\t" : : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - } - } else { - for(faddr = 0; faddr < 0x10000; faddr += 0x20) { - __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" : - "=r" (tagval) : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - - /* If modified and valid, kick it. */ - if((tagval & 0x60) == 0x60) - cypress_sucks = *(unsigned long *) - (0xf0020000 + faddr); - } - } - - /* And one more, for our good neighbor, Mr. Broken Cypress. */ - clear = srmmu_get_faddr(); - clear = srmmu_get_fstatus(); - - mreg |= (CYPRESS_CENABLE | CYPRESS_CMODE); - srmmu_set_mmureg(mreg); -} - -static void __init init_cypress_common(void) -{ - init_vac_layout(); - - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_all, cypress_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, cypress_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, cypress_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, cypress_flush_cache_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, cypress_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, cypress_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, cypress_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, cypress_flush_tlb_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(__flush_page_to_ram, cypress_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, cypress_flush_sig_insns, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_page_for_dma, cypress_flush_page_for_dma, BTFIXUPCALL_NOP); - - poke_srmmu = poke_cypress; -} - -static void __init init_cypress_604(void) -{ - srmmu_name = "ROSS Cypress-604(UP)"; - srmmu_modtype = Cypress; - init_cypress_common(); -} - -static void __init init_cypress_605(unsigned long mrev) -{ - srmmu_name = "ROSS Cypress-605(MP)"; - if(mrev == 0xe) { - srmmu_modtype = Cypress_vE; - hwbug_bitmask |= HWBUG_COPYBACK_BROKEN; - } else { - if(mrev == 0xd) { - srmmu_modtype = Cypress_vD; - hwbug_bitmask |= HWBUG_ASIFLUSH_BROKEN; - } else { - srmmu_modtype = Cypress; - } - } - init_cypress_common(); -} - -static void __cpuinit poke_swift(void) +static void poke_swift(void) { unsigned long mreg; @@ -1617,6 +1133,20 @@ static void __cpuinit poke_swift(void) srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops swift_ops = { + .cache_all = swift_flush_cache_all, + .cache_mm = swift_flush_cache_mm, + .cache_page = swift_flush_cache_page, + .cache_range = swift_flush_cache_range, + .tlb_all = swift_flush_tlb_all, + .tlb_mm = swift_flush_tlb_mm, + .tlb_page = swift_flush_tlb_page, + .tlb_range = swift_flush_tlb_range, + .page_to_ram = swift_flush_page_to_ram, + .sig_insns = swift_flush_sig_insns, + .page_for_dma = swift_flush_page_for_dma, +}; + #define SWIFT_MASKID_ADDR 0x10003018 static void __init init_swift(void) { @@ -1627,7 +1157,7 @@ static void __init init_swift(void) "=r" (swift_rev) : "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); srmmu_name = "Fujitsu Swift"; - switch(swift_rev) { + switch (swift_rev) { case 0x11: case 0x20: case 0x23: @@ -1665,25 +1195,9 @@ static void __init init_swift(void) default: srmmu_modtype = Swift_ok; break; - }; - - BTFIXUPSET_CALL(flush_cache_all, swift_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, swift_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, swift_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, swift_flush_cache_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(flush_tlb_all, swift_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, swift_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, swift_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, swift_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, swift_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, swift_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, swift_flush_page_for_dma, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(update_mmu_cache, swift_update_mmu_cache, BTFIXUPCALL_NORM); + } + sparc32_cachetlb_ops = &swift_ops; flush_page_for_dma_global = 0; /* @@ -1734,7 +1248,7 @@ static void turbosparc_flush_page_to_ram(unsigned long page) #ifdef TURBOSPARC_WRITEBACK volatile unsigned long clear; - if (srmmu_hwprobe(page)) + if (srmmu_probe(page)) turbosparc_flush_page_cache(page); clear = srmmu_get_fstatus(); #endif @@ -1776,17 +1290,18 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long } -static void __cpuinit poke_turbosparc(void) +static void poke_turbosparc(void) { unsigned long mreg = srmmu_get_mmureg(); unsigned long ccreg; /* Clear any crap from the cache or else... */ turbosparc_flush_cache_all(); - mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */ + /* Temporarily disable I & D caches */ + mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ srmmu_set_mmureg(mreg); - + ccreg = turbosparc_get_ccreg(); #ifdef TURBOSPARC_WRITEBACK @@ -1809,37 +1324,36 @@ static void __cpuinit poke_turbosparc(void) default: ccreg |= (TURBOSPARC_SCENABLE); } - turbosparc_set_ccreg (ccreg); + turbosparc_set_ccreg(ccreg); mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops turbosparc_ops = { + .cache_all = turbosparc_flush_cache_all, + .cache_mm = turbosparc_flush_cache_mm, + .cache_page = turbosparc_flush_cache_page, + .cache_range = turbosparc_flush_cache_range, + .tlb_all = turbosparc_flush_tlb_all, + .tlb_mm = turbosparc_flush_tlb_mm, + .tlb_page = turbosparc_flush_tlb_page, + .tlb_range = turbosparc_flush_tlb_range, + .page_to_ram = turbosparc_flush_page_to_ram, + .sig_insns = turbosparc_flush_sig_insns, + .page_for_dma = turbosparc_flush_page_for_dma, +}; + static void __init init_turbosparc(void) { srmmu_name = "Fujitsu TurboSparc"; srmmu_modtype = TurboSparc; - - BTFIXUPSET_CALL(flush_cache_all, turbosparc_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, turbosparc_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, turbosparc_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, turbosparc_flush_cache_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, turbosparc_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, turbosparc_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, turbosparc_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, turbosparc_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, turbosparc_flush_page_to_ram, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_sig_insns, turbosparc_flush_sig_insns, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_page_for_dma, turbosparc_flush_page_for_dma, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = &turbosparc_ops; poke_srmmu = poke_turbosparc; } -static void __cpuinit poke_tsunami(void) +static void poke_tsunami(void) { unsigned long mreg = srmmu_get_mmureg(); @@ -1850,6 +1364,20 @@ static void __cpuinit poke_tsunami(void) srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops tsunami_ops = { + .cache_all = tsunami_flush_cache_all, + .cache_mm = tsunami_flush_cache_mm, + .cache_page = tsunami_flush_cache_page, + .cache_range = tsunami_flush_cache_range, + .tlb_all = tsunami_flush_tlb_all, + .tlb_mm = tsunami_flush_tlb_mm, + .tlb_page = tsunami_flush_tlb_page, + .tlb_range = tsunami_flush_tlb_range, + .page_to_ram = tsunami_flush_page_to_ram, + .sig_insns = tsunami_flush_sig_insns, + .page_for_dma = tsunami_flush_page_for_dma, +}; + static void __init init_tsunami(void) { /* @@ -1860,33 +1388,18 @@ static void __init init_tsunami(void) srmmu_name = "TI Tsunami"; srmmu_modtype = Tsunami; - - BTFIXUPSET_CALL(flush_cache_all, tsunami_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, tsunami_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, tsunami_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, tsunami_flush_cache_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(flush_tlb_all, tsunami_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, tsunami_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, tsunami_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, tsunami_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, tsunami_flush_page_to_ram, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, tsunami_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, tsunami_flush_page_for_dma, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = &tsunami_ops; poke_srmmu = poke_tsunami; tsunami_setup_blockops(); } -static void __cpuinit poke_viking(void) +static void poke_viking(void) { unsigned long mreg = srmmu_get_mmureg(); static int smp_catch; - if(viking_mxcc_present) { + if (viking_mxcc_present) { unsigned long mxcc_control = mxcc_get_creg(); mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE); @@ -1905,7 +1418,7 @@ static void __cpuinit poke_viking(void) unsigned long bpreg; mreg &= ~(VIKING_TCENABLE); - if(smp_catch++) { + if (smp_catch++) { /* Must disable mixed-cmd mode here for other cpu's. */ bpreg = viking_get_bpreg(); bpreg &= ~(VIKING_ACTION_MIX); @@ -1923,20 +1436,62 @@ static void __cpuinit poke_viking(void) srmmu_set_mmureg(mreg); } +static struct sparc32_cachetlb_ops viking_ops = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = viking_flush_tlb_all, + .tlb_mm = viking_flush_tlb_mm, + .tlb_page = viking_flush_tlb_page, + .tlb_range = viking_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; + +#ifdef CONFIG_SMP +/* On sun4d the cpu broadcasts local TLB flushes, so we can just + * perform the local TLB flush and all the other cpus will see it. + * But, unfortunately, there is a bug in the sun4d XBUS backplane + * that requires that we add some synchronization to these flushes. + * + * The bug is that the fifo which keeps track of all the pending TLB + * broadcasts in the system is an entry or two too small, so if we + * have too many going at once we'll overflow that fifo and lose a TLB + * flush resulting in corruption. + * + * Our workaround is to take a global spinlock around the TLB flushes, + * which guarentees we won't ever have too many pending. It's a big + * hammer, but a semaphore like system to make sure we only have N TLB + * flushes going at once will require SMP locking anyways so there's + * no real value in trying any harder than this. + */ +static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = sun4dsmp_flush_tlb_all, + .tlb_mm = sun4dsmp_flush_tlb_mm, + .tlb_page = sun4dsmp_flush_tlb_page, + .tlb_range = sun4dsmp_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; +#endif + static void __init init_viking(void) { unsigned long mreg = srmmu_get_mmureg(); /* Ahhh, the viking. SRMMU VLSI abortion number two... */ - if(mreg & VIKING_MMODE) { + if (mreg & VIKING_MMODE) { srmmu_name = "TI Viking"; viking_mxcc_present = 0; msi_set_sync(); - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - /* * We need this to make sure old viking takes no hits * on it's cache for dma snoops to workaround the @@ -1944,84 +1499,28 @@ static void __init init_viking(void) * This is only necessary because of the new way in * which we use the IOMMU. */ - BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page, BTFIXUPCALL_NORM); - + viking_ops.page_for_dma = viking_flush_page; +#ifdef CONFIG_SMP + viking_sun4d_smp_ops.page_for_dma = viking_flush_page; +#endif flush_page_for_dma_global = 0; } else { srmmu_name = "TI Viking/MXCC"; viking_mxcc_present = 1; - srmmu_cache_pagetables = 1; - - /* MXCC vikings lack the DMA snooping bug. */ - BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page_for_dma, BTFIXUPCALL_NOP); } - BTFIXUPSET_CALL(flush_cache_all, viking_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, viking_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, viking_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, viking_flush_cache_range, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_ops; #ifdef CONFIG_SMP - if (sparc_cpu_model == sun4d) { - BTFIXUPSET_CALL(flush_tlb_all, sun4dsmp_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, sun4dsmp_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, sun4dsmp_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, sun4dsmp_flush_tlb_range, BTFIXUPCALL_NORM); - } else + if (sparc_cpu_model == sun4d) + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_sun4d_smp_ops; #endif - { - BTFIXUPSET_CALL(flush_tlb_all, viking_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, viking_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, viking_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, viking_flush_tlb_range, BTFIXUPCALL_NORM); - } - - BTFIXUPSET_CALL(__flush_page_to_ram, viking_flush_page_to_ram, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, viking_flush_sig_insns, BTFIXUPCALL_NOP); poke_srmmu = poke_viking; } -#ifdef CONFIG_SPARC_LEON - -void __init poke_leonsparc(void) -{ -} - -void __init init_leon(void) -{ - - srmmu_name = "LEON"; - - BTFIXUPSET_CALL(flush_cache_all, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, leon_flush_pcache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, leon_flush_dcache_all, - BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, leon_flush_tlb_all, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, leon_flush_cache_all, - BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, leon_flush_cache_all, BTFIXUPCALL_NOP); - - poke_srmmu = poke_leonsparc; - - srmmu_cache_pagetables = 0; - - leon_flush_during_switch = leon_flush_needed(); -} -#endif - /* Probe for the srmmu chip version. */ static void __init get_srmmu_type(void) { @@ -2044,37 +1543,29 @@ static void __init get_srmmu_type(void) } /* Second, check for HyperSparc or Cypress. */ - if(mod_typ == 1) { - switch(mod_rev) { + if (mod_typ == 1) { + switch (mod_rev) { case 7: /* UP or MP Hypersparc */ init_hypersparc(); break; case 0: case 2: - /* Uniprocessor Cypress */ - init_cypress_604(); - break; case 10: case 11: case 12: - /* _REALLY OLD_ Cypress MP chips... */ case 13: case 14: case 15: - /* MP Cypress mmu/cache-controller */ - init_cypress_605(mod_rev); - break; default: - /* Some other Cypress revision, assume a 605. */ - init_cypress_605(mod_rev); + prom_printf("Sparc-Linux Cypress support does not longer exit.\n"); + prom_halt(); break; - }; + } return; } - - /* - * Now Fujitsu TurboSparc. It might happen that it is + + /* Now Fujitsu TurboSparc. It might happen that it is * in Swift emulation mode, so we will check later... */ if (psr_typ == 0 && psr_vers == 5) { @@ -2083,15 +1574,15 @@ static void __init get_srmmu_type(void) } /* Next check for Fujitsu Swift. */ - if(psr_typ == 0 && psr_vers == 4) { + if (psr_typ == 0 && psr_vers == 4) { phandle cpunode; char node_str[128]; /* Look if it is not a TurboSparc emulating Swift... */ cpunode = prom_getchild(prom_root_node); - while((cpunode = prom_getsibling(cpunode)) != 0) { + while ((cpunode = prom_getsibling(cpunode)) != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { if (!prom_getintdefault(cpunode, "psr-implementation", 1) && prom_getintdefault(cpunode, "psr-version", 1) == 5) { init_turbosparc(); @@ -2100,13 +1591,13 @@ static void __init get_srmmu_type(void) break; } } - + init_swift(); return; } /* Now the Viking family of srmmu. */ - if(psr_typ == 4 && + if (psr_typ == 4 && ((psr_vers == 0) || ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { init_viking(); @@ -2114,7 +1605,7 @@ static void __init get_srmmu_type(void) } /* Finally the Tsunami. */ - if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { + if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { init_tsunami(); return; } @@ -2123,203 +1614,190 @@ static void __init get_srmmu_type(void) srmmu_is_bad(); } -/* don't laugh, static pagetables */ -static void srmmu_check_pgt_cache(int low, int high) +#ifdef CONFIG_SMP +/* Local cross-calls. */ +static void smp_flush_page_for_dma(unsigned long page) { + xc1((smpfunc_t) local_ops->page_for_dma, page); + local_ops->page_for_dma(page); } -extern unsigned long spwin_mmu_patchme, fwin_mmu_patchme, - tsetup_mmu_patchme, rtrap_mmu_patchme; - -extern unsigned long spwin_srmmu_stackchk, srmmu_fwin_stackchk, - tsetup_srmmu_stackchk, srmmu_rett_stackchk; - -extern unsigned long srmmu_fault; - -#define PATCH_BRANCH(insn, dest) do { \ - iaddr = &(insn); \ - daddr = &(dest); \ - *iaddr = SPARC_BRANCH((unsigned long) daddr, (unsigned long) iaddr); \ - } while(0) - -static void __init patch_window_trap_handlers(void) +static void smp_flush_cache_all(void) { - unsigned long *iaddr, *daddr; - - PATCH_BRANCH(spwin_mmu_patchme, spwin_srmmu_stackchk); - PATCH_BRANCH(fwin_mmu_patchme, srmmu_fwin_stackchk); - PATCH_BRANCH(tsetup_mmu_patchme, tsetup_srmmu_stackchk); - PATCH_BRANCH(rtrap_mmu_patchme, srmmu_rett_stackchk); - PATCH_BRANCH(sparc_ttable[SP_TRAP_TFLT].inst_three, srmmu_fault); - PATCH_BRANCH(sparc_ttable[SP_TRAP_DFLT].inst_three, srmmu_fault); - PATCH_BRANCH(sparc_ttable[SP_TRAP_DACC].inst_three, srmmu_fault); + xc0((smpfunc_t) local_ops->cache_all); + local_ops->cache_all(); } -#ifdef CONFIG_SMP -/* Local cross-calls. */ -static void smp_flush_page_for_dma(unsigned long page) +static void smp_flush_tlb_all(void) { - xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_for_dma), page); - local_flush_page_for_dma(page); + xc0((smpfunc_t) local_ops->tlb_all); + local_ops->tlb_all(); } -#endif +static void smp_flush_cache_mm(struct mm_struct *mm) +{ + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm); + local_ops->cache_mm(mm); + } +} -static pte_t srmmu_pgoff_to_pte(unsigned long pgoff) +static void smp_flush_tlb_mm(struct mm_struct *mm) { - return __pte((pgoff << SRMMU_PTE_FILE_SHIFT) | SRMMU_FILE); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) { + xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm); + if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) + cpumask_copy(mm_cpumask(mm), + cpumask_of(smp_processor_id())); + } + local_ops->tlb_mm(mm); + } } -static unsigned long srmmu_pte_to_pgoff(pte_t pte) +static void smp_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) { - return pte_val(pte) >> SRMMU_PTE_FILE_SHIFT; + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc3((smpfunc_t) local_ops->cache_range, + (unsigned long) vma, start, end); + local_ops->cache_range(vma, start, end); + } } -static pgprot_t srmmu_pgprot_noncached(pgprot_t prot) +static void smp_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) { - prot &= ~__pgprot(SRMMU_CACHE); + struct mm_struct *mm = vma->vm_mm; - return prot; + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc3((smpfunc_t) local_ops->tlb_range, + (unsigned long) vma, start, end); + local_ops->tlb_range(vma, start, end); + } } -/* Load up routines and constants for sun4m and sun4d mmu */ -void __init ld_mmu_srmmu(void) +static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) { - extern void ld_mmu_iommu(void); - extern void ld_mmu_iounit(void); - extern void ___xchg32_sun4md(void); + struct mm_struct *mm = vma->vm_mm; - BTFIXUPSET_SIMM13(pgdir_shift, SRMMU_PGDIR_SHIFT); - BTFIXUPSET_SETHI(pgdir_size, SRMMU_PGDIR_SIZE); - BTFIXUPSET_SETHI(pgdir_mask, SRMMU_PGDIR_MASK); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->cache_page, + (unsigned long) vma, page); + local_ops->cache_page(vma, page); + } +} - BTFIXUPSET_SIMM13(ptrs_per_pmd, SRMMU_PTRS_PER_PMD); - BTFIXUPSET_SIMM13(ptrs_per_pgd, SRMMU_PTRS_PER_PGD); +static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; - BTFIXUPSET_INT(page_none, pgprot_val(SRMMU_PAGE_NONE)); - PAGE_SHARED = pgprot_val(SRMMU_PAGE_SHARED); - BTFIXUPSET_INT(page_copy, pgprot_val(SRMMU_PAGE_COPY)); - BTFIXUPSET_INT(page_readonly, pgprot_val(SRMMU_PAGE_RDONLY)); - BTFIXUPSET_INT(page_kernel, pgprot_val(SRMMU_PAGE_KERNEL)); - page_kernel = pgprot_val(SRMMU_PAGE_KERNEL); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->tlb_page, + (unsigned long) vma, page); + local_ops->tlb_page(vma, page); + } +} - /* Functions */ - BTFIXUPSET_CALL(pgprot_noncached, srmmu_pgprot_noncached, BTFIXUPCALL_NORM); -#ifndef CONFIG_SMP - BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4md, BTFIXUPCALL_SWAPG1G2); +static void smp_flush_page_to_ram(unsigned long page) +{ + /* Current theory is that those who call this are the one's + * who have just dirtied their cache with the pages contents + * in kernel space, therefore we only run this on local cpu. + * + * XXX This experiment failed, research further... -DaveM + */ +#if 1 + xc1((smpfunc_t) local_ops->page_to_ram, page); +#endif + local_ops->page_to_ram(page); +} + +static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) +{ + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->sig_insns, + (unsigned long) mm, insn_addr); + local_ops->sig_insns(mm, insn_addr); +} + +static struct sparc32_cachetlb_ops smp_cachetlb_ops = { + .cache_all = smp_flush_cache_all, + .cache_mm = smp_flush_cache_mm, + .cache_page = smp_flush_cache_page, + .cache_range = smp_flush_cache_range, + .tlb_all = smp_flush_tlb_all, + .tlb_mm = smp_flush_tlb_mm, + .tlb_page = smp_flush_tlb_page, + .tlb_range = smp_flush_tlb_range, + .page_to_ram = smp_flush_page_to_ram, + .sig_insns = smp_flush_sig_insns, + .page_for_dma = smp_flush_page_for_dma, +}; #endif - BTFIXUPSET_CALL(do_check_pgt_cache, srmmu_check_pgt_cache, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(set_pte, srmmu_set_pte, BTFIXUPCALL_SWAPO0O1); - BTFIXUPSET_CALL(switch_mm, srmmu_switch_mm, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_pfn, srmmu_pte_pfn, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_page, srmmu_pmd_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_page_vaddr, srmmu_pgd_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_present, srmmu_pte_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(pmd_bad, srmmu_pmd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_present, srmmu_pmd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(pgd_none, srmmu_pgd_none, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_bad, srmmu_pgd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_present, srmmu_pgd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(mk_pte, srmmu_mk_pte, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_phys, srmmu_mk_pte_phys, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_io, srmmu_mk_pte_io, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_set, srmmu_pgd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_set, srmmu_pmd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_populate, srmmu_pmd_populate, BTFIXUPCALL_NORM); - - BTFIXUPSET_INT(pte_modify_mask, SRMMU_CHG_MASK); - BTFIXUPSET_CALL(pmd_offset, srmmu_pmd_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_offset_kernel, srmmu_pte_offset, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(free_pte_fast, srmmu_free_pte_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_free, srmmu_pte_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one_kernel, srmmu_pte_alloc_one_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one, srmmu_pte_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pmd_fast, srmmu_pmd_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_alloc_one, srmmu_pmd_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pgd_fast, srmmu_free_pgd_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(get_pgd_fast, srmmu_get_pgd_fast, BTFIXUPCALL_NORM); - - BTFIXUPSET_HALF(pte_writei, SRMMU_WRITE); - BTFIXUPSET_HALF(pte_dirtyi, SRMMU_DIRTY); - BTFIXUPSET_HALF(pte_youngi, SRMMU_REF); - BTFIXUPSET_HALF(pte_filei, SRMMU_FILE); - BTFIXUPSET_HALF(pte_wrprotecti, SRMMU_WRITE); - BTFIXUPSET_HALF(pte_mkcleani, SRMMU_DIRTY); - BTFIXUPSET_HALF(pte_mkoldi, SRMMU_REF); - BTFIXUPSET_CALL(pte_mkwrite, srmmu_pte_mkwrite, BTFIXUPCALL_ORINT(SRMMU_WRITE)); - BTFIXUPSET_CALL(pte_mkdirty, srmmu_pte_mkdirty, BTFIXUPCALL_ORINT(SRMMU_DIRTY)); - BTFIXUPSET_CALL(pte_mkyoung, srmmu_pte_mkyoung, BTFIXUPCALL_ORINT(SRMMU_REF)); - BTFIXUPSET_CALL(update_mmu_cache, srmmu_update_mmu_cache, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(destroy_context, srmmu_destroy_context, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(sparc_mapiorange, srmmu_mapiorange, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sparc_unmapiorange, srmmu_unmapiorange, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__swp_type, srmmu_swp_type, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_offset, srmmu_swp_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_entry, srmmu_swp_entry, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(alloc_thread_info, srmmu_alloc_thread_info, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_thread_info, srmmu_free_thread_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_to_pgoff, srmmu_pte_to_pgoff, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgoff_to_pte, srmmu_pgoff_to_pte, BTFIXUPCALL_NORM); +/* Load up routines and constants for sun4m and sun4d mmu */ +void __init load_mmu(void) +{ + /* Functions */ get_srmmu_type(); - patch_window_trap_handlers(); #ifdef CONFIG_SMP /* El switcheroo... */ + local_ops = sparc32_cachetlb_ops; - BTFIXUPCOPY_CALL(local_flush_cache_all, flush_cache_all); - BTFIXUPCOPY_CALL(local_flush_cache_mm, flush_cache_mm); - BTFIXUPCOPY_CALL(local_flush_cache_range, flush_cache_range); - BTFIXUPCOPY_CALL(local_flush_cache_page, flush_cache_page); - BTFIXUPCOPY_CALL(local_flush_tlb_all, flush_tlb_all); - BTFIXUPCOPY_CALL(local_flush_tlb_mm, flush_tlb_mm); - BTFIXUPCOPY_CALL(local_flush_tlb_range, flush_tlb_range); - BTFIXUPCOPY_CALL(local_flush_tlb_page, flush_tlb_page); - BTFIXUPCOPY_CALL(local_flush_page_to_ram, __flush_page_to_ram); - BTFIXUPCOPY_CALL(local_flush_sig_insns, flush_sig_insns); - BTFIXUPCOPY_CALL(local_flush_page_for_dma, flush_page_for_dma); - - BTFIXUPSET_CALL(flush_cache_all, smp_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, smp_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, smp_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, smp_flush_cache_page, BTFIXUPCALL_NORM); - if (sparc_cpu_model != sun4d && - sparc_cpu_model != sparc_leon) { - BTFIXUPSET_CALL(flush_tlb_all, smp_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, smp_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, smp_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, smp_flush_tlb_page, BTFIXUPCALL_NORM); + if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) { + smp_cachetlb_ops.tlb_all = local_ops->tlb_all; + smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm; + smp_cachetlb_ops.tlb_range = local_ops->tlb_range; + smp_cachetlb_ops.tlb_page = local_ops->tlb_page; } - BTFIXUPSET_CALL(__flush_page_to_ram, smp_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, smp_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, smp_flush_page_for_dma, BTFIXUPCALL_NORM); if (poke_srmmu == poke_viking) { /* Avoid unnecessary cross calls. */ - BTFIXUPCOPY_CALL(flush_cache_all, local_flush_cache_all); - BTFIXUPCOPY_CALL(flush_cache_mm, local_flush_cache_mm); - BTFIXUPCOPY_CALL(flush_cache_range, local_flush_cache_range); - BTFIXUPCOPY_CALL(flush_cache_page, local_flush_cache_page); - BTFIXUPCOPY_CALL(__flush_page_to_ram, local_flush_page_to_ram); - BTFIXUPCOPY_CALL(flush_sig_insns, local_flush_sig_insns); - BTFIXUPCOPY_CALL(flush_page_for_dma, local_flush_page_for_dma); + smp_cachetlb_ops.cache_all = local_ops->cache_all; + smp_cachetlb_ops.cache_mm = local_ops->cache_mm; + smp_cachetlb_ops.cache_range = local_ops->cache_range; + smp_cachetlb_ops.cache_page = local_ops->cache_page; + + smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram; + smp_cachetlb_ops.sig_insns = local_ops->sig_insns; + smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma; } + + /* It really is const after this point. */ + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &smp_cachetlb_ops; #endif if (sparc_cpu_model == sun4d) diff --git a/arch/sparc/mm/srmmu_access.S b/arch/sparc/mm/srmmu_access.S new file mode 100644 index 00000000000..d0a67b2c238 --- /dev/null +++ b/arch/sparc/mm/srmmu_access.S @@ -0,0 +1,82 @@ +/* Assembler variants of srmmu access functions. + * Implemented in assembler to allow run-time patching. + * LEON uses a different ASI for MMUREGS than SUN. + * + * The leon_1insn_patch infrastructure is used + * for the run-time patching. + */ + +#include <linux/linkage.h> + +#include <asm/asmmacro.h> +#include <asm/pgtsrmmu.h> +#include <asm/asi.h> + +/* unsigned int srmmu_get_mmureg(void) */ +ENTRY(srmmu_get_mmureg) +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_mmureg) + +/* void srmmu_set_mmureg(unsigned long regval) */ +ENTRY(srmmu_set_mmureg) +LEON_PI(sta %o0, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g0] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_mmureg) + +/* void srmmu_set_ctable_ptr(unsigned long paddr) */ +ENTRY(srmmu_set_ctable_ptr) + /* paddr = ((paddr >> 4) & SRMMU_CTX_PMASK); */ + srl %o0, 4, %g1 + and %g1, SRMMU_CTX_PMASK, %g1 + + mov SRMMU_CTXTBL_PTR, %g2 +LEON_PI(sta %g1, [%g2] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g2] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_ctable_ptr) + + +/* void srmmu_set_context(int context) */ +ENTRY(srmmu_set_context) + mov SRMMU_CTX_REG, %g1 +LEON_PI(sta %o0, [%g1] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g1] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_context) + + +/* int srmmu_get_context(void) */ +ENTRY(srmmu_get_context) + mov SRMMU_CTX_REG, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_context) + + +/* unsigned int srmmu_get_fstatus(void) */ +ENTRY(srmmu_get_fstatus) + mov SRMMU_FAULT_STATUS, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_fstatus) + + +/* unsigned int srmmu_get_faddr(void) */ +ENTRY(srmmu_get_faddr) + mov SRMMU_FAULT_ADDR, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_faddr) diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c deleted file mode 100644 index ddd0d86e508..00000000000 --- a/arch/sparc/mm/sun4c.c +++ /dev/null @@ -1,2168 +0,0 @@ -/* sun4c.c: Doing in software what should be done in hardware. - * - * Copyright (C) 1996 David S. Miller (davem@davemloft.net) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au) - * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org) - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define NR_TASK_BUCKETS 512 - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <linux/slab.h> -#include <linux/bootmem.h> -#include <linux/highmem.h> -#include <linux/fs.h> -#include <linux/seq_file.h> -#include <linux/scatterlist.h> -#include <linux/bitmap.h> - -#include <asm/sections.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/vaddrs.h> -#include <asm/idprom.h> -#include <asm/machines.h> -#include <asm/memreg.h> -#include <asm/processor.h> -#include <asm/auxio.h> -#include <asm/io.h> -#include <asm/oplib.h> -#include <asm/openprom.h> -#include <asm/mmu_context.h> -#include <asm/highmem.h> -#include <asm/btfixup.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> - -/* Because of our dynamic kernel TLB miss strategy, and how - * our DVMA mapping allocation works, you _MUST_: - * - * 1) Disable interrupts _and_ not touch any dynamic kernel - * memory while messing with kernel MMU state. By - * dynamic memory I mean any object which is not in - * the kernel image itself or a thread_union (both of - * which are locked into the MMU). - * 2) Disable interrupts while messing with user MMU state. - */ - -extern int num_segmaps, num_contexts; - -extern unsigned long page_kernel; - -/* That's it, we prom_halt() on sun4c if the cache size is something other than 65536. - * So let's save some cycles and just use that everywhere except for that bootup - * sanity check. - */ -#define SUN4C_VAC_SIZE 65536 - -#define SUN4C_KERNEL_BUCKETS 32 - -/* Flushing the cache. */ -struct sun4c_vac_props sun4c_vacinfo; -unsigned long sun4c_kernel_faults; - -/* Invalidate every sun4c cache line tag. */ -static void __init sun4c_flush_all(void) -{ - unsigned long begin, end; - - if (sun4c_vacinfo.on) - panic("SUN4C: AIEEE, trying to invalidate vac while it is on."); - - /* Clear 'valid' bit in all cache line tags */ - begin = AC_CACHETAGS; - end = (AC_CACHETAGS + SUN4C_VAC_SIZE); - while (begin < end) { - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (begin), "i" (ASI_CONTROL)); - begin += sun4c_vacinfo.linesize; - } -} - -static void sun4c_flush_context_hw(void) -{ - unsigned long end = SUN4C_VAC_SIZE; - - __asm__ __volatile__( - "1: addcc %0, -4096, %0\n\t" - " bne 1b\n\t" - " sta %%g0, [%0] %2" - : "=&r" (end) - : "0" (end), "i" (ASI_HWFLUSHCONTEXT) - : "cc"); -} - -/* Must be called minimally with IRQs disabled. */ -static void sun4c_flush_segment_hw(unsigned long addr) -{ - if (sun4c_get_segmap(addr) != invalid_segment) { - unsigned long vac_size = SUN4C_VAC_SIZE; - - __asm__ __volatile__( - "1: addcc %0, -4096, %0\n\t" - " bne 1b\n\t" - " sta %%g0, [%2 + %0] %3" - : "=&r" (vac_size) - : "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG) - : "cc"); - } -} - -/* File local boot time fixups. */ -BTFIXUPDEF_CALL(void, sun4c_flush_page, unsigned long) -BTFIXUPDEF_CALL(void, sun4c_flush_segment, unsigned long) -BTFIXUPDEF_CALL(void, sun4c_flush_context, void) - -#define sun4c_flush_page(addr) BTFIXUP_CALL(sun4c_flush_page)(addr) -#define sun4c_flush_segment(addr) BTFIXUP_CALL(sun4c_flush_segment)(addr) -#define sun4c_flush_context() BTFIXUP_CALL(sun4c_flush_context)() - -/* Must be called minimally with interrupts disabled. */ -static void sun4c_flush_page_hw(unsigned long addr) -{ - addr &= PAGE_MASK; - if ((int)sun4c_get_pte(addr) < 0) - __asm__ __volatile__("sta %%g0, [%0] %1" - : : "r" (addr), "i" (ASI_HWFLUSHPAGE)); -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_context_sw(void) -{ - unsigned long nbytes = SUN4C_VAC_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %0, %%o5, %0\n\t" - "sta %%g0, [%0] %3\n\t" - "sta %%g0, [%0 + %2] %3\n\t" - "sta %%g0, [%0 + %%g1] %3\n\t" - "sta %%g0, [%0 + %%g2] %3\n\t" - "sta %%g0, [%0 + %%g3] %3\n\t" - "sta %%g0, [%0 + %%g4] %3\n\t" - "sta %%g0, [%0 + %%g5] %3\n\t" - "bg 1b\n\t" - " sta %%g0, [%1 + %%o4] %3\n" - : "=&r" (nbytes) - : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_segment_sw(unsigned long addr) -{ - if (sun4c_get_segmap(addr) != invalid_segment) { - unsigned long nbytes = SUN4C_VAC_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %1, %%o5, %1\n\t" - "sta %%g0, [%0] %6\n\t" - "sta %%g0, [%0 + %2] %6\n\t" - "sta %%g0, [%0 + %%g1] %6\n\t" - "sta %%g0, [%0 + %%g2] %6\n\t" - "sta %%g0, [%0 + %%g3] %6\n\t" - "sta %%g0, [%0 + %%g4] %6\n\t" - "sta %%g0, [%0 + %%g5] %6\n\t" - "sta %%g0, [%0 + %%o4] %6\n\t" - "bg 1b\n\t" - " add %0, %%o5, %0\n" - : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize) - : "0" (addr), "1" (nbytes), "2" (lsize), - "i" (ASI_FLUSHSEG) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); - } -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_page_sw(unsigned long addr) -{ - addr &= PAGE_MASK; - if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) == - _SUN4C_PAGE_VALID) { - unsigned long left = PAGE_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %1, %%o5, %1\n\t" - "sta %%g0, [%0] %6\n\t" - "sta %%g0, [%0 + %2] %6\n\t" - "sta %%g0, [%0 + %%g1] %6\n\t" - "sta %%g0, [%0 + %%g2] %6\n\t" - "sta %%g0, [%0 + %%g3] %6\n\t" - "sta %%g0, [%0 + %%g4] %6\n\t" - "sta %%g0, [%0 + %%g5] %6\n\t" - "sta %%g0, [%0 + %%o4] %6\n\t" - "bg 1b\n\t" - " add %0, %%o5, %0\n" - : "=&r" (addr), "=&r" (left), "=&r" (lsize) - : "0" (addr), "1" (left), "2" (lsize), - "i" (ASI_FLUSHPG) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); - } -} - -/* The sun4c's do have an on chip store buffer. And the way you - * clear them out isn't so obvious. The only way I can think of - * to accomplish this is to read the current context register, - * store the same value there, then read an external hardware - * register. - */ -void sun4c_complete_all_stores(void) -{ - volatile int _unused; - - _unused = sun4c_get_context(); - sun4c_set_context(_unused); - _unused = get_auxio(); -} - -/* Bootup utility functions. */ -static inline void sun4c_init_clean_segmap(unsigned char pseg) -{ - unsigned long vaddr; - - sun4c_put_segmap(0, pseg); - for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE) - sun4c_put_pte(vaddr, 0); - sun4c_put_segmap(0, invalid_segment); -} - -static inline void sun4c_init_clean_mmu(unsigned long kernel_end) -{ - unsigned long vaddr; - unsigned char savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - } - sun4c_set_context(savectx); -} - -void __init sun4c_probe_vac(void) -{ - sun4c_disable_vac(); - - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { - /* PROM on SS1 lacks this info, to be super safe we - * hard code it here since this arch is cast in stone. - */ - sun4c_vacinfo.num_bytes = 65536; - sun4c_vacinfo.linesize = 16; - } else { - sun4c_vacinfo.num_bytes = - prom_getintdefault(prom_root_node, "vac-size", 65536); - sun4c_vacinfo.linesize = - prom_getintdefault(prom_root_node, "vac-linesize", 16); - } - sun4c_vacinfo.do_hwflushes = - prom_getintdefault(prom_root_node, "vac-hwflush", 0); - - if (sun4c_vacinfo.do_hwflushes == 0) - sun4c_vacinfo.do_hwflushes = - prom_getintdefault(prom_root_node, "vac_hwflush", 0); - - if (sun4c_vacinfo.num_bytes != 65536) { - prom_printf("WEIRD Sun4C VAC cache size, " - "tell sparclinux@vger.kernel.org"); - prom_halt(); - } - - switch (sun4c_vacinfo.linesize) { - case 16: - sun4c_vacinfo.log2lsize = 4; - break; - case 32: - sun4c_vacinfo.log2lsize = 5; - break; - default: - prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n", - sun4c_vacinfo.linesize); - prom_halt(); - }; - - sun4c_flush_all(); - sun4c_enable_vac(); -} - -/* Patch instructions for the low level kernel fault handler. */ -extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff; -extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff; -extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff; -extern unsigned long num_context_patch1, num_context_patch1_16; -extern unsigned long num_context_patch2_16; -extern unsigned long vac_linesize_patch, vac_linesize_patch_32; -extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on; -extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on; - -#define PATCH_INSN(src, dst) do { \ - daddr = &(dst); \ - iaddr = &(src); \ - *daddr = *iaddr; \ - } while (0) - -static void __init patch_kernel_fault_handler(void) -{ - unsigned long *iaddr, *daddr; - - switch (num_segmaps) { - case 128: - /* Default, nothing to do. */ - break; - case 256: - PATCH_INSN(invalid_segment_patch1_ff, - invalid_segment_patch1); - PATCH_INSN(invalid_segment_patch2_ff, - invalid_segment_patch2); - break; - case 512: - PATCH_INSN(invalid_segment_patch1_1ff, - invalid_segment_patch1); - PATCH_INSN(invalid_segment_patch2_1ff, - invalid_segment_patch2); - break; - default: - prom_printf("Unhandled number of segmaps: %d\n", - num_segmaps); - prom_halt(); - }; - switch (num_contexts) { - case 8: - /* Default, nothing to do. */ - break; - case 16: - PATCH_INSN(num_context_patch1_16, - num_context_patch1); - break; - default: - prom_printf("Unhandled number of contexts: %d\n", - num_contexts); - prom_halt(); - }; - - if (sun4c_vacinfo.do_hwflushes != 0) { - PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1); - PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2); - } else { - switch (sun4c_vacinfo.linesize) { - case 16: - /* Default, nothing to do. */ - break; - case 32: - PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch); - break; - default: - prom_printf("Impossible VAC linesize %d, halting...\n", - sun4c_vacinfo.linesize); - prom_halt(); - }; - } -} - -static void __init sun4c_probe_mmu(void) -{ - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { - /* Hardcode these just to be safe, PROM on SS1 does - * not have this info available in the root node. - */ - num_segmaps = 128; - num_contexts = 8; - } else { - num_segmaps = - prom_getintdefault(prom_root_node, "mmu-npmg", 128); - num_contexts = - prom_getintdefault(prom_root_node, "mmu-nctx", 0x8); - } - patch_kernel_fault_handler(); -} - -volatile unsigned long __iomem *sun4c_memerr_reg = NULL; - -void __init sun4c_probe_memerr_reg(void) -{ - phandle node; - struct linux_prom_registers regs[1]; - - node = prom_getchild(prom_root_node); - node = prom_searchsiblings(prom_root_node, "memory-error"); - if (!node) - return; - if (prom_getproperty(node, "reg", (char *)regs, sizeof(regs)) <= 0) - return; - /* hmm I think regs[0].which_io is zero here anyways */ - sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size); -} - -static inline void sun4c_init_ss2_cache_bug(void) -{ - extern unsigned long start; - - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) { - /* Whee.. */ - printk("SS2 cache bug detected, uncaching trap table page\n"); - sun4c_flush_page((unsigned int) &start); - sun4c_put_pte(((unsigned long) &start), - (sun4c_get_pte((unsigned long) &start) | _SUN4C_PAGE_NOCACHE)); - } -} - -/* Addr is always aligned on a page boundary for us already. */ -static int sun4c_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, - unsigned long addr, int len) -{ - unsigned long page, end; - - *pba = addr; - - end = PAGE_ALIGN((addr + len)); - while (addr < end) { - page = va; - sun4c_flush_page(page); - page -= PAGE_OFFSET; - page >>= PAGE_SHIFT; - page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY | - _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV); - sun4c_put_pte(addr, page); - addr += PAGE_SIZE; - va += PAGE_SIZE; - } - - return 0; -} - -static void sun4c_unmap_dma_area(struct device *dev, unsigned long busa, int len) -{ - /* Fortunately for us, bus_addr == uncached_virt in sun4c. */ - /* XXX Implement this */ -} - -/* TLB management. */ - -/* Don't change this struct without changing entry.S. This is used - * in the in-window kernel fault handler, and you don't want to mess - * with that. (See sun4c_fault in entry.S). - */ -struct sun4c_mmu_entry { - struct sun4c_mmu_entry *next; - struct sun4c_mmu_entry *prev; - unsigned long vaddr; - unsigned char pseg; - unsigned char locked; - - /* For user mappings only, and completely hidden from kernel - * TLB miss code. - */ - unsigned char ctx; - struct sun4c_mmu_entry *lru_next; - struct sun4c_mmu_entry *lru_prev; -}; - -static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS]; - -static void __init sun4c_init_mmu_entry_pool(void) -{ - int i; - - for (i=0; i < SUN4C_MAX_SEGMAPS; i++) { - mmu_entry_pool[i].pseg = i; - mmu_entry_pool[i].next = NULL; - mmu_entry_pool[i].prev = NULL; - mmu_entry_pool[i].vaddr = 0; - mmu_entry_pool[i].locked = 0; - mmu_entry_pool[i].ctx = 0; - mmu_entry_pool[i].lru_next = NULL; - mmu_entry_pool[i].lru_prev = NULL; - } - mmu_entry_pool[invalid_segment].locked = 1; -} - -static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on, - unsigned long bits_off) -{ - unsigned long start, end; - - end = vaddr + SUN4C_REAL_PGDIR_SIZE; - for (start = vaddr; start < end; start += PAGE_SIZE) - if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID) - sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) & - ~bits_off); -} - -static inline void sun4c_init_map_kernelprom(unsigned long kernel_end) -{ - unsigned long vaddr; - unsigned char pseg, ctx; - - for (vaddr = KADB_DEBUGGER_BEGVM; - vaddr < LINUX_OPPROM_ENDVM; - vaddr += SUN4C_REAL_PGDIR_SIZE) { - pseg = sun4c_get_segmap(vaddr); - if (pseg != invalid_segment) { - mmu_entry_pool[pseg].locked = 1; - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, vaddr, pseg); - fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0); - } - } - - for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) { - pseg = sun4c_get_segmap(vaddr); - mmu_entry_pool[pseg].locked = 1; - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, vaddr, pseg); - fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE); - } -} - -static void __init sun4c_init_lock_area(unsigned long start, unsigned long end) -{ - int i, ctx; - - while (start < end) { - for (i = 0; i < invalid_segment; i++) - if (!mmu_entry_pool[i].locked) - break; - mmu_entry_pool[i].locked = 1; - sun4c_init_clean_segmap(i); - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, start, mmu_entry_pool[i].pseg); - start += SUN4C_REAL_PGDIR_SIZE; - } -} - -/* Don't change this struct without changing entry.S. This is used - * in the in-window kernel fault handler, and you don't want to mess - * with that. (See sun4c_fault in entry.S). - */ -struct sun4c_mmu_ring { - struct sun4c_mmu_entry ringhd; - int num_entries; -}; - -static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */ -static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */ -static struct sun4c_mmu_ring sun4c_ulru_ring; /* LRU user entries */ -struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */ -struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */ - -static inline void sun4c_init_rings(void) -{ - int i; - - for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) { - sun4c_context_ring[i].ringhd.next = - sun4c_context_ring[i].ringhd.prev = - &sun4c_context_ring[i].ringhd; - sun4c_context_ring[i].num_entries = 0; - } - sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev = - &sun4c_ufree_ring.ringhd; - sun4c_ufree_ring.num_entries = 0; - sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev = - &sun4c_ulru_ring.ringhd; - sun4c_ulru_ring.num_entries = 0; - sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev = - &sun4c_kernel_ring.ringhd; - sun4c_kernel_ring.num_entries = 0; - sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev = - &sun4c_kfree_ring.ringhd; - sun4c_kfree_ring.num_entries = 0; -} - -static void add_ring(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *head = &ring->ringhd; - - entry->prev = head; - (entry->next = head->next)->prev = entry; - head->next = entry; - ring->num_entries++; -} - -static inline void add_lru(struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_ring *ring = &sun4c_ulru_ring; - struct sun4c_mmu_entry *head = &ring->ringhd; - - entry->lru_next = head; - (entry->lru_prev = head->lru_prev)->lru_next = entry; - head->lru_prev = entry; -} - -static void add_ring_ordered(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *head = &ring->ringhd; - unsigned long addr = entry->vaddr; - - while ((head->next != &ring->ringhd) && (head->next->vaddr < addr)) - head = head->next; - - entry->prev = head; - (entry->next = head->next)->prev = entry; - head->next = entry; - ring->num_entries++; - - add_lru(entry); -} - -static inline void remove_ring(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *next = entry->next; - - (next->prev = entry->prev)->next = next; - ring->num_entries--; -} - -static void remove_lru(struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *next = entry->lru_next; - - (next->lru_prev = entry->lru_prev)->lru_next = next; -} - -static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry) -{ - remove_ring(sun4c_context_ring+ctx, entry); - remove_lru(entry); - add_ring(&sun4c_ufree_ring, entry); -} - -static void free_kernel_entry(struct sun4c_mmu_entry *entry, - struct sun4c_mmu_ring *ring) -{ - remove_ring(ring, entry); - add_ring(&sun4c_kfree_ring, entry); -} - -static void __init sun4c_init_fill_kernel_ring(int howmany) -{ - int i; - - while (howmany) { - for (i = 0; i < invalid_segment; i++) - if (!mmu_entry_pool[i].locked) - break; - mmu_entry_pool[i].locked = 1; - sun4c_init_clean_segmap(i); - add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]); - howmany--; - } -} - -static void __init sun4c_init_fill_user_ring(void) -{ - int i; - - for (i = 0; i < invalid_segment; i++) { - if (mmu_entry_pool[i].locked) - continue; - sun4c_init_clean_segmap(i); - add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]); - } -} - -static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry) -{ - int savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(kentry->vaddr, invalid_segment); - } - sun4c_set_context(savectx); -} - -static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry) -{ - int savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(kentry->vaddr, kentry->pseg); - } - sun4c_set_context(savectx); -} - -#define sun4c_user_unmap(__entry) \ - sun4c_put_segmap((__entry)->vaddr, invalid_segment) - -static void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx) -{ - struct sun4c_mmu_entry *head = &crp->ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - flush_user_windows(); - sun4c_set_context(ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); -} - -static int sun4c_user_taken_entries; /* This is how much we have. */ -static int max_user_taken_entries; /* This limits us and prevents deadlock. */ - -static struct sun4c_mmu_entry *sun4c_kernel_strategy(void) -{ - struct sun4c_mmu_entry *this_entry; - - /* If some are free, return first one. */ - if (sun4c_kfree_ring.num_entries) { - this_entry = sun4c_kfree_ring.ringhd.next; - return this_entry; - } - - /* Else free one up. */ - this_entry = sun4c_kernel_ring.ringhd.prev; - sun4c_flush_segment(this_entry->vaddr); - sun4c_kernel_unmap(this_entry); - free_kernel_entry(this_entry, &sun4c_kernel_ring); - this_entry = sun4c_kfree_ring.ringhd.next; - - return this_entry; -} - -/* Using this method to free up mmu entries eliminates a lot of - * potential races since we have a kernel that incurs tlb - * replacement faults. There may be performance penalties. - * - * NOTE: Must be called with interrupts disabled. - */ -static struct sun4c_mmu_entry *sun4c_user_strategy(void) -{ - struct sun4c_mmu_entry *entry; - unsigned char ctx; - int savectx; - - /* If some are free, return first one. */ - if (sun4c_ufree_ring.num_entries) { - entry = sun4c_ufree_ring.ringhd.next; - goto unlink_out; - } - - if (sun4c_user_taken_entries) { - entry = sun4c_kernel_strategy(); - sun4c_user_taken_entries--; - goto kunlink_out; - } - - /* Grab from the beginning of the LRU list. */ - entry = sun4c_ulru_ring.ringhd.lru_next; - ctx = entry->ctx; - - savectx = sun4c_get_context(); - flush_user_windows(); - sun4c_set_context(ctx); - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - remove_ring(sun4c_context_ring + ctx, entry); - remove_lru(entry); - sun4c_set_context(savectx); - - return entry; - -unlink_out: - remove_ring(&sun4c_ufree_ring, entry); - return entry; -kunlink_out: - remove_ring(&sun4c_kfree_ring, entry); - return entry; -} - -/* NOTE: Must be called with interrupts disabled. */ -void sun4c_grow_kernel_ring(void) -{ - struct sun4c_mmu_entry *entry; - - /* Prevent deadlock condition. */ - if (sun4c_user_taken_entries >= max_user_taken_entries) - return; - - if (sun4c_ufree_ring.num_entries) { - entry = sun4c_ufree_ring.ringhd.next; - remove_ring(&sun4c_ufree_ring, entry); - add_ring(&sun4c_kfree_ring, entry); - sun4c_user_taken_entries++; - } -} - -/* 2 page buckets for task struct and kernel stack allocation. - * - * TASK_STACK_BEGIN - * bucket[0] - * bucket[1] - * [ ... ] - * bucket[NR_TASK_BUCKETS-1] - * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS) - * - * Each slot looks like: - * - * page 1 -- task struct + beginning of kernel stack - * page 2 -- rest of kernel stack - */ - -union task_union *sun4c_bucket[NR_TASK_BUCKETS]; - -static int sun4c_lowbucket_avail; - -#define BUCKET_EMPTY ((union task_union *) 0) -#define BUCKET_SHIFT (PAGE_SHIFT + 1) /* log2(sizeof(struct task_bucket)) */ -#define BUCKET_SIZE (1 << BUCKET_SHIFT) -#define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT)) -#define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR) -#define BUCKET_PTE(page) \ - ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL)) -#define BUCKET_PTE_PAGE(pte) \ - (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT)) - -static void get_locked_segment(unsigned long addr) -{ - struct sun4c_mmu_entry *stolen; - unsigned long flags; - - local_irq_save(flags); - addr &= SUN4C_REAL_PGDIR_MASK; - stolen = sun4c_user_strategy(); - max_user_taken_entries--; - stolen->vaddr = addr; - flush_user_windows(); - sun4c_kernel_map(stolen); - local_irq_restore(flags); -} - -static void free_locked_segment(unsigned long addr) -{ - struct sun4c_mmu_entry *entry; - unsigned long flags; - unsigned char pseg; - - local_irq_save(flags); - addr &= SUN4C_REAL_PGDIR_MASK; - pseg = sun4c_get_segmap(addr); - entry = &mmu_entry_pool[pseg]; - - flush_user_windows(); - sun4c_flush_segment(addr); - sun4c_kernel_unmap(entry); - add_ring(&sun4c_ufree_ring, entry); - max_user_taken_entries++; - local_irq_restore(flags); -} - -static inline void garbage_collect(int entry) -{ - int start, end; - - /* 32 buckets per segment... */ - entry &= ~31; - start = entry; - for (end = (start + 32); start < end; start++) - if (sun4c_bucket[start] != BUCKET_EMPTY) - return; - - /* Entire segment empty, release it. */ - free_locked_segment(BUCKET_ADDR(entry)); -} - -static struct thread_info *sun4c_alloc_thread_info(void) -{ - unsigned long addr, pages; - int entry; - - pages = __get_free_pages(GFP_KERNEL, THREAD_INFO_ORDER); - if (!pages) - return NULL; - - for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++) - if (sun4c_bucket[entry] == BUCKET_EMPTY) - break; - if (entry == NR_TASK_BUCKETS) { - free_pages(pages, THREAD_INFO_ORDER); - return NULL; - } - if (entry >= sun4c_lowbucket_avail) - sun4c_lowbucket_avail = entry + 1; - - addr = BUCKET_ADDR(entry); - sun4c_bucket[entry] = (union task_union *) addr; - if(sun4c_get_segmap(addr) == invalid_segment) - get_locked_segment(addr); - - /* We are changing the virtual color of the page(s) - * so we must flush the cache to guarantee consistency. - */ - sun4c_flush_page(pages); - sun4c_flush_page(pages + PAGE_SIZE); - - sun4c_put_pte(addr, BUCKET_PTE(pages)); - sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE)); - -#ifdef CONFIG_DEBUG_STACK_USAGE - memset((void *)addr, 0, PAGE_SIZE << THREAD_INFO_ORDER); -#endif /* DEBUG_STACK_USAGE */ - - return (struct thread_info *) addr; -} - -static void sun4c_free_thread_info(struct thread_info *ti) -{ - unsigned long tiaddr = (unsigned long) ti; - unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tiaddr)); - int entry = BUCKET_NUM(tiaddr); - - /* We are deleting a mapping, so the flush here is mandatory. */ - sun4c_flush_page(tiaddr); - sun4c_flush_page(tiaddr + PAGE_SIZE); - - sun4c_put_pte(tiaddr, 0); - sun4c_put_pte(tiaddr + PAGE_SIZE, 0); - - sun4c_bucket[entry] = BUCKET_EMPTY; - if (entry < sun4c_lowbucket_avail) - sun4c_lowbucket_avail = entry; - - free_pages(pages, THREAD_INFO_ORDER); - garbage_collect(entry); -} - -static void __init sun4c_init_buckets(void) -{ - int entry; - - if (sizeof(union thread_union) != (PAGE_SIZE << THREAD_INFO_ORDER)) { - extern void thread_info_size_is_bolixed_pete(void); - thread_info_size_is_bolixed_pete(); - } - - for (entry = 0; entry < NR_TASK_BUCKETS; entry++) - sun4c_bucket[entry] = BUCKET_EMPTY; - sun4c_lowbucket_avail = 0; -} - -static unsigned long sun4c_iobuffer_start; -static unsigned long sun4c_iobuffer_end; -static unsigned long sun4c_iobuffer_high; -static unsigned long *sun4c_iobuffer_map; -static int iobuffer_map_size; - -/* - * Alias our pages so they do not cause a trap. - * Also one page may be aliased into several I/O areas and we may - * finish these I/O separately. - */ -static char *sun4c_lockarea(char *vaddr, unsigned long size) -{ - unsigned long base, scan; - unsigned long npages; - unsigned long vpage; - unsigned long pte; - unsigned long apage; - unsigned long high; - unsigned long flags; - - npages = (((unsigned long)vaddr & ~PAGE_MASK) + - size + (PAGE_SIZE-1)) >> PAGE_SHIFT; - - local_irq_save(flags); - base = bitmap_find_next_zero_area(sun4c_iobuffer_map, iobuffer_map_size, - 0, npages, 0); - if (base >= iobuffer_map_size) - goto abend; - - high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start; - high = SUN4C_REAL_PGDIR_ALIGN(high); - while (high > sun4c_iobuffer_high) { - get_locked_segment(sun4c_iobuffer_high); - sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE; - } - - vpage = ((unsigned long) vaddr) & PAGE_MASK; - for (scan = base; scan < base+npages; scan++) { - pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT); - pte |= pgprot_val(SUN4C_PAGE_KERNEL); - pte |= _SUN4C_PAGE_NOCACHE; - set_bit(scan, sun4c_iobuffer_map); - apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start; - - /* Flush original mapping so we see the right things later. */ - sun4c_flush_page(vpage); - - sun4c_put_pte(apage, pte); - vpage += PAGE_SIZE; - } - local_irq_restore(flags); - return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start + - (((unsigned long) vaddr) & ~PAGE_MASK)); - -abend: - local_irq_restore(flags); - printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size); - panic("Out of iobuffer table"); - return NULL; -} - -static void sun4c_unlockarea(char *vaddr, unsigned long size) -{ - unsigned long vpage, npages; - unsigned long flags; - int scan, high; - - vpage = (unsigned long)vaddr & PAGE_MASK; - npages = (((unsigned long)vaddr & ~PAGE_MASK) + - size + (PAGE_SIZE-1)) >> PAGE_SHIFT; - - local_irq_save(flags); - while (npages != 0) { - --npages; - - /* This mapping is marked non-cachable, no flush necessary. */ - sun4c_put_pte(vpage, 0); - clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT, - sun4c_iobuffer_map); - vpage += PAGE_SIZE; - } - - /* garbage collect */ - scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT; - while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5]) - scan -= 32; - scan += 32; - high = sun4c_iobuffer_start + (scan << PAGE_SHIFT); - high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE; - while (high < sun4c_iobuffer_high) { - sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE; - free_locked_segment(sun4c_iobuffer_high); - } - local_irq_restore(flags); -} - -/* Note the scsi code at init time passes to here buffers - * which sit on the kernel stack, those are already locked - * by implication and fool the page locking code above - * if passed to by mistake. - */ -static __u32 sun4c_get_scsi_one(struct device *dev, char *bufptr, unsigned long len) -{ - unsigned long page; - - page = ((unsigned long)bufptr) & PAGE_MASK; - if (!virt_addr_valid(page)) { - sun4c_flush_page(page); - return (__u32)bufptr; /* already locked */ - } - return (__u32)sun4c_lockarea(bufptr, len); -} - -static void sun4c_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) -{ - while (sz != 0) { - --sz; - sg->dma_address = (__u32)sun4c_lockarea(sg_virt(sg), sg->length); - sg->dma_length = sg->length; - sg = sg_next(sg); - } -} - -static void sun4c_release_scsi_one(struct device *dev, __u32 bufptr, unsigned long len) -{ - if (bufptr < sun4c_iobuffer_start) - return; /* On kernel stack or similar, see above */ - sun4c_unlockarea((char *)bufptr, len); -} - -static void sun4c_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) -{ - while (sz != 0) { - --sz; - sun4c_unlockarea((char *)sg->dma_address, sg->length); - sg = sg_next(sg); - } -} - -#define TASK_ENTRY_SIZE BUCKET_SIZE /* see above */ -#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) - -struct vm_area_struct sun4c_kstack_vma; - -static void __init sun4c_init_lock_areas(void) -{ - unsigned long sun4c_taskstack_start; - unsigned long sun4c_taskstack_end; - int bitmap_size; - - sun4c_init_buckets(); - sun4c_taskstack_start = SUN4C_LOCK_VADDR; - sun4c_taskstack_end = (sun4c_taskstack_start + - (TASK_ENTRY_SIZE * NR_TASK_BUCKETS)); - if (sun4c_taskstack_end >= SUN4C_LOCK_END) { - prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n"); - prom_halt(); - } - - sun4c_iobuffer_start = sun4c_iobuffer_high = - SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end); - sun4c_iobuffer_end = SUN4C_LOCK_END; - bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT; - bitmap_size = (bitmap_size + 7) >> 3; - bitmap_size = LONG_ALIGN(bitmap_size); - iobuffer_map_size = bitmap_size << 3; - sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL); - memset((void *) sun4c_iobuffer_map, 0, bitmap_size); - - sun4c_kstack_vma.vm_mm = &init_mm; - sun4c_kstack_vma.vm_start = sun4c_taskstack_start; - sun4c_kstack_vma.vm_end = sun4c_taskstack_end; - sun4c_kstack_vma.vm_page_prot = PAGE_SHARED; - sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC; - insert_vm_struct(&init_mm, &sun4c_kstack_vma); -} - -/* Cache flushing on the sun4c. */ -static void sun4c_flush_cache_all(void) -{ - unsigned long begin, end; - - flush_user_windows(); - begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE); - end = (begin + SUN4C_VAC_SIZE); - - if (sun4c_vacinfo.linesize == 32) { - while (begin < end) { - __asm__ __volatile__( - "ld [%0 + 0x00], %%g0\n\t" - "ld [%0 + 0x20], %%g0\n\t" - "ld [%0 + 0x40], %%g0\n\t" - "ld [%0 + 0x60], %%g0\n\t" - "ld [%0 + 0x80], %%g0\n\t" - "ld [%0 + 0xa0], %%g0\n\t" - "ld [%0 + 0xc0], %%g0\n\t" - "ld [%0 + 0xe0], %%g0\n\t" - "ld [%0 + 0x100], %%g0\n\t" - "ld [%0 + 0x120], %%g0\n\t" - "ld [%0 + 0x140], %%g0\n\t" - "ld [%0 + 0x160], %%g0\n\t" - "ld [%0 + 0x180], %%g0\n\t" - "ld [%0 + 0x1a0], %%g0\n\t" - "ld [%0 + 0x1c0], %%g0\n\t" - "ld [%0 + 0x1e0], %%g0\n" - : : "r" (begin)); - begin += 512; - } - } else { - while (begin < end) { - __asm__ __volatile__( - "ld [%0 + 0x00], %%g0\n\t" - "ld [%0 + 0x10], %%g0\n\t" - "ld [%0 + 0x20], %%g0\n\t" - "ld [%0 + 0x30], %%g0\n\t" - "ld [%0 + 0x40], %%g0\n\t" - "ld [%0 + 0x50], %%g0\n\t" - "ld [%0 + 0x60], %%g0\n\t" - "ld [%0 + 0x70], %%g0\n\t" - "ld [%0 + 0x80], %%g0\n\t" - "ld [%0 + 0x90], %%g0\n\t" - "ld [%0 + 0xa0], %%g0\n\t" - "ld [%0 + 0xb0], %%g0\n\t" - "ld [%0 + 0xc0], %%g0\n\t" - "ld [%0 + 0xd0], %%g0\n\t" - "ld [%0 + 0xe0], %%g0\n\t" - "ld [%0 + 0xf0], %%g0\n" - : : "r" (begin)); - begin += 256; - } - } -} - -static void sun4c_flush_cache_mm(struct mm_struct *mm) -{ - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - flush_user_windows(); - - if (sun4c_context_ring[new_ctx].num_entries) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); - } - } -} - -static void sun4c_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - struct sun4c_mmu_entry *entry; - unsigned long flags; - - flush_user_windows(); - - local_irq_save(flags); - /* All user segmap chains are ordered on entry->vaddr. */ - for (entry = head->next; - (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); - entry = entry->next) - ; - - /* Tracing various job mixtures showed that this conditional - * only passes ~35% of the time for most worse case situations, - * therefore we avoid all of this gross overhead ~65% of the time. - */ - if ((entry != head) && (entry->vaddr < end)) { - int octx = sun4c_get_context(); - sun4c_set_context(new_ctx); - - /* At this point, always, (start >= entry->vaddr) and - * (entry->vaddr < end), once the latter condition - * ceases to hold, or we hit the end of the list, we - * exit the loop. The ordering of all user allocated - * segmaps makes this all work out so beautifully. - */ - do { - struct sun4c_mmu_entry *next = entry->next; - unsigned long realend; - - /* "realstart" is always >= entry->vaddr */ - realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE; - if (end < realend) - realend = end; - if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) { - unsigned long page = entry->vaddr; - while (page < realend) { - sun4c_flush_page(page); - page += PAGE_SIZE; - } - } else { - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - } - entry = next; - } while ((entry != head) && (entry->vaddr < end)); - sun4c_set_context(octx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - /* Sun4c has no separate I/D caches so cannot optimize for non - * text page flushes. - */ - if (new_ctx != NO_CONTEXT) { - int octx = sun4c_get_context(); - unsigned long flags; - - flush_user_windows(); - local_irq_save(flags); - sun4c_set_context(new_ctx); - sun4c_flush_page(page); - sun4c_set_context(octx); - local_irq_restore(flags); - } -} - -static void sun4c_flush_page_to_ram(unsigned long page) -{ - unsigned long flags; - - local_irq_save(flags); - sun4c_flush_page(page); - local_irq_restore(flags); -} - -/* Sun4c cache is unified, both instructions and data live there, so - * no need to flush the on-stack instructions for new signal handlers. - */ -static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) -{ -} - -/* TLB flushing on the sun4c. These routines count on the cache - * flushing code to flush the user register windows so that we need - * not do so when we get here. - */ - -static void sun4c_flush_tlb_all(void) -{ - struct sun4c_mmu_entry *this_entry, *next_entry; - unsigned long flags; - int savectx, ctx; - - local_irq_save(flags); - this_entry = sun4c_kernel_ring.ringhd.next; - savectx = sun4c_get_context(); - flush_user_windows(); - while (sun4c_kernel_ring.num_entries) { - next_entry = this_entry->next; - sun4c_flush_segment(this_entry->vaddr); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(this_entry->vaddr, invalid_segment); - } - free_kernel_entry(this_entry, &sun4c_kernel_ring); - this_entry = next_entry; - } - sun4c_set_context(savectx); - local_irq_restore(flags); -} - -static void sun4c_flush_tlb_mm(struct mm_struct *mm) -{ - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - struct sun4c_mmu_entry *entry; - unsigned long flags; - - local_irq_save(flags); - /* See commentary in sun4c_flush_cache_range(). */ - for (entry = head->next; - (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); - entry = entry->next) - ; - - if ((entry != head) && (entry->vaddr < end)) { - int octx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while ((entry != head) && (entry->vaddr < end)); - sun4c_set_context(octx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - int savectx = sun4c_get_context(); - unsigned long flags; - - local_irq_save(flags); - sun4c_set_context(new_ctx); - page &= PAGE_MASK; - sun4c_flush_page(page); - sun4c_put_pte(page, 0); - sun4c_set_context(savectx); - local_irq_restore(flags); - } -} - -static inline void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr) -{ - unsigned long page_entry, pg_iobits; - - pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE | - _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE; - - page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK); - page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT)); - sun4c_put_pte(virt_addr, page_entry); -} - -static void sun4c_mapiorange(unsigned int bus, unsigned long xpa, - unsigned long xva, unsigned int len) -{ - while (len != 0) { - len -= PAGE_SIZE; - sun4c_mapioaddr(xpa, xva); - xva += PAGE_SIZE; - xpa += PAGE_SIZE; - } -} - -static void sun4c_unmapiorange(unsigned long virt_addr, unsigned int len) -{ - while (len != 0) { - len -= PAGE_SIZE; - sun4c_put_pte(virt_addr, 0); - virt_addr += PAGE_SIZE; - } -} - -static void sun4c_alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) -{ - struct ctx_list *ctxp; - - ctxp = ctx_free.next; - if (ctxp != &ctx_free) { - remove_from_ctx_list(ctxp); - add_to_used_ctxlist(ctxp); - mm->context = ctxp->ctx_number; - ctxp->ctx_mm = mm; - return; - } - ctxp = ctx_used.next; - if (ctxp->ctx_mm == old_mm) - ctxp = ctxp->next; - remove_from_ctx_list(ctxp); - add_to_used_ctxlist(ctxp); - ctxp->ctx_mm->context = NO_CONTEXT; - ctxp->ctx_mm = mm; - mm->context = ctxp->ctx_number; - sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number], - ctxp->ctx_number); -} - -/* Switch the current MM context. */ -static void sun4c_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu) -{ - struct ctx_list *ctx; - int dirty = 0; - - if (mm->context == NO_CONTEXT) { - dirty = 1; - sun4c_alloc_context(old_mm, mm); - } else { - /* Update the LRU ring of contexts. */ - ctx = ctx_list_pool + mm->context; - remove_from_ctx_list(ctx); - add_to_used_ctxlist(ctx); - } - if (dirty || old_mm != mm) - sun4c_set_context(mm->context); -} - -static void sun4c_destroy_context(struct mm_struct *mm) -{ - struct ctx_list *ctx_old; - - if (mm->context != NO_CONTEXT) { - sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context); - ctx_old = ctx_list_pool + mm->context; - remove_from_ctx_list(ctx_old); - add_to_free_ctxlist(ctx_old); - mm->context = NO_CONTEXT; - } -} - -static void sun4c_mmu_info(struct seq_file *m) -{ - int used_user_entries, i; - - used_user_entries = 0; - for (i = 0; i < num_contexts; i++) - used_user_entries += sun4c_context_ring[i].num_entries; - - seq_printf(m, - "vacsize\t\t: %d bytes\n" - "vachwflush\t: %s\n" - "vaclinesize\t: %d bytes\n" - "mmuctxs\t\t: %d\n" - "mmupsegs\t: %d\n" - "kernelpsegs\t: %d\n" - "kfreepsegs\t: %d\n" - "usedpsegs\t: %d\n" - "ufreepsegs\t: %d\n" - "user_taken\t: %d\n" - "max_taken\t: %d\n", - sun4c_vacinfo.num_bytes, - (sun4c_vacinfo.do_hwflushes ? "yes" : "no"), - sun4c_vacinfo.linesize, - num_contexts, - (invalid_segment + 1), - sun4c_kernel_ring.num_entries, - sun4c_kfree_ring.num_entries, - used_user_entries, - sun4c_ufree_ring.num_entries, - sun4c_user_taken_entries, - max_user_taken_entries); -} - -/* Nothing below here should touch the mmu hardware nor the mmu_entry - * data structures. - */ - -/* First the functions which the mid-level code uses to directly - * manipulate the software page tables. Some defines since we are - * emulating the i386 page directory layout. - */ -#define PGD_PRESENT 0x001 -#define PGD_RW 0x002 -#define PGD_USER 0x004 -#define PGD_ACCESSED 0x020 -#define PGD_DIRTY 0x040 -#define PGD_TABLE (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY) - -static void sun4c_set_pte(pte_t *ptep, pte_t pte) -{ - *ptep = pte; -} - -static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp) -{ -} - -static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep) -{ - pmdp->pmdv[0] = PGD_TABLE | (unsigned long) ptep; -} - -static void sun4c_pmd_populate(pmd_t * pmdp, struct page * ptep) -{ - if (page_address(ptep) == NULL) BUG(); /* No highmem on sun4c */ - pmdp->pmdv[0] = PGD_TABLE | (unsigned long) page_address(ptep); -} - -static int sun4c_pte_present(pte_t pte) -{ - return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0); -} -static void sun4c_pte_clear(pte_t *ptep) { *ptep = __pte(0); } - -static int sun4c_pmd_bad(pmd_t pmd) -{ - return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) || - (!virt_addr_valid(pmd_val(pmd)))); -} - -static int sun4c_pmd_present(pmd_t pmd) -{ - return ((pmd_val(pmd) & PGD_PRESENT) != 0); -} - -#if 0 /* if PMD takes one word */ -static void sun4c_pmd_clear(pmd_t *pmdp) { *pmdp = __pmd(0); } -#else /* if pmd_t is a longish aggregate */ -static void sun4c_pmd_clear(pmd_t *pmdp) { - memset((void *)pmdp, 0, sizeof(pmd_t)); -} -#endif - -static int sun4c_pgd_none(pgd_t pgd) { return 0; } -static int sun4c_pgd_bad(pgd_t pgd) { return 0; } -static int sun4c_pgd_present(pgd_t pgd) { return 1; } -static void sun4c_pgd_clear(pgd_t * pgdp) { } - -/* - * The following only work if pte_present() is true. - * Undefined behaviour if not.. - */ -static pte_t sun4c_pte_mkwrite(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE); - if (pte_val(pte) & _SUN4C_PAGE_MODIFIED) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); - return pte; -} - -static pte_t sun4c_pte_mkdirty(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED); - if (pte_val(pte) & _SUN4C_PAGE_WRITE) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); - return pte; -} - -static pte_t sun4c_pte_mkyoung(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED); - if (pte_val(pte) & _SUN4C_PAGE_READ) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ); - return pte; -} - -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot) -{ - return __pte(page_to_pfn(page) | pgprot_val(pgprot)); -} - -static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot) -{ - return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot)); -} - -static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space) -{ - return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot)); -} - -static unsigned long sun4c_pte_pfn(pte_t pte) -{ - return pte_val(pte) & SUN4C_PFN_MASK; -} - -static pte_t sun4c_pgoff_to_pte(unsigned long pgoff) -{ - return __pte(pgoff | _SUN4C_PAGE_FILE); -} - -static unsigned long sun4c_pte_to_pgoff(pte_t pte) -{ - return pte_val(pte) & ((1UL << PTE_FILE_MAX_BITS) - 1); -} - - -static inline unsigned long sun4c_pmd_page_v(pmd_t pmd) -{ - return (pmd_val(pmd) & PAGE_MASK); -} - -static struct page *sun4c_pmd_page(pmd_t pmd) -{ - return virt_to_page(sun4c_pmd_page_v(pmd)); -} - -static unsigned long sun4c_pgd_page(pgd_t pgd) { return 0; } - -/* to find an entry in a page-table-directory */ -static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address) -{ - return mm->pgd + (address >> SUN4C_PGDIR_SHIFT); -} - -/* Find an entry in the second-level page table.. */ -static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address) -{ - return (pmd_t *) dir; -} - -/* Find an entry in the third-level page table.. */ -pte_t *sun4c_pte_offset_kernel(pmd_t * dir, unsigned long address) -{ - return (pte_t *) sun4c_pmd_page_v(*dir) + - ((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1)); -} - -static unsigned long sun4c_swp_type(swp_entry_t entry) -{ - return (entry.val & SUN4C_SWP_TYPE_MASK); -} - -static unsigned long sun4c_swp_offset(swp_entry_t entry) -{ - return (entry.val >> SUN4C_SWP_OFF_SHIFT) & SUN4C_SWP_OFF_MASK; -} - -static swp_entry_t sun4c_swp_entry(unsigned long type, unsigned long offset) -{ - return (swp_entry_t) { - (offset & SUN4C_SWP_OFF_MASK) << SUN4C_SWP_OFF_SHIFT - | (type & SUN4C_SWP_TYPE_MASK) }; -} - -static void sun4c_free_pte_slow(pte_t *pte) -{ - free_page((unsigned long)pte); -} - -static void sun4c_free_pgd_slow(pgd_t *pgd) -{ - free_page((unsigned long)pgd); -} - -static pgd_t *sun4c_get_pgd_fast(void) -{ - unsigned long *ret; - - if ((ret = pgd_quicklist) != NULL) { - pgd_quicklist = (unsigned long *)(*ret); - ret[0] = ret[1]; - pgtable_cache_size--; - } else { - pgd_t *init; - - ret = (unsigned long *)__get_free_page(GFP_KERNEL); - memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t)); - init = sun4c_pgd_offset(&init_mm, 0); - memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, - (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); - } - return (pgd_t *)ret; -} - -static void sun4c_free_pgd_fast(pgd_t *pgd) -{ - *(unsigned long *)pgd = (unsigned long) pgd_quicklist; - pgd_quicklist = (unsigned long *) pgd; - pgtable_cache_size++; -} - - -static inline pte_t * -sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address) -{ - unsigned long *ret; - - if ((ret = (unsigned long *)pte_quicklist) != NULL) { - pte_quicklist = (unsigned long *)(*ret); - ret[0] = ret[1]; - pgtable_cache_size--; - } - return (pte_t *)ret; -} - -static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) -{ - pte_t *pte; - - if ((pte = sun4c_pte_alloc_one_fast(mm, address)) != NULL) - return pte; - - pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); - return pte; -} - -static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address) -{ - pte_t *pte; - struct page *page; - - pte = sun4c_pte_alloc_one_kernel(mm, address); - if (pte == NULL) - return NULL; - page = virt_to_page(pte); - pgtable_page_ctor(page); - return page; -} - -static inline void sun4c_free_pte_fast(pte_t *pte) -{ - *(unsigned long *)pte = (unsigned long) pte_quicklist; - pte_quicklist = (unsigned long *) pte; - pgtable_cache_size++; -} - -static void sun4c_pte_free(pgtable_t pte) -{ - pgtable_page_dtor(pte); - sun4c_free_pte_fast(page_address(pte)); -} - -/* - * allocating and freeing a pmd is trivial: the 1-entry pmd is - * inside the pgd, so has no extra memory associated with it. - */ -static pmd_t *sun4c_pmd_alloc_one(struct mm_struct *mm, unsigned long address) -{ - BUG(); - return NULL; -} - -static void sun4c_free_pmd_fast(pmd_t * pmd) { } - -static void sun4c_check_pgt_cache(int low, int high) -{ - if (pgtable_cache_size > high) { - do { - if (pgd_quicklist) - sun4c_free_pgd_slow(sun4c_get_pgd_fast()); - if (pte_quicklist) - sun4c_free_pte_slow(sun4c_pte_alloc_one_fast(NULL, 0)); - } while (pgtable_cache_size > low); - } -} - -/* An experiment, turn off by default for now... -DaveM */ -#define SUN4C_PRELOAD_PSEG - -void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) -{ - unsigned long flags; - int pseg; - - if (vma->vm_mm->context == NO_CONTEXT) - return; - - local_irq_save(flags); - address &= PAGE_MASK; - if ((pseg = sun4c_get_segmap(address)) == invalid_segment) { - struct sun4c_mmu_entry *entry = sun4c_user_strategy(); - struct mm_struct *mm = vma->vm_mm; - unsigned long start, end; - - entry->vaddr = start = (address & SUN4C_REAL_PGDIR_MASK); - entry->ctx = mm->context; - add_ring_ordered(sun4c_context_ring + mm->context, entry); - sun4c_put_segmap(entry->vaddr, entry->pseg); - end = start + SUN4C_REAL_PGDIR_SIZE; - while (start < end) { -#ifdef SUN4C_PRELOAD_PSEG - pgd_t *pgdp = sun4c_pgd_offset(mm, start); - pte_t *ptep; - - if (!pgdp) - goto no_mapping; - ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, start); - if (!ptep || !(pte_val(*ptep) & _SUN4C_PAGE_PRESENT)) - goto no_mapping; - sun4c_put_pte(start, pte_val(*ptep)); - goto next; - - no_mapping: -#endif - sun4c_put_pte(start, 0); -#ifdef SUN4C_PRELOAD_PSEG - next: -#endif - start += PAGE_SIZE; - } -#ifndef SUN4C_PRELOAD_PSEG - sun4c_put_pte(address, pte_val(*ptep)); -#endif - local_irq_restore(flags); - return; - } else { - struct sun4c_mmu_entry *entry = &mmu_entry_pool[pseg]; - - remove_lru(entry); - add_lru(entry); - } - - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); -} - -extern void sparc_context_init(int); -extern unsigned long bootmem_init(unsigned long *pages_avail); -extern unsigned long last_valid_pfn; - -void __init sun4c_paging_init(void) -{ - int i, cnt; - unsigned long kernel_end, vaddr; - extern struct resource sparc_iomap; - unsigned long end_pfn, pages_avail; - - kernel_end = (unsigned long) &_end; - kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end); - - pages_avail = 0; - last_valid_pfn = bootmem_init(&pages_avail); - end_pfn = last_valid_pfn; - - sun4c_probe_mmu(); - invalid_segment = (num_segmaps - 1); - sun4c_init_mmu_entry_pool(); - sun4c_init_rings(); - sun4c_init_map_kernelprom(kernel_end); - sun4c_init_clean_mmu(kernel_end); - sun4c_init_fill_kernel_ring(SUN4C_KERNEL_BUCKETS); - sun4c_init_lock_area(sparc_iomap.start, IOBASE_END); - sun4c_init_lock_area(DVMA_VADDR, DVMA_END); - sun4c_init_lock_areas(); - sun4c_init_fill_user_ring(); - - sun4c_set_context(0); - memset(swapper_pg_dir, 0, PAGE_SIZE); - memset(pg0, 0, PAGE_SIZE); - memset(pg1, 0, PAGE_SIZE); - memset(pg2, 0, PAGE_SIZE); - memset(pg3, 0, PAGE_SIZE); - - /* Save work later. */ - vaddr = VMALLOC_START; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg0); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg1); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg2); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg3); - sun4c_init_ss2_cache_bug(); - sparc_context_init(num_contexts); - - { - unsigned long zones_size[MAX_NR_ZONES]; - unsigned long zholes_size[MAX_NR_ZONES]; - unsigned long npages; - int znum; - - for (znum = 0; znum < MAX_NR_ZONES; znum++) - zones_size[znum] = zholes_size[znum] = 0; - - npages = max_low_pfn - pfn_base; - - zones_size[ZONE_DMA] = npages; - zholes_size[ZONE_DMA] = npages - pages_avail; - - npages = highend_pfn - max_low_pfn; - zones_size[ZONE_HIGHMEM] = npages; - zholes_size[ZONE_HIGHMEM] = npages - calc_highpages(); - - free_area_init_node(0, zones_size, pfn_base, zholes_size); - } - - cnt = 0; - for (i = 0; i < num_segmaps; i++) - if (mmu_entry_pool[i].locked) - cnt++; - - max_user_taken_entries = num_segmaps - cnt - 40 - 1; - - printk("SUN4C: %d mmu entries for the kernel\n", cnt); -} - -static pgprot_t sun4c_pgprot_noncached(pgprot_t prot) -{ - prot |= __pgprot(_SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE); - - return prot; -} - -/* Load up routines and constants for sun4c mmu */ -void __init ld_mmu_sun4c(void) -{ - extern void ___xchg32_sun4c(void); - - printk("Loading sun4c MMU routines\n"); - - /* First the constants */ - BTFIXUPSET_SIMM13(pgdir_shift, SUN4C_PGDIR_SHIFT); - BTFIXUPSET_SETHI(pgdir_size, SUN4C_PGDIR_SIZE); - BTFIXUPSET_SETHI(pgdir_mask, SUN4C_PGDIR_MASK); - - BTFIXUPSET_SIMM13(ptrs_per_pmd, SUN4C_PTRS_PER_PMD); - BTFIXUPSET_SIMM13(ptrs_per_pgd, SUN4C_PTRS_PER_PGD); - BTFIXUPSET_SIMM13(user_ptrs_per_pgd, KERNBASE / SUN4C_PGDIR_SIZE); - - BTFIXUPSET_INT(page_none, pgprot_val(SUN4C_PAGE_NONE)); - PAGE_SHARED = pgprot_val(SUN4C_PAGE_SHARED); - BTFIXUPSET_INT(page_copy, pgprot_val(SUN4C_PAGE_COPY)); - BTFIXUPSET_INT(page_readonly, pgprot_val(SUN4C_PAGE_READONLY)); - BTFIXUPSET_INT(page_kernel, pgprot_val(SUN4C_PAGE_KERNEL)); - page_kernel = pgprot_val(SUN4C_PAGE_KERNEL); - - /* Functions */ - BTFIXUPSET_CALL(pgprot_noncached, sun4c_pgprot_noncached, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4c, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(do_check_pgt_cache, sun4c_check_pgt_cache, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_cache_all, sun4c_flush_cache_all, BTFIXUPCALL_NORM); - - if (sun4c_vacinfo.do_hwflushes) { - BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_hw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_hw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_hw, BTFIXUPCALL_NORM); - } else { - BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_sw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_sw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_sw, BTFIXUPCALL_NORM); - } - - BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_all, sun4c_flush_tlb_all, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_sig_insns, sun4c_flush_sig_insns, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(set_pte, sun4c_set_pte, BTFIXUPCALL_STO1O0); - - BTFIXUPSET_CALL(pte_pfn, sun4c_pte_pfn, BTFIXUPCALL_NORM); -#if 0 /* PAGE_SHIFT <= 12 */ /* Eek. Investigate. XXX */ - BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_ANDNINT(PAGE_SIZE - 1)); -#else - BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_NORM); -#endif - BTFIXUPSET_CALL(pmd_set, sun4c_pmd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_populate, sun4c_pmd_populate, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_present, sun4c_pte_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_clear, sun4c_pte_clear, BTFIXUPCALL_STG0O0); - - BTFIXUPSET_CALL(pmd_bad, sun4c_pmd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_present, sun4c_pmd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, sun4c_pmd_clear, BTFIXUPCALL_STG0O0); - - BTFIXUPSET_CALL(pgd_none, sun4c_pgd_none, BTFIXUPCALL_RETINT(0)); - BTFIXUPSET_CALL(pgd_bad, sun4c_pgd_bad, BTFIXUPCALL_RETINT(0)); - BTFIXUPSET_CALL(pgd_present, sun4c_pgd_present, BTFIXUPCALL_RETINT(1)); - BTFIXUPSET_CALL(pgd_clear, sun4c_pgd_clear, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(mk_pte, sun4c_mk_pte, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_phys, sun4c_mk_pte_phys, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_io, sun4c_mk_pte_io, BTFIXUPCALL_NORM); - - BTFIXUPSET_INT(pte_modify_mask, _SUN4C_PAGE_CHG_MASK); - BTFIXUPSET_CALL(pmd_offset, sun4c_pmd_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_offset_kernel, sun4c_pte_offset_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pte_fast, sun4c_free_pte_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_free, sun4c_pte_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one_kernel, sun4c_pte_alloc_one_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one, sun4c_pte_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pmd_fast, sun4c_free_pmd_fast, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(pmd_alloc_one, sun4c_pmd_alloc_one, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(free_pgd_fast, sun4c_free_pgd_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(get_pgd_fast, sun4c_get_pgd_fast, BTFIXUPCALL_NORM); - - BTFIXUPSET_HALF(pte_writei, _SUN4C_PAGE_WRITE); - BTFIXUPSET_HALF(pte_dirtyi, _SUN4C_PAGE_MODIFIED); - BTFIXUPSET_HALF(pte_youngi, _SUN4C_PAGE_ACCESSED); - BTFIXUPSET_HALF(pte_filei, _SUN4C_PAGE_FILE); - BTFIXUPSET_HALF(pte_wrprotecti, _SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE); - BTFIXUPSET_HALF(pte_mkcleani, _SUN4C_PAGE_MODIFIED|_SUN4C_PAGE_SILENT_WRITE); - BTFIXUPSET_HALF(pte_mkoldi, _SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_SILENT_READ); - BTFIXUPSET_CALL(pte_mkwrite, sun4c_pte_mkwrite, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_mkdirty, sun4c_pte_mkdirty, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_mkyoung, sun4c_pte_mkyoung, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(update_mmu_cache, sun4c_update_mmu_cache, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_to_pgoff, sun4c_pte_to_pgoff, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgoff_to_pte, sun4c_pgoff_to_pte, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_lockarea, sun4c_lockarea, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unlockarea, sun4c_unlockarea, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_get_scsi_one, sun4c_get_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, sun4c_get_scsi_sgl, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_one, sun4c_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, sun4c_release_scsi_sgl, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_map_dma_area, sun4c_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, sun4c_unmap_dma_area, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(sparc_mapiorange, sun4c_mapiorange, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sparc_unmapiorange, sun4c_unmapiorange, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__swp_type, sun4c_swp_type, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_offset, sun4c_swp_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_entry, sun4c_swp_entry, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(alloc_thread_info, sun4c_alloc_thread_info, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_thread_info, sun4c_free_thread_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM); - - /* These should _never_ get called with two level tables. */ - BTFIXUPSET_CALL(pgd_set, sun4c_pgd_set, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(pgd_page_vaddr, sun4c_pgd_page, BTFIXUPCALL_RETO0); -} diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S index c801c3953a0..5d2b88d3942 100644 --- a/arch/sparc/mm/swift.S +++ b/arch/sparc/mm/swift.S @@ -105,7 +105,7 @@ swift_flush_cache_mm_out: .globl swift_flush_cache_range swift_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 sub %o2, %o1, %o2 sethi %hi(4096), %o3 cmp %o2, %o3 @@ -116,7 +116,7 @@ swift_flush_cache_range: .globl swift_flush_cache_page swift_flush_cache_page: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 70: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -219,7 +219,7 @@ swift_flush_sig_insns: .globl swift_flush_tlb_range .globl swift_flush_tlb_all swift_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 swift_flush_tlb_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -233,7 +233,7 @@ swift_flush_tlb_all_out: .globl swift_flush_tlb_page swift_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index d8f21e24a82..b89aba217e3 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -4,7 +4,6 @@ */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/percpu.h> #include <linux/mm.h> #include <linux/swap.h> @@ -19,39 +18,92 @@ /* Heavily inspired by the ppc64 code. */ -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); +static DEFINE_PER_CPU(struct tlb_batch, tlb_batch); void flush_tlb_pending(void) { - struct mmu_gather *mp = &get_cpu_var(mmu_gathers); + struct tlb_batch *tb = &get_cpu_var(tlb_batch); + struct mm_struct *mm = tb->mm; - if (mp->tlb_nr) { - flush_tsb_user(mp); + if (!tb->tlb_nr) + goto out; - if (CTX_VALID(mp->mm->context)) { + flush_tsb_user(tb); + + if (CTX_VALID(mm->context)) { + if (tb->tlb_nr == 1) { + global_flush_tlb_page(mm, tb->vaddrs[0]); + } else { #ifdef CONFIG_SMP - smp_flush_tlb_pending(mp->mm, mp->tlb_nr, - &mp->vaddrs[0]); + smp_flush_tlb_pending(tb->mm, tb->tlb_nr, + &tb->vaddrs[0]); #else - __flush_tlb_pending(CTX_HWBITS(mp->mm->context), - mp->tlb_nr, &mp->vaddrs[0]); + __flush_tlb_pending(CTX_HWBITS(tb->mm->context), + tb->tlb_nr, &tb->vaddrs[0]); #endif } - mp->tlb_nr = 0; } - put_cpu_var(mmu_gathers); + tb->tlb_nr = 0; + +out: + put_cpu_var(tlb_batch); +} + +void arch_enter_lazy_mmu_mode(void) +{ + struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + + tb->active = 1; } -void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t orig) +void arch_leave_lazy_mmu_mode(void) { - struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); + struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + + if (tb->tlb_nr) + flush_tlb_pending(); + tb->active = 0; +} + +static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, + bool exec) +{ + struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; vaddr &= PAGE_MASK; - if (pte_exec(orig)) + if (exec) vaddr |= 0x1UL; + nr = tb->tlb_nr; + + if (unlikely(nr != 0 && mm != tb->mm)) { + flush_tlb_pending(); + nr = 0; + } + + if (!tb->active) { + flush_tsb_user_page(mm, vaddr); + global_flush_tlb_page(mm, vaddr); + goto out; + } + + if (nr == 0) + tb->mm = mm; + + tb->vaddrs[nr] = vaddr; + tb->tlb_nr = ++nr; + if (nr >= TLB_BATCH_NR) + flush_tlb_pending(); + +out: + put_cpu_var(tlb_batch); +} + +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm) +{ if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); @@ -76,22 +128,116 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t } no_cache_flush: + if (!fullmm) + tlb_batch_add_one(mm, vaddr, pte_exec(orig)); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, + pmd_t pmd) +{ + unsigned long end; + pte_t *pte; - if (mp->fullmm) + pte = pte_offset_map(&pmd, vaddr); + end = vaddr + HPAGE_SIZE; + while (vaddr < end) { + if (pte_val(*pte) & _PAGE_VALID) { + bool exec = pte_exec(*pte); + + tlb_batch_add_one(mm, vaddr, exec); + } + pte++; + vaddr += PAGE_SIZE; + } + pte_unmap(pte); +} + +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + pmd_t orig = *pmdp; + + *pmdp = pmd; + + if (mm == &init_mm) return; - nr = mp->tlb_nr; + if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { + if (pmd_val(pmd) & _PAGE_PMD_HUGE) + mm->context.huge_pte_count++; + else + mm->context.huge_pte_count--; + + /* Do not try to allocate the TSB hash table if we + * don't have one already. We have various locks held + * and thus we'll end up doing a GFP_KERNEL allocation + * in an atomic context. + * + * Instead, we let the first TLB miss on a hugepage + * take care of this. + */ + } - if (unlikely(nr != 0 && mm != mp->mm)) { - flush_tlb_pending(); - nr = 0; + if (!pmd_none(orig)) { + addr &= HPAGE_MASK; + if (pmd_trans_huge(orig)) { + pte_t orig_pte = __pte(pmd_val(orig)); + bool exec = pte_exec(orig_pte); + + tlb_batch_add_one(mm, addr, exec); + tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); + } else { + tlb_batch_pmd_scan(mm, addr, orig); + } } +} - if (nr == 0) - mp->mm = mm; +void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp) +{ + pmd_t entry = *pmdp; - mp->vaddrs[nr] = vaddr; - mp->tlb_nr = ++nr; - if (nr >= TLB_BATCH_NR) - flush_tlb_pending(); + pmd_val(entry) &= ~_PAGE_VALID; + + set_pmd_at(vma->vm_mm, address, pmdp, entry); + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); +} + +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable) +{ + struct list_head *lh = (struct list_head *) pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + if (!pmd_huge_pte(mm, pmdp)) + INIT_LIST_HEAD(lh); + else + list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); + pmd_huge_pte(mm, pmdp) = pgtable; +} + +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) +{ + struct list_head *lh; + pgtable_t pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + pgtable = pmd_huge_pte(mm, pmdp); + lh = (struct list_head *) pgtable; + if (list_empty(lh)) + pmd_huge_pte(mm, pmdp) = NULL; + else { + pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; + list_del(lh); + } + pte_val(pgtable[0]) = 0; + pte_val(pgtable[1]) = 0; + + return pgtable; } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 101d7c82870..a06576683c3 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -6,13 +6,12 @@ #include <linux/kernel.h> #include <linux/preempt.h> #include <linux/slab.h> -#include <asm/system.h> #include <asm/page.h> -#include <asm/tlbflush.h> -#include <asm/tlb.h> -#include <asm/mmu_context.h> #include <asm/pgtable.h> +#include <asm/mmu_context.h> +#include <asm/setup.h> #include <asm/tsb.h> +#include <asm/tlb.h> #include <asm/oplib.h> extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; @@ -47,27 +46,56 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) } } -static void __flush_tsb_one(struct mmu_gather *mp, unsigned long hash_shift, unsigned long tsb, unsigned long nentries) +static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries) +{ + unsigned long tag, ent, hash; + + v &= ~0x1UL; + hash = tsb_hash(v, hash_shift, nentries); + ent = tsb + (hash * sizeof(struct tsb)); + tag = (v >> 22UL); + + tsb_flush(ent, tag); +} + +static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries) { unsigned long i; - for (i = 0; i < mp->tlb_nr; i++) { - unsigned long v = mp->vaddrs[i]; - unsigned long tag, ent, hash; + for (i = 0; i < tb->tlb_nr; i++) + __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); +} + +void flush_tsb_user(struct tlb_batch *tb) +{ + struct mm_struct *mm = tb->mm; + unsigned long nentries, base, flags; - v &= ~0x1UL; + spin_lock_irqsave(&mm->context.lock, flags); - hash = tsb_hash(v, hash_shift, nentries); - ent = tsb + (hash * sizeof(struct tsb)); - tag = (v >> 22UL); + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); - tsb_flush(ent, tag); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; + nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); } +#endif + spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_tsb_user(struct mmu_gather *mp) +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) { - struct mm_struct *mm = mp->mm; unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); @@ -76,43 +104,26 @@ void flush_tsb_user(struct mmu_gather *mp) nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(mp, PAGE_SHIFT, base, nentries); + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(mp, HPAGE_SHIFT, base, nentries); + __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); } -#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB) #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K -#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB) -#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K -#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K -#else -#error Broken base page size setting... -#endif -#ifdef CONFIG_HUGETLB_PAGE -#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K -#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K -#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB -#else -#error Broken huge page size setting... -#endif #endif static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) @@ -123,7 +134,19 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign mm->context.tsb_block[tsb_idx].tsb_nentries = tsb_bytes / sizeof(struct tsb); - base = TSBMAP_BASE; + switch (tsb_idx) { + case MM_TSB_BASE: + base = TSBMAP_8K_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + base = TSBMAP_4M_BASE; + break; +#endif + default: + BUG(); + } + tte = pgprot_val(PAGE_KERNEL_LOCKED); tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); @@ -179,7 +202,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", current->comm, current->pid, tsb_bytes); do_exit(SIGSEGV); - }; + } tte |= pte_sz_bits(page_sz); if (tlb_type == cheetah_plus || tlb_type == hypervisor) { @@ -207,14 +230,14 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign case MM_TSB_BASE: hp->pgsz_idx = HV_PGSZ_IDX_BASE; break; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) case MM_TSB_HUGE: hp->pgsz_idx = HV_PGSZ_IDX_HUGE; break; #endif default: BUG(); - }; + } hp->assoc = 1; hp->num_ttes = tsb_bytes / 16; hp->ctx_idx = 0; @@ -222,19 +245,21 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign case MM_TSB_BASE: hp->pgsz_mask = HV_PGSZ_MASK_BASE; break; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) case MM_TSB_HUGE: hp->pgsz_mask = HV_PGSZ_MASK_HUGE; break; #endif default: BUG(); - }; + } hp->tsb_base = tsb_paddr; hp->resv = 0; } } +struct kmem_cache *pgtable_cache __read_mostly; + static struct kmem_cache *tsb_caches[8] __read_mostly; static const char *tsb_cache_names[8] = { @@ -252,7 +277,16 @@ void __init pgtable_cache_init(void) { unsigned long i; - for (i = 0; i < 8; i++) { + pgtable_cache = kmem_cache_create("pgtable_cache", + PAGE_SIZE, PAGE_SIZE, + 0, + _clear_page); + if (!pgtable_cache) { + prom_printf("pgtable_cache_init(): Could not create!\n"); + prom_halt(); + } + + for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) { unsigned long size = 8192 << i; const char *name = tsb_cache_names[i]; @@ -320,7 +354,7 @@ void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) retry_tsb_alloc: gfp_flags = GFP_KERNEL; if (new_size > (PAGE_SIZE * 2)) - gfp_flags = __GFP_NOWARN | __GFP_NORETRY; + gfp_flags |= __GFP_NOWARN | __GFP_NORETRY; new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index], gfp_flags, numa_node_id()); @@ -433,7 +467,7 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) unsigned long huge_pte_count; #endif unsigned int i; @@ -442,7 +476,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) /* We reset it to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. @@ -463,7 +497,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) */ tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if (unlikely(huge_pte_count)) tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); #endif diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S index 4e55e8f7664..bf10a345fa8 100644 --- a/arch/sparc/mm/tsunami.S +++ b/arch/sparc/mm/tsunami.S @@ -24,7 +24,7 @@ /* Sliiick... */ tsunami_flush_cache_page: tsunami_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 tsunami_flush_cache_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -46,7 +46,7 @@ tsunami_flush_sig_insns: /* More slick stuff... */ tsunami_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 tsunami_flush_tlb_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -65,7 +65,7 @@ tsunami_flush_tlb_out: /* This one can be done in a fine grained manner... */ tsunami_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S index b57a5942ba6..b4f4733abc6 100644 --- a/arch/sparc/mm/ultra.S +++ b/arch/sparc/mm/ultra.S @@ -53,6 +53,33 @@ __flush_tlb_mm: /* 18 insns */ nop .align 32 + .globl __flush_tlb_page +__flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, %pstate + mov SECONDARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + andn %o1, 1, %o3 + be,pn %icc, 1f + or %o3, 0x10, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + retl + wrpr %g7, 0x0, %pstate + nop + nop + nop + nop + + .align 32 .globl __flush_tlb_pending __flush_tlb_pending: /* 26 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ @@ -126,10 +153,10 @@ __spitfire_flush_tlb_mm_slow: .globl __flush_icache_page __flush_icache_page: /* %o0 = phys_page */ srlx %o0, PAGE_SHIFT, %o0 - sethi %uhi(PAGE_OFFSET), %g1 + sethi %hi(PAGE_OFFSET), %g1 sllx %o0, PAGE_SHIFT, %o0 sethi %hi(PAGE_SIZE), %g2 - sllx %g1, 32, %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 add %o0, %g1, %o0 1: subcc %g2, 32, %g2 bne,pt %icc, 1b @@ -151,8 +178,8 @@ __flush_icache_page: /* %o0 = phys_page */ .align 64 .globl __flush_dcache_page __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 ! physical address srlx %o0, 11, %o0 ! make D-cache TAG sethi %hi(1 << 14), %o2 ! D-cache size @@ -203,6 +230,31 @@ __cheetah_flush_tlb_mm: /* 19 insns */ retl wrpr %g7, 0x0, %pstate +__cheetah_flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 + sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 + or %o0, %o3, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + be,pn %icc, 1f + andn %o1, 1, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + wrpr %g0, 0, %tl + retl + wrpr %g7, 0x0, %pstate + __cheetah_flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ rdpr %pstate, %g7 @@ -235,8 +287,8 @@ __cheetah_flush_tlb_pending: /* 27 insns */ #ifdef DCACHE_ALIASING_POSSIBLE __cheetah_flush_dcache_page: /* 11 insns */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 sethi %hi(PAGE_SIZE), %o4 1: subcc %o4, (1 << 5), %o4 @@ -269,6 +321,20 @@ __hypervisor_flush_tlb_mm: /* 10 insns */ retl nop +__hypervisor_flush_tlb_page: /* 11 insns */ + /* %o0 = context, %o1 = vaddr */ + mov %o0, %g2 + mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ + mov %g2, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, __hypervisor_tlb_tl0_error + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + retl + nop + __hypervisor_flush_tlb_pending: /* 16 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ sllx %o1, 3, %g1 @@ -339,6 +405,13 @@ cheetah_patch_cachetlbops: call tlb_patch_one mov 19, %o2 + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__cheetah_flush_tlb_pending), %o1 @@ -397,10 +470,9 @@ xcall_flush_tlb_mm: /* 21 insns */ nop nop - .globl xcall_flush_tlb_pending -xcall_flush_tlb_pending: /* 21 insns */ - /* %g5=context, %g1=nr, %g7=vaddrs[] */ - sllx %g1, 3, %g1 + .globl xcall_flush_tlb_page +xcall_flush_tlb_page: /* 17 insns */ + /* %g5=context, %g1=vaddr */ mov PRIMARY_CONTEXT, %g4 ldxa [%g4] ASI_DMMU, %g2 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 @@ -408,20 +480,16 @@ xcall_flush_tlb_pending: /* 21 insns */ or %g5, %g4, %g5 mov PRIMARY_CONTEXT, %g4 stxa %g5, [%g4] ASI_DMMU -1: sub %g1, (1 << 3), %g1 - ldx [%g7 + %g1], %g5 - andcc %g5, 0x1, %g0 + andcc %g1, 0x1, %g0 be,pn %icc, 2f - - andn %g5, 0x1, %g5 + andn %g1, 0x1, %g5 stxa %g0, [%g5] ASI_IMMU_DEMAP 2: stxa %g0, [%g5] ASI_DMMU_DEMAP membar #Sync - brnz,pt %g1, 1b - nop stxa %g2, [%g4] ASI_DMMU retry nop + nop .globl xcall_flush_tlb_kernel_range xcall_flush_tlb_kernel_range: /* 25 insns */ @@ -481,8 +549,8 @@ xcall_sync_tick: .globl xcall_fetch_glob_regs xcall_fetch_glob_regs: - sethi %hi(global_reg_snapshot), %g1 - or %g1, %lo(global_reg_snapshot), %g1 + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 __GET_CPUID(%g2) sllx %g2, 6, %g3 add %g1, %g3, %g1 @@ -495,11 +563,11 @@ xcall_fetch_glob_regs: stx %o7, [%g1 + GR_SNAP_O7] stx %i7, [%g1 + GR_SNAP_I7] /* Don't try this at home kids... */ - rdpr %cwp, %g2 - sub %g2, 1, %g7 + rdpr %cwp, %g3 + sub %g3, 1, %g7 wrpr %g7, %cwp mov %i7, %g7 - wrpr %g2, %cwp + wrpr %g3, %cwp stx %g7, [%g1 + GR_SNAP_RPC] sethi %hi(trap_block), %g7 or %g7, %lo(trap_block), %g7 @@ -509,6 +577,66 @@ xcall_fetch_glob_regs: stx %g3, [%g1 + GR_SNAP_THREAD] retry + .globl xcall_fetch_glob_pmu +xcall_fetch_glob_pmu: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + rd %pic, %g7 + stx %g7, [%g1 + (4 * 8)] + rd %pcr, %g7 + stx %g7, [%g1 + (0 * 8)] + retry + + .globl xcall_fetch_glob_pmu_n4 +xcall_fetch_glob_pmu_n4: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + + ldxa [%g0] ASI_PIC, %g7 + stx %g7, [%g1 + (4 * 8)] + mov 0x08, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (5 * 8)] + mov 0x10, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (6 * 8)] + mov 0x18, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (7 * 8)] + + mov %o0, %g2 + mov %o1, %g3 + mov %o5, %g7 + + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 3, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (3 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 2, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (2 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 1, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (1 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 0, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (0 * 8)] + + mov %g2, %o0 + mov %g3, %o1 + mov %g7, %o5 + + retry + #ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah @@ -596,15 +724,13 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ membar #Sync retry - .globl __hypervisor_xcall_flush_tlb_pending -__hypervisor_xcall_flush_tlb_pending: /* 21 insns */ - /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ - sllx %g1, 3, %g1 + .globl __hypervisor_xcall_flush_tlb_page +__hypervisor_xcall_flush_tlb_page: /* 17 insns */ + /* %g5=ctx, %g1=vaddr */ mov %o0, %g2 mov %o1, %g3 mov %o2, %g4 -1: sub %g1, (1 << 3), %g1 - ldx [%g7 + %g1], %o0 /* ARG0: virtual address */ + mov %g1, %o0 /* ARG0: virtual address */ mov %g5, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ srlx %o0, PAGE_SHIFT, %o0 @@ -613,8 +739,6 @@ __hypervisor_xcall_flush_tlb_pending: /* 21 insns */ mov HV_MMU_UNMAP_ADDR_TRAP, %g6 brnz,a,pn %o0, __hypervisor_tlb_xcall_error mov %o0, %g5 - brnz,pt %g1, 1b - nop mov %g2, %o0 mov %g3, %o1 mov %g4, %o2 @@ -697,6 +821,13 @@ hypervisor_patch_cachetlbops: call tlb_patch_one mov 10, %o2 + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__hypervisor_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_flush_tlb_page), %o1 + call tlb_patch_one + mov 11, %o2 + sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__hypervisor_flush_tlb_pending), %o1 @@ -728,12 +859,12 @@ hypervisor_patch_cachetlbops: call tlb_patch_one mov 21, %o2 - sethi %hi(xcall_flush_tlb_pending), %o0 - or %o0, %lo(xcall_flush_tlb_pending), %o0 - sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 - or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 + sethi %hi(xcall_flush_tlb_page), %o0 + or %o0, %lo(xcall_flush_tlb_page), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 call tlb_patch_one - mov 21, %o2 + mov 17, %o2 sethi %hi(xcall_flush_tlb_kernel_range), %o0 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S index 6dfcc13d310..852257fcc82 100644 --- a/arch/sparc/mm/viking.S +++ b/arch/sparc/mm/viking.S @@ -14,7 +14,6 @@ #include <asm/page.h> #include <asm/pgtsrmmu.h> #include <asm/viking.h> -#include <asm/btfixup.h> #ifdef CONFIG_SMP .data @@ -109,7 +108,7 @@ viking_mxcc_flush_page: viking_flush_cache_page: viking_flush_cache_range: #ifndef CONFIG_SMP - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 #endif viking_flush_cache_mm: #ifndef CONFIG_SMP @@ -149,7 +148,7 @@ viking_flush_tlb_mm: #endif viking_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -174,7 +173,7 @@ viking_flush_tlb_range: #endif viking_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -240,7 +239,7 @@ sun4dsmp_flush_tlb_range: tst %g5 bne 3f mov SRMMU_CTX_REG, %g1 - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 @@ -266,7 +265,7 @@ sun4dsmp_flush_tlb_page: tst %g5 bne 2f mov SRMMU_CTX_REG, %g1 - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 and %o1, PAGE_MASK, %o1 |