diff options
Diffstat (limited to 'arch/mips/mm/tlbex.c')
| -rw-r--r-- | arch/mips/mm/tlbex.c | 2774 |
1 files changed, 1610 insertions, 1164 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 0f9485806ba..e80e10bafc8 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -5,704 +5,398 @@ * * Synthesize TLB refill handlers at runtime. * - * Copyright (C) 2004,2005 by Thiemo Seufer - * Copyright (C) 2005 Maciej W. Rozycki + * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer + * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki + * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2008, 2009 Cavium Networks, Inc. + * Copyright (C) 2011 MIPS Technologies, Inc. + * + * ... and the days got worse and worse and now you see + * I've gone completly out of my mind. + * + * They're coming to take me a away haha + * they're coming to take me a away hoho hihi haha + * to the funny farm where code is beautiful all the time ... + * + * (Condolences to Napoleon XIV) */ -#include <stdarg.h> - -#include <linux/config.h> -#include <linux/mm.h> +#include <linux/bug.h> #include <linux/kernel.h> #include <linux/types.h> +#include <linux/smp.h> #include <linux/string.h> -#include <linux/init.h> +#include <linux/cache.h> -#include <asm/pgtable.h> #include <asm/cacheflush.h> -#include <asm/mmu_context.h> -#include <asm/inst.h> -#include <asm/elf.h> -#include <asm/smp.h> +#include <asm/cpu-type.h> +#include <asm/pgtable.h> #include <asm/war.h> +#include <asm/uasm.h> +#include <asm/setup.h> + +/* + * TLB load/store/modify handlers. + * + * Only the fastpath gets synthesized at runtime, the slowpath for + * do_page_fault remains normal asm. + */ +extern void tlb_do_page_fault_0(void); +extern void tlb_do_page_fault_1(void); + +struct work_registers { + int r1; + int r2; + int r3; +}; -/* #define DEBUG_TLB */ +struct tlb_reg_save { + unsigned long a; + unsigned long b; +} ____cacheline_aligned_in_smp; -static __init int __attribute__((unused)) r45k_bvahwbug(void) +static struct tlb_reg_save handler_reg_save[NR_CPUS]; + +static inline int r45k_bvahwbug(void) { /* XXX: We should probe for the presence of this bug, but we don't. */ return 0; } -static __init int __attribute__((unused)) r4k_250MHZhwbug(void) +static inline int r4k_250MHZhwbug(void) { /* XXX: We should probe for the presence of this bug, but we don't. */ return 0; } -static __init int __attribute__((unused)) bcm1250_m3_war(void) +static inline int __maybe_unused bcm1250_m3_war(void) { return BCM1250_M3_WAR; } -static __init int __attribute__((unused)) r10000_llsc_war(void) +static inline int __maybe_unused r10000_llsc_war(void) { return R10000_LLSC_WAR; } -/* - * A little micro-assembler, intended for TLB refill handler - * synthesizing. It is intentionally kept simple, does only support - * a subset of instructions, and does not try to hide pipeline effects - * like branch delay slots. - */ - -enum fields -{ - RS = 0x001, - RT = 0x002, - RD = 0x004, - RE = 0x008, - SIMM = 0x010, - UIMM = 0x020, - BIMM = 0x040, - JIMM = 0x080, - FUNC = 0x100, -}; - -#define OP_MASK 0x2f -#define OP_SH 26 -#define RS_MASK 0x1f -#define RS_SH 21 -#define RT_MASK 0x1f -#define RT_SH 16 -#define RD_MASK 0x1f -#define RD_SH 11 -#define RE_MASK 0x1f -#define RE_SH 6 -#define IMM_MASK 0xffff -#define IMM_SH 0 -#define JIMM_MASK 0x3ffffff -#define JIMM_SH 0 -#define FUNC_MASK 0x2f -#define FUNC_SH 0 - -enum opcode { - insn_invalid, - insn_addu, insn_addiu, insn_and, insn_andi, insn_beq, - insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl, - insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0, - insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, - insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld, - insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0, - insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll, - insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi, - insn_tlbwr, insn_xor, insn_xori -}; - -struct insn { - enum opcode opcode; - u32 match; - enum fields fields; -}; - -/* This macro sets the non-variable bits of an instruction. */ -#define M(a, b, c, d, e, f) \ - ((a) << OP_SH \ - | (b) << RS_SH \ - | (c) << RT_SH \ - | (d) << RD_SH \ - | (e) << RE_SH \ - | (f) << FUNC_SH) - -static __initdata struct insn insn_table[] = { - { insn_addiu, M(addiu_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_addu, M(spec_op,0,0,0,0,addu_op), RS | RT | RD }, - { insn_and, M(spec_op,0,0,0,0,and_op), RS | RT | RD }, - { insn_andi, M(andi_op,0,0,0,0,0), RS | RT | UIMM }, - { insn_beq, M(beq_op,0,0,0,0,0), RS | RT | BIMM }, - { insn_beql, M(beql_op,0,0,0,0,0), RS | RT | BIMM }, - { insn_bgez, M(bcond_op,0,bgez_op,0,0,0), RS | BIMM }, - { insn_bgezl, M(bcond_op,0,bgezl_op,0,0,0), RS | BIMM }, - { insn_bltz, M(bcond_op,0,bltz_op,0,0,0), RS | BIMM }, - { insn_bltzl, M(bcond_op,0,bltzl_op,0,0,0), RS | BIMM }, - { insn_bne, M(bne_op,0,0,0,0,0), RS | RT | BIMM }, - { insn_daddiu, M(daddiu_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_daddu, M(spec_op,0,0,0,0,daddu_op), RS | RT | RD }, - { insn_dmfc0, M(cop0_op,dmfc_op,0,0,0,0), RT | RD }, - { insn_dmtc0, M(cop0_op,dmtc_op,0,0,0,0), RT | RD }, - { insn_dsll, M(spec_op,0,0,0,0,dsll_op), RT | RD | RE }, - { insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE }, - { insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE }, - { insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE }, - { insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD }, - { insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 }, - { insn_j, M(j_op,0,0,0,0,0), JIMM }, - { insn_jal, M(jal_op,0,0,0,0,0), JIMM }, - { insn_jr, M(spec_op,0,0,0,0,jr_op), RS }, - { insn_ld, M(ld_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_ll, M(ll_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_lld, M(lld_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_lui, M(lui_op,0,0,0,0,0), RT | SIMM }, - { insn_lw, M(lw_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_mfc0, M(cop0_op,mfc_op,0,0,0,0), RT | RD }, - { insn_mtc0, M(cop0_op,mtc_op,0,0,0,0), RT | RD }, - { insn_ori, M(ori_op,0,0,0,0,0), RS | RT | UIMM }, - { insn_rfe, M(cop0_op,cop_op,0,0,0,rfe_op), 0 }, - { insn_sc, M(sc_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_scd, M(scd_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_sd, M(sd_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_sll, M(spec_op,0,0,0,0,sll_op), RT | RD | RE }, - { insn_sra, M(spec_op,0,0,0,0,sra_op), RT | RD | RE }, - { insn_srl, M(spec_op,0,0,0,0,srl_op), RT | RD | RE }, - { insn_subu, M(spec_op,0,0,0,0,subu_op), RS | RT | RD }, - { insn_sw, M(sw_op,0,0,0,0,0), RS | RT | SIMM }, - { insn_tlbp, M(cop0_op,cop_op,0,0,0,tlbp_op), 0 }, - { insn_tlbwi, M(cop0_op,cop_op,0,0,0,tlbwi_op), 0 }, - { insn_tlbwr, M(cop0_op,cop_op,0,0,0,tlbwr_op), 0 }, - { insn_xor, M(spec_op,0,0,0,0,xor_op), RS | RT | RD }, - { insn_xori, M(xori_op,0,0,0,0,0), RS | RT | UIMM }, - { insn_invalid, 0, 0 } -}; - -#undef M - -static __init u32 build_rs(u32 arg) -{ - if (arg & ~RS_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return (arg & RS_MASK) << RS_SH; -} - -static __init u32 build_rt(u32 arg) +static int use_bbit_insns(void) { - if (arg & ~RT_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return (arg & RT_MASK) << RT_SH; -} - -static __init u32 build_rd(u32 arg) -{ - if (arg & ~RD_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return (arg & RD_MASK) << RD_SH; + switch (current_cpu_type()) { + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + case CPU_CAVIUM_OCTEON3: + return 1; + default: + return 0; + } } -static __init u32 build_re(u32 arg) +static int use_lwx_insns(void) { - if (arg & ~RE_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return (arg & RE_MASK) << RE_SH; + switch (current_cpu_type()) { + case CPU_CAVIUM_OCTEON2: + case CPU_CAVIUM_OCTEON3: + return 1; + default: + return 0; + } } - -static __init u32 build_simm(s32 arg) +#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ + CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 +static bool scratchpad_available(void) { - if (arg > 0x7fff || arg < -0x8000) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return arg & 0xffff; + return true; } - -static __init u32 build_uimm(u32 arg) +static int scratchpad_offset(int i) { - if (arg & ~IMM_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return arg & IMM_MASK; -} - -static __init u32 build_bimm(s32 arg) -{ - if (arg > 0x1ffff || arg < -0x20000) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - if (arg & 0x3) - printk(KERN_WARNING "Invalid TLB synthesizer branch target\n"); - - return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff); + /* + * CVMSEG starts at address -32768 and extends for + * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. + */ + i += 1; /* Kernel use starts at the top and works down. */ + return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; } - -static __init u32 build_jimm(u32 arg) +#else +static bool scratchpad_available(void) { - if (arg & ~((JIMM_MASK) << 2)) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return (arg >> 2) & JIMM_MASK; + return false; } - -static __init u32 build_func(u32 arg) +static int scratchpad_offset(int i) { - if (arg & ~FUNC_MASK) - printk(KERN_WARNING "TLB synthesizer field overflow\n"); - - return arg & FUNC_MASK; + BUG(); + /* Really unreachable, but evidently some GCC want this. */ + return 0; } - +#endif /* - * The order of opcode arguments is implicitly left to right, - * starting with RS and ending with FUNC or IMM. + * Found by experiment: At least some revisions of the 4kc throw under + * some circumstances a machine check exception, triggered by invalid + * values in the index register. Delaying the tlbp instruction until + * after the next branch, plus adding an additional nop in front of + * tlbwi/tlbwr avoids the invalid index register values. Nobody knows + * why; it's not an issue caused by the core RTL. + * */ -static void __init build_insn(u32 **buf, enum opcode opc, ...) +static int m4kc_tlbp_war(void) { - struct insn *ip = NULL; - unsigned int i; - va_list ap; - u32 op; - - for (i = 0; insn_table[i].opcode != insn_invalid; i++) - if (insn_table[i].opcode == opc) { - ip = &insn_table[i]; - break; - } - - if (!ip) - panic("Unsupported TLB synthesizer instruction %d", opc); - - op = ip->match; - va_start(ap, opc); - if (ip->fields & RS) op |= build_rs(va_arg(ap, u32)); - if (ip->fields & RT) op |= build_rt(va_arg(ap, u32)); - if (ip->fields & RD) op |= build_rd(va_arg(ap, u32)); - if (ip->fields & RE) op |= build_re(va_arg(ap, u32)); - if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32)); - if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32)); - if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32)); - if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32)); - if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32)); - va_end(ap); - - **buf = op; - (*buf)++; + return (current_cpu_data.processor_id & 0xffff00) == + (PRID_COMP_MIPS | PRID_IMP_4KC); } -#define I_u1u2u3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b, unsigned int c) \ - { \ - build_insn(buf, insn##op, a, b, c); \ - } - -#define I_u2u1u3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b, unsigned int c) \ - { \ - build_insn(buf, insn##op, b, a, c); \ - } - -#define I_u3u1u2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b, unsigned int c) \ - { \ - build_insn(buf, insn##op, b, c, a); \ - } - -#define I_u1u2s3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b, signed int c) \ - { \ - build_insn(buf, insn##op, a, b, c); \ - } - -#define I_u2s3u1(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - signed int b, unsigned int c) \ - { \ - build_insn(buf, insn##op, c, a, b); \ - } - -#define I_u2u1s3(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b, signed int c) \ - { \ - build_insn(buf, insn##op, b, a, c); \ - } - -#define I_u1u2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - unsigned int b) \ - { \ - build_insn(buf, insn##op, a, b); \ - } - -#define I_u1s2(op) \ - static inline void i##op(u32 **buf, unsigned int a, \ - signed int b) \ - { \ - build_insn(buf, insn##op, a, b); \ - } - -#define I_u1(op) \ - static inline void i##op(u32 **buf, unsigned int a) \ - { \ - build_insn(buf, insn##op, a); \ - } - -#define I_0(op) \ - static inline void i##op(u32 **buf) \ - { \ - build_insn(buf, insn##op); \ - } - -I_u2u1s3(_addiu); -I_u3u1u2(_addu); -I_u2u1u3(_andi); -I_u3u1u2(_and); -I_u1u2s3(_beq); -I_u1u2s3(_beql); -I_u1s2(_bgez); -I_u1s2(_bgezl); -I_u1s2(_bltz); -I_u1s2(_bltzl); -I_u1u2s3(_bne); -I_u1u2(_dmfc0); -I_u1u2(_dmtc0); -I_u2u1s3(_daddiu); -I_u3u1u2(_daddu); -I_u2u1u3(_dsll); -I_u2u1u3(_dsll32); -I_u2u1u3(_dsra); -I_u2u1u3(_dsrl); -I_u3u1u2(_dsubu); -I_0(_eret); -I_u1(_j); -I_u1(_jal); -I_u1(_jr); -I_u2s3u1(_ld); -I_u2s3u1(_ll); -I_u2s3u1(_lld); -I_u1s2(_lui); -I_u2s3u1(_lw); -I_u1u2(_mfc0); -I_u1u2(_mtc0); -I_u2u1u3(_ori); -I_0(_rfe); -I_u2s3u1(_sc); -I_u2s3u1(_scd); -I_u2s3u1(_sd); -I_u2u1u3(_sll); -I_u2u1u3(_sra); -I_u2u1u3(_srl); -I_u3u1u2(_subu); -I_u2s3u1(_sw); -I_0(_tlbp); -I_0(_tlbwi); -I_0(_tlbwr); -I_u3u1u2(_xor) -I_u2u1u3(_xori); - -/* - * handling labels - */ - +/* Handle labels (which must be positive integers). */ enum label_id { - label_invalid, - label_second_part, + label_second_part = 1, label_leave, label_vmalloc, label_vmalloc_done, - label_tlbw_hazard, - label_split, + label_tlbw_hazard_0, + label_split = label_tlbw_hazard_0 + 8, + label_tlbl_goaround1, + label_tlbl_goaround2, label_nopage_tlbl, label_nopage_tlbs, label_nopage_tlbm, label_smp_pgtable_change, label_r3000_write_probe_fail, + label_large_segbits_fault, +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + label_tlb_huge_update, +#endif }; -struct label { - u32 *addr; - enum label_id lab; -}; - -static __init void build_label(struct label **lab, u32 *addr, - enum label_id l) -{ - (*lab)->addr = addr; - (*lab)->lab = l; - (*lab)++; -} - -#define L_LA(lb) \ - static inline void l##lb(struct label **lab, u32 *addr) \ - { \ - build_label(lab, addr, label##lb); \ - } - -L_LA(_second_part) -L_LA(_leave) -L_LA(_vmalloc) -L_LA(_vmalloc_done) -L_LA(_tlbw_hazard) -L_LA(_split) -L_LA(_nopage_tlbl) -L_LA(_nopage_tlbs) -L_LA(_nopage_tlbm) -L_LA(_smp_pgtable_change) -L_LA(_r3000_write_probe_fail) - -/* convenience macros for instructions */ -#ifdef CONFIG_64BIT -# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off) -# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off) -# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh) -# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh) -# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh) -# define i_MFC0(buf, rt, rd) i_dmfc0(buf, rt, rd) -# define i_MTC0(buf, rt, rd) i_dmtc0(buf, rt, rd) -# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val) -# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd) -# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd) -# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off) -# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off) -#else -# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off) -# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off) -# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh) -# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh) -# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh) -# define i_MFC0(buf, rt, rd) i_mfc0(buf, rt, rd) -# define i_MTC0(buf, rt, rd) i_mtc0(buf, rt, rd) -# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val) -# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd) -# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd) -# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off) -# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off) +UASM_L_LA(_second_part) +UASM_L_LA(_leave) +UASM_L_LA(_vmalloc) +UASM_L_LA(_vmalloc_done) +/* _tlbw_hazard_x is handled differently. */ +UASM_L_LA(_split) +UASM_L_LA(_tlbl_goaround1) +UASM_L_LA(_tlbl_goaround2) +UASM_L_LA(_nopage_tlbl) +UASM_L_LA(_nopage_tlbs) +UASM_L_LA(_nopage_tlbm) +UASM_L_LA(_smp_pgtable_change) +UASM_L_LA(_r3000_write_probe_fail) +UASM_L_LA(_large_segbits_fault) +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT +UASM_L_LA(_tlb_huge_update) #endif -#define i_b(buf, off) i_beq(buf, 0, 0, off) -#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off) -#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off) -#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off) -#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off) -#define i_move(buf, a, b) i_ADDU(buf, a, 0, b) -#define i_nop(buf) i_sll(buf, 0, 0, 0) -#define i_ssnop(buf) i_sll(buf, 0, 0, 1) -#define i_ehb(buf) i_sll(buf, 0, 0, 3) +static int hazard_instance; -#ifdef CONFIG_64BIT -static __init int __attribute__((unused)) in_compat_space_p(long addr) +static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance) { - /* Is this address in 32bit compat space? */ - return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L); + switch (instance) { + case 0 ... 7: + uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance); + return; + default: + BUG(); + } } -static __init int __attribute__((unused)) rel_highest(long val) +static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance) { - return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000; + switch (instance) { + case 0 ... 7: + uasm_build_label(l, *p, label_tlbw_hazard_0 + instance); + break; + default: + BUG(); + } } -static __init int __attribute__((unused)) rel_higher(long val) -{ - return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000; -} +/* + * pgtable bits are assigned dynamically depending on processor feature + * and statically based on kernel configuration. This spits out the actual + * values the kernel is using. Required to make sense from disassembled + * TLB exception handlers. + */ +static void output_pgtable_bits_defines(void) +{ +#define pr_define(fmt, ...) \ + pr_debug("#define " fmt, ##__VA_ARGS__) + + pr_debug("#include <asm/asm.h>\n"); + pr_debug("#include <asm/regdef.h>\n"); + pr_debug("\n"); + + pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); + pr_define("_PAGE_READ_SHIFT %d\n", _PAGE_READ_SHIFT); + pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); + pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); + pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); + pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT); #endif - -static __init int rel_hi(long val) -{ - return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000; + if (cpu_has_rixi) { +#ifdef _PAGE_NO_EXEC_SHIFT + pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); +#endif +#ifdef _PAGE_NO_READ_SHIFT + pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); +#endif + } + pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); + pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); + pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); + pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT); + pr_debug("\n"); } -static __init int rel_lo(long val) +static inline void dump_handler(const char *symbol, const u32 *handler, int count) { - return ((val & 0xffff) ^ 0x8000) - 0x8000; -} + int i; -static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr) -{ -#ifdef CONFIG_64BIT - if (!in_compat_space_p(addr)) { - i_lui(buf, rs, rel_highest(addr)); - if (rel_higher(addr)) - i_daddiu(buf, rs, rs, rel_higher(addr)); - if (rel_hi(addr)) { - i_dsll(buf, rs, rs, 16); - i_daddiu(buf, rs, rs, rel_hi(addr)); - i_dsll(buf, rs, rs, 16); - } else - i_dsll32(buf, rs, rs, 0); - } else -#endif - i_lui(buf, rs, rel_hi(addr)); -} + pr_debug("LEAF(%s)\n", symbol); -static __init void __attribute__((unused)) i_LA(u32 **buf, unsigned int rs, - long addr) -{ - i_LA_mostly(buf, rs, addr); - if (rel_lo(addr)) - i_ADDIU(buf, rs, rs, rel_lo(addr)); -} + pr_debug("\t.set push\n"); + pr_debug("\t.set noreorder\n"); -/* - * handle relocations - */ + for (i = 0; i < count; i++) + pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]); -struct reloc { - u32 *addr; - unsigned int type; - enum label_id lab; -}; + pr_debug("\t.set\tpop\n"); -static __init void r_mips_pc16(struct reloc **rel, u32 *addr, - enum label_id l) -{ - (*rel)->addr = addr; - (*rel)->type = R_MIPS_PC16; - (*rel)->lab = l; - (*rel)++; + pr_debug("\tEND(%s)\n", symbol); } -static inline void __resolve_relocs(struct reloc *rel, struct label *lab) -{ - long laddr = (long)lab->addr; - long raddr = (long)rel->addr; +/* The only general purpose registers allowed in TLB handlers. */ +#define K0 26 +#define K1 27 - switch (rel->type) { - case R_MIPS_PC16: - *rel->addr |= build_bimm(laddr - (raddr + 4)); - break; +/* Some CP0 registers */ +#define C0_INDEX 0, 0 +#define C0_ENTRYLO0 2, 0 +#define C0_TCBIND 2, 2 +#define C0_ENTRYLO1 3, 0 +#define C0_CONTEXT 4, 0 +#define C0_PAGEMASK 5, 0 +#define C0_BADVADDR 8, 0 +#define C0_ENTRYHI 10, 0 +#define C0_EPC 14, 0 +#define C0_XCONTEXT 20, 0 - default: - panic("Unsupported TLB synthesizer relocation %d", - rel->type); - } -} +#ifdef CONFIG_64BIT +# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT) +#else +# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT) +#endif -static __init void resolve_relocs(struct reloc *rel, struct label *lab) -{ - struct label *l; +/* The worst case length of the handler is around 18 instructions for + * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. + * Maximum space available is 32 instructions for R3000 and 64 + * instructions for R4000. + * + * We deliberately chose a buffer size of 128, so we won't scribble + * over anything important on overflow before we panic. + */ +static u32 tlb_handler[128]; - for (; rel->lab != label_invalid; rel++) - for (l = lab; l->lab != label_invalid; l++) - if (rel->lab == l->lab) - __resolve_relocs(rel, l); -} +/* simply assume worst case size for labels and relocs */ +static struct uasm_label labels[128]; +static struct uasm_reloc relocs[128]; -static __init void move_relocs(struct reloc *rel, u32 *first, u32 *end, - long off) -{ - for (; rel->lab != label_invalid; rel++) - if (rel->addr >= first && rel->addr < end) - rel->addr += off; -} +static int check_for_high_segbits; -static __init void move_labels(struct label *lab, u32 *first, u32 *end, - long off) +static unsigned int kscratch_used_mask; + +static inline int __maybe_unused c0_kscratch(void) { - for (; lab->lab != label_invalid; lab++) - if (lab->addr >= first && lab->addr < end) - lab->addr += off; + switch (current_cpu_type()) { + case CPU_XLP: + case CPU_XLR: + return 22; + default: + return 31; + } } -static __init void copy_handler(struct reloc *rel, struct label *lab, - u32 *first, u32 *end, u32 *target) +static int allocate_kscratch(void) { - long off = (long)(target - first); + int r; + unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; - memcpy(target, first, (end - first) * sizeof(u32)); + r = ffs(a); - move_relocs(rel, first, end, off); - move_labels(lab, first, end, off); -} + if (r == 0) + return -1; -static __init int __attribute__((unused)) insn_has_bdelay(struct reloc *rel, - u32 *addr) -{ - for (; rel->lab != label_invalid; rel++) { - if (rel->addr == addr - && (rel->type == R_MIPS_PC16 - || rel->type == R_MIPS_26)) - return 1; - } + r--; /* make it zero based */ - return 0; -} + kscratch_used_mask |= (1 << r); -/* convenience functions for labeled branches */ -static void __attribute__((unused)) il_bltz(u32 **p, struct reloc **r, - unsigned int reg, enum label_id l) -{ - r_mips_pc16(r, *p, l); - i_bltz(p, reg, 0); + return r; } -static void __attribute__((unused)) il_b(u32 **p, struct reloc **r, - enum label_id l) -{ - r_mips_pc16(r, *p, l); - i_b(p, 0); -} +static int scratch_reg; +static int pgd_reg; +enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; -static void il_beqz(u32 **p, struct reloc **r, unsigned int reg, - enum label_id l) +static struct work_registers build_get_work_registers(u32 **p) { - r_mips_pc16(r, *p, l); - i_beqz(p, reg, 0); -} + struct work_registers r; -static void __attribute__((unused)) -il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) -{ - r_mips_pc16(r, *p, l); - i_beqzl(p, reg, 0); -} + if (scratch_reg >= 0) { + /* Save in CPU local C0_KScratch? */ + UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg); + r.r1 = K0; + r.r2 = K1; + r.r3 = 1; + return r; + } -static void il_bnez(u32 **p, struct reloc **r, unsigned int reg, - enum label_id l) -{ - r_mips_pc16(r, *p, l); - i_bnez(p, reg, 0); + if (num_possible_cpus() > 1) { + /* Get smp_processor_id */ + UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG); + UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT); + + /* handler_reg_save index in K0 */ + UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save))); + + UASM_i_LA(p, K1, (long)&handler_reg_save); + UASM_i_ADDU(p, K0, K0, K1); + } else { + UASM_i_LA(p, K0, (long)&handler_reg_save); + } + /* K0 now points to save area, save $1 and $2 */ + UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0); + UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0); + + r.r1 = K1; + r.r2 = 1; + r.r3 = 2; + return r; } -static void il_bgezl(u32 **p, struct reloc **r, unsigned int reg, - enum label_id l) +static void build_restore_work_registers(u32 **p) { - r_mips_pc16(r, *p, l); - i_bgezl(p, reg, 0); + if (scratch_reg >= 0) { + UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); + return; + } + /* K0 already points to save area, restore $1 and $2 */ + UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0); + UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0); } -/* The only general purpose registers allowed in TLB handlers. */ -#define K0 26 -#define K1 27 - -/* Some CP0 registers */ -#define C0_INDEX 0 -#define C0_ENTRYLO0 2 -#define C0_ENTRYLO1 3 -#define C0_CONTEXT 4 -#define C0_BADVADDR 8 -#define C0_ENTRYHI 10 -#define C0_EPC 14 -#define C0_XCONTEXT 20 +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT -#ifdef CONFIG_64BIT -# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT) -#else -# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT) -#endif - -/* The worst case length of the handler is around 18 instructions for - * R3000-style TLBs and up to 63 instructions for R4000-style TLBs. - * Maximum space available is 32 instructions for R3000 and 64 - * instructions for R4000. +/* + * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, + * we cannot do r3000 under these circumstances. * - * We deliberately chose a buffer size of 128, so we won't scribble - * over anything important on overflow before we panic. + * Declare pgd_current here instead of including mmu_context.h to avoid type + * conflicts for tlbmiss_handler_setup_pgd */ -static __initdata u32 tlb_handler[128]; - -/* simply assume worst case size for labels and relocs */ -static __initdata struct label labels[128]; -static __initdata struct reloc relocs[128]; +extern unsigned long pgd_current[]; /* * The R3000 TLB handler is simple. */ -static void __init build_r3000_tlb_refill_handler(void) +static void build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; @@ -710,40 +404,35 @@ static void __init build_r3000_tlb_refill_handler(void) memset(tlb_handler, 0, sizeof(tlb_handler)); p = tlb_handler; - i_mfc0(&p, K0, C0_BADVADDR); - i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */ - i_lw(&p, K1, rel_lo(pgdc), K1); - i_srl(&p, K0, K0, 22); /* load delay */ - i_sll(&p, K0, K0, 2); - i_addu(&p, K1, K1, K0); - i_mfc0(&p, K0, C0_CONTEXT); - i_lw(&p, K1, 0, K1); /* cp0 delay */ - i_andi(&p, K0, K0, 0xffc); /* load delay */ - i_addu(&p, K1, K1, K0); - i_lw(&p, K0, 0, K1); - i_nop(&p); /* load delay */ - i_mtc0(&p, K0, C0_ENTRYLO0); - i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ - i_tlbwr(&p); /* cp0 delay */ - i_jr(&p, K1); - i_rfe(&p); /* branch delay */ + uasm_i_mfc0(&p, K0, C0_BADVADDR); + uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */ + uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1); + uasm_i_srl(&p, K0, K0, 22); /* load delay */ + uasm_i_sll(&p, K0, K0, 2); + uasm_i_addu(&p, K1, K1, K0); + uasm_i_mfc0(&p, K0, C0_CONTEXT); + uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */ + uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */ + uasm_i_addu(&p, K1, K1, K0); + uasm_i_lw(&p, K0, 0, K1); + uasm_i_nop(&p); /* load delay */ + uasm_i_mtc0(&p, K0, C0_ENTRYLO0); + uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */ + uasm_i_tlbwr(&p); /* cp0 delay */ + uasm_i_jr(&p, K1); + uasm_i_rfe(&p); /* branch delay */ if (p > tlb_handler + 32) panic("TLB refill handler space exceeded"); - printk("Synthesized TLB refill handler (%u instructions).\n", - (unsigned int)(p - tlb_handler)); -#ifdef DEBUG_TLB - { - int i; + pr_debug("Wrote TLB refill handler (%u instructions).\n", + (unsigned int)(p - tlb_handler)); - for (i = 0; i < (p - tlb_handler); i++) - printk("%08x\n", tlb_handler[i]); - } -#endif + memcpy((void *)ebase, tlb_handler, 0x80); - memcpy((void *)CAC_BASE, tlb_handler, 0x80); + dump_handler("r3000_tlb_refill", (u32 *)ebase, 32); } +#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ /* * The R4000 TLB handler is much more complicated. We have two @@ -752,7 +441,7 @@ static void __init build_r3000_tlb_refill_handler(void) * other one.To keep things simple, we first assume linear space, * then we relocate it to the final handler layout as needed. */ -static __initdata u32 final_handler[64]; +static u32 final_handler[64]; /* * Hazards @@ -760,8 +449,8 @@ static __initdata u32 final_handler[64]; * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: * 2. A timing hazard exists for the TLBP instruction. * - * stalling_instruction - * TLBP + * stalling_instruction + * TLBP * * The JTLB is being read for the TLBP throughout the stall generated by the * previous instruction. This is not really correct as the stalling instruction @@ -772,46 +461,69 @@ static __initdata u32 final_handler[64]; * The software work-around is to not allow the instruction preceding the TLBP * to stall - make it an NOP or some other instruction guaranteed not to stall. * - * Errata 2 will not be fixed. This errata is also on the R5000. + * Errata 2 will not be fixed. This errata is also on the R5000. * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ -static __init void __attribute__((unused)) build_tlb_probe_entry(u32 **p) +static void __maybe_unused build_tlb_probe_entry(u32 **p) { - switch (current_cpu_data.cputype) { - /* Found by experiment: R4600 v2.0 needs this, too. */ + switch (current_cpu_type()) { + /* Found by experiment: R4600 v2.0/R4700 needs this, too. */ case CPU_R4600: + case CPU_R4700: case CPU_R5000: - case CPU_R5000A: case CPU_NEVADA: - i_nop(p); - i_tlbp(p); + uasm_i_nop(p); + uasm_i_tlbp(p); break; default: - i_tlbp(p); + uasm_i_tlbp(p); break; } } /* * Write random or indexed TLB entry, and care about the hazards from - * the preceeding mtc0 and for the following eret. + * the preceding mtc0 and for the following eret. */ enum tlb_write_entry { tlb_random, tlb_indexed }; -static __init void build_tlb_write_entry(u32 **p, struct label **l, - struct reloc **r, - enum tlb_write_entry wmode) +static void build_tlb_write_entry(u32 **p, struct uasm_label **l, + struct uasm_reloc **r, + enum tlb_write_entry wmode) { void(*tlbw)(u32 **) = NULL; switch (wmode) { - case tlb_random: tlbw = i_tlbwr; break; - case tlb_indexed: tlbw = i_tlbwi; break; + case tlb_random: tlbw = uasm_i_tlbwr; break; + case tlb_indexed: tlbw = uasm_i_tlbwi; break; } - switch (current_cpu_data.cputype) { + if (cpu_has_mips_r2) { + /* + * The architecture spec says an ehb is required here, + * but a number of cores do not have the hazard and + * using an ehb causes an expensive pipeline stall. + */ + switch (current_cpu_type()) { + case CPU_M14KC: + case CPU_74K: + case CPU_1074K: + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_M5150: + break; + + default: + uasm_i_ehb(p); + break; + } + tlbw(p); + return; + } + + switch (current_cpu_type()) { case CPU_R4000PC: case CPU_R4000SC: case CPU_R4000MC: @@ -822,195 +534,385 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, * This branch uses up a mtc0 hazard nop slot and saves * two nops after the tlbw instruction. */ - il_bgezl(p, r, 0, label_tlbw_hazard); + uasm_bgezl_hazard(p, r, hazard_instance); tlbw(p); - l_tlbw_hazard(l, *p); - i_nop(p); + uasm_bgezl_label(l, p, hazard_instance); + hazard_instance++; + uasm_i_nop(p); break; case CPU_R4600: case CPU_R4700: + uasm_i_nop(p); + tlbw(p); + uasm_i_nop(p); + break; + case CPU_R5000: - case CPU_R5000A: - i_nop(p); + case CPU_NEVADA: + uasm_i_nop(p); /* QED specifies 2 nops hazard */ + uasm_i_nop(p); /* QED specifies 2 nops hazard */ tlbw(p); - i_nop(p); break; case CPU_R4300: case CPU_5KC: case CPU_TX49XX: - case CPU_AU1000: - case CPU_AU1100: - case CPU_AU1500: - case CPU_AU1550: - case CPU_AU1200: case CPU_PR4450: - i_nop(p); + case CPU_XLR: + uasm_i_nop(p); tlbw(p); break; case CPU_R10000: case CPU_R12000: + case CPU_R14000: case CPU_4KC: + case CPU_4KEC: + case CPU_M14KC: + case CPU_M14KEC: case CPU_SB1: case CPU_SB1A: case CPU_4KSC: case CPU_20KC: case CPU_25KF: + case CPU_BMIPS32: + case CPU_BMIPS3300: + case CPU_BMIPS4350: + case CPU_BMIPS4380: + case CPU_BMIPS5000: + case CPU_LOONGSON2: + case CPU_LOONGSON3: + case CPU_R5500: + if (m4kc_tlbp_war()) + uasm_i_nop(p); + case CPU_ALCHEMY: tlbw(p); break; - case CPU_NEVADA: - i_nop(p); /* QED specifies 2 nops hazard */ - /* - * This branch uses up a mtc0 hazard nop slot and saves - * a nop after the tlbw instruction. - */ - il_bgezl(p, r, 0, label_tlbw_hazard); - tlbw(p); - l_tlbw_hazard(l, *p); - break; - case CPU_RM7000: - i_nop(p); - i_nop(p); - i_nop(p); - i_nop(p); - tlbw(p); - break; - - case CPU_4KEC: - case CPU_24K: - case CPU_34K: - i_ehb(p); + uasm_i_nop(p); + uasm_i_nop(p); + uasm_i_nop(p); + uasm_i_nop(p); tlbw(p); break; - case CPU_RM9000: - /* - * When the JTLB is updated by tlbwi or tlbwr, a subsequent - * use of the JTLB for instructions should not occur for 4 - * cpu cycles and use for data translations should not occur - * for 3 cpu cycles. - */ - i_ssnop(p); - i_ssnop(p); - i_ssnop(p); - i_ssnop(p); - tlbw(p); - i_ssnop(p); - i_ssnop(p); - i_ssnop(p); - i_ssnop(p); - break; - case CPU_VR4111: case CPU_VR4121: case CPU_VR4122: case CPU_VR4181: case CPU_VR4181A: - i_nop(p); - i_nop(p); + uasm_i_nop(p); + uasm_i_nop(p); tlbw(p); - i_nop(p); - i_nop(p); + uasm_i_nop(p); + uasm_i_nop(p); break; case CPU_VR4131: case CPU_VR4133: case CPU_R5432: - i_nop(p); - i_nop(p); + uasm_i_nop(p); + uasm_i_nop(p); + tlbw(p); + break; + + case CPU_JZRISC: tlbw(p); + uasm_i_nop(p); break; default: panic("No TLB refill handler yet (CPU type: %d)", - current_cpu_data.cputype); + current_cpu_type()); break; } } +static __maybe_unused void build_convert_pte_to_entrylo(u32 **p, + unsigned int reg) +{ + if (cpu_has_rixi) { + UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL)); + } else { +#ifdef CONFIG_64BIT_PHYS_ADDR + uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL)); +#else + UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL)); +#endif + } +} + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + +static void build_restore_pagemask(u32 **p, struct uasm_reloc **r, + unsigned int tmp, enum label_id lid, + int restore_scratch) +{ + if (restore_scratch) { + /* Reset default page size */ + if (PM_DEFAULT_MASK >> 16) { + uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } else if (PM_DEFAULT_MASK) { + uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } else { + uasm_i_mtc0(p, 0, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } + if (scratch_reg >= 0) + UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); + else + UASM_i_LW(p, 1, scratchpad_offset(0), 0); + } else { + /* Reset default page size */ + if (PM_DEFAULT_MASK >> 16) { + uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else if (PM_DEFAULT_MASK) { + uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else { + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, 0, C0_PAGEMASK); + } + } +} + +static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, + struct uasm_reloc **r, + unsigned int tmp, + enum tlb_write_entry wmode, + int restore_scratch) +{ + /* Set huge page tlb entry size */ + uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + + build_tlb_write_entry(p, l, r, wmode); + + build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); +} + +/* + * Check if Huge PTE is present, if so then jump to LABEL. + */ +static void +build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, + unsigned int pmd, int lid) +{ + UASM_i_LW(p, tmp, 0, pmd); + if (use_bbit_insns()) { + uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); + } else { + uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); + uasm_il_bnez(p, r, tmp, lid); + } +} + +static void build_huge_update_entries(u32 **p, unsigned int pte, + unsigned int tmp) +{ + int small_sequence; + + /* + * A huge PTE describes an area the size of the + * configured huge page size. This is twice the + * of the large TLB entry size we intend to use. + * A TLB entry half the size of the configured + * huge page size is configured into entrylo0 + * and entrylo1 to cover the contiguous huge PTE + * address space. + */ + small_sequence = (HPAGE_SIZE >> 7) < 0x10000; + + /* We can clobber tmp. It isn't used after this.*/ + if (!small_sequence) + uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16)); + + build_convert_pte_to_entrylo(p, pte); + UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */ + /* convert to entrylo1 */ + if (small_sequence) + UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7); + else + UASM_i_ADDU(p, pte, pte, tmp); + + UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */ +} + +static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r, + struct uasm_label **l, + unsigned int pte, + unsigned int ptr) +{ +#ifdef CONFIG_SMP + UASM_i_SC(p, pte, 0, ptr); + uasm_il_beqz(p, r, pte, label_tlb_huge_update); + UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */ +#else + UASM_i_SW(p, pte, 0, ptr); +#endif + build_huge_update_entries(p, pte, ptr); + build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); +} +#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ + #ifdef CONFIG_64BIT /* * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pmd entry. */ -static __init void -build_get_pmde64(u32 **p, struct label **l, struct reloc **r, +static void +build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) { +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT long pgdc = (long)pgd_current; - +#endif /* * The vmalloc handling is not in the hotpath. */ - i_dmfc0(p, tmp, C0_BADVADDR); - il_bltz(p, r, tmp, label_vmalloc); - /* No i_nop needed here, since the next insn doesn't touch TMP. */ + uasm_i_dmfc0(p, tmp, C0_BADVADDR); -#ifdef CONFIG_SMP -# ifdef CONFIG_BUILD_ELF64 - /* - * 64 bit SMP running in XKPHYS has smp_processor_id() << 3 - * stored in CONTEXT. - */ - i_dmfc0(p, ptr, C0_CONTEXT); - i_dsrl(p, ptr, ptr, 23); - i_LA_mostly(p, tmp, pgdc); - i_daddu(p, ptr, ptr, tmp); - i_dmfc0(p, tmp, C0_BADVADDR); - i_ld(p, ptr, rel_lo(pgdc), ptr); -# else - /* - * 64 bit SMP running in compat space has the lower part of - * &pgd_current[smp_processor_id()] stored in CONTEXT. - */ - if (!in_compat_space_p(pgdc)) - panic("Invalid page directory address!"); + if (check_for_high_segbits) { + /* + * The kernel currently implicitely assumes that the + * MIPS SEGBITS parameter for the processor is + * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never + * allocate virtual addresses outside the maximum + * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But + * that doesn't prevent user code from accessing the + * higher xuseg addresses. Here, we make sure that + * everything but the lower xuseg addresses goes down + * the module_alloc/vmalloc path. + */ + uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); + uasm_il_bnez(p, r, ptr, label_vmalloc); + } else { + uasm_il_bltz(p, r, tmp, label_vmalloc); + } + /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ - i_dmfc0(p, ptr, C0_CONTEXT); - i_dsra(p, ptr, ptr, 23); - i_ld(p, ptr, 0, ptr); -# endif + if (pgd_reg != -1) { + /* pgd is in pgd_reg */ + UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); + } else { +#if defined(CONFIG_MIPS_PGD_C0_CONTEXT) + /* + * &pgd << 11 stored in CONTEXT [23..63]. + */ + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + + /* 1 0 1 0 1 << 6 xkphys cached */ + uasm_i_ori(p, ptr, ptr, 0x540); + uasm_i_drotr(p, ptr, ptr, 11); +#elif defined(CONFIG_SMP) + UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG); + uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT); + UASM_i_LA_mostly(p, tmp, pgdc); + uasm_i_daddu(p, ptr, ptr, tmp); + uasm_i_dmfc0(p, tmp, C0_BADVADDR); + uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); #else - i_LA_mostly(p, ptr, pgdc); - i_ld(p, ptr, rel_lo(pgdc), ptr); + UASM_i_LA_mostly(p, ptr, pgdc); + uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr); #endif + } + + uasm_l_vmalloc_done(l, *p); - l_vmalloc_done(l, *p); - i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); /* get pgd offset in bytes */ - i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); - i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ - i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ - i_ld(p, ptr, 0, ptr); /* get pmd pointer */ - i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ - i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); - i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ + /* get pgd offset in bytes */ + uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); + + uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); + uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */ +#ifndef __PAGETABLE_PMD_FOLDED + uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */ + uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */ + uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ + uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); + uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */ +#endif } /* * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ -static __init void -build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, - unsigned int bvaddr, unsigned int ptr) +static void +build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, + unsigned int bvaddr, unsigned int ptr, + enum vmalloc64_mode mode) { long swpd = (long)swapper_pg_dir; + int single_insn_swpd; + int did_vmalloc_branch = 0; - l_vmalloc(l, *p); - i_LA(p, ptr, VMALLOC_START); - i_dsubu(p, bvaddr, bvaddr, ptr); + single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd); - if (in_compat_space_p(swpd) && !rel_lo(swpd)) { - il_b(p, r, label_vmalloc_done); - i_lui(p, ptr, rel_hi(swpd)); - } else { - i_LA_mostly(p, ptr, swpd); - il_b(p, r, label_vmalloc_done); - i_daddiu(p, ptr, ptr, rel_lo(swpd)); + uasm_l_vmalloc(l, *p); + + if (mode != not_refill && check_for_high_segbits) { + if (single_insn_swpd) { + uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); + uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); + did_vmalloc_branch = 1; + /* fall through */ + } else { + uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault); + } + } + if (!did_vmalloc_branch) { + if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) { + uasm_il_b(p, r, label_vmalloc_done); + uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); + } else { + UASM_i_LA_mostly(p, ptr, swpd); + uasm_il_b(p, r, label_vmalloc_done); + if (uasm_in_compat_space_p(swpd)) + uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd)); + else + uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); + } + } + if (mode != not_refill && check_for_high_segbits) { + uasm_l_large_segbits_fault(l, *p); + /* + * We get here if we are an xsseg address, or if we are + * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary. + * + * Ignoring xsseg (assume disabled so would generate + * (address errors?), the only remaining possibility + * is the upper xuseg addresses. On processors with + * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these + * addresses would have taken an address error. We try + * to mimic that here by taking a load/istream page + * fault. + */ + UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); + uasm_i_jr(p, ptr); + + if (mode == refill_scratch) { + if (scratch_reg >= 0) + UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg); + else + UASM_i_LW(p, 1, scratchpad_offset(0), 0); + } else { + uasm_i_nop(p); + } } } @@ -1020,35 +922,41 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pgd entry. */ -static __init void __attribute__((unused)) +static void __maybe_unused build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) { - long pgdc = (long)pgd_current; + if (pgd_reg != -1) { + /* pgd is in pgd_reg */ + uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg); + uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ + } else { + long pgdc = (long)pgd_current; - /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ + /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */ #ifdef CONFIG_SMP - i_mfc0(p, ptr, C0_CONTEXT); - i_LA_mostly(p, tmp, pgdc); - i_srl(p, ptr, ptr, 23); - i_addu(p, ptr, tmp, ptr); + uasm_i_mfc0(p, ptr, SMP_CPUID_REG); + UASM_i_LA_mostly(p, tmp, pgdc); + uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT); + uasm_i_addu(p, ptr, tmp, ptr); #else - i_LA_mostly(p, ptr, pgdc); + UASM_i_LA_mostly(p, ptr, pgdc); #endif - i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ - i_lw(p, ptr, rel_lo(pgdc), ptr); - i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ - i_sll(p, tmp, tmp, PGD_T_LOG2); - i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ + uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */ + uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); + } + uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */ + uasm_i_sll(p, tmp, tmp, PGD_T_LOG2); + uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */ } #endif /* !CONFIG_64BIT */ -static __init void build_adjust_context(u32 **p, unsigned int ctx) +static void build_adjust_context(u32 **p, unsigned int ctx) { - unsigned int shift = 4 - (PTE_T_LOG2 + 1); + unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); - switch (current_cpu_data.cputype) { + switch (current_cpu_type()) { case CPU_VR41XX: case CPU_VR4111: case CPU_VR4121: @@ -1065,11 +973,11 @@ static __init void build_adjust_context(u32 **p, unsigned int ctx) } if (shift) - i_SRL(p, ctx, ctx, shift); - i_andi(p, ctx, ctx, mask); + UASM_i_SRL(p, ctx, ctx, shift); + uasm_i_andi(p, ctx, ctx, mask); } -static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) +static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) { /* * Bug workaround for the Nevada. It seems as if under certain @@ -1078,24 +986,23 @@ static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) * in a different cacheline or a load instruction, probably any * memory reference, is between them. */ - switch (current_cpu_data.cputype) { + switch (current_cpu_type()) { case CPU_NEVADA: - i_LW(p, ptr, 0, ptr); + UASM_i_LW(p, ptr, 0, ptr); GET_CONTEXT(p, tmp); /* get context reg */ break; default: GET_CONTEXT(p, tmp); /* get context reg */ - i_LW(p, ptr, 0, ptr); + UASM_i_LW(p, ptr, 0, ptr); break; } build_adjust_context(p, tmp); - i_ADDU(p, ptr, ptr, tmp); /* add in offset */ + UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */ } -static __init void build_update_entries(u32 **p, unsigned int tmp, - unsigned int ptep) +static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) { /* * 64bit address support (36bit on a 32bit CPU) in a 32bit @@ -1103,258 +1010,565 @@ static __init void build_update_entries(u32 **p, unsigned int tmp, */ #ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) { - i_ld(p, tmp, 0, ptep); /* get even pte */ - i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ - i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */ - i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ - i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */ - i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ + uasm_i_ld(p, tmp, 0, ptep); /* get even pte */ + uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ + if (cpu_has_rixi) { + UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); + UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ + UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); + } else { + uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ + UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ + uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ + } + UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ } else { int pte_off_even = sizeof(pte_t) / 2; int pte_off_odd = pte_off_even + sizeof(pte_t); /* The pte entries are pre-shifted */ - i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ - i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ - i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ - i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ + uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */ + UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ + uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */ + UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ } #else - i_LW(p, tmp, 0, ptep); /* get even pte */ - i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ + UASM_i_LW(p, tmp, 0, ptep); /* get even pte */ + UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */ if (r45k_bvahwbug()) build_tlb_probe_entry(p); - i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */ - if (r4k_250MHZhwbug()) - i_mtc0(p, 0, C0_ENTRYLO0); - i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */ - i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */ - if (r45k_bvahwbug()) - i_mfc0(p, tmp, C0_INDEX); + if (cpu_has_rixi) { + UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); + if (r4k_250MHZhwbug()) + UASM_i_MTC0(p, 0, C0_ENTRYLO0); + UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ + UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); + } else { + UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */ + if (r4k_250MHZhwbug()) + UASM_i_MTC0(p, 0, C0_ENTRYLO0); + UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */ + UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */ + if (r45k_bvahwbug()) + uasm_i_mfc0(p, tmp, C0_INDEX); + } if (r4k_250MHZhwbug()) - i_mtc0(p, 0, C0_ENTRYLO1); - i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */ + UASM_i_MTC0(p, 0, C0_ENTRYLO1); + UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */ #endif } -static void __init build_r4000_tlb_refill_handler(void) +struct mips_huge_tlb_info { + int huge_pte; + int restore_scratch; +}; + +static struct mips_huge_tlb_info +build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, + struct uasm_reloc **r, unsigned int tmp, + unsigned int ptr, int c0_scratch_reg) +{ + struct mips_huge_tlb_info rv; + unsigned int even, odd; + int vmalloc_branch_delay_filled = 0; + const int scratch = 1; /* Our extra working register */ + + rv.huge_pte = scratch; + rv.restore_scratch = 0; + + if (check_for_high_segbits) { + UASM_i_MFC0(p, tmp, C0_BADVADDR); + + if (pgd_reg != -1) + UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); + else + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + if (c0_scratch_reg >= 0) + UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); + else + UASM_i_SW(p, scratch, scratchpad_offset(0), 0); + + uasm_i_dsrl_safe(p, scratch, tmp, + PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); + uasm_il_bnez(p, r, scratch, label_vmalloc); + + if (pgd_reg == -1) { + vmalloc_branch_delay_filled = 1; + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + } + } else { + if (pgd_reg != -1) + UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg); + else + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + UASM_i_MFC0(p, tmp, C0_BADVADDR); + + if (c0_scratch_reg >= 0) + UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg); + else + UASM_i_SW(p, scratch, scratchpad_offset(0), 0); + + if (pgd_reg == -1) + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + + uasm_il_bltz(p, r, tmp, label_vmalloc); + } + + if (pgd_reg == -1) { + vmalloc_branch_delay_filled = 1; + /* 1 0 1 0 1 << 6 xkphys cached */ + uasm_i_ori(p, ptr, ptr, 0x540); + uasm_i_drotr(p, ptr, ptr, 11); + } + +#ifdef __PAGETABLE_PMD_FOLDED +#define LOC_PTEP scratch +#else +#define LOC_PTEP ptr +#endif + + if (!vmalloc_branch_delay_filled) + /* get pgd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); + + uasm_l_vmalloc_done(l, *p); + + /* + * tmp ptr + * fall-through case = badvaddr *pgd_current + * vmalloc case = badvaddr swapper_pg_dir + */ + + if (vmalloc_branch_delay_filled) + /* get pgd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); + +#ifdef __PAGETABLE_PMD_FOLDED + GET_CONTEXT(p, tmp); /* get context reg */ +#endif + uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); + + if (use_lwx_insns()) { + UASM_i_LWX(p, LOC_PTEP, scratch, ptr); + } else { + uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ + uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ + } + +#ifndef __PAGETABLE_PMD_FOLDED + /* get pmd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); + uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); + GET_CONTEXT(p, tmp); /* get context reg */ + + if (use_lwx_insns()) { + UASM_i_LWX(p, scratch, scratch, ptr); + } else { + uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ + UASM_i_LW(p, scratch, 0, ptr); + } +#endif + /* Adjust the context during the load latency. */ + build_adjust_context(p, tmp); + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); + /* + * The in the LWX case we don't want to do the load in the + * delay slot. It cannot issue in the same cycle and may be + * speculative and unneeded. + */ + if (use_lwx_insns()) + uasm_i_nop(p); +#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */ + + + /* build_update_entries */ + if (use_lwx_insns()) { + even = ptr; + odd = tmp; + UASM_i_LWX(p, even, scratch, tmp); + UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); + UASM_i_LWX(p, odd, scratch, tmp); + } else { + UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ + even = tmp; + odd = ptr; + UASM_i_LW(p, even, 0, ptr); /* get even pte */ + UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ + } + if (cpu_has_rixi) { + uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL)); + UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ + uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL)); + } else { + uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); + UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ + uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); + } + UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ + + if (c0_scratch_reg >= 0) { + UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg); + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + rv.restore_scratch = 1; + } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + UASM_i_LW(p, scratch, scratchpad_offset(0), 0); + } else { + UASM_i_LW(p, scratch, scratchpad_offset(0), 0); + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + rv.restore_scratch = 1; + } + + uasm_i_eret(p); /* return from trap */ + + return rv; +} + +/* + * For a 64-bit kernel, we are using the 64-bit XTLB refill exception + * because EXL == 0. If we wrap, we can also use the 32 instruction + * slots before the XTLB refill exception handler which belong to the + * unused TLB refill exception. + */ +#define MIPS64_REFILL_INSNS 32 + +static void build_r4000_tlb_refill_handler(void) { u32 *p = tlb_handler; - struct label *l = labels; - struct reloc *r = relocs; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; u32 *f; unsigned int final_len; + struct mips_huge_tlb_info htlb_info __maybe_unused; + enum vmalloc64_mode vmalloc_mode __maybe_unused; memset(tlb_handler, 0, sizeof(tlb_handler)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); memset(final_handler, 0, sizeof(final_handler)); - /* - * create the plain linear handler - */ - if (bcm1250_m3_war()) { - i_MFC0(&p, K0, C0_BADVADDR); - i_MFC0(&p, K1, C0_ENTRYHI); - i_xor(&p, K0, K0, K1); - i_SRL(&p, K0, K0, PAGE_SHIFT + 1); - il_bnez(&p, &r, K0, label_leave); - /* No need for i_nop */ - } + if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) { + htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, + scratch_reg); + vmalloc_mode = refill_scratch; + } else { + htlb_info.huge_pte = K0; + htlb_info.restore_scratch = 0; + vmalloc_mode = refill_noscratch; + /* + * create the plain linear handler + */ + if (bcm1250_m3_war()) { + unsigned int segbits = 44; + + uasm_i_dmfc0(&p, K0, C0_BADVADDR); + uasm_i_dmfc0(&p, K1, C0_ENTRYHI); + uasm_i_xor(&p, K0, K0, K1); + uasm_i_dsrl_safe(&p, K1, K0, 62); + uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); + uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); + uasm_i_or(&p, K0, K0, K1); + uasm_il_bnez(&p, &r, K0, label_leave); + /* No need for uasm_i_nop */ + } #ifdef CONFIG_64BIT - build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ + build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ #else - build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ + build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ +#endif + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); #endif - build_get_ptep(&p, K0, K1); - build_update_entries(&p, K0, K1); - build_tlb_write_entry(&p, &l, &r, tlb_random); - l_leave(&l, p); - i_eret(&p); /* return from trap */ + build_get_ptep(&p, K0, K1); + build_update_entries(&p, K0, K1); + build_tlb_write_entry(&p, &l, &r, tlb_random); + uasm_l_leave(&l, p); + uasm_i_eret(&p); /* return from trap */ + } +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + uasm_l_tlb_huge_update(&l, p); + build_huge_update_entries(&p, htlb_info.huge_pte, K1); + build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, + htlb_info.restore_scratch); +#endif #ifdef CONFIG_64BIT - build_get_pgd_vmalloc64(&p, &l, &r, K0, K1); + build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); #endif /* * Overflow check: For the 64bit handler, we need at least one * free instruction slot for the wrap-around branch. In worst * case, if the intended insertion point is a delay slot, we - * need three, with the the second nop'ed and the third being + * need three, with the second nop'ed and the third being * unused. */ -#ifdef CONFIG_32BIT - if ((p - tlb_handler) > 64) - panic("TLB refill handler space exceeded"); + switch (boot_cpu_type()) { + default: + if (sizeof(long) == 4) { + case CPU_LOONGSON2: + /* Loongson2 ebase is different than r4k, we have more space */ + if ((p - tlb_handler) > 64) + panic("TLB refill handler space exceeded"); + /* + * Now fold the handler in the TLB refill handler space. + */ + f = final_handler; + /* Simplest case, just copy the handler. */ + uasm_copy_handler(relocs, labels, tlb_handler, p, f); + final_len = p - tlb_handler; + break; + } else { + if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) + || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) + && uasm_insn_has_bdelay(relocs, + tlb_handler + MIPS64_REFILL_INSNS - 3))) + panic("TLB refill handler space exceeded"); + /* + * Now fold the handler in the TLB refill handler space. + */ + f = final_handler + MIPS64_REFILL_INSNS; + if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { + /* Just copy the handler. */ + uasm_copy_handler(relocs, labels, tlb_handler, p, f); + final_len = p - tlb_handler; + } else { +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + const enum label_id ls = label_tlb_huge_update; #else - if (((p - tlb_handler) > 63) - || (((p - tlb_handler) > 61) - && insn_has_bdelay(relocs, tlb_handler + 29))) - panic("TLB refill handler space exceeded"); + const enum label_id ls = label_vmalloc; #endif - - /* - * Now fold the handler in the TLB refill handler space. - */ -#ifdef CONFIG_32BIT - f = final_handler; - /* Simplest case, just copy the handler. */ - copy_handler(relocs, labels, tlb_handler, p, f); - final_len = p - tlb_handler; -#else /* CONFIG_64BIT */ - f = final_handler + 32; - if ((p - tlb_handler) <= 32) { - /* Just copy the handler. */ - copy_handler(relocs, labels, tlb_handler, p, f); - final_len = p - tlb_handler; - } else { - u32 *split = tlb_handler + 30; - - /* - * Find the split point. - */ - if (insn_has_bdelay(relocs, split - 1)) - split--; - - /* Copy first part of the handler. */ - copy_handler(relocs, labels, tlb_handler, split, f); - f += split - tlb_handler; - - /* Insert branch. */ - l_split(&l, final_handler); - il_b(&f, &r, label_split); - if (insn_has_bdelay(relocs, split)) - i_nop(&f); - else { - copy_handler(relocs, labels, split, split + 1, f); - move_labels(labels, f, f + 1, -1); - f++; - split++; + u32 *split; + int ov = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++) + ; + BUG_ON(i == ARRAY_SIZE(labels)); + split = labels[i].addr; + + /* + * See if we have overflown one way or the other. + */ + if (split > tlb_handler + MIPS64_REFILL_INSNS || + split < p - MIPS64_REFILL_INSNS) + ov = 1; + + if (ov) { + /* + * Split two instructions before the end. One + * for the branch and one for the instruction + * in the delay slot. + */ + split = tlb_handler + MIPS64_REFILL_INSNS - 2; + + /* + * If the branch would fall in a delay slot, + * we must back up an additional instruction + * so that it is no longer in a delay slot. + */ + if (uasm_insn_has_bdelay(relocs, split - 1)) + split--; + } + /* Copy first part of the handler. */ + uasm_copy_handler(relocs, labels, tlb_handler, split, f); + f += split - tlb_handler; + + if (ov) { + /* Insert branch. */ + uasm_l_split(&l, final_handler); + uasm_il_b(&f, &r, label_split); + if (uasm_insn_has_bdelay(relocs, split)) + uasm_i_nop(&f); + else { + uasm_copy_handler(relocs, labels, + split, split + 1, f); + uasm_move_labels(labels, f, f + 1, -1); + f++; + split++; + } + } + + /* Copy the rest of the handler. */ + uasm_copy_handler(relocs, labels, split, p, final_handler); + final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + + (p - split); + } } - - /* Copy the rest of the handler. */ - copy_handler(relocs, labels, split, p, final_handler); - final_len = (f - (final_handler + 32)) + (p - split); + break; } -#endif /* CONFIG_64BIT */ - resolve_relocs(relocs, labels); - printk("Synthesized TLB refill handler (%u instructions).\n", - final_len); + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB refill handler (%u instructions).\n", + final_len); -#ifdef DEBUG_TLB - { - int i; + memcpy((void *)ebase, final_handler, 0x100); - f = final_handler; -#ifdef CONFIG_64BIT - if (final_len > 32) - final_len = 64; - else - f = final_handler + 32; -#endif /* CONFIG_64BIT */ - for (i = 0; i < final_len; i++) - printk("%08x\n", f[i]); - } -#endif - - memcpy((void *)CAC_BASE, final_handler, 0x100); + dump_handler("r4000_tlb_refill", (u32 *)ebase, 64); } -/* - * TLB load/store/modify handlers. - * - * Only the fastpath gets synthesized at runtime, the slowpath for - * do_page_fault remains normal asm. - */ -extern void tlb_do_page_fault_0(void); -extern void tlb_do_page_fault_1(void); +extern u32 handle_tlbl[], handle_tlbl_end[]; +extern u32 handle_tlbs[], handle_tlbs_end[]; +extern u32 handle_tlbm[], handle_tlbm_end[]; +extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[]; +extern u32 tlbmiss_handler_setup_pgd_end[]; -#define __tlb_handler_align \ - __attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT))) +static void build_setup_pgd(void) +{ + const int a0 = 4; + const int __maybe_unused a1 = 5; + const int __maybe_unused a2 = 6; + u32 *p = tlbmiss_handler_setup_pgd_start; + const int tlbmiss_handler_setup_pgd_size = + tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start; +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT + long pgdc = (long)pgd_current; +#endif -/* - * 128 instructions for the fastpath handler is generous and should - * never be exceeded. - */ -#define FASTPATH_SIZE 128 + memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size * + sizeof(tlbmiss_handler_setup_pgd[0])); + memset(labels, 0, sizeof(labels)); + memset(relocs, 0, sizeof(relocs)); + pgd_reg = allocate_kscratch(); +#ifdef CONFIG_MIPS_PGD_C0_CONTEXT + if (pgd_reg == -1) { + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + + /* PGD << 11 in c0_Context */ + /* + * If it is a ckseg0 address, convert to a physical + * address. Shifting right by 29 and adding 4 will + * result in zero for these addresses. + * + */ + UASM_i_SRA(&p, a1, a0, 29); + UASM_i_ADDIU(&p, a1, a1, 4); + uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); + uasm_i_nop(&p); + uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); + uasm_l_tlbl_goaround1(&l, p); + UASM_i_SLL(&p, a0, a0, 11); + uasm_i_jr(&p, 31); + UASM_i_MTC0(&p, a0, C0_CONTEXT); + } else { + /* PGD in c0_KScratch */ + uasm_i_jr(&p, 31); + UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); + } +#else +#ifdef CONFIG_SMP + /* Save PGD to pgd_current[smp_processor_id()] */ + UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG); + UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT); + UASM_i_LA_mostly(&p, a2, pgdc); + UASM_i_ADDU(&p, a2, a2, a1); + UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); +#else + UASM_i_LA_mostly(&p, a2, pgdc); + UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2); +#endif /* SMP */ + uasm_i_jr(&p, 31); + + /* if pgd_reg is allocated, save PGD also to scratch register */ + if (pgd_reg != -1) + UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg); + else + uasm_i_nop(&p); +#endif + if (p >= tlbmiss_handler_setup_pgd_end) + panic("tlbmiss_handler_setup_pgd space exceeded"); + + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", + (unsigned int)(p - tlbmiss_handler_setup_pgd)); -u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE]; -u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE]; -u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE]; + dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd, + tlbmiss_handler_setup_pgd_size); +} -static void __init -iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr) +static void +iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) { #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_lld(p, pte, 0, ptr); + uasm_i_lld(p, pte, 0, ptr); else # endif - i_LL(p, pte, 0, ptr); + UASM_i_LL(p, pte, 0, ptr); #else # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_ld(p, pte, 0, ptr); + uasm_i_ld(p, pte, 0, ptr); else # endif - i_LW(p, pte, 0, ptr); + UASM_i_LW(p, pte, 0, ptr); #endif } -static void __init -iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr, +static void +iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, unsigned int mode) { #ifdef CONFIG_64BIT_PHYS_ADDR unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY); #endif - i_ori(p, pte, pte, mode); + uasm_i_ori(p, pte, pte, mode); #ifdef CONFIG_SMP # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_scd(p, pte, 0, ptr); + uasm_i_scd(p, pte, 0, ptr); else # endif - i_SC(p, pte, 0, ptr); + UASM_i_SC(p, pte, 0, ptr); if (r10000_llsc_war()) - il_beqzl(p, r, pte, label_smp_pgtable_change); + uasm_il_beqzl(p, r, pte, label_smp_pgtable_change); else - il_beqz(p, r, pte, label_smp_pgtable_change); + uasm_il_beqz(p, r, pte, label_smp_pgtable_change); # ifdef CONFIG_64BIT_PHYS_ADDR if (!cpu_has_64bits) { - /* no i_nop needed */ - i_ll(p, pte, sizeof(pte_t) / 2, ptr); - i_ori(p, pte, pte, hwmode); - i_sc(p, pte, sizeof(pte_t) / 2, ptr); - il_beqz(p, r, pte, label_smp_pgtable_change); - /* no i_nop needed */ - i_lw(p, pte, 0, ptr); + /* no uasm_i_nop needed */ + uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr); + uasm_i_ori(p, pte, pte, hwmode); + uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr); + uasm_il_beqz(p, r, pte, label_smp_pgtable_change); + /* no uasm_i_nop needed */ + uasm_i_lw(p, pte, 0, ptr); } else - i_nop(p); + uasm_i_nop(p); # else - i_nop(p); + uasm_i_nop(p); # endif #else # ifdef CONFIG_64BIT_PHYS_ADDR if (cpu_has_64bits) - i_sd(p, pte, 0, ptr); + uasm_i_sd(p, pte, 0, ptr); else # endif - i_SW(p, pte, 0, ptr); + UASM_i_SW(p, pte, 0, ptr); # ifdef CONFIG_64BIT_PHYS_ADDR if (!cpu_has_64bits) { - i_lw(p, pte, sizeof(pte_t) / 2, ptr); - i_ori(p, pte, pte, hwmode); - i_sw(p, pte, sizeof(pte_t) / 2, ptr); - i_lw(p, pte, 0, ptr); + uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr); + uasm_i_ori(p, pte, pte, hwmode); + uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr); + uasm_i_lw(p, pte, 0, ptr); } # endif #endif @@ -1365,19 +1579,36 @@ iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr, * the page table where this PTE is located, PTE will be re-loaded * with it's original value. */ -static void __init -build_pte_present(u32 **p, struct label **l, struct reloc **r, - unsigned int pte, unsigned int ptr, enum label_id lid) -{ - i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); - i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); - il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); +static void +build_pte_present(u32 **p, struct uasm_reloc **r, + int pte, int ptr, int scratch, enum label_id lid) +{ + int t = scratch >= 0 ? scratch : pte; + + if (cpu_has_rixi) { + if (use_bbit_insns()) { + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); + uasm_i_nop(p); + } else { + uasm_i_andi(p, t, pte, _PAGE_PRESENT); + uasm_il_beqz(p, r, t, lid); + if (pte == t) + /* You lose the SMP race :-(*/ + iPTE_LW(p, pte, ptr); + } + } else { + uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ); + uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ); + uasm_il_bnez(p, r, t, lid); + if (pte == t) + /* You lose the SMP race :-(*/ + iPTE_LW(p, pte, ptr); + } } /* Make PTE valid, store result in PTR. */ -static void __init -build_make_valid(u32 **p, struct reloc **r, unsigned int pte, +static void +build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED; @@ -1389,21 +1620,28 @@ build_make_valid(u32 **p, struct reloc **r, unsigned int pte, * Check if PTE can be written to, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __init -build_pte_writable(u32 **p, struct label **l, struct reloc **r, - unsigned int pte, unsigned int ptr, enum label_id lid) -{ - i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); - i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); - il_bnez(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); +static void +build_pte_writable(u32 **p, struct uasm_reloc **r, + unsigned int pte, unsigned int ptr, int scratch, + enum label_id lid) +{ + int t = scratch >= 0 ? scratch : pte; + + uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE); + uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE); + uasm_il_bnez(p, r, t, lid); + if (pte == t) + /* You lose the SMP race :-(*/ + iPTE_LW(p, pte, ptr); + else + uasm_i_nop(p); } /* Make PTE writable, update software status bits as well, then store * at PTR. */ -static void __init -build_make_write(u32 **p, struct reloc **r, unsigned int pte, +static void +build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr) { unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID @@ -1416,15 +1654,27 @@ build_make_write(u32 **p, struct reloc **r, unsigned int pte, * Check if PTE can be modified, if not branch to LABEL. Regardless * restore PTE with value from PTR when done. */ -static void __init -build_pte_modifiable(u32 **p, struct label **l, struct reloc **r, - unsigned int pte, unsigned int ptr, enum label_id lid) -{ - i_andi(p, pte, pte, _PAGE_WRITE); - il_beqz(p, r, pte, lid); - iPTE_LW(p, l, pte, ptr); +static void +build_pte_modifiable(u32 **p, struct uasm_reloc **r, + unsigned int pte, unsigned int ptr, int scratch, + enum label_id lid) +{ + if (use_bbit_insns()) { + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); + uasm_i_nop(p); + } else { + int t = scratch >= 0 ? scratch : pte; + uasm_i_andi(p, t, pte, _PAGE_WRITE); + uasm_il_beqz(p, r, t, lid); + if (pte == t) + /* You lose the SMP race :-(*/ + iPTE_LW(p, pte, ptr); + } } +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT + + /* * R3000 style TLB load/store/modify handlers. */ @@ -1433,14 +1683,14 @@ build_pte_modifiable(u32 **p, struct label **l, struct reloc **r, * This places the pte into ENTRYLO0 and writes it with tlbwi. * Then it returns. */ -static void __init +static void build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) { - i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ - i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ - i_tlbwi(p); - i_jr(p, tmp); - i_rfe(p); /* branch delay */ + uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ + uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */ + uasm_i_tlbwi(p); + uasm_i_jr(p, tmp); + uasm_i_rfe(p); /* branch delay */ } /* @@ -1449,317 +1699,502 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp) * may have the probe fail bit set as a result of a trap on a * kseg2 access, i.e. without refill. Then it returns. */ -static void __init -build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r, - unsigned int pte, unsigned int tmp) -{ - i_mfc0(p, tmp, C0_INDEX); - i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ - il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ - i_mfc0(p, tmp, C0_EPC); /* branch delay */ - i_tlbwi(p); /* cp0 delay */ - i_jr(p, tmp); - i_rfe(p); /* branch delay */ - l_r3000_write_probe_fail(l, *p); - i_tlbwr(p); /* cp0 delay */ - i_jr(p, tmp); - i_rfe(p); /* branch delay */ -} - -static void __init +static void +build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l, + struct uasm_reloc **r, unsigned int pte, + unsigned int tmp) +{ + uasm_i_mfc0(p, tmp, C0_INDEX); + uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */ + uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */ + uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */ + uasm_i_tlbwi(p); /* cp0 delay */ + uasm_i_jr(p, tmp); + uasm_i_rfe(p); /* branch delay */ + uasm_l_r3000_write_probe_fail(l, *p); + uasm_i_tlbwr(p); /* cp0 delay */ + uasm_i_jr(p, tmp); + uasm_i_rfe(p); /* branch delay */ +} + +static void build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte, unsigned int ptr) { long pgdc = (long)pgd_current; - i_mfc0(p, pte, C0_BADVADDR); - i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */ - i_lw(p, ptr, rel_lo(pgdc), ptr); - i_srl(p, pte, pte, 22); /* load delay */ - i_sll(p, pte, pte, 2); - i_addu(p, ptr, ptr, pte); - i_mfc0(p, pte, C0_CONTEXT); - i_lw(p, ptr, 0, ptr); /* cp0 delay */ - i_andi(p, pte, pte, 0xffc); /* load delay */ - i_addu(p, ptr, ptr, pte); - i_lw(p, pte, 0, ptr); - i_tlbp(p); /* load delay */ + uasm_i_mfc0(p, pte, C0_BADVADDR); + uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */ + uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr); + uasm_i_srl(p, pte, pte, 22); /* load delay */ + uasm_i_sll(p, pte, pte, 2); + uasm_i_addu(p, ptr, ptr, pte); + uasm_i_mfc0(p, pte, C0_CONTEXT); + uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */ + uasm_i_andi(p, pte, pte, 0xffc); /* load delay */ + uasm_i_addu(p, ptr, ptr, pte); + uasm_i_lw(p, pte, 0, ptr); + uasm_i_tlbp(p); /* load delay */ } -static void __init build_r3000_tlb_load_handler(void) +static void build_r3000_tlb_load_handler(void) { u32 *p = handle_tlbl; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; - memset(handle_tlbl, 0, sizeof(handle_tlbl)); + memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); - i_nop(&p); /* load delay */ + build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl); + uasm_i_nop(&p); /* load delay */ build_make_valid(&p, &r, K0, K1); build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); - l_nopage_tlbl(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); - i_nop(&p); + uasm_l_nopage_tlbl(&l, p); + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); + uasm_i_nop(&p); - if ((p - handle_tlbl) > FASTPATH_SIZE) + if (p >= handle_tlbl_end) panic("TLB load handler fastpath space exceeded"); - resolve_relocs(relocs, labels); - printk("Synthesized TLB load handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbl)); + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbl)); -#ifdef DEBUG_TLB - { - int i; - - for (i = 0; i < (p - handle_tlbl); i++) - printk("%08x\n", handle_tlbl[i]); - } -#endif + dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size); } -static void __init build_r3000_tlb_store_handler(void) +static void build_r3000_tlb_store_handler(void) { u32 *p = handle_tlbs; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; - memset(handle_tlbs, 0, sizeof(handle_tlbs)); + memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); - i_nop(&p); /* load delay */ + build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs); + uasm_i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); build_r3000_tlb_reload_write(&p, &l, &r, K0, K1); - l_nopage_tlbs(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); - i_nop(&p); + uasm_l_nopage_tlbs(&l, p); + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); + uasm_i_nop(&p); - if ((p - handle_tlbs) > FASTPATH_SIZE) + if (p >= handle_tlbs_end) panic("TLB store handler fastpath space exceeded"); - resolve_relocs(relocs, labels); - printk("Synthesized TLB store handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbs)); - -#ifdef DEBUG_TLB - { - int i; + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbs)); - for (i = 0; i < (p - handle_tlbs); i++) - printk("%08x\n", handle_tlbs[i]); - } -#endif + dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size); } -static void __init build_r3000_tlb_modify_handler(void) +static void build_r3000_tlb_modify_handler(void) { u32 *p = handle_tlbm; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; - memset(handle_tlbm, 0, sizeof(handle_tlbm)); + memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); build_r3000_tlbchange_handler_head(&p, K0, K1); - build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); - i_nop(&p); /* load delay */ + build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm); + uasm_i_nop(&p); /* load delay */ build_make_write(&p, &r, K0, K1); build_r3000_pte_reload_tlbwi(&p, K0, K1); - l_nopage_tlbm(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); - i_nop(&p); + uasm_l_nopage_tlbm(&l, p); + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); + uasm_i_nop(&p); - if ((p - handle_tlbm) > FASTPATH_SIZE) + if (p >= handle_tlbm_end) panic("TLB modify handler fastpath space exceeded"); - resolve_relocs(relocs, labels); - printk("Synthesized TLB modify handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbm)); + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbm)); -#ifdef DEBUG_TLB - { - int i; - - for (i = 0; i < (p - handle_tlbm); i++) - printk("%08x\n", handle_tlbm[i]); - } -#endif + dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size); } +#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */ /* * R4000 style TLB load/store/modify handlers. */ -static void __init -build_r4000_tlbchange_handler_head(u32 **p, struct label **l, - struct reloc **r, unsigned int pte, - unsigned int ptr) +static struct work_registers +build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l, + struct uasm_reloc **r) { + struct work_registers wr = build_get_work_registers(p); + #ifdef CONFIG_64BIT - build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */ + build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */ #else - build_get_pgde32(p, pte, ptr); /* get pgd in ptr */ + build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */ +#endif + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + /* + * For huge tlb entries, pmd doesn't contain an address but + * instead contains the tlb pte. Check the PAGE_HUGE bit and + * see if we need to jump to huge tlb processing. + */ + build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update); #endif - i_MFC0(p, pte, C0_BADVADDR); - i_LW(p, ptr, 0, ptr); - i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); - i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2); - i_ADDU(p, ptr, ptr, pte); + UASM_i_MFC0(p, wr.r1, C0_BADVADDR); + UASM_i_LW(p, wr.r2, 0, wr.r2); + UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2); + uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); + UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1); #ifdef CONFIG_SMP - l_smp_pgtable_change(l, *p); -# endif - iPTE_LW(p, l, pte, ptr); /* get even pte */ - build_tlb_probe_entry(p); + uasm_l_smp_pgtable_change(l, *p); +#endif + iPTE_LW(p, wr.r1, wr.r2); /* get even pte */ + if (!m4kc_tlbp_war()) + build_tlb_probe_entry(p); + return wr; } -static void __init -build_r4000_tlbchange_handler_tail(u32 **p, struct label **l, - struct reloc **r, unsigned int tmp, +static void +build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l, + struct uasm_reloc **r, unsigned int tmp, unsigned int ptr) { - i_ori(p, ptr, ptr, sizeof(pte_t)); - i_xori(p, ptr, ptr, sizeof(pte_t)); + uasm_i_ori(p, ptr, ptr, sizeof(pte_t)); + uasm_i_xori(p, ptr, ptr, sizeof(pte_t)); build_update_entries(p, tmp, ptr); build_tlb_write_entry(p, l, r, tlb_indexed); - l_leave(l, *p); - i_eret(p); /* return from trap */ + uasm_l_leave(l, *p); + build_restore_work_registers(p); + uasm_i_eret(p); /* return from trap */ #ifdef CONFIG_64BIT - build_get_pgd_vmalloc64(p, l, r, tmp, ptr); + build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill); #endif } -static void __init build_r4000_tlb_load_handler(void) +static void build_r4000_tlb_load_handler(void) { u32 *p = handle_tlbl; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbl_size = handle_tlbl_end - handle_tlbl; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + struct work_registers wr; - memset(handle_tlbl, 0, sizeof(handle_tlbl)); + memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); if (bcm1250_m3_war()) { - i_MFC0(&p, K0, C0_BADVADDR); - i_MFC0(&p, K1, C0_ENTRYHI); - i_xor(&p, K0, K0, K1); - i_SRL(&p, K0, K0, PAGE_SHIFT + 1); - il_bnez(&p, &r, K0, label_leave); - /* No need for i_nop */ + unsigned int segbits = 44; + + uasm_i_dmfc0(&p, K0, C0_BADVADDR); + uasm_i_dmfc0(&p, K1, C0_ENTRYHI); + uasm_i_xor(&p, K0, K0, K1); + uasm_i_dsrl_safe(&p, K1, K0, 62); + uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); + uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); + uasm_i_or(&p, K0, K0, K1); + uasm_il_bnez(&p, &r, K0, label_leave); + /* No need for uasm_i_nop */ } - build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl); - build_make_valid(&p, &r, K0, K1); - build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); + wr = build_r4000_tlbchange_handler_head(&p, &l, &r); + build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); + if (m4kc_tlbp_war()) + build_tlb_probe_entry(&p); - l_nopage_tlbl(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); - i_nop(&p); + if (cpu_has_rixi) { + /* + * If the page is not _PAGE_VALID, RI or XI could not + * have triggered it. Skip the expensive test.. + */ + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), + label_tlbl_goaround1); + } else { + uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); + uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1); + } + uasm_i_nop(&p); - if ((p - handle_tlbl) > FASTPATH_SIZE) - panic("TLB load handler fastpath space exceeded"); + uasm_i_tlbr(&p); - resolve_relocs(relocs, labels); - printk("Synthesized TLB load handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbl)); + switch (current_cpu_type()) { + default: + if (cpu_has_mips_r2) { + uasm_i_ehb(&p); -#ifdef DEBUG_TLB - { - int i; + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + break; + } + } - for (i = 0; i < (p - handle_tlbl); i++) - printk("%08x\n", handle_tlbl[i]); + /* Examine entrylo 0 or 1 based on ptr. */ + if (use_bbit_insns()) { + uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); + } else { + uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); + uasm_i_beqz(&p, wr.r3, 8); + } + /* load it in the delay slot*/ + UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); + /* load it if ptr is odd */ + UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); + /* + * If the entryLo (now in wr.r3) is valid (bit 1), RI or + * XI must have triggered it. + */ + if (use_bbit_insns()) { + uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl); + uasm_i_nop(&p); + uasm_l_tlbl_goaround1(&l, p); + } else { + uasm_i_andi(&p, wr.r3, wr.r3, 2); + uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl); + uasm_i_nop(&p); + } + uasm_l_tlbl_goaround1(&l, p); } + build_make_valid(&p, &r, wr.r1, wr.r2); + build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + /* + * This is the entry point when build_r4000_tlbchange_handler_head + * spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, wr.r1, wr.r2); + build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl); + build_tlb_probe_entry(&p); + + if (cpu_has_rixi) { + /* + * If the page is not _PAGE_VALID, RI or XI could not + * have triggered it. Skip the expensive test.. + */ + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID), + label_tlbl_goaround2); + } else { + uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID); + uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); + } + uasm_i_nop(&p); + + uasm_i_tlbr(&p); + + switch (current_cpu_type()) { + default: + if (cpu_has_mips_r2) { + uasm_i_ehb(&p); + + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + break; + } + } + + /* Examine entrylo 0 or 1 based on ptr. */ + if (use_bbit_insns()) { + uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8); + } else { + uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t)); + uasm_i_beqz(&p, wr.r3, 8); + } + /* load it in the delay slot*/ + UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0); + /* load it if ptr is odd */ + UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1); + /* + * If the entryLo (now in wr.r3) is valid (bit 1), RI or + * XI must have triggered it. + */ + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2); + } else { + uasm_i_andi(&p, wr.r3, wr.r3, 2); + uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2); + } + if (PM_DEFAULT_MASK == 0) + uasm_i_nop(&p); + /* + * We clobbered C0_PAGEMASK, restore it. On the other branch + * it is restored in build_huge_tlb_write_entry. + */ + build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0); + + uasm_l_tlbl_goaround2(&l, p); + } + uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID)); + build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); +#endif + + uasm_l_nopage_tlbl(&l, p); + build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_0 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0)); + uasm_i_jr(&p, K0); + } else #endif + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff); + uasm_i_nop(&p); + + if (p >= handle_tlbl_end) + panic("TLB load handler fastpath space exceeded"); + + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbl)); + + dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size); } -static void __init build_r4000_tlb_store_handler(void) +static void build_r4000_tlb_store_handler(void) { u32 *p = handle_tlbs; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbs_size = handle_tlbs_end - handle_tlbs; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + struct work_registers wr; - memset(handle_tlbs, 0, sizeof(handle_tlbs)); + memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); - build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs); - build_make_write(&p, &r, K0, K1); - build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); + wr = build_r4000_tlbchange_handler_head(&p, &l, &r); + build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); + if (m4kc_tlbp_war()) + build_tlb_probe_entry(&p); + build_make_write(&p, &r, wr.r1, wr.r2); + build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); - l_nopage_tlbs(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); - i_nop(&p); +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + /* + * This is the entry point when + * build_r4000_tlbchange_handler_head spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, wr.r1, wr.r2); + build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs); + build_tlb_probe_entry(&p); + uasm_i_ori(&p, wr.r1, wr.r1, + _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); + build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); +#endif - if ((p - handle_tlbs) > FASTPATH_SIZE) - panic("TLB store handler fastpath space exceeded"); + uasm_l_nopage_tlbs(&l, p); + build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); + uasm_i_nop(&p); - resolve_relocs(relocs, labels); - printk("Synthesized TLB store handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbs)); + if (p >= handle_tlbs_end) + panic("TLB store handler fastpath space exceeded"); -#ifdef DEBUG_TLB - { - int i; + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbs)); - for (i = 0; i < (p - handle_tlbs); i++) - printk("%08x\n", handle_tlbs[i]); - } -#endif + dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size); } -static void __init build_r4000_tlb_modify_handler(void) +static void build_r4000_tlb_modify_handler(void) { u32 *p = handle_tlbm; - struct label *l = labels; - struct reloc *r = relocs; + const int handle_tlbm_size = handle_tlbm_end - handle_tlbm; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + struct work_registers wr; - memset(handle_tlbm, 0, sizeof(handle_tlbm)); + memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0])); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); - build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1); - build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm); + wr = build_r4000_tlbchange_handler_head(&p, &l, &r); + build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); + if (m4kc_tlbp_war()) + build_tlb_probe_entry(&p); /* Present and writable bits set, set accessed and dirty bits. */ - build_make_write(&p, &r, K0, K1); - build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); + build_make_write(&p, &r, wr.r1, wr.r2); + build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2); + +#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT + /* + * This is the entry point when + * build_r4000_tlbchange_handler_head spots a huge page. + */ + uasm_l_tlb_huge_update(&l, p); + iPTE_LW(&p, wr.r1, wr.r2); + build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm); + build_tlb_probe_entry(&p); + uasm_i_ori(&p, wr.r1, wr.r1, + _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY); + build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2); +#endif - l_nopage_tlbm(&l, p); - i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); - i_nop(&p); + uasm_l_nopage_tlbm(&l, p); + build_restore_work_registers(&p); +#ifdef CONFIG_CPU_MICROMIPS + if ((unsigned long)tlb_do_page_fault_1 & 1) { + uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1)); + uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1)); + uasm_i_jr(&p, K0); + } else +#endif + uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff); + uasm_i_nop(&p); - if ((p - handle_tlbm) > FASTPATH_SIZE) + if (p >= handle_tlbm_end) panic("TLB modify handler fastpath space exceeded"); - resolve_relocs(relocs, labels); - printk("Synthesized TLB modify handler fastpath (%u instructions).\n", - (unsigned int)(p - handle_tlbm)); + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", + (unsigned int)(p - handle_tlbm)); -#ifdef DEBUG_TLB - { - int i; + dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size); +} - for (i = 0; i < (p - handle_tlbm); i++) - printk("%08x\n", handle_tlbm[i]); - } -#endif +static void flush_tlb_handlers(void) +{ + local_flush_icache_range((unsigned long)handle_tlbl, + (unsigned long)handle_tlbl_end); + local_flush_icache_range((unsigned long)handle_tlbs, + (unsigned long)handle_tlbs_end); + local_flush_icache_range((unsigned long)handle_tlbm, + (unsigned long)handle_tlbm_end); + local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, + (unsigned long)tlbmiss_handler_setup_pgd_end); } -void __init build_tlb_refill_handler(void) +void build_tlb_refill_handler(void) { /* * The refill handler is generated per-CPU, multi-node systems @@ -1768,7 +2203,13 @@ void __init build_tlb_refill_handler(void) */ static int run_once = 0; - switch (current_cpu_data.cputype) { + output_pgtable_bits_defines(); + +#ifdef CONFIG_64BIT + check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); +#endif + + switch (current_cpu_type()) { case CPU_R2000: case CPU_R3000: case CPU_R3000A: @@ -1776,13 +2217,22 @@ void __init build_tlb_refill_handler(void) case CPU_TX3912: case CPU_TX3922: case CPU_TX3927: - build_r3000_tlb_refill_handler(); +#ifndef CONFIG_MIPS_PGD_C0_CONTEXT + if (cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); if (!run_once) { + if (!cpu_has_local_ebase) + build_r3000_tlb_refill_handler(); + build_setup_pgd(); build_r3000_tlb_load_handler(); build_r3000_tlb_store_handler(); build_r3000_tlb_modify_handler(); + flush_tlb_handlers(); run_once++; } +#else + panic("No R3000 TLB refill handler"); +#endif break; case CPU_R6000: @@ -1795,22 +2245,18 @@ void __init build_tlb_refill_handler(void) break; default: - build_r4000_tlb_refill_handler(); if (!run_once) { + scratch_reg = allocate_kscratch(); + build_setup_pgd(); build_r4000_tlb_load_handler(); build_r4000_tlb_store_handler(); build_r4000_tlb_modify_handler(); + if (!cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); + flush_tlb_handlers(); run_once++; } + if (cpu_has_local_ebase) + build_r4000_tlb_refill_handler(); } } - -void __init flush_tlb_handlers(void) -{ - flush_icache_range((unsigned long)handle_tlbl, - (unsigned long)handle_tlbl + sizeof(handle_tlbl)); - flush_icache_range((unsigned long)handle_tlbs, - (unsigned long)handle_tlbs + sizeof(handle_tlbs)); - flush_icache_range((unsigned long)handle_tlbm, - (unsigned long)handle_tlbm + sizeof(handle_tlbm)); -} |