aboutsummaryrefslogtreecommitdiff
path: root/arch/mips/mm/tlbex.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm/tlbex.c')
-rw-r--r--arch/mips/mm/tlbex.c1130
1 files changed, 666 insertions, 464 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 083d3412d0b..e80e10bafc8 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -5,10 +5,11 @@
*
* Synthesize TLB refill handlers at runtime.
*
- * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
- * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
+ * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
+ * Copyright (C) 2005, 2007, 2008, 2009 Maciej W. Rozycki
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
* Copyright (C) 2008, 2009 Cavium Networks, Inc.
+ * Copyright (C) 2011 MIPS Technologies, Inc.
*
* ... and the days got worse and worse and now you see
* I've gone completly out of my mind.
@@ -25,13 +26,14 @@
#include <linux/types.h>
#include <linux/smp.h>
#include <linux/string.h>
-#include <linux/init.h>
#include <linux/cache.h>
#include <asm/cacheflush.h>
+#include <asm/cpu-type.h>
#include <asm/pgtable.h>
#include <asm/war.h>
#include <asm/uasm.h>
+#include <asm/setup.h>
/*
* TLB load/store/modify handlers.
@@ -42,6 +44,18 @@
extern void tlb_do_page_fault_0(void);
extern void tlb_do_page_fault_1(void);
+struct work_registers {
+ int r1;
+ int r2;
+ int r3;
+};
+
+struct tlb_reg_save {
+ unsigned long a;
+ unsigned long b;
+} ____cacheline_aligned_in_smp;
+
+static struct tlb_reg_save handler_reg_save[NR_CPUS];
static inline int r45k_bvahwbug(void)
{
@@ -71,6 +85,7 @@ static int use_bbit_insns(void)
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
case CPU_CAVIUM_OCTEON2:
+ case CPU_CAVIUM_OCTEON3:
return 1;
default:
return 0;
@@ -81,6 +96,7 @@ static int use_lwx_insns(void)
{
switch (current_cpu_type()) {
case CPU_CAVIUM_OCTEON2:
+ case CPU_CAVIUM_OCTEON3:
return 1;
default:
return 0;
@@ -109,6 +125,8 @@ static bool scratchpad_available(void)
static int scratchpad_offset(int i)
{
BUG();
+ /* Really unreachable, but evidently some GCC want this. */
+ return 0;
}
#endif
/*
@@ -120,7 +138,7 @@ static int scratchpad_offset(int i)
* why; it's not an issue caused by the core RTL.
*
*/
-static int __cpuinit m4kc_tlbp_war(void)
+static int m4kc_tlbp_war(void)
{
return (current_cpu_data.processor_id & 0xffff00) ==
(PRID_COMP_MIPS | PRID_IMP_4KC);
@@ -132,8 +150,8 @@ enum label_id {
label_leave,
label_vmalloc,
label_vmalloc_done,
- label_tlbw_hazard,
- label_split,
+ label_tlbw_hazard_0,
+ label_split = label_tlbw_hazard_0 + 8,
label_tlbl_goaround1,
label_tlbl_goaround2,
label_nopage_tlbl,
@@ -142,7 +160,7 @@ enum label_id {
label_smp_pgtable_change,
label_r3000_write_probe_fail,
label_large_segbits_fault,
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
label_tlb_huge_update,
#endif
};
@@ -151,7 +169,7 @@ UASM_L_LA(_second_part)
UASM_L_LA(_leave)
UASM_L_LA(_vmalloc)
UASM_L_LA(_vmalloc_done)
-UASM_L_LA(_tlbw_hazard)
+/* _tlbw_hazard_x is handled differently. */
UASM_L_LA(_split)
UASM_L_LA(_tlbl_goaround1)
UASM_L_LA(_tlbl_goaround2)
@@ -161,24 +179,88 @@ UASM_L_LA(_nopage_tlbm)
UASM_L_LA(_smp_pgtable_change)
UASM_L_LA(_r3000_write_probe_fail)
UASM_L_LA(_large_segbits_fault)
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
UASM_L_LA(_tlb_huge_update)
#endif
+static int hazard_instance;
+
+static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
+{
+ switch (instance) {
+ case 0 ... 7:
+ uasm_il_bgezl(p, r, 0, label_tlbw_hazard_0 + instance);
+ return;
+ default:
+ BUG();
+ }
+}
+
+static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
+{
+ switch (instance) {
+ case 0 ... 7:
+ uasm_build_label(l, *p, label_tlbw_hazard_0 + instance);
+ break;
+ default:
+ BUG();
+ }
+}
+
/*
- * For debug purposes.
+ * pgtable bits are assigned dynamically depending on processor feature
+ * and statically based on kernel configuration. This spits out the actual
+ * values the kernel is using. Required to make sense from disassembled
+ * TLB exception handlers.
*/
-static inline void dump_handler(const u32 *handler, int count)
+static void output_pgtable_bits_defines(void)
+{
+#define pr_define(fmt, ...) \
+ pr_debug("#define " fmt, ##__VA_ARGS__)
+
+ pr_debug("#include <asm/asm.h>\n");
+ pr_debug("#include <asm/regdef.h>\n");
+ pr_debug("\n");
+
+ pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT);
+ pr_define("_PAGE_READ_SHIFT %d\n", _PAGE_READ_SHIFT);
+ pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT);
+ pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT);
+ pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT);
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
+ pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT);
+ pr_define("_PAGE_SPLITTING_SHIFT %d\n", _PAGE_SPLITTING_SHIFT);
+#endif
+ if (cpu_has_rixi) {
+#ifdef _PAGE_NO_EXEC_SHIFT
+ pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT);
+#endif
+#ifdef _PAGE_NO_READ_SHIFT
+ pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT);
+#endif
+ }
+ pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT);
+ pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT);
+ pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT);
+ pr_define("_PFN_SHIFT %d\n", _PFN_SHIFT);
+ pr_debug("\n");
+}
+
+static inline void dump_handler(const char *symbol, const u32 *handler, int count)
{
int i;
+ pr_debug("LEAF(%s)\n", symbol);
+
pr_debug("\t.set push\n");
pr_debug("\t.set noreorder\n");
for (i = 0; i < count; i++)
- pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
+ pr_debug("\t.word\t0x%08x\t\t# %p\n", handler[i], &handler[i]);
- pr_debug("\t.set pop\n");
+ pr_debug("\t.set\tpop\n");
+
+ pr_debug("\tEND(%s)\n", symbol);
}
/* The only general purpose registers allowed in TLB handlers. */
@@ -211,21 +293,28 @@ static inline void dump_handler(const u32 *handler, int count)
* We deliberately chose a buffer size of 128, so we won't scribble
* over anything important on overflow before we panic.
*/
-static u32 tlb_handler[128] __cpuinitdata;
+static u32 tlb_handler[128];
/* simply assume worst case size for labels and relocs */
-static struct uasm_label labels[128] __cpuinitdata;
-static struct uasm_reloc relocs[128] __cpuinitdata;
+static struct uasm_label labels[128];
+static struct uasm_reloc relocs[128];
-#ifdef CONFIG_64BIT
-static int check_for_high_segbits __cpuinitdata;
-#endif
+static int check_for_high_segbits;
-static int check_for_high_segbits __cpuinitdata;
+static unsigned int kscratch_used_mask;
-static unsigned int kscratch_used_mask __cpuinitdata;
+static inline int __maybe_unused c0_kscratch(void)
+{
+ switch (current_cpu_type()) {
+ case CPU_XLP:
+ case CPU_XLR:
+ return 22;
+ default:
+ return 31;
+ }
+}
-static int __cpuinit allocate_kscratch(void)
+static int allocate_kscratch(void)
{
int r;
unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
@@ -242,10 +331,57 @@ static int __cpuinit allocate_kscratch(void)
return r;
}
-static int scratch_reg __cpuinitdata;
-static int pgd_reg __cpuinitdata;
+static int scratch_reg;
+static int pgd_reg;
enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
+static struct work_registers build_get_work_registers(u32 **p)
+{
+ struct work_registers r;
+
+ if (scratch_reg >= 0) {
+ /* Save in CPU local C0_KScratch? */
+ UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
+ r.r1 = K0;
+ r.r2 = K1;
+ r.r3 = 1;
+ return r;
+ }
+
+ if (num_possible_cpus() > 1) {
+ /* Get smp_processor_id */
+ UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
+ UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
+
+ /* handler_reg_save index in K0 */
+ UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
+
+ UASM_i_LA(p, K1, (long)&handler_reg_save);
+ UASM_i_ADDU(p, K0, K0, K1);
+ } else {
+ UASM_i_LA(p, K0, (long)&handler_reg_save);
+ }
+ /* K0 now points to save area, save $1 and $2 */
+ UASM_i_SW(p, 1, offsetof(struct tlb_reg_save, a), K0);
+ UASM_i_SW(p, 2, offsetof(struct tlb_reg_save, b), K0);
+
+ r.r1 = K1;
+ r.r2 = 1;
+ r.r3 = 2;
+ return r;
+}
+
+static void build_restore_work_registers(u32 **p)
+{
+ if (scratch_reg >= 0) {
+ UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
+ return;
+ }
+ /* K0 already points to save area, restore $1 and $2 */
+ UASM_i_LW(p, 1, offsetof(struct tlb_reg_save, a), K0);
+ UASM_i_LW(p, 2, offsetof(struct tlb_reg_save, b), K0);
+}
+
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
/*
@@ -260,7 +396,7 @@ extern unsigned long pgd_current[];
/*
* The R3000 TLB handler is simple.
*/
-static void __cpuinit build_r3000_tlb_refill_handler(void)
+static void build_r3000_tlb_refill_handler(void)
{
long pgdc = (long)pgd_current;
u32 *p;
@@ -294,7 +430,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
memcpy((void *)ebase, tlb_handler, 0x80);
- dump_handler((u32 *)ebase, 32);
+ dump_handler("r3000_tlb_refill", (u32 *)ebase, 32);
}
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
@@ -305,7 +441,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
* other one.To keep things simple, we first assume linear space,
* then we relocate it to the final handler layout as needed.
*/
-static u32 final_handler[64] __cpuinitdata;
+static u32 final_handler[64];
/*
* Hazards
@@ -313,8 +449,8 @@ static u32 final_handler[64] __cpuinitdata;
* From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
* 2. A timing hazard exists for the TLBP instruction.
*
- * stalling_instruction
- * TLBP
+ * stalling_instruction
+ * TLBP
*
* The JTLB is being read for the TLBP throughout the stall generated by the
* previous instruction. This is not really correct as the stalling instruction
@@ -325,18 +461,17 @@ static u32 final_handler[64] __cpuinitdata;
* The software work-around is to not allow the instruction preceding the TLBP
* to stall - make it an NOP or some other instruction guaranteed not to stall.
*
- * Errata 2 will not be fixed. This errata is also on the R5000.
+ * Errata 2 will not be fixed. This errata is also on the R5000.
*
* As if we MIPS hackers wouldn't know how to nop pipelines happy ...
*/
-static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
+static void __maybe_unused build_tlb_probe_entry(u32 **p)
{
switch (current_cpu_type()) {
/* Found by experiment: R4600 v2.0/R4700 needs this, too. */
case CPU_R4600:
case CPU_R4700:
case CPU_R5000:
- case CPU_R5000A:
case CPU_NEVADA:
uasm_i_nop(p);
uasm_i_tlbp(p);
@@ -350,13 +485,13 @@ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
/*
* Write random or indexed TLB entry, and care about the hazards from
- * the preceeding mtc0 and for the following eret.
+ * the preceding mtc0 and for the following eret.
*/
enum tlb_write_entry { tlb_random, tlb_indexed };
-static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
- struct uasm_reloc **r,
- enum tlb_write_entry wmode)
+static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r,
+ enum tlb_write_entry wmode)
{
void(*tlbw)(u32 **) = NULL;
@@ -366,8 +501,24 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
}
if (cpu_has_mips_r2) {
- if (cpu_has_mips_r2_exec_hazard)
+ /*
+ * The architecture spec says an ehb is required here,
+ * but a number of cores do not have the hazard and
+ * using an ehb causes an expensive pipeline stall.
+ */
+ switch (current_cpu_type()) {
+ case CPU_M14KC:
+ case CPU_74K:
+ case CPU_1074K:
+ case CPU_PROAPTIV:
+ case CPU_P5600:
+ case CPU_M5150:
+ break;
+
+ default:
uasm_i_ehb(p);
+ break;
+ }
tlbw(p);
return;
}
@@ -383,25 +534,32 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
* This branch uses up a mtc0 hazard nop slot and saves
* two nops after the tlbw instruction.
*/
- uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
+ uasm_bgezl_hazard(p, r, hazard_instance);
tlbw(p);
- uasm_l_tlbw_hazard(l, *p);
+ uasm_bgezl_label(l, p, hazard_instance);
+ hazard_instance++;
uasm_i_nop(p);
break;
case CPU_R4600:
case CPU_R4700:
- case CPU_R5000:
- case CPU_R5000A:
uasm_i_nop(p);
tlbw(p);
uasm_i_nop(p);
break;
+ case CPU_R5000:
+ case CPU_NEVADA:
+ uasm_i_nop(p); /* QED specifies 2 nops hazard */
+ uasm_i_nop(p); /* QED specifies 2 nops hazard */
+ tlbw(p);
+ break;
+
case CPU_R4300:
case CPU_5KC:
case CPU_TX49XX:
case CPU_PR4450:
+ case CPU_XLR:
uasm_i_nop(p);
tlbw(p);
break;
@@ -411,6 +569,8 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
case CPU_R14000:
case CPU_4KC:
case CPU_4KEC:
+ case CPU_M14KC:
+ case CPU_M14KEC:
case CPU_SB1:
case CPU_SB1A:
case CPU_4KSC:
@@ -422,6 +582,7 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
case CPU_BMIPS4380:
case CPU_BMIPS5000:
case CPU_LOONGSON2:
+ case CPU_LOONGSON3:
case CPU_R5500:
if (m4kc_tlbp_war())
uasm_i_nop(p);
@@ -429,17 +590,6 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
tlbw(p);
break;
- case CPU_NEVADA:
- uasm_i_nop(p); /* QED specifies 2 nops hazard */
- /*
- * This branch uses up a mtc0 hazard nop slot and saves
- * a nop after the tlbw instruction.
- */
- uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
- tlbw(p);
- uasm_l_tlbw_hazard(l, *p);
- break;
-
case CPU_RM7000:
uasm_i_nop(p);
uasm_i_nop(p);
@@ -448,24 +598,6 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
tlbw(p);
break;
- case CPU_RM9000:
- /*
- * When the JTLB is updated by tlbwi or tlbwr, a subsequent
- * use of the JTLB for instructions should not occur for 4
- * cpu cycles and use for data translations should not occur
- * for 3 cpu cycles.
- */
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- tlbw(p);
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- uasm_i_ssnop(p);
- break;
-
case CPU_VR4111:
case CPU_VR4121:
case CPU_VR4122:
@@ -493,17 +625,16 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
default:
panic("No TLB refill handler yet (CPU type: %d)",
- current_cpu_data.cputype);
+ current_cpu_type());
break;
}
}
-static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
- unsigned int reg)
+static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
+ unsigned int reg)
{
- if (kernel_uses_smartmips_rixi) {
- UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
- UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ if (cpu_has_rixi) {
+ UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
} else {
#ifdef CONFIG_64BIT_PHYS_ADDR
uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
@@ -513,13 +644,11 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
}
}
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
-static __cpuinit void build_restore_pagemask(u32 **p,
- struct uasm_reloc **r,
- unsigned int tmp,
- enum label_id lid,
- int restore_scratch)
+static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
+ unsigned int tmp, enum label_id lid,
+ int restore_scratch)
{
if (restore_scratch) {
/* Reset default page size */
@@ -536,8 +665,8 @@ static __cpuinit void build_restore_pagemask(u32 **p,
uasm_i_mtc0(p, 0, C0_PAGEMASK);
uasm_il_b(p, r, lid);
}
- if (scratch_reg > 0)
- UASM_i_MFC0(p, 1, 31, scratch_reg);
+ if (scratch_reg >= 0)
+ UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
else
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
} else {
@@ -558,12 +687,11 @@ static __cpuinit void build_restore_pagemask(u32 **p,
}
}
-static __cpuinit void build_huge_tlb_write_entry(u32 **p,
- struct uasm_label **l,
- struct uasm_reloc **r,
- unsigned int tmp,
- enum tlb_write_entry wmode,
- int restore_scratch)
+static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r,
+ unsigned int tmp,
+ enum tlb_write_entry wmode,
+ int restore_scratch)
{
/* Set huge page tlb entry size */
uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
@@ -578,9 +706,9 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p,
/*
* Check if Huge PTE is present, if so then jump to LABEL.
*/
-static void __cpuinit
+static void
build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
- unsigned int pmd, int lid)
+ unsigned int pmd, int lid)
{
UASM_i_LW(p, tmp, 0, pmd);
if (use_bbit_insns()) {
@@ -591,9 +719,8 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
}
}
-static __cpuinit void build_huge_update_entries(u32 **p,
- unsigned int pte,
- unsigned int tmp)
+static void build_huge_update_entries(u32 **p, unsigned int pte,
+ unsigned int tmp)
{
int small_sequence;
@@ -608,7 +735,7 @@ static __cpuinit void build_huge_update_entries(u32 **p,
*/
small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
- /* We can clobber tmp. It isn't used after this.*/
+ /* We can clobber tmp. It isn't used after this.*/
if (!small_sequence)
uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
@@ -623,11 +750,10 @@ static __cpuinit void build_huge_update_entries(u32 **p,
UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
}
-static __cpuinit void build_huge_handler_tail(u32 **p,
- struct uasm_reloc **r,
- struct uasm_label **l,
- unsigned int pte,
- unsigned int ptr)
+static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
+ struct uasm_label **l,
+ unsigned int pte,
+ unsigned int ptr)
{
#ifdef CONFIG_SMP
UASM_i_SC(p, pte, 0, ptr);
@@ -639,14 +765,14 @@ static __cpuinit void build_huge_handler_tail(u32 **p,
build_huge_update_entries(p, pte, ptr);
build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0);
}
-#endif /* CONFIG_HUGETLB_PAGE */
+#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
#ifdef CONFIG_64BIT
/*
* TMP and PTR are scratch.
* TMP will be clobbered, PTR will hold the pmd entry.
*/
-static void __cpuinit
+static void
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int tmp, unsigned int ptr)
{
@@ -677,11 +803,11 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
}
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg != -1) {
/* pgd is in pgd_reg */
- UASM_i_MFC0(p, ptr, 31, pgd_reg);
+ UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
} else {
+#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
/*
* &pgd << 11 stored in CONTEXT [23..63].
*/
@@ -690,33 +816,21 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
/* Clear lower 23 bits of context. */
uasm_i_dins(p, ptr, 0, 0, 23);
- /* 1 0 1 0 1 << 6 xkphys cached */
+ /* 1 0 1 0 1 << 6 xkphys cached */
uasm_i_ori(p, ptr, ptr, 0x540);
uasm_i_drotr(p, ptr, ptr, 11);
- }
#elif defined(CONFIG_SMP)
-# ifdef CONFIG_MIPS_MT_SMTC
- /*
- * SMTC uses TCBind value as "CPU" index
- */
- uasm_i_mfc0(p, ptr, C0_TCBIND);
- uasm_i_dsrl_safe(p, ptr, ptr, 19);
-# else
- /*
- * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
- * stored in CONTEXT.
- */
- uasm_i_dmfc0(p, ptr, C0_CONTEXT);
- uasm_i_dsrl_safe(p, ptr, ptr, 23);
-# endif
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_daddu(p, ptr, ptr, tmp);
- uasm_i_dmfc0(p, tmp, C0_BADVADDR);
- uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
+ UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
+ uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_daddu(p, ptr, ptr, tmp);
+ uasm_i_dmfc0(p, tmp, C0_BADVADDR);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#else
- UASM_i_LA_mostly(p, ptr, pgdc);
- uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
+ UASM_i_LA_mostly(p, ptr, pgdc);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#endif
+ }
uasm_l_vmalloc_done(l, *p);
@@ -738,7 +852,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* BVADDR is the faulting address, PTR is scratch.
* PTR will hold the pgd for vmalloc.
*/
-static void __cpuinit
+static void
build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int bvaddr, unsigned int ptr,
enum vmalloc64_mode mode)
@@ -792,8 +906,8 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
uasm_i_jr(p, ptr);
if (mode == refill_scratch) {
- if (scratch_reg > 0)
- UASM_i_MFC0(p, 1, 31, scratch_reg);
+ if (scratch_reg >= 0)
+ UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
else
UASM_i_LW(p, 1, scratchpad_offset(0), 0);
} else {
@@ -808,34 +922,28 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* TMP and PTR are scratch.
* TMP will be clobbered, PTR will hold the pgd entry.
*/
-static void __cpuinit __maybe_unused
+static void __maybe_unused
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{
- long pgdc = (long)pgd_current;
+ if (pgd_reg != -1) {
+ /* pgd is in pgd_reg */
+ uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
+ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ } else {
+ long pgdc = (long)pgd_current;
- /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
+ /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
#ifdef CONFIG_SMP
-#ifdef CONFIG_MIPS_MT_SMTC
- /*
- * SMTC uses TCBind value as "CPU" index
- */
- uasm_i_mfc0(p, ptr, C0_TCBIND);
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_srl(p, ptr, ptr, 19);
-#else
- /*
- * smp_processor_id() << 3 is stored in CONTEXT.
- */
- uasm_i_mfc0(p, ptr, C0_CONTEXT);
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_srl(p, ptr, ptr, 23);
-#endif
- uasm_i_addu(p, ptr, tmp, ptr);
+ uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
+ uasm_i_addu(p, ptr, tmp, ptr);
#else
- UASM_i_LA_mostly(p, ptr, pgdc);
+ UASM_i_LA_mostly(p, ptr, pgdc);
#endif
- uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
- uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ }
uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
@@ -843,7 +951,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
#endif /* !CONFIG_64BIT */
-static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
+static void build_adjust_context(u32 **p, unsigned int ctx)
{
unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
@@ -869,7 +977,7 @@ static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
uasm_i_andi(p, ctx, ctx, mask);
}
-static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
+static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
{
/*
* Bug workaround for the Nevada. It seems as if under certain
@@ -894,8 +1002,7 @@ static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr
UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}
-static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
- unsigned int ptep)
+static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
{
/*
* 64bit address support (36bit on a 32bit CPU) in a 32bit
@@ -905,12 +1012,10 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
if (cpu_has_64bits) {
uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
- if (kernel_uses_smartmips_rixi) {
- UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
- UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
- UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ if (cpu_has_rixi) {
+ UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
- UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL));
} else {
uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
@@ -932,14 +1037,12 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
if (r45k_bvahwbug())
build_tlb_probe_entry(p);
- if (kernel_uses_smartmips_rixi) {
- UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
- UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
- UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ if (cpu_has_rixi) {
+ UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL));
if (r4k_250MHZhwbug())
UASM_i_MTC0(p, 0, C0_ENTRYLO0);
UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
- UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL));
} else {
UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
if (r4k_250MHZhwbug())
@@ -960,10 +1063,10 @@ struct mips_huge_tlb_info {
int restore_scratch;
};
-static struct mips_huge_tlb_info __cpuinit
+static struct mips_huge_tlb_info
build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int tmp,
- unsigned int ptr, int c0_scratch)
+ unsigned int ptr, int c0_scratch_reg)
{
struct mips_huge_tlb_info rv;
unsigned int even, odd;
@@ -977,12 +1080,12 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
UASM_i_MFC0(p, tmp, C0_BADVADDR);
if (pgd_reg != -1)
- UASM_i_MFC0(p, ptr, 31, pgd_reg);
+ UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
else
UASM_i_MFC0(p, ptr, C0_CONTEXT);
- if (c0_scratch >= 0)
- UASM_i_MTC0(p, scratch, 31, c0_scratch);
+ if (c0_scratch_reg >= 0)
+ UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
else
UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
@@ -997,14 +1100,14 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
}
} else {
if (pgd_reg != -1)
- UASM_i_MFC0(p, ptr, 31, pgd_reg);
+ UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
else
UASM_i_MFC0(p, ptr, C0_CONTEXT);
UASM_i_MFC0(p, tmp, C0_BADVADDR);
- if (c0_scratch >= 0)
- UASM_i_MTC0(p, scratch, 31, c0_scratch);
+ if (c0_scratch_reg >= 0)
+ UASM_i_MTC0(p, scratch, c0_kscratch(), c0_scratch_reg);
else
UASM_i_SW(p, scratch, scratchpad_offset(0), 0);
@@ -1017,7 +1120,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
if (pgd_reg == -1) {
vmalloc_branch_delay_filled = 1;
- /* 1 0 1 0 1 << 6 xkphys cached */
+ /* 1 0 1 0 1 << 6 xkphys cached */
uasm_i_ori(p, ptr, ptr, 0x540);
uasm_i_drotr(p, ptr, ptr, 11);
}
@@ -1035,9 +1138,9 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
uasm_l_vmalloc_done(l, *p);
/*
- * tmp ptr
- * fall-through case = badvaddr *pgd_current
- * vmalloc case = badvaddr swapper_pg_dir
+ * tmp ptr
+ * fall-through case = badvaddr *pgd_current
+ * vmalloc case = badvaddr swapper_pg_dir
*/
if (vmalloc_branch_delay_filled)
@@ -1072,16 +1175,16 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
/* Adjust the context during the load latency. */
build_adjust_context(p, tmp);
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update);
/*
* The in the LWX case we don't want to do the load in the
- * delay slot. It cannot issue in the same cycle and may be
+ * delay slot. It cannot issue in the same cycle and may be
* speculative and unneeded.
*/
if (use_lwx_insns())
uasm_i_nop(p);
-#endif /* CONFIG_HUGETLB_PAGE */
+#endif /* CONFIG_MIPS_HUGE_TLB_SUPPORT */
/* build_update_entries */
@@ -1098,14 +1201,10 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
UASM_i_LW(p, even, 0, ptr); /* get even pte */
UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */
}
- if (kernel_uses_smartmips_rixi) {
- uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC));
- uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC));
- uasm_i_drotr(p, even, even,
- ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ if (cpu_has_rixi) {
+ uasm_i_drotr(p, even, even, ilog2(_PAGE_GLOBAL));
UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
- uasm_i_drotr(p, odd, odd,
- ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ uasm_i_drotr(p, odd, odd, ilog2(_PAGE_GLOBAL));
} else {
uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL));
UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */
@@ -1113,8 +1212,8 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
}
UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */
- if (c0_scratch >= 0) {
- UASM_i_MFC0(p, scratch, 31, c0_scratch);
+ if (c0_scratch_reg >= 0) {
+ UASM_i_MFC0(p, scratch, c0_kscratch(), c0_scratch_reg);
build_tlb_write_entry(p, l, r, tlb_random);
uasm_l_leave(l, *p);
rv.restore_scratch = 1;
@@ -1142,25 +1241,22 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
*/
#define MIPS64_REFILL_INSNS 32
-static void __cpuinit build_r4000_tlb_refill_handler(void)
+static void build_r4000_tlb_refill_handler(void)
{
u32 *p = tlb_handler;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
u32 *f;
unsigned int final_len;
- struct mips_huge_tlb_info htlb_info;
- enum vmalloc64_mode vmalloc_mode;
+ struct mips_huge_tlb_info htlb_info __maybe_unused;
+ enum vmalloc64_mode vmalloc_mode __maybe_unused;
memset(tlb_handler, 0, sizeof(tlb_handler));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
memset(final_handler, 0, sizeof(final_handler));
- if (scratch_reg == 0)
- scratch_reg = allocate_kscratch();
-
- if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) {
+ if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
scratch_reg);
vmalloc_mode = refill_scratch;
@@ -1191,7 +1287,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
#endif
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
#endif
@@ -1201,7 +1297,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
uasm_l_leave(&l, p);
uasm_i_eret(&p); /* return from trap */
}
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
uasm_l_tlb_huge_update(&l, p);
build_huge_update_entries(&p, htlb_info.huge_pte, K1);
build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random,
@@ -1219,95 +1315,100 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
* need three, with the second nop'ed and the third being
* unused.
*/
- /* Loongson2 ebase is different than r4k, we have more space */
-#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
- if ((p - tlb_handler) > 64)
- panic("TLB refill handler space exceeded");
-#else
- if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
- || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
- && uasm_insn_has_bdelay(relocs,
- tlb_handler + MIPS64_REFILL_INSNS - 3)))
- panic("TLB refill handler space exceeded");
-#endif
-
- /*
- * Now fold the handler in the TLB refill handler space.
- */
-#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
- f = final_handler;
- /* Simplest case, just copy the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, p, f);
- final_len = p - tlb_handler;
-#else /* CONFIG_64BIT */
- f = final_handler + MIPS64_REFILL_INSNS;
- if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
- /* Just copy the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, p, f);
- final_len = p - tlb_handler;
- } else {
-#if defined(CONFIG_HUGETLB_PAGE)
- const enum label_id ls = label_tlb_huge_update;
-#else
- const enum label_id ls = label_vmalloc;
-#endif
- u32 *split;
- int ov = 0;
- int i;
-
- for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
- ;
- BUG_ON(i == ARRAY_SIZE(labels));
- split = labels[i].addr;
-
- /*
- * See if we have overflown one way or the other.
- */
- if (split > tlb_handler + MIPS64_REFILL_INSNS ||
- split < p - MIPS64_REFILL_INSNS)
- ov = 1;
-
- if (ov) {
+ switch (boot_cpu_type()) {
+ default:
+ if (sizeof(long) == 4) {
+ case CPU_LOONGSON2:
+ /* Loongson2 ebase is different than r4k, we have more space */
+ if ((p - tlb_handler) > 64)
+ panic("TLB refill handler space exceeded");
/*
- * Split two instructions before the end. One
- * for the branch and one for the instruction
- * in the delay slot.
+ * Now fold the handler in the TLB refill handler space.
*/
- split = tlb_handler + MIPS64_REFILL_INSNS - 2;
-
+ f = final_handler;
+ /* Simplest case, just copy the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
+ final_len = p - tlb_handler;
+ break;
+ } else {
+ if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
+ || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
+ && uasm_insn_has_bdelay(relocs,
+ tlb_handler + MIPS64_REFILL_INSNS - 3)))
+ panic("TLB refill handler space exceeded");
/*
- * If the branch would fall in a delay slot,
- * we must back up an additional instruction
- * so that it is no longer in a delay slot.
+ * Now fold the handler in the TLB refill handler space.
*/
- if (uasm_insn_has_bdelay(relocs, split - 1))
- split--;
- }
- /* Copy first part of the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, split, f);
- f += split - tlb_handler;
-
- if (ov) {
- /* Insert branch. */
- uasm_l_split(&l, final_handler);
- uasm_il_b(&f, &r, label_split);
- if (uasm_insn_has_bdelay(relocs, split))
- uasm_i_nop(&f);
- else {
- uasm_copy_handler(relocs, labels,
- split, split + 1, f);
- uasm_move_labels(labels, f, f + 1, -1);
- f++;
- split++;
+ f = final_handler + MIPS64_REFILL_INSNS;
+ if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
+ /* Just copy the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
+ final_len = p - tlb_handler;
+ } else {
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
+ const enum label_id ls = label_tlb_huge_update;
+#else
+ const enum label_id ls = label_vmalloc;
+#endif
+ u32 *split;
+ int ov = 0;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
+ ;
+ BUG_ON(i == ARRAY_SIZE(labels));
+ split = labels[i].addr;
+
+ /*
+ * See if we have overflown one way or the other.
+ */
+ if (split > tlb_handler + MIPS64_REFILL_INSNS ||
+ split < p - MIPS64_REFILL_INSNS)
+ ov = 1;
+
+ if (ov) {
+ /*
+ * Split two instructions before the end. One
+ * for the branch and one for the instruction
+ * in the delay slot.
+ */
+ split = tlb_handler + MIPS64_REFILL_INSNS - 2;
+
+ /*
+ * If the branch would fall in a delay slot,
+ * we must back up an additional instruction
+ * so that it is no longer in a delay slot.
+ */
+ if (uasm_insn_has_bdelay(relocs, split - 1))
+ split--;
+ }
+ /* Copy first part of the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, split, f);
+ f += split - tlb_handler;
+
+ if (ov) {
+ /* Insert branch. */
+ uasm_l_split(&l, final_handler);
+ uasm_il_b(&f, &r, label_split);
+ if (uasm_insn_has_bdelay(relocs, split))
+ uasm_i_nop(&f);
+ else {
+ uasm_copy_handler(relocs, labels,
+ split, split + 1, f);
+ uasm_move_labels(labels, f, f + 1, -1);
+ f++;
+ split++;
+ }
+ }
+
+ /* Copy the rest of the handler. */
+ uasm_copy_handler(relocs, labels, split, p, final_handler);
+ final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
+ (p - split);
}
}
-
- /* Copy the rest of the handler. */
- uasm_copy_handler(relocs, labels, split, p, final_handler);
- final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
- (p - split);
+ break;
}
-#endif /* CONFIG_64BIT */
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB refill handler (%u instructions).\n",
@@ -1315,36 +1416,37 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
memcpy((void *)ebase, final_handler, 0x100);
- dump_handler((u32 *)ebase, 64);
+ dump_handler("r4000_tlb_refill", (u32 *)ebase, 64);
}
-/*
- * 128 instructions for the fastpath handler is generous and should
- * never be exceeded.
- */
-#define FASTPATH_SIZE 128
-
-u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
-u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
-u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
-u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned;
+extern u32 handle_tlbl[], handle_tlbl_end[];
+extern u32 handle_tlbs[], handle_tlbs_end[];
+extern u32 handle_tlbm[], handle_tlbm_end[];
+extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[];
+extern u32 tlbmiss_handler_setup_pgd_end[];
-static void __cpuinit build_r4000_setup_pgd(void)
+static void build_setup_pgd(void)
{
const int a0 = 4;
- const int a1 = 5;
- u32 *p = tlbmiss_handler_setup_pgd;
- struct uasm_label *l = labels;
- struct uasm_reloc *r = relocs;
+ const int __maybe_unused a1 = 5;
+ const int __maybe_unused a2 = 6;
+ u32 *p = tlbmiss_handler_setup_pgd_start;
+ const int tlbmiss_handler_setup_pgd_size =
+ tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+ long pgdc = (long)pgd_current;
+#endif
- memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd));
+ memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
+ sizeof(tlbmiss_handler_setup_pgd[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
-
pgd_reg = allocate_kscratch();
-
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg == -1) {
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
+
/* PGD << 11 in c0_Context */
/*
* If it is a ckseg0 address, convert to a physical
@@ -1364,20 +1466,40 @@ static void __cpuinit build_r4000_setup_pgd(void)
} else {
/* PGD in c0_KScratch */
uasm_i_jr(&p, 31);
- UASM_i_MTC0(&p, a0, 31, pgd_reg);
+ UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
}
- if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd))
+#else
+#ifdef CONFIG_SMP
+ /* Save PGD to pgd_current[smp_processor_id()] */
+ UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
+ UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
+ UASM_i_LA_mostly(&p, a2, pgdc);
+ UASM_i_ADDU(&p, a2, a2, a1);
+ UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
+#else
+ UASM_i_LA_mostly(&p, a2, pgdc);
+ UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
+#endif /* SMP */
+ uasm_i_jr(&p, 31);
+
+ /* if pgd_reg is allocated, save PGD also to scratch register */
+ if (pgd_reg != -1)
+ UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
+ else
+ uasm_i_nop(&p);
+#endif
+ if (p >= tlbmiss_handler_setup_pgd_end)
panic("tlbmiss_handler_setup_pgd space exceeded");
+
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n",
(unsigned int)(p - tlbmiss_handler_setup_pgd));
- dump_handler(tlbmiss_handler_setup_pgd,
- ARRAY_SIZE(tlbmiss_handler_setup_pgd));
+ dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
+ tlbmiss_handler_setup_pgd_size);
}
-#endif
-static void __cpuinit
+static void
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
@@ -1397,7 +1519,7 @@ iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
#endif
}
-static void __cpuinit
+static void
iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
unsigned int mode)
{
@@ -1457,29 +1579,35 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
* the page table where this PTE is located, PTE will be re-loaded
* with it's original value.
*/
-static void __cpuinit
+static void
build_pte_present(u32 **p, struct uasm_reloc **r,
- unsigned int pte, unsigned int ptr, enum label_id lid)
+ int pte, int ptr, int scratch, enum label_id lid)
{
- if (kernel_uses_smartmips_rixi) {
+ int t = scratch >= 0 ? scratch : pte;
+
+ if (cpu_has_rixi) {
if (use_bbit_insns()) {
uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
uasm_i_nop(p);
} else {
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
- uasm_il_beqz(p, r, pte, lid);
- iPTE_LW(p, pte, ptr);
+ uasm_i_andi(p, t, pte, _PAGE_PRESENT);
+ uasm_il_beqz(p, r, t, lid);
+ if (pte == t)
+ /* You lose the SMP race :-(*/
+ iPTE_LW(p, pte, ptr);
}
} else {
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- uasm_il_bnez(p, r, pte, lid);
- iPTE_LW(p, pte, ptr);
+ uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_READ);
+ uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_READ);
+ uasm_il_bnez(p, r, t, lid);
+ if (pte == t)
+ /* You lose the SMP race :-(*/
+ iPTE_LW(p, pte, ptr);
}
}
/* Make PTE valid, store result in PTR. */
-static void __cpuinit
+static void
build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
@@ -1492,27 +1620,27 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
* Check if PTE can be written to, if not branch to LABEL. Regardless
* restore PTE with value from PTR when done.
*/
-static void __cpuinit
+static void
build_pte_writable(u32 **p, struct uasm_reloc **r,
- unsigned int pte, unsigned int ptr, enum label_id lid)
+ unsigned int pte, unsigned int ptr, int scratch,
+ enum label_id lid)
{
- if (use_bbit_insns()) {
- uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid);
- uasm_i_nop(p);
- uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
- uasm_i_nop(p);
- } else {
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
- uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
- uasm_il_bnez(p, r, pte, lid);
+ int t = scratch >= 0 ? scratch : pte;
+
+ uasm_i_andi(p, t, pte, _PAGE_PRESENT | _PAGE_WRITE);
+ uasm_i_xori(p, t, t, _PAGE_PRESENT | _PAGE_WRITE);
+ uasm_il_bnez(p, r, t, lid);
+ if (pte == t)
+ /* You lose the SMP race :-(*/
iPTE_LW(p, pte, ptr);
- }
+ else
+ uasm_i_nop(p);
}
/* Make PTE writable, update software status bits as well, then store
* at PTR.
*/
-static void __cpuinit
+static void
build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
@@ -1526,17 +1654,21 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
* Check if PTE can be modified, if not branch to LABEL. Regardless
* restore PTE with value from PTR when done.
*/
-static void __cpuinit
+static void
build_pte_modifiable(u32 **p, struct uasm_reloc **r,
- unsigned int pte, unsigned int ptr, enum label_id lid)
+ unsigned int pte, unsigned int ptr, int scratch,
+ enum label_id lid)
{
if (use_bbit_insns()) {
uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid);
uasm_i_nop(p);
} else {
- uasm_i_andi(p, pte, pte, _PAGE_WRITE);
- uasm_il_beqz(p, r, pte, lid);
- iPTE_LW(p, pte, ptr);
+ int t = scratch >= 0 ? scratch : pte;
+ uasm_i_andi(p, t, pte, _PAGE_WRITE);
+ uasm_il_beqz(p, r, t, lid);
+ if (pte == t)
+ /* You lose the SMP race :-(*/
+ iPTE_LW(p, pte, ptr);
}
}
@@ -1551,7 +1683,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
* This places the pte into ENTRYLO0 and writes it with tlbwi.
* Then it returns.
*/
-static void __cpuinit
+static void
build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
{
uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
@@ -1567,7 +1699,7 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
* may have the probe fail bit set as a result of a trap on a
* kseg2 access, i.e. without refill. Then it returns.
*/
-static void __cpuinit
+static void
build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int pte,
unsigned int tmp)
@@ -1585,7 +1717,7 @@ build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
uasm_i_rfe(p); /* branch delay */
}
-static void __cpuinit
+static void
build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
unsigned int ptr)
{
@@ -1605,18 +1737,19 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
uasm_i_tlbp(p); /* load delay */
}
-static void __cpuinit build_r3000_tlb_load_handler(void)
+static void build_r3000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
+ const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
- memset(handle_tlbl, 0, sizeof(handle_tlbl));
+ memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
+ build_pte_present(&p, &r, K0, K1, -1, label_nopage_tlbl);
uasm_i_nop(&p); /* load delay */
build_make_valid(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
@@ -1625,28 +1758,29 @@ static void __cpuinit build_r3000_tlb_load_handler(void)
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbl) > FASTPATH_SIZE)
+ if (p >= handle_tlbl_end)
panic("TLB load handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbl));
- dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
+ dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
}
-static void __cpuinit build_r3000_tlb_store_handler(void)
+static void build_r3000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
+ const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
- memset(handle_tlbs, 0, sizeof(handle_tlbs));
+ memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
+ build_pte_writable(&p, &r, K0, K1, -1, label_nopage_tlbs);
uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
@@ -1655,28 +1789,29 @@ static void __cpuinit build_r3000_tlb_store_handler(void)
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbs) > FASTPATH_SIZE)
+ if (p >= handle_tlbs_end)
panic("TLB store handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbs));
- dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
+ dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
}
-static void __cpuinit build_r3000_tlb_modify_handler(void)
+static void build_r3000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
+ const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
- memset(handle_tlbm, 0, sizeof(handle_tlbm));
+ memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
build_r3000_tlbchange_handler_head(&p, K0, K1);
- build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
+ build_pte_modifiable(&p, &r, K0, K1, -1, label_nopage_tlbm);
uasm_i_nop(&p); /* load delay */
build_make_write(&p, &r, K0, K1);
build_r3000_pte_reload_tlbwi(&p, K0, K1);
@@ -1685,55 +1820,57 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbm) > FASTPATH_SIZE)
+ if (p >= handle_tlbm_end)
panic("TLB modify handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbm));
- dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
+ dump_handler("r3000_tlb_modify", handle_tlbm, handle_tlbm_size);
}
#endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
/*
* R4000 style TLB load/store/modify handlers.
*/
-static void __cpuinit
+static struct work_registers
build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
- struct uasm_reloc **r, unsigned int pte,
- unsigned int ptr)
+ struct uasm_reloc **r)
{
+ struct work_registers wr = build_get_work_registers(p);
+
#ifdef CONFIG_64BIT
- build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
+ build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */
#else
- build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
+ build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */
#endif
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/*
* For huge tlb entries, pmd doesn't contain an address but
* instead contains the tlb pte. Check the PAGE_HUGE bit and
* see if we need to jump to huge tlb processing.
*/
- build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update);
+ build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);
#endif
- UASM_i_MFC0(p, pte, C0_BADVADDR);
- UASM_i_LW(p, ptr, 0, ptr);
- UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
- uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
- UASM_i_ADDU(p, ptr, ptr, pte);
+ UASM_i_MFC0(p, wr.r1, C0_BADVADDR);
+ UASM_i_LW(p, wr.r2, 0, wr.r2);
+ UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
+ uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
+ UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);
#ifdef CONFIG_SMP
uasm_l_smp_pgtable_change(l, *p);
#endif
- iPTE_LW(p, pte, ptr); /* get even pte */
+ iPTE_LW(p, wr.r1, wr.r2); /* get even pte */
if (!m4kc_tlbp_war())
build_tlb_probe_entry(p);
+ return wr;
}
-static void __cpuinit
+static void
build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int tmp,
unsigned int ptr)
@@ -1743,6 +1880,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
build_update_entries(p, tmp, ptr);
build_tlb_write_entry(p, l, r, tlb_indexed);
uasm_l_leave(l, *p);
+ build_restore_work_registers(p);
uasm_i_eret(p); /* return from trap */
#ifdef CONFIG_64BIT
@@ -1750,13 +1888,15 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
#endif
}
-static void __cpuinit build_r4000_tlb_load_handler(void)
+static void build_r4000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
+ const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
+ struct work_registers wr;
- memset(handle_tlbl, 0, sizeof(handle_tlbl));
+ memset(handle_tlbl, 0, handle_tlbl_size * sizeof(handle_tlbl[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
@@ -1774,221 +1914,287 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
/* No need for uasm_i_nop */
}
- build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
+ wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+ build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
- if (kernel_uses_smartmips_rixi) {
+ if (cpu_has_rixi) {
/*
* If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test..
*/
if (use_bbit_insns()) {
- uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
+ uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
label_tlbl_goaround1);
} else {
- uasm_i_andi(&p, K0, K0, _PAGE_VALID);
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
+ uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
+ uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);
}
uasm_i_nop(&p);
uasm_i_tlbr(&p);
+
+ switch (current_cpu_type()) {
+ default:
+ if (cpu_has_mips_r2) {
+ uasm_i_ehb(&p);
+
+ case CPU_CAVIUM_OCTEON:
+ case CPU_CAVIUM_OCTEON_PLUS:
+ case CPU_CAVIUM_OCTEON2:
+ break;
+ }
+ }
+
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
- uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
+ uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
} else {
- uasm_i_andi(&p, K0, K1, sizeof(pte_t));
- uasm_i_beqz(&p, K0, 8);
+ uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
+ uasm_i_beqz(&p, wr.r3, 8);
}
-
- UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
- UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+ /* load it in the delay slot*/
+ UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
+ /* load it if ptr is odd */
+ UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
/*
- * If the entryLo (now in K0) is valid (bit 1), RI or
+ * If the entryLo (now in wr.r3) is valid (bit 1), RI or
* XI must have triggered it.
*/
if (use_bbit_insns()) {
- uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl);
- /* Reload the PTE value */
- iPTE_LW(&p, K0, K1);
+ uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);
+ uasm_i_nop(&p);
uasm_l_tlbl_goaround1(&l, p);
} else {
- uasm_i_andi(&p, K0, K0, 2);
- uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
- uasm_l_tlbl_goaround1(&l, p);
- /* Reload the PTE value */
- iPTE_LW(&p, K0, K1);
+ uasm_i_andi(&p, wr.r3, wr.r3, 2);
+ uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);
+ uasm_i_nop(&p);
}
+ uasm_l_tlbl_goaround1(&l, p);
}
- build_make_valid(&p, &r, K0, K1);
- build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+ build_make_valid(&p, &r, wr.r1, wr.r2);
+ build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/*
* This is the entry point when build_r4000_tlbchange_handler_head
* spots a huge page.
*/
uasm_l_tlb_huge_update(&l, p);
- iPTE_LW(&p, K0, K1);
- build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
+ iPTE_LW(&p, wr.r1, wr.r2);
+ build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);
build_tlb_probe_entry(&p);
- if (kernel_uses_smartmips_rixi) {
+ if (cpu_has_rixi) {
/*
* If the page is not _PAGE_VALID, RI or XI could not
* have triggered it. Skip the expensive test..
*/
if (use_bbit_insns()) {
- uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID),
+ uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),
label_tlbl_goaround2);
} else {
- uasm_i_andi(&p, K0, K0, _PAGE_VALID);
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+ uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);
+ uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
}
uasm_i_nop(&p);
uasm_i_tlbr(&p);
+
+ switch (current_cpu_type()) {
+ default:
+ if (cpu_has_mips_r2) {
+ uasm_i_ehb(&p);
+
+ case CPU_CAVIUM_OCTEON:
+ case CPU_CAVIUM_OCTEON_PLUS:
+ case CPU_CAVIUM_OCTEON2:
+ break;
+ }
+ }
+
/* Examine entrylo 0 or 1 based on ptr. */
if (use_bbit_insns()) {
- uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8);
+ uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);
} else {
- uasm_i_andi(&p, K0, K1, sizeof(pte_t));
- uasm_i_beqz(&p, K0, 8);
+ uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));
+ uasm_i_beqz(&p, wr.r3, 8);
}
- UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
- UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+ /* load it in the delay slot*/
+ UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);
+ /* load it if ptr is odd */
+ UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);
/*
- * If the entryLo (now in K0) is valid (bit 1), RI or
+ * If the entryLo (now in wr.r3) is valid (bit 1), RI or
* XI must have triggered it.
*/
if (use_bbit_insns()) {
- uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2);
+ uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);
} else {
- uasm_i_andi(&p, K0, K0, 2);
- uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+ uasm_i_andi(&p, wr.r3, wr.r3, 2);
+ uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);
}
- /* Reload the PTE value */
- iPTE_LW(&p, K0, K1);
-
+ if (PM_DEFAULT_MASK == 0)
+ uasm_i_nop(&p);
/*
* We clobbered C0_PAGEMASK, restore it. On the other branch
* it is restored in build_huge_tlb_write_entry.
*/
- build_restore_pagemask(&p, &r, K0, label_nopage_tlbl, 0);
+ build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);
uasm_l_tlbl_goaround2(&l, p);
}
- uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
- build_huge_handler_tail(&p, &r, &l, K0, K1);
+ uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));
+ build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
#endif
uasm_l_nopage_tlbl(&l, p);
+ build_restore_work_registers(&p);
+#ifdef CONFIG_CPU_MICROMIPS
+ if ((unsigned long)tlb_do_page_fault_0 & 1) {
+ uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_0));
+ uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_0));
+ uasm_i_jr(&p, K0);
+ } else
+#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbl) > FASTPATH_SIZE)
+ if (p >= handle_tlbl_end)
panic("TLB load handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbl));
- dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
+ dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
}
-static void __cpuinit build_r4000_tlb_store_handler(void)
+static void build_r4000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
+ const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
+ struct work_registers wr;
- memset(handle_tlbs, 0, sizeof(handle_tlbs));
+ memset(handle_tlbs, 0, handle_tlbs_size * sizeof(handle_tlbs[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
- build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
+ wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+ build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
- build_make_write(&p, &r, K0, K1);
- build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+ build_make_write(&p, &r, wr.r1, wr.r2);
+ build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/*
* This is the entry point when
* build_r4000_tlbchange_handler_head spots a huge page.
*/
uasm_l_tlb_huge_update(&l, p);
- iPTE_LW(&p, K0, K1);
- build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
+ iPTE_LW(&p, wr.r1, wr.r2);
+ build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);
build_tlb_probe_entry(&p);
- uasm_i_ori(&p, K0, K0,
+ uasm_i_ori(&p, wr.r1, wr.r1,
_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
- build_huge_handler_tail(&p, &r, &l, K0, K1);
+ build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
#endif
uasm_l_nopage_tlbs(&l, p);
+ build_restore_work_registers(&p);
+#ifdef CONFIG_CPU_MICROMIPS
+ if ((unsigned long)tlb_do_page_fault_1 & 1) {
+ uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
+ uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
+ uasm_i_jr(&p, K0);
+ } else
+#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbs) > FASTPATH_SIZE)
+ if (p >= handle_tlbs_end)
panic("TLB store handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbs));
- dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
+ dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
}
-static void __cpuinit build_r4000_tlb_modify_handler(void)
+static void build_r4000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
+ const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
struct uasm_label *l = labels;
struct uasm_reloc *r = relocs;
+ struct work_registers wr;
- memset(handle_tlbm, 0, sizeof(handle_tlbm));
+ memset(handle_tlbm, 0, handle_tlbm_size * sizeof(handle_tlbm[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
- build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
- build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
+ wr = build_r4000_tlbchange_handler_head(&p, &l, &r);
+ build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
/* Present and writable bits set, set accessed and dirty bits. */
- build_make_write(&p, &r, K0, K1);
- build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
+ build_make_write(&p, &r, wr.r1, wr.r2);
+ build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);
-#ifdef CONFIG_HUGETLB_PAGE
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
/*
* This is the entry point when
* build_r4000_tlbchange_handler_head spots a huge page.
*/
uasm_l_tlb_huge_update(&l, p);
- iPTE_LW(&p, K0, K1);
- build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
+ iPTE_LW(&p, wr.r1, wr.r2);
+ build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);
build_tlb_probe_entry(&p);
- uasm_i_ori(&p, K0, K0,
+ uasm_i_ori(&p, wr.r1, wr.r1,
_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
- build_huge_handler_tail(&p, &r, &l, K0, K1);
+ build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2);
#endif
uasm_l_nopage_tlbm(&l, p);
+ build_restore_work_registers(&p);
+#ifdef CONFIG_CPU_MICROMIPS
+ if ((unsigned long)tlb_do_page_fault_1 & 1) {
+ uasm_i_lui(&p, K0, uasm_rel_hi((long)tlb_do_page_fault_1));
+ uasm_i_addiu(&p, K0, K0, uasm_rel_lo((long)tlb_do_page_fault_1));
+ uasm_i_jr(&p, K0);
+ } else
+#endif
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
- if ((p - handle_tlbm) > FASTPATH_SIZE)
+ if (p >= handle_tlbm_end)
panic("TLB modify handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbm));
- dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
+ dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
+}
+
+static void flush_tlb_handlers(void)
+{
+ local_flush_icache_range((unsigned long)handle_tlbl,
+ (unsigned long)handle_tlbl_end);
+ local_flush_icache_range((unsigned long)handle_tlbs,
+ (unsigned long)handle_tlbs_end);
+ local_flush_icache_range((unsigned long)handle_tlbm,
+ (unsigned long)handle_tlbm_end);
+ local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
+ (unsigned long)tlbmiss_handler_setup_pgd_end);
}
-void __cpuinit build_tlb_refill_handler(void)
+void build_tlb_refill_handler(void)
{
/*
* The refill handler is generated per-CPU, multi-node systems
@@ -1997,6 +2203,8 @@ void __cpuinit build_tlb_refill_handler(void)
*/
static int run_once = 0;
+ output_pgtable_bits_defines();
+
#ifdef CONFIG_64BIT
check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
#endif
@@ -2010,11 +2218,16 @@ void __cpuinit build_tlb_refill_handler(void)
case CPU_TX3922:
case CPU_TX3927:
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
- build_r3000_tlb_refill_handler();
+ if (cpu_has_local_ebase)
+ build_r3000_tlb_refill_handler();
if (!run_once) {
+ if (!cpu_has_local_ebase)
+ build_r3000_tlb_refill_handler();
+ build_setup_pgd();
build_r3000_tlb_load_handler();
build_r3000_tlb_store_handler();
build_r3000_tlb_modify_handler();
+ flush_tlb_handlers();
run_once++;
}
#else
@@ -2033,28 +2246,17 @@ void __cpuinit build_tlb_refill_handler(void)
default:
if (!run_once) {
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
- build_r4000_setup_pgd();
-#endif
+ scratch_reg = allocate_kscratch();
+ build_setup_pgd();
build_r4000_tlb_load_handler();
build_r4000_tlb_store_handler();
build_r4000_tlb_modify_handler();
+ if (!cpu_has_local_ebase)
+ build_r4000_tlb_refill_handler();
+ flush_tlb_handlers();
run_once++;
}
- build_r4000_tlb_refill_handler();
+ if (cpu_has_local_ebase)
+ build_r4000_tlb_refill_handler();
}
}
-
-void __cpuinit flush_tlb_handlers(void)
-{
- local_flush_icache_range((unsigned long)handle_tlbl,
- (unsigned long)handle_tlbl + sizeof(handle_tlbl));
- local_flush_icache_range((unsigned long)handle_tlbs,
- (unsigned long)handle_tlbs + sizeof(handle_tlbs));
- local_flush_icache_range((unsigned long)handle_tlbm,
- (unsigned long)handle_tlbm + sizeof(handle_tlbm));
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
- local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
- (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm));
-#endif
-}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-09 16:31:48 +0000