aboutsummaryrefslogtreecommitdiff
path: root/arch/mips/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/c-octeon.c29
-rw-r--r--arch/mips/mm/c-r3k.c9
-rw-r--r--arch/mips/mm/c-r4k.c391
-rw-r--r--arch/mips/mm/c-tx39.c2
-rw-r--r--arch/mips/mm/cache.c12
-rw-r--r--arch/mips/mm/cex-sb1.S5
-rw-r--r--arch/mips/mm/dma-default.c35
-rw-r--r--arch/mips/mm/fault.c8
-rw-r--r--arch/mips/mm/gup.c3
-rw-r--r--arch/mips/mm/hugetlbpage.c1
-rw-r--r--arch/mips/mm/init.c100
-rw-r--r--arch/mips/mm/page.c46
-rw-r--r--arch/mips/mm/sc-ip22.c2
-rw-r--r--arch/mips/mm/sc-mips.c9
-rw-r--r--arch/mips/mm/sc-r5k.c2
-rw-r--r--arch/mips/mm/sc-rm7k.c13
-rw-r--r--arch/mips/mm/tlb-funcs.S4
-rw-r--r--arch/mips/mm/tlb-r3k.c3
-rw-r--r--arch/mips/mm/tlb-r4k.c179
-rw-r--r--arch/mips/mm/tlb-r8k.c5
-rw-r--r--arch/mips/mm/tlbex.c471
-rw-r--r--arch/mips/mm/uasm-micromips.c26
-rw-r--r--arch/mips/mm/uasm-mips.c27
-rw-r--r--arch/mips/mm/uasm.c165
24 files changed, 924 insertions, 623 deletions
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 8557fb55286..05b1d7cf951 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -6,7 +6,6 @@
* Copyright (C) 2005-2007 Cavium Networks
*/
#include <linux/export.h>
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
@@ -19,6 +18,7 @@
#include <asm/bootinfo.h>
#include <asm/cacheops.h>
#include <asm/cpu-features.h>
+#include <asm/cpu-type.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/r4kcache.h>
@@ -137,8 +137,10 @@ static void octeon_flush_cache_sigtramp(unsigned long addr)
{
struct vm_area_struct *vma;
+ down_read(&current->mm->mmap_sem);
vma = find_vma(current->mm, addr);
octeon_flush_icache_all_cores(vma);
+ up_read(&current->mm->mmap_sem);
}
@@ -180,15 +182,16 @@ static void octeon_flush_kernel_vmap_range(unsigned long vaddr, int size)
* Probe Octeon's caches
*
*/
-static void __cpuinit probe_octeon(void)
+static void probe_octeon(void)
{
unsigned long icache_size;
unsigned long dcache_size;
unsigned int config1;
struct cpuinfo_mips *c = &current_cpu_data;
+ int cputype = current_cpu_type();
config1 = read_c0_config1();
- switch (c->cputype) {
+ switch (cputype) {
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
c->icache.linesz = 2 << ((config1 >> 19) & 7);
@@ -199,7 +202,7 @@ static void __cpuinit probe_octeon(void)
c->icache.sets * c->icache.ways * c->icache.linesz;
c->icache.waybit = ffs(icache_size / c->icache.ways) - 1;
c->dcache.linesz = 128;
- if (c->cputype == CPU_CAVIUM_OCTEON_PLUS)
+ if (cputype == CPU_CAVIUM_OCTEON_PLUS)
c->dcache.sets = 2; /* CN5XXX has two Dcache sets */
else
c->dcache.sets = 1; /* CN3XXX has one Dcache set */
@@ -224,6 +227,20 @@ static void __cpuinit probe_octeon(void)
c->options |= MIPS_CPU_PREFETCH;
break;
+ case CPU_CAVIUM_OCTEON3:
+ c->icache.linesz = 128;
+ c->icache.sets = 16;
+ c->icache.ways = 39;
+ c->icache.flags |= MIPS_CACHE_VTAG;
+ icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
+
+ c->dcache.linesz = 128;
+ c->dcache.ways = 32;
+ c->dcache.sets = 8;
+ dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
+ c->options |= MIPS_CPU_PREFETCH;
+ break;
+
default:
panic("Unsupported Cavium Networks CPU type");
break;
@@ -251,7 +268,7 @@ static void __cpuinit probe_octeon(void)
}
}
-static void __cpuinit octeon_cache_error_setup(void)
+static void octeon_cache_error_setup(void)
{
extern char except_vec2_octeon;
set_handler(0x100, &except_vec2_octeon, 0x80);
@@ -261,7 +278,7 @@ static void __cpuinit octeon_cache_error_setup(void)
* Setup the Octeon cache flush routines
*
*/
-void __cpuinit octeon_cache_init(void)
+void octeon_cache_init(void)
{
probe_octeon();
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index 704dc735a59..135ec313c1f 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -9,7 +9,6 @@
* Copyright (C) 1998 Gleb Raiko & Vladimir Roganov
* Copyright (C) 2001, 2004, 2007 Maciej W. Rozycki
*/
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
@@ -26,7 +25,7 @@
static unsigned long icache_size, dcache_size; /* Size in bytes */
static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */
-unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags)
+unsigned long r3k_cache_size(unsigned long ca_flags)
{
unsigned long flags, status, dummy, size;
volatile unsigned long *p;
@@ -61,7 +60,7 @@ unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags)
return size * sizeof(*p);
}
-unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags)
+unsigned long r3k_cache_lsize(unsigned long ca_flags)
{
unsigned long flags, status, lsize, i;
volatile unsigned long *p;
@@ -90,7 +89,7 @@ unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags)
return lsize * sizeof(*p);
}
-static void __cpuinit r3k_probe_cache(void)
+static void r3k_probe_cache(void)
{
dcache_size = r3k_cache_size(ST0_ISC);
if (dcache_size)
@@ -312,7 +311,7 @@ static void r3k_dma_cache_wback_inv(unsigned long start, unsigned long size)
r3k_flush_dcache_range(start, start + size);
}
-void __cpuinit r3k_cache_init(void)
+void r3k_cache_init(void)
{
extern void build_clear_page(void);
extern void build_copy_page(void);
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index 21813beec7a..f2e8302fa70 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -7,11 +7,13 @@
* Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org)
* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
*/
+#include <linux/cpu_pm.h>
#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/highmem.h>
#include <linux/kernel.h>
#include <linux/linkage.h>
+#include <linux/preempt.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
@@ -24,6 +26,7 @@
#include <asm/cacheops.h>
#include <asm/cpu.h>
#include <asm/cpu-features.h>
+#include <asm/cpu-type.h>
#include <asm/io.h>
#include <asm/page.h>
#include <asm/pgtable.h>
@@ -48,14 +51,14 @@ static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
{
preempt_disable();
-#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC)
+#ifndef CONFIG_MIPS_MT_SMP
smp_call_function(func, info, 1);
#endif
func(info);
preempt_enable();
}
-#if defined(CONFIG_MIPS_CMP)
+#if defined(CONFIG_MIPS_CMP) || defined(CONFIG_MIPS_CPS)
#define cpu_has_safe_index_cacheops 0
#else
#define cpu_has_safe_index_cacheops 1
@@ -103,27 +106,64 @@ static inline void r4k_blast_dcache_page_dc32(unsigned long addr)
static inline void r4k_blast_dcache_page_dc64(unsigned long addr)
{
- R4600_HIT_CACHEOP_WAR_IMPL;
blast_dcache64_page(addr);
}
-static void __cpuinit r4k_blast_dcache_page_setup(void)
+static inline void r4k_blast_dcache_page_dc128(unsigned long addr)
+{
+ blast_dcache128_page(addr);
+}
+
+static void r4k_blast_dcache_page_setup(void)
{
unsigned long dc_lsize = cpu_dcache_line_size();
- if (dc_lsize == 0)
+ switch (dc_lsize) {
+ case 0:
r4k_blast_dcache_page = (void *)cache_noop;
- else if (dc_lsize == 16)
+ break;
+ case 16:
r4k_blast_dcache_page = blast_dcache16_page;
- else if (dc_lsize == 32)
+ break;
+ case 32:
r4k_blast_dcache_page = r4k_blast_dcache_page_dc32;
- else if (dc_lsize == 64)
+ break;
+ case 64:
r4k_blast_dcache_page = r4k_blast_dcache_page_dc64;
+ break;
+ case 128:
+ r4k_blast_dcache_page = r4k_blast_dcache_page_dc128;
+ break;
+ default:
+ break;
+ }
+}
+
+#ifndef CONFIG_EVA
+#define r4k_blast_dcache_user_page r4k_blast_dcache_page
+#else
+
+static void (*r4k_blast_dcache_user_page)(unsigned long addr);
+
+static void r4k_blast_dcache_user_page_setup(void)
+{
+ unsigned long dc_lsize = cpu_dcache_line_size();
+
+ if (dc_lsize == 0)
+ r4k_blast_dcache_user_page = (void *)cache_noop;
+ else if (dc_lsize == 16)
+ r4k_blast_dcache_user_page = blast_dcache16_user_page;
+ else if (dc_lsize == 32)
+ r4k_blast_dcache_user_page = blast_dcache32_user_page;
+ else if (dc_lsize == 64)
+ r4k_blast_dcache_user_page = blast_dcache64_user_page;
}
+#endif
+
static void (* r4k_blast_dcache_page_indexed)(unsigned long addr);
-static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
+static void r4k_blast_dcache_page_indexed_setup(void)
{
unsigned long dc_lsize = cpu_dcache_line_size();
@@ -135,12 +175,14 @@ static void __cpuinit r4k_blast_dcache_page_indexed_setup(void)
r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed;
else if (dc_lsize == 64)
r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed;
+ else if (dc_lsize == 128)
+ r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed;
}
void (* r4k_blast_dcache)(void);
EXPORT_SYMBOL(r4k_blast_dcache);
-static void __cpuinit r4k_blast_dcache_setup(void)
+static void r4k_blast_dcache_setup(void)
{
unsigned long dc_lsize = cpu_dcache_line_size();
@@ -152,6 +194,8 @@ static void __cpuinit r4k_blast_dcache_setup(void)
r4k_blast_dcache = blast_dcache32;
else if (dc_lsize == 64)
r4k_blast_dcache = blast_dcache64;
+ else if (dc_lsize == 128)
+ r4k_blast_dcache = blast_dcache128;
}
/* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */
@@ -227,7 +271,7 @@ static inline void tx49_blast_icache32_page_indexed(unsigned long page)
static void (* r4k_blast_icache_page)(unsigned long addr);
-static void __cpuinit r4k_blast_icache_page_setup(void)
+static void r4k_blast_icache_page_setup(void)
{
unsigned long ic_lsize = cpu_icache_line_size();
@@ -235,16 +279,41 @@ static void __cpuinit r4k_blast_icache_page_setup(void)
r4k_blast_icache_page = (void *)cache_noop;
else if (ic_lsize == 16)
r4k_blast_icache_page = blast_icache16_page;
+ else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2)
+ r4k_blast_icache_page = loongson2_blast_icache32_page;
else if (ic_lsize == 32)
r4k_blast_icache_page = blast_icache32_page;
else if (ic_lsize == 64)
r4k_blast_icache_page = blast_icache64_page;
+ else if (ic_lsize == 128)
+ r4k_blast_icache_page = blast_icache128_page;
}
+#ifndef CONFIG_EVA
+#define r4k_blast_icache_user_page r4k_blast_icache_page
+#else
+
+static void (*r4k_blast_icache_user_page)(unsigned long addr);
+
+static void __cpuinit r4k_blast_icache_user_page_setup(void)
+{
+ unsigned long ic_lsize = cpu_icache_line_size();
+
+ if (ic_lsize == 0)
+ r4k_blast_icache_user_page = (void *)cache_noop;
+ else if (ic_lsize == 16)
+ r4k_blast_icache_user_page = blast_icache16_user_page;
+ else if (ic_lsize == 32)
+ r4k_blast_icache_user_page = blast_icache32_user_page;
+ else if (ic_lsize == 64)
+ r4k_blast_icache_user_page = blast_icache64_user_page;
+}
+
+#endif
static void (* r4k_blast_icache_page_indexed)(unsigned long addr);
-static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
+static void r4k_blast_icache_page_indexed_setup(void)
{
unsigned long ic_lsize = cpu_icache_line_size();
@@ -259,6 +328,9 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
else if (TX49XX_ICACHE_INDEX_INV_WAR)
r4k_blast_icache_page_indexed =
tx49_blast_icache32_page_indexed;
+ else if (current_cpu_type() == CPU_LOONGSON2)
+ r4k_blast_icache_page_indexed =
+ loongson2_blast_icache32_page_indexed;
else
r4k_blast_icache_page_indexed =
blast_icache32_page_indexed;
@@ -269,7 +341,7 @@ static void __cpuinit r4k_blast_icache_page_indexed_setup(void)
void (* r4k_blast_icache)(void);
EXPORT_SYMBOL(r4k_blast_icache);
-static void __cpuinit r4k_blast_icache_setup(void)
+static void r4k_blast_icache_setup(void)
{
unsigned long ic_lsize = cpu_icache_line_size();
@@ -282,15 +354,19 @@ static void __cpuinit r4k_blast_icache_setup(void)
r4k_blast_icache = blast_r4600_v1_icache32;
else if (TX49XX_ICACHE_INDEX_INV_WAR)
r4k_blast_icache = tx49_blast_icache32;
+ else if (current_cpu_type() == CPU_LOONGSON2)
+ r4k_blast_icache = loongson2_blast_icache32;
else
r4k_blast_icache = blast_icache32;
} else if (ic_lsize == 64)
r4k_blast_icache = blast_icache64;
+ else if (ic_lsize == 128)
+ r4k_blast_icache = blast_icache128;
}
static void (* r4k_blast_scache_page)(unsigned long addr);
-static void __cpuinit r4k_blast_scache_page_setup(void)
+static void r4k_blast_scache_page_setup(void)
{
unsigned long sc_lsize = cpu_scache_line_size();
@@ -308,7 +384,7 @@ static void __cpuinit r4k_blast_scache_page_setup(void)
static void (* r4k_blast_scache_page_indexed)(unsigned long addr);
-static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
+static void r4k_blast_scache_page_indexed_setup(void)
{
unsigned long sc_lsize = cpu_scache_line_size();
@@ -326,7 +402,7 @@ static void __cpuinit r4k_blast_scache_page_indexed_setup(void)
static void (* r4k_blast_scache)(void);
-static void __cpuinit r4k_blast_scache_setup(void)
+static void r4k_blast_scache_setup(void)
{
unsigned long sc_lsize = cpu_scache_line_size();
@@ -344,14 +420,9 @@ static void __cpuinit r4k_blast_scache_setup(void)
static inline void local_r4k___flush_cache_all(void * args)
{
-#if defined(CONFIG_CPU_LOONGSON2)
- r4k_blast_scache();
- return;
-#endif
- r4k_blast_dcache();
- r4k_blast_icache();
-
switch (current_cpu_type()) {
+ case CPU_LOONGSON2:
+ case CPU_LOONGSON3:
case CPU_R4000SC:
case CPU_R4000MC:
case CPU_R4400SC:
@@ -359,7 +430,18 @@ static inline void local_r4k___flush_cache_all(void * args)
case CPU_R10000:
case CPU_R12000:
case CPU_R14000:
+ /*
+ * These caches are inclusive caches, that is, if something
+ * is not cached in the S-cache, we know it also won't be
+ * in one of the primary caches.
+ */
r4k_blast_scache();
+ break;
+
+ default:
+ r4k_blast_dcache();
+ r4k_blast_icache();
+ break;
}
}
@@ -370,7 +452,7 @@ static void r4k___flush_cache_all(void)
static inline int has_valid_asid(const struct mm_struct *mm)
{
-#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC)
+#ifdef CONFIG_MIPS_MT_SMP
int i;
for_each_online_cpu(i)
@@ -505,7 +587,8 @@ static inline void local_r4k_flush_cache_page(void *args)
}
if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) {
- r4k_blast_dcache_page(addr);
+ vaddr ? r4k_blast_dcache_page(addr) :
+ r4k_blast_dcache_user_page(addr);
if (exec && !cpu_icache_snoops_remote_store)
r4k_blast_scache_page(addr);
}
@@ -516,7 +599,8 @@ static inline void local_r4k_flush_cache_page(void *args)
if (cpu_context(cpu, mm) != 0)
drop_mmu_context(mm, cpu);
} else
- r4k_blast_icache_page(addr);
+ vaddr ? r4k_blast_icache_page(addr) :
+ r4k_blast_icache_user_page(addr);
}
if (vaddr) {
@@ -570,8 +654,28 @@ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned lo
if (end - start > icache_size)
r4k_blast_icache();
- else
- protected_blast_icache_range(start, end);
+ else {
+ switch (boot_cpu_type()) {
+ case CPU_LOONGSON2:
+ protected_loongson2_blast_icache_range(start, end);
+ break;
+
+ default:
+ protected_blast_icache_range(start, end);
+ break;
+ }
+ }
+#ifdef CONFIG_EVA
+ /*
+ * Due to all possible segment mappings, there might cache aliases
+ * caused by the bootloader being in non-EVA mode, and the CPU switching
+ * to EVA during early kernel init. It's best to flush the scache
+ * to avoid having secondary cores fetching stale data and lead to
+ * kernel crashes.
+ */
+ bc_wback_inv(start, (end - start));
+ __sync();
+#endif
}
static inline void local_r4k_flush_icache_range_ipi(void *args)
@@ -594,18 +698,20 @@ static void r4k_flush_icache_range(unsigned long start, unsigned long end)
instruction_hazard();
}
-#ifdef CONFIG_DMA_NONCOHERENT
+#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
{
/* Catch bad driver code */
BUG_ON(size == 0);
+ preempt_disable();
if (cpu_has_inclusive_pcaches) {
if (size >= scache_size)
r4k_blast_scache();
else
blast_scache_range(addr, addr + size);
+ preempt_enable();
__sync();
return;
}
@@ -621,6 +727,7 @@ static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size)
R4600_HIT_CACHEOP_WAR_IMPL;
blast_dcache_range(addr, addr + size);
}
+ preempt_enable();
bc_wback_inv(addr, size);
__sync();
@@ -631,6 +738,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
/* Catch bad driver code */
BUG_ON(size == 0);
+ preempt_disable();
if (cpu_has_inclusive_pcaches) {
if (size >= scache_size)
r4k_blast_scache();
@@ -645,6 +753,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
*/
blast_inv_scache_range(addr, addr + size);
}
+ preempt_enable();
__sync();
return;
}
@@ -655,11 +764,12 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size)
R4600_HIT_CACHEOP_WAR_IMPL;
blast_inv_dcache_range(addr, addr + size);
}
+ preempt_enable();
bc_inv(addr, size);
__sync();
}
-#endif /* CONFIG_DMA_NONCOHERENT */
+#endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */
/*
* While we're protected against bad userland addresses we don't care
@@ -780,28 +890,38 @@ static inline void rm7k_erratum31(void)
static inline void alias_74k_erratum(struct cpuinfo_mips *c)
{
+ unsigned int imp = c->processor_id & PRID_IMP_MASK;
+ unsigned int rev = c->processor_id & PRID_REV_MASK;
+
/*
* Early versions of the 74K do not update the cache tags on a
* vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG
* aliases. In this case it is better to treat the cache as always
* having aliases.
*/
- if ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(2, 4, 0))
- c->dcache.flags |= MIPS_CACHE_VTAG;
- if ((c->processor_id & 0xff) == PRID_REV_ENCODE_332(2, 4, 0))
- write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
- if (((c->processor_id & 0xff00) == PRID_IMP_1074K) &&
- ((c->processor_id & 0xff) <= PRID_REV_ENCODE_332(1, 1, 0))) {
- c->dcache.flags |= MIPS_CACHE_VTAG;
- write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
+ switch (imp) {
+ case PRID_IMP_74K:
+ if (rev <= PRID_REV_ENCODE_332(2, 4, 0))
+ c->dcache.flags |= MIPS_CACHE_VTAG;
+ if (rev == PRID_REV_ENCODE_332(2, 4, 0))
+ write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
+ break;
+ case PRID_IMP_1074K:
+ if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) {
+ c->dcache.flags |= MIPS_CACHE_VTAG;
+ write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND);
+ }
+ break;
+ default:
+ BUG();
}
}
-static char *way_string[] __cpuinitdata = { NULL, "direct mapped", "2-way",
+static char *way_string[] = { NULL, "direct mapped", "2-way",
"3-way", "4-way", "5-way", "6-way", "7-way", "8-way"
};
-static void __cpuinit probe_pcache(void)
+static void probe_pcache(void)
{
struct cpuinfo_mips *c = &current_cpu_data;
unsigned int config = read_c0_config();
@@ -809,7 +929,7 @@ static void __cpuinit probe_pcache(void)
unsigned long config1;
unsigned int lsize;
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_R4600: /* QED style two way caches? */
case CPU_R4700:
case CPU_R5000:
@@ -971,6 +1091,48 @@ static void __cpuinit probe_pcache(void)
c->dcache.waybit = 0;
break;
+ case CPU_LOONGSON3:
+ config1 = read_c0_config1();
+ lsize = (config1 >> 19) & 7;
+ if (lsize)
+ c->icache.linesz = 2 << lsize;
+ else
+ c->icache.linesz = 0;
+ c->icache.sets = 64 << ((config1 >> 22) & 7);
+ c->icache.ways = 1 + ((config1 >> 16) & 7);
+ icache_size = c->icache.sets *
+ c->icache.ways *
+ c->icache.linesz;
+ c->icache.waybit = 0;
+
+ lsize = (config1 >> 10) & 7;
+ if (lsize)
+ c->dcache.linesz = 2 << lsize;
+ else
+ c->dcache.linesz = 0;
+ c->dcache.sets = 64 << ((config1 >> 13) & 7);
+ c->dcache.ways = 1 + ((config1 >> 7) & 7);
+ dcache_size = c->dcache.sets *
+ c->dcache.ways *
+ c->dcache.linesz;
+ c->dcache.waybit = 0;
+ break;
+
+ case CPU_CAVIUM_OCTEON3:
+ /* For now lie about the number of ways. */
+ c->icache.linesz = 128;
+ c->icache.sets = 16;
+ c->icache.ways = 8;
+ c->icache.flags |= MIPS_CACHE_VTAG;
+ icache_size = c->icache.sets * c->icache.ways * c->icache.linesz;
+
+ c->dcache.linesz = 128;
+ c->dcache.ways = 8;
+ c->dcache.sets = 8;
+ dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz;
+ c->options |= MIPS_CPU_PREFETCH;
+ break;
+
default:
if (!(config & MIPS_CONF_M))
panic("Don't know how to probe P-caches on this cpu.");
@@ -981,10 +1143,14 @@ static void __cpuinit probe_pcache(void)
*/
config1 = read_c0_config1();
- if ((lsize = ((config1 >> 19) & 7)))
- c->icache.linesz = 2 << lsize;
- else
- c->icache.linesz = lsize;
+ lsize = (config1 >> 19) & 7;
+
+ /* IL == 7 is reserved */
+ if (lsize == 7)
+ panic("Invalid icache line size");
+
+ c->icache.linesz = lsize ? 2 << lsize : 0;
+
c->icache.sets = 32 << (((config1 >> 22) + 1) & 7);
c->icache.ways = 1 + ((config1 >> 16) & 7);
@@ -1001,10 +1167,14 @@ static void __cpuinit probe_pcache(void)
*/
c->dcache.flags = 0;
- if ((lsize = ((config1 >> 10) & 7)))
- c->dcache.linesz = 2 << lsize;
- else
- c->dcache.linesz= lsize;
+ lsize = (config1 >> 10) & 7;
+
+ /* DL == 7 is reserved */
+ if (lsize == 7)
+ panic("Invalid dcache line size");
+
+ c->dcache.linesz = lsize ? 2 << lsize : 0;
+
c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7);
c->dcache.ways = 1 + ((config1 >> 7) & 7);
@@ -1025,7 +1195,8 @@ static void __cpuinit probe_pcache(void)
* presumably no vendor is shipping his hardware in the "bad"
* configuration.
*/
- if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 &&
+ if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 &&
+ (prid & PRID_REV_MASK) < PRID_REV_R4400 &&
!(config & CONF_SC) && c->icache.linesz != 16 &&
PAGE_SIZE <= 0x8000)
panic("Improper R4000SC processor configuration detected");
@@ -1045,7 +1216,7 @@ static void __cpuinit probe_pcache(void)
* normally they'd suffer from aliases but magic in the hardware deals
* with that for us so we don't need to take care ourselves.
*/
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_20KC:
case CPU_25KF:
case CPU_SB1:
@@ -1065,11 +1236,21 @@ static void __cpuinit probe_pcache(void)
case CPU_34K:
case CPU_74K:
case CPU_1004K:
- if (c->cputype == CPU_74K)
+ case CPU_1074K:
+ case CPU_INTERAPTIV:
+ case CPU_P5600:
+ case CPU_PROAPTIV:
+ case CPU_M5150:
+ if ((c->cputype == CPU_74K) || (c->cputype == CPU_1074K))
alias_74k_erratum(c);
- if ((read_c0_config7() & (1 << 16))) {
- /* effectively physically indexed dcache,
- thus no virtual aliases. */
+ if (!(read_c0_config7() & MIPS_CONF7_IAR) &&
+ (c->icache.waysize > PAGE_SIZE))
+ c->icache.flags |= MIPS_CACHE_ALIASES;
+ if (read_c0_config7() & MIPS_CONF7_AR) {
+ /*
+ * Effectively physically indexed dcache,
+ * thus no virtual aliases.
+ */
c->dcache.flags |= MIPS_CACHE_PINDEX;
break;
}
@@ -1078,7 +1259,7 @@ static void __cpuinit probe_pcache(void)
c->dcache.flags |= MIPS_CACHE_ALIASES;
}
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_20KC:
/*
* Some older 20Kc chips doesn't have the 'VI' bit in
@@ -1090,15 +1271,14 @@ static void __cpuinit probe_pcache(void)
case CPU_ALCHEMY:
c->icache.flags |= MIPS_CACHE_IC_F_DC;
break;
- }
-#ifdef CONFIG_CPU_LOONGSON2
- /*
- * LOONGSON2 has 4 way icache, but when using indexed cache op,
- * one op will act on all 4 ways
- */
- c->icache.ways = 1;
-#endif
+ case CPU_LOONGSON2:
+ /*
+ * LOONGSON2 has 4 way icache, but when using indexed cache op,
+ * one op will act on all 4 ways
+ */
+ c->icache.ways = 1;
+ }
printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n",
icache_size >> 10,
@@ -1119,7 +1299,7 @@ static void __cpuinit probe_pcache(void)
* executes in KSEG1 space or else you will crash and burn badly. You have
* been warned.
*/
-static int __cpuinit probe_scache(void)
+static int probe_scache(void)
{
unsigned long flags, addr, begin, end, pow2;
unsigned int config = read_c0_config();
@@ -1174,7 +1354,6 @@ static int __cpuinit probe_scache(void)
return 1;
}
-#if defined(CONFIG_CPU_LOONGSON2)
static void __init loongson2_sc_init(void)
{
struct cpuinfo_mips *c = &current_cpu_data;
@@ -1190,13 +1369,39 @@ static void __init loongson2_sc_init(void)
c->options |= MIPS_CPU_INCLUSIVE_CACHES;
}
-#endif
+
+static void __init loongson3_sc_init(void)
+{
+ struct cpuinfo_mips *c = &current_cpu_data;
+ unsigned int config2, lsize;
+
+ config2 = read_c0_config2();
+ lsize = (config2 >> 4) & 15;
+ if (lsize)
+ c->scache.linesz = 2 << lsize;
+ else
+ c->scache.linesz = 0;
+ c->scache.sets = 64 << ((config2 >> 8) & 15);
+ c->scache.ways = 1 + (config2 & 15);
+
+ scache_size = c->scache.sets *
+ c->scache.ways *
+ c->scache.linesz;
+ /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */
+ scache_size *= 4;
+ c->scache.waybit = 0;
+ pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n",
+ scache_size >> 10, way_string[c->scache.ways], c->scache.linesz);
+ if (scache_size)
+ c->options |= MIPS_CPU_INCLUSIVE_CACHES;
+ return;
+}
extern int r5k_sc_init(void);
extern int rm7k_sc_init(void);
extern int mips_sc_init(void);
-static void __cpuinit setup_scache(void)
+static void setup_scache(void)
{
struct cpuinfo_mips *c = &current_cpu_data;
unsigned int config = read_c0_config();
@@ -1207,7 +1412,7 @@ static void __cpuinit setup_scache(void)
* processors don't have a S-cache that would be relevant to the
* Linux memory management.
*/
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_R4000SC:
case CPU_R4000MC:
case CPU_R4400SC:
@@ -1240,11 +1445,15 @@ static void __cpuinit setup_scache(void)
#endif
return;
-#if defined(CONFIG_CPU_LOONGSON2)
case CPU_LOONGSON2:
loongson2_sc_init();
return;
-#endif
+
+ case CPU_LOONGSON3:
+ loongson3_sc_init();
+ return;
+
+ case CPU_CAVIUM_OCTEON3:
case CPU_XLP:
/* don't need to worry about L2, fully coherent */
return;
@@ -1329,7 +1538,7 @@ static void nxp_pr4450_fixup_config(void)
NXP_BARRIER();
}
-static int __cpuinitdata cca = -1;
+static int cca = -1;
static int __init cca_setup(char *str)
{
@@ -1340,7 +1549,7 @@ static int __init cca_setup(char *str)
early_param("cca", cca_setup);
-static void __cpuinit coherency_setup(void)
+static void coherency_setup(void)
{
if (cca < 0 || cca > 7)
cca = read_c0_config() & CONF_CM_CMASK;
@@ -1380,13 +1589,12 @@ static void __cpuinit coherency_setup(void)
}
}
-static void __cpuinit r4k_cache_error_setup(void)
+static void r4k_cache_error_setup(void)
{
extern char __weak except_vec2_generic;
extern char __weak except_vec2_sb1;
- struct cpuinfo_mips *c = &current_cpu_data;
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_SB1:
case CPU_SB1A:
set_uncached_handler(0x100, &except_vec2_sb1, 0x80);
@@ -1398,7 +1606,7 @@ static void __cpuinit r4k_cache_error_setup(void)
}
}
-void __cpuinit r4k_cache_init(void)
+void r4k_cache_init(void)
{
extern void build_clear_page(void);
extern void build_copy_page(void);
@@ -1416,6 +1624,10 @@ void __cpuinit r4k_cache_init(void)
r4k_blast_scache_page_setup();
r4k_blast_scache_page_indexed_setup();
r4k_blast_scache_setup();
+#ifdef CONFIG_EVA
+ r4k_blast_dcache_user_page_setup();
+ r4k_blast_icache_user_page_setup();
+#endif
/*
* Some MIPS32 and MIPS64 processors have physically indexed caches.
@@ -1447,7 +1659,7 @@ void __cpuinit r4k_cache_init(void)
flush_icache_range = r4k_flush_icache_range;
local_flush_icache_range = local_r4k_flush_icache_range;
-#if defined(CONFIG_DMA_NONCOHERENT)
+#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
if (coherentio) {
_dma_cache_wback_inv = (void *)cache_noop;
_dma_cache_wback = (void *)cache_noop;
@@ -1472,3 +1684,26 @@ void __cpuinit r4k_cache_init(void)
coherency_setup();
board_cache_error_setup = r4k_cache_error_setup;
}
+
+static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd,
+ void *v)
+{
+ switch (cmd) {
+ case CPU_PM_ENTER_FAILED:
+ case CPU_PM_EXIT:
+ coherency_setup();
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block r4k_cache_pm_notifier_block = {
+ .notifier_call = r4k_cache_pm_notifier,
+};
+
+int __init r4k_cache_init_pm(void)
+{
+ return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block);
+}
+arch_initcall(r4k_cache_init_pm);
diff --git a/arch/mips/mm/c-tx39.c b/arch/mips/mm/c-tx39.c
index ba9da270289..8d909dbbf37 100644
--- a/arch/mips/mm/c-tx39.c
+++ b/arch/mips/mm/c-tx39.c
@@ -344,7 +344,7 @@ static __init void tx39_probe_cache(void)
}
}
-void __cpuinit tx39_cache_init(void)
+void tx39_cache_init(void)
{
extern void build_clear_page(void);
extern void build_copy_page(void);
diff --git a/arch/mips/mm/cache.c b/arch/mips/mm/cache.c
index 5aeb3eb0b72..f7b91d3a371 100644
--- a/arch/mips/mm/cache.c
+++ b/arch/mips/mm/cache.c
@@ -8,7 +8,6 @@
*/
#include <linux/fs.h>
#include <linux/fcntl.h>
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/linkage.h>
#include <linux/module.h>
@@ -30,15 +29,16 @@ void (*flush_cache_range)(struct vm_area_struct *vma, unsigned long start,
void (*flush_cache_page)(struct vm_area_struct *vma, unsigned long page,
unsigned long pfn);
void (*flush_icache_range)(unsigned long start, unsigned long end);
+EXPORT_SYMBOL_GPL(flush_icache_range);
void (*local_flush_icache_range)(unsigned long start, unsigned long end);
+EXPORT_SYMBOL_GPL(local_flush_icache_range);
void (*__flush_cache_vmap)(void);
void (*__flush_cache_vunmap)(void);
void (*__flush_kernel_vmap_range)(unsigned long vaddr, int size);
-void (*__invalidate_kernel_vmap_range)(unsigned long vaddr, int size);
-
EXPORT_SYMBOL_GPL(__flush_kernel_vmap_range);
+void (*__invalidate_kernel_vmap_range)(unsigned long vaddr, int size);
/* MIPS specific cache operations */
void (*flush_cache_sigtramp)(unsigned long addr);
@@ -50,7 +50,7 @@ EXPORT_SYMBOL_GPL(local_flush_data_cache_page);
EXPORT_SYMBOL(flush_data_cache_page);
EXPORT_SYMBOL(flush_icache_all);
-#ifdef CONFIG_DMA_NONCOHERENT
+#if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT)
/* DMA cache operations. */
void (*_dma_cache_wback_inv)(unsigned long start, unsigned long size);
@@ -59,7 +59,7 @@ void (*_dma_cache_inv)(unsigned long start, unsigned long size);
EXPORT_SYMBOL(_dma_cache_wback_inv);
-#endif /* CONFIG_DMA_NONCOHERENT */
+#endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */
/*
* We could optimize the case where the cache argument is not BCACHE but
@@ -182,7 +182,7 @@ static inline void setup_protection_map(void)
}
}
-void __cpuinit cpu_cache_init(void)
+void cpu_cache_init(void)
{
if (cpu_has_3k_cache) {
extern void __weak r3k_cache_init(void);
diff --git a/arch/mips/mm/cex-sb1.S b/arch/mips/mm/cex-sb1.S
index fe1d887e8d7..5d5f29681a2 100644
--- a/arch/mips/mm/cex-sb1.S
+++ b/arch/mips/mm/cex-sb1.S
@@ -15,7 +15,6 @@
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
-#include <linux/init.h>
#include <asm/asm.h>
#include <asm/regdef.h>
@@ -49,8 +48,6 @@
* (0x170-0x17f) are used to preserve k0, k1, and ra.
*/
- __CPUINIT
-
LEAF(except_vec2_sb1)
/*
* If this error is recoverable, we need to exit the handler
@@ -142,8 +139,6 @@ unrecoverable:
END(except_vec2_sb1)
- __FINIT
-
LEAF(handle_vec2_sb1)
mfc0 k0,CP0_CONFIG
li k1,~CONF_CM_CMASK
diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c
index aaccf1c1069..44b6dff5aba 100644
--- a/arch/mips/mm/dma-default.c
+++ b/arch/mips/mm/dma-default.c
@@ -18,10 +18,12 @@
#include <linux/highmem.h>
#include <asm/cache.h>
+#include <asm/cpu-type.h>
#include <asm/io.h>
#include <dma-coherence.h>
+#ifdef CONFIG_DMA_MAYBE_COHERENT
int coherentio = 0; /* User defined DMA coherency from command line. */
EXPORT_SYMBOL_GPL(coherentio);
int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
@@ -41,6 +43,7 @@ static int __init setnocoherentio(char *str)
return 0;
}
early_param("nocoherentio", setnocoherentio);
+#endif
static inline struct page *dma_addr_to_page(struct device *dev,
dma_addr_t dma_addr)
@@ -50,16 +53,20 @@ static inline struct page *dma_addr_to_page(struct device *dev,
}
/*
+ * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
+ * speculatively fill random cachelines with stale data at any time,
+ * requiring an extra flush post-DMA.
+ *
* Warning on the terminology - Linux calls an uncached area coherent;
* MIPS terminology calls memory areas with hardware maintained coherency
* coherent.
*/
-
-static inline int cpu_is_noncoherent_r10000(struct device *dev)
+static inline int cpu_needs_post_dma_flush(struct device *dev)
{
return !plat_device_is_coherent(dev) &&
- (current_cpu_type() == CPU_R10000 ||
- current_cpu_type() == CPU_R12000);
+ (boot_cpu_type() == CPU_R10000 ||
+ boot_cpu_type() == CPU_R12000 ||
+ boot_cpu_type() == CPU_BMIPS5000);
}
static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
@@ -230,7 +237,7 @@ static inline void __dma_sync(struct page *page,
static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
{
- if (cpu_is_noncoherent_r10000(dev))
+ if (cpu_needs_post_dma_flush(dev))
__dma_sync(dma_addr_to_page(dev, dma_addr),
dma_addr & ~PAGE_MASK, size, direction);
@@ -284,7 +291,7 @@ static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
static void mips_dma_sync_single_for_cpu(struct device *dev,
dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
- if (cpu_is_noncoherent_r10000(dev))
+ if (cpu_needs_post_dma_flush(dev))
__dma_sync(dma_addr_to_page(dev, dma_handle),
dma_handle & ~PAGE_MASK, size, direction);
}
@@ -292,7 +299,6 @@ static void mips_dma_sync_single_for_cpu(struct device *dev,
static void mips_dma_sync_single_for_device(struct device *dev,
dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
- plat_extra_sync_for_device(dev);
if (!plat_device_is_coherent(dev))
__dma_sync(dma_addr_to_page(dev, dma_handle),
dma_handle & ~PAGE_MASK, size, direction);
@@ -303,12 +309,10 @@ static void mips_dma_sync_sg_for_cpu(struct device *dev,
{
int i;
- /* Make sure that gcc doesn't leave the empty loop body. */
- for (i = 0; i < nelems; i++, sg++) {
- if (cpu_is_noncoherent_r10000(dev))
+ if (cpu_needs_post_dma_flush(dev))
+ for (i = 0; i < nelems; i++, sg++)
__dma_sync(sg_page(sg), sg->offset, sg->length,
direction);
- }
}
static void mips_dma_sync_sg_for_device(struct device *dev,
@@ -316,17 +320,15 @@ static void mips_dma_sync_sg_for_device(struct device *dev,
{
int i;
- /* Make sure that gcc doesn't leave the empty loop body. */
- for (i = 0; i < nelems; i++, sg++) {
- if (!plat_device_is_coherent(dev))
+ if (!plat_device_is_coherent(dev))
+ for (i = 0; i < nelems; i++, sg++)
__dma_sync(sg_page(sg), sg->offset, sg->length,
direction);
- }
}
int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
- return plat_dma_mapping_error(dev, dma_addr);
+ return 0;
}
int mips_dma_supported(struct device *dev, u64 mask)
@@ -339,7 +341,6 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
{
BUG_ON(direction == DMA_NONE);
- plat_extra_sync_for_device(dev);
if (!plat_device_is_coherent(dev))
__dma_sync_virtual(vaddr, size, direction);
}
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 85df1cd8d44..becc42bb184 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -42,8 +42,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
const int field = sizeof(unsigned long) * 2;
siginfo_t info;
int fault;
- unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
- (write ? FAULT_FLAG_WRITE : 0);
+ unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
#if 0
printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", raw_smp_processor_id(),
@@ -93,6 +92,8 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
if (in_atomic() || !mm)
goto bad_area_nosemaphore;
+ if (user_mode(regs))
+ flags |= FAULT_FLAG_USER;
retry:
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
@@ -114,6 +115,7 @@ good_area:
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
+ flags |= FAULT_FLAG_WRITE;
} else {
if (cpu_has_rixi) {
if (address == regs->cp0_epc && !(vma->vm_flags & VM_EXEC)) {
@@ -241,6 +243,8 @@ out_of_memory:
* (which will retry the fault, or kill us if we got oom-killed).
*/
up_read(&mm->mmap_sem);
+ if (!user_mode(regs))
+ goto no_context;
pagefault_out_of_memory();
return;
diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c
index d4ea5c9c4a9..06ce17c2a90 100644
--- a/arch/mips/mm/gup.c
+++ b/arch/mips/mm/gup.c
@@ -12,6 +12,7 @@
#include <linux/swap.h>
#include <linux/hugetlb.h>
+#include <asm/cpu-features.h>
#include <asm/pgtable.h>
static inline pte_t gup_get_pte(pte_t *ptep)
@@ -273,7 +274,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
len = (unsigned long) nr_pages << PAGE_SHIFT;
end = start + len;
- if (end < start)
+ if (end < start || cpu_has_dc_aliases)
goto slow_irqon;
/* XXX: batch / limit 'nr' */
diff --git a/arch/mips/mm/hugetlbpage.c b/arch/mips/mm/hugetlbpage.c
index a7fee0dfb7a..4ec8ee10d37 100644
--- a/arch/mips/mm/hugetlbpage.c
+++ b/arch/mips/mm/hugetlbpage.c
@@ -11,7 +11,6 @@
* Copyright (C) 2008, 2009 Cavium Networks, Inc.
*/
-#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 4e73f10a751..6e4413330e3 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -44,27 +44,6 @@
#include <asm/tlb.h>
#include <asm/fixmap.h>
-/* Atomicity and interruptability */
-#ifdef CONFIG_MIPS_MT_SMTC
-
-#include <asm/mipsmtregs.h>
-
-#define ENTER_CRITICAL(flags) \
- { \
- unsigned int mvpflags; \
- local_irq_save(flags);\
- mvpflags = dvpe()
-#define EXIT_CRITICAL(flags) \
- evpe(mvpflags); \
- local_irq_restore(flags); \
- }
-#else
-
-#define ENTER_CRITICAL(flags) local_irq_save(flags)
-#define EXIT_CRITICAL(flags) local_irq_restore(flags)
-
-#endif /* CONFIG_MIPS_MT_SMTC */
-
/*
* We have up to 8 empty zeroed pages so we can map one of the right colour
* when needed. This is necessary only on R4000 / R4400 SC and MC versions
@@ -100,21 +79,7 @@ void setup_zero_pages(void)
zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
}
-#ifdef CONFIG_MIPS_MT_SMTC
-static pte_t *kmap_coherent_pte;
-static void __init kmap_coherent_init(void)
-{
- unsigned long vaddr;
-
- /* cache the first coherent kmap pte */
- vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
- kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
-}
-#else
-static inline void kmap_coherent_init(void) {}
-#endif
-
-void *kmap_coherent(struct page *page, unsigned long addr)
+static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
{
enum fixed_addresses idx;
unsigned long vaddr, flags, entrylo;
@@ -124,62 +89,50 @@ void *kmap_coherent(struct page *page, unsigned long addr)
BUG_ON(Page_dcache_dirty(page));
- inc_preempt_count();
+ pagefault_disable();
idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
-#ifdef CONFIG_MIPS_MT_SMTC
- idx += FIX_N_COLOURS * smp_processor_id() +
- (in_interrupt() ? (FIX_N_COLOURS * NR_CPUS) : 0);
-#else
idx += in_interrupt() ? FIX_N_COLOURS : 0;
-#endif
vaddr = __fix_to_virt(FIX_CMAP_END - idx);
- pte = mk_pte(page, PAGE_KERNEL);
+ pte = mk_pte(page, prot);
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
entrylo = pte.pte_high;
#else
entrylo = pte_to_entrylo(pte_val(pte));
#endif
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
old_ctx = read_c0_entryhi();
write_c0_entryhi(vaddr & (PAGE_MASK << 1));
write_c0_entrylo0(entrylo);
write_c0_entrylo1(entrylo);
-#ifdef CONFIG_MIPS_MT_SMTC
- set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
- /* preload TLB instead of local_flush_tlb_one() */
- mtc0_tlbw_hazard();
- tlb_probe();
- tlb_probe_hazard();
- tlbidx = read_c0_index();
- mtc0_tlbw_hazard();
- if (tlbidx < 0)
- tlb_write_random();
- else
- tlb_write_indexed();
-#else
tlbidx = read_c0_wired();
write_c0_wired(tlbidx + 1);
write_c0_index(tlbidx);
mtc0_tlbw_hazard();
tlb_write_indexed();
-#endif
tlbw_use_hazard();
write_c0_entryhi(old_ctx);
- EXIT_CRITICAL(flags);
+ local_irq_restore(flags);
return (void*) vaddr;
}
-#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
+void *kmap_coherent(struct page *page, unsigned long addr)
+{
+ return __kmap_pgprot(page, addr, PAGE_KERNEL);
+}
+
+void *kmap_noncoherent(struct page *page, unsigned long addr)
+{
+ return __kmap_pgprot(page, addr, PAGE_KERNEL_NC);
+}
void kunmap_coherent(void)
{
-#ifndef CONFIG_MIPS_MT_SMTC
unsigned int wired;
unsigned long flags, old_ctx;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
old_ctx = read_c0_entryhi();
wired = read_c0_wired() - 1;
write_c0_wired(wired);
@@ -191,10 +144,8 @@ void kunmap_coherent(void)
tlb_write_indexed();
tlbw_use_hazard();
write_c0_entryhi(old_ctx);
- EXIT_CRITICAL(flags);
-#endif
- dec_preempt_count();
- preempt_check_resched();
+ local_irq_restore(flags);
+ pagefault_enable();
}
void copy_user_highpage(struct page *to, struct page *from,
@@ -254,11 +205,12 @@ void copy_from_user_page(struct vm_area_struct *vma,
SetPageDcacheDirty(page);
}
}
+EXPORT_SYMBOL_GPL(copy_from_user_page);
void __init fixrange_init(unsigned long start, unsigned long end,
pgd_t *pgd_base)
{
-#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
+#ifdef CONFIG_HIGHMEM
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
@@ -329,8 +281,6 @@ void __init paging_init(void)
#ifdef CONFIG_HIGHMEM
kmap_init();
#endif
- kmap_coherent_init();
-
#ifdef CONFIG_ZONE_DMA
max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
#endif
@@ -424,10 +374,20 @@ void free_initrd_mem(unsigned long start, unsigned long end)
}
#endif
+void (*free_init_pages_eva)(void *begin, void *end) = NULL;
+
void __init_refok free_initmem(void)
{
prom_free_prom_memory();
- free_initmem_default(POISON_FREE_INITMEM);
+ /*
+ * Let the platform define a specific function to free the
+ * init section since EVA may have used any possible mapping
+ * between virtual and physical addresses.
+ */
+ if (free_init_pages_eva)
+ free_init_pages_eva((void *)&__init_begin, (void *)&__init_end);
+ else
+ free_initmem_default(POISON_FREE_INITMEM);
}
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index 2c0bd580b9d..b611102e23b 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -8,7 +8,6 @@
* Copyright (C) 2008 Thiemo Seufer
* Copyright (C) 2012 MIPS Technologies, Inc.
*/
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
@@ -18,6 +17,7 @@
#include <asm/bugs.h>
#include <asm/cacheops.h>
+#include <asm/cpu-type.h>
#include <asm/inst.h>
#include <asm/io.h>
#include <asm/page.h>
@@ -66,29 +66,29 @@ UASM_L_LA(_copy_pref_both)
UASM_L_LA(_copy_pref_store)
/* We need one branch and therefore one relocation per target label. */
-static struct uasm_label __cpuinitdata labels[5];
-static struct uasm_reloc __cpuinitdata relocs[5];
+static struct uasm_label labels[5];
+static struct uasm_reloc relocs[5];
#define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010)
#define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020)
-static int pref_bias_clear_store __cpuinitdata;
-static int pref_bias_copy_load __cpuinitdata;
-static int pref_bias_copy_store __cpuinitdata;
+static int pref_bias_clear_store;
+static int pref_bias_copy_load;
+static int pref_bias_copy_store;
-static u32 pref_src_mode __cpuinitdata;
-static u32 pref_dst_mode __cpuinitdata;
+static u32 pref_src_mode;
+static u32 pref_dst_mode;
-static int clear_word_size __cpuinitdata;
-static int copy_word_size __cpuinitdata;
+static int clear_word_size;
+static int copy_word_size;
-static int half_clear_loop_size __cpuinitdata;
-static int half_copy_loop_size __cpuinitdata;
+static int half_clear_loop_size;
+static int half_copy_loop_size;
-static int cache_line_size __cpuinitdata;
+static int cache_line_size;
#define cache_line_mask() (cache_line_size - 1)
-static inline void __cpuinit
+static inline void
pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
{
if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
@@ -108,7 +108,7 @@ pg_addiu(u32 **buf, unsigned int reg1, unsigned int reg2, unsigned int off)
}
}
-static void __cpuinit set_prefetch_parameters(void)
+static void set_prefetch_parameters(void)
{
if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg)
clear_word_size = 8;
@@ -199,7 +199,7 @@ static void __cpuinit set_prefetch_parameters(void)
4 * copy_word_size));
}
-static void __cpuinit build_clear_store(u32 **buf, int off)
+static void build_clear_store(u32 **buf, int off)
{
if (cpu_has_64bit_gp_regs || cpu_has_64bit_zero_reg) {
uasm_i_sd(buf, ZERO, off, A0);
@@ -208,7 +208,7 @@ static void __cpuinit build_clear_store(u32 **buf, int off)
}
}
-static inline void __cpuinit build_clear_pref(u32 **buf, int off)
+static inline void build_clear_pref(u32 **buf, int off)
{
if (off & cache_line_mask())
return;
@@ -240,7 +240,7 @@ extern u32 __clear_page_end;
extern u32 __copy_page_start;
extern u32 __copy_page_end;
-void __cpuinit build_clear_page(void)
+void build_clear_page(void)
{
int off;
u32 *buf = &__clear_page_start;
@@ -273,7 +273,7 @@ void __cpuinit build_clear_page(void)
uasm_i_ori(&buf, A2, A0, off);
if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
- uasm_i_lui(&buf, AT, 0xa000);
+ uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
off = cache_line_size ? min(8, pref_bias_clear_store / cache_line_size)
* cache_line_size : 0;
@@ -333,7 +333,7 @@ void __cpuinit build_clear_page(void)
pr_debug("\t.set pop\n");
}
-static void __cpuinit build_copy_load(u32 **buf, int reg, int off)
+static void build_copy_load(u32 **buf, int reg, int off)
{
if (cpu_has_64bit_gp_regs) {
uasm_i_ld(buf, reg, off, A1);
@@ -342,7 +342,7 @@ static void __cpuinit build_copy_load(u32 **buf, int reg, int off)
}
}
-static void __cpuinit build_copy_store(u32 **buf, int reg, int off)
+static void build_copy_store(u32 **buf, int reg, int off)
{
if (cpu_has_64bit_gp_regs) {
uasm_i_sd(buf, reg, off, A0);
@@ -387,7 +387,7 @@ static inline void build_copy_store_pref(u32 **buf, int off)
}
}
-void __cpuinit build_copy_page(void)
+void build_copy_page(void)
{
int off;
u32 *buf = &__copy_page_start;
@@ -424,7 +424,7 @@ void __cpuinit build_copy_page(void)
uasm_i_ori(&buf, A2, A0, off);
if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
- uasm_i_lui(&buf, AT, 0xa000);
+ uasm_i_lui(&buf, AT, uasm_rel_hi(0xa0000000));
off = cache_line_size ? min(8, pref_bias_copy_load / cache_line_size) *
cache_line_size : 0;
diff --git a/arch/mips/mm/sc-ip22.c b/arch/mips/mm/sc-ip22.c
index c6aaed934d5..dc7c5a5214a 100644
--- a/arch/mips/mm/sc-ip22.c
+++ b/arch/mips/mm/sc-ip22.c
@@ -167,7 +167,7 @@ static struct bcache_ops indy_sc_ops = {
.bc_inv = indy_sc_wback_invalidate
};
-void __cpuinit indy_sc_init(void)
+void indy_sc_init(void)
{
if (indy_sc_probe()) {
indy_sc_enable();
diff --git a/arch/mips/mm/sc-mips.c b/arch/mips/mm/sc-mips.c
index df96da7e939..99eb8fabab6 100644
--- a/arch/mips/mm/sc-mips.c
+++ b/arch/mips/mm/sc-mips.c
@@ -6,6 +6,7 @@
#include <linux/sched.h>
#include <linux/mm.h>
+#include <asm/cpu-type.h>
#include <asm/mipsregs.h>
#include <asm/bcache.h>
#include <asm/cacheops.h>
@@ -71,10 +72,14 @@ static inline int mips_sc_is_activated(struct cpuinfo_mips *c)
unsigned int tmp;
/* Check the bypass bit (L2B) */
- switch (c->cputype) {
+ switch (current_cpu_type()) {
case CPU_34K:
case CPU_74K:
case CPU_1004K:
+ case CPU_1074K:
+ case CPU_INTERAPTIV:
+ case CPU_PROAPTIV:
+ case CPU_P5600:
case CPU_BMIPS5000:
if (config2 & (1 << 12))
return 0;
@@ -132,7 +137,7 @@ static inline int __init mips_sc_probe(void)
return 1;
}
-int __cpuinit mips_sc_init(void)
+int mips_sc_init(void)
{
int found = mips_sc_probe();
if (found) {
diff --git a/arch/mips/mm/sc-r5k.c b/arch/mips/mm/sc-r5k.c
index 8bc67720e14..0216ed6eaa2 100644
--- a/arch/mips/mm/sc-r5k.c
+++ b/arch/mips/mm/sc-r5k.c
@@ -98,7 +98,7 @@ static struct bcache_ops r5k_sc_ops = {
.bc_inv = r5k_dma_cache_inv_sc
};
-void __cpuinit r5k_sc_init(void)
+void r5k_sc_init(void)
{
if (r5k_sc_probe()) {
r5k_sc_enable();
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
index 274af3be144..9ac1efcfbcc 100644
--- a/arch/mips/mm/sc-rm7k.c
+++ b/arch/mips/mm/sc-rm7k.c
@@ -6,7 +6,6 @@
#undef DEBUG
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/bitops.h>
@@ -104,7 +103,7 @@ static void blast_rm7k_tcache(void)
/*
* This function is executed in uncached address space.
*/
-static __cpuinit void __rm7k_tc_enable(void)
+static void __rm7k_tc_enable(void)
{
int i;
@@ -117,7 +116,7 @@ static __cpuinit void __rm7k_tc_enable(void)
cache_op(Index_Store_Tag_T, CKSEG0ADDR(i));
}
-static __cpuinit void rm7k_tc_enable(void)
+static void rm7k_tc_enable(void)
{
if (read_c0_config() & RM7K_CONF_TE)
return;
@@ -130,7 +129,7 @@ static __cpuinit void rm7k_tc_enable(void)
/*
* This function is executed in uncached address space.
*/
-static __cpuinit void __rm7k_sc_enable(void)
+static void __rm7k_sc_enable(void)
{
int i;
@@ -143,7 +142,7 @@ static __cpuinit void __rm7k_sc_enable(void)
cache_op(Index_Store_Tag_SD, CKSEG0ADDR(i));
}
-static __cpuinit void rm7k_sc_enable(void)
+static void rm7k_sc_enable(void)
{
if (read_c0_config() & RM7K_CONF_SE)
return;
@@ -184,7 +183,7 @@ static struct bcache_ops rm7k_sc_ops = {
* This is a probing function like the one found in c-r4k.c, we look for the
* wrap around point with different addresses.
*/
-static __cpuinit void __probe_tcache(void)
+static void __probe_tcache(void)
{
unsigned long flags, addr, begin, end, pow2;
@@ -226,7 +225,7 @@ static __cpuinit void __probe_tcache(void)
local_irq_restore(flags);
}
-void __cpuinit rm7k_sc_init(void)
+void rm7k_sc_init(void)
{
struct cpuinfo_mips *c = &current_cpu_data;
unsigned int config = read_c0_config();
diff --git a/arch/mips/mm/tlb-funcs.S b/arch/mips/mm/tlb-funcs.S
index 30a494db99c..a5427c6e975 100644
--- a/arch/mips/mm/tlb-funcs.S
+++ b/arch/mips/mm/tlb-funcs.S
@@ -16,8 +16,10 @@
#define FASTPATH_SIZE 128
+EXPORT(tlbmiss_handler_setup_pgd_start)
LEAF(tlbmiss_handler_setup_pgd)
- .space 16 * 4
+1: j 1b /* Dummy, will be replaced. */
+ .space 64
END(tlbmiss_handler_setup_pgd)
EXPORT(tlbmiss_handler_setup_pgd_end)
diff --git a/arch/mips/mm/tlb-r3k.c b/arch/mips/mm/tlb-r3k.c
index a63d1ed0827..d657493ef56 100644
--- a/arch/mips/mm/tlb-r3k.c
+++ b/arch/mips/mm/tlb-r3k.c
@@ -10,7 +10,6 @@
* Copyright (C) 2002 Ralf Baechle
* Copyright (C) 2002 Maciej W. Rozycki
*/
-#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
@@ -276,7 +275,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
}
}
-void __cpuinit tlb_init(void)
+void tlb_init(void)
{
local_flush_tlb_all();
diff --git a/arch/mips/mm/tlb-r4k.c b/arch/mips/mm/tlb-r4k.c
index c643de4c473..3914e27456f 100644
--- a/arch/mips/mm/tlb-r4k.c
+++ b/arch/mips/mm/tlb-r4k.c
@@ -8,6 +8,7 @@
* Carsten Langgaard, carstenl@mips.com
* Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
*/
+#include <linux/cpu_pm.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/smp.h>
@@ -16,64 +17,44 @@
#include <linux/module.h>
#include <asm/cpu.h>
+#include <asm/cpu-type.h>
#include <asm/bootinfo.h>
#include <asm/mmu_context.h>
#include <asm/pgtable.h>
+#include <asm/tlb.h>
#include <asm/tlbmisc.h>
extern void build_tlb_refill_handler(void);
/*
- * Make sure all entries differ. If they're not different
- * MIPS32 will take revenge ...
+ * LOONGSON2/3 has a 4 entry itlb which is a subset of dtlb,
+ * unfortunately, itlb is not totally transparent to software.
*/
-#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
-
-/* Atomicity and interruptability */
-#ifdef CONFIG_MIPS_MT_SMTC
-
-#include <asm/smtc.h>
-#include <asm/mipsmtregs.h>
-
-#define ENTER_CRITICAL(flags) \
- { \
- unsigned int mvpflags; \
- local_irq_save(flags);\
- mvpflags = dvpe()
-#define EXIT_CRITICAL(flags) \
- evpe(mvpflags); \
- local_irq_restore(flags); \
+static inline void flush_itlb(void)
+{
+ switch (current_cpu_type()) {
+ case CPU_LOONGSON2:
+ case CPU_LOONGSON3:
+ write_c0_diag(4);
+ break;
+ default:
+ break;
}
-#else
-
-#define ENTER_CRITICAL(flags) local_irq_save(flags)
-#define EXIT_CRITICAL(flags) local_irq_restore(flags)
-
-#endif /* CONFIG_MIPS_MT_SMTC */
-
-#if defined(CONFIG_CPU_LOONGSON2)
-/*
- * LOONGSON2 has a 4 entry itlb which is a subset of dtlb,
- * unfortrunately, itlb is not totally transparent to software.
- */
-#define FLUSH_ITLB write_c0_diag(4);
-
-#define FLUSH_ITLB_VM(vma) { if ((vma)->vm_flags & VM_EXEC) write_c0_diag(4); }
-
-#else
-
-#define FLUSH_ITLB
-#define FLUSH_ITLB_VM(vma)
+}
-#endif
+static inline void flush_itlb_vm(struct vm_area_struct *vma)
+{
+ if (vma->vm_flags & VM_EXEC)
+ flush_itlb();
+}
void local_flush_tlb_all(void)
{
unsigned long flags;
unsigned long old_ctx;
- int entry;
+ int entry, ftlbhighset;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
/* Save old context and create impossible VPN2 value */
old_ctx = read_c0_entryhi();
write_c0_entrylo0(0);
@@ -82,18 +63,35 @@ void local_flush_tlb_all(void)
entry = read_c0_wired();
/* Blast 'em all away. */
- while (entry < current_cpu_data.tlbsize) {
- /* Make sure all entries differ. */
- write_c0_entryhi(UNIQUE_ENTRYHI(entry));
- write_c0_index(entry);
- mtc0_tlbw_hazard();
- tlb_write_indexed();
- entry++;
+ if (cpu_has_tlbinv) {
+ if (current_cpu_data.tlbsizevtlb) {
+ write_c0_index(0);
+ mtc0_tlbw_hazard();
+ tlbinvf(); /* invalidate VTLB */
+ }
+ ftlbhighset = current_cpu_data.tlbsizevtlb +
+ current_cpu_data.tlbsizeftlbsets;
+ for (entry = current_cpu_data.tlbsizevtlb;
+ entry < ftlbhighset;
+ entry++) {
+ write_c0_index(entry);
+ mtc0_tlbw_hazard();
+ tlbinvf(); /* invalidate one FTLB set */
+ }
+ } else {
+ while (entry < current_cpu_data.tlbsize) {
+ /* Make sure all entries differ. */
+ write_c0_entryhi(UNIQUE_ENTRYHI(entry));
+ write_c0_index(entry);
+ mtc0_tlbw_hazard();
+ tlb_write_indexed();
+ entry++;
+ }
}
tlbw_use_hazard();
write_c0_entryhi(old_ctx);
- FLUSH_ITLB;
- EXIT_CRITICAL(flags);
+ flush_itlb();
+ local_irq_restore(flags);
}
EXPORT_SYMBOL(local_flush_tlb_all);
@@ -123,11 +121,13 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
if (cpu_context(cpu, mm) != 0) {
unsigned long size, flags;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
start = round_down(start, PAGE_SIZE << 1);
end = round_up(end, PAGE_SIZE << 1);
size = (end - start) >> (PAGE_SHIFT + 1);
- if (size <= current_cpu_data.tlbsize/2) {
+ if (size <= (current_cpu_data.tlbsizeftlbsets ?
+ current_cpu_data.tlbsize / 8 :
+ current_cpu_data.tlbsize / 2)) {
int oldpid = read_c0_entryhi();
int newpid = cpu_asid(cpu, mm);
@@ -154,8 +154,8 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
} else {
drop_mmu_context(mm, cpu);
}
- FLUSH_ITLB;
- EXIT_CRITICAL(flags);
+ flush_itlb();
+ local_irq_restore(flags);
}
}
@@ -163,10 +163,12 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
unsigned long size, flags;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
size = (size + 1) >> 1;
- if (size <= current_cpu_data.tlbsize / 2) {
+ if (size <= (current_cpu_data.tlbsizeftlbsets ?
+ current_cpu_data.tlbsize / 8 :
+ current_cpu_data.tlbsize / 2)) {
int pid = read_c0_entryhi();
start &= (PAGE_MASK << 1);
@@ -196,8 +198,8 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
} else {
local_flush_tlb_all();
}
- FLUSH_ITLB;
- EXIT_CRITICAL(flags);
+ flush_itlb();
+ local_irq_restore(flags);
}
void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
@@ -210,7 +212,7 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
newpid = cpu_asid(cpu, vma->vm_mm);
page &= (PAGE_MASK << 1);
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
oldpid = read_c0_entryhi();
write_c0_entryhi(page | newpid);
mtc0_tlbw_hazard();
@@ -229,8 +231,8 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
finish:
write_c0_entryhi(oldpid);
- FLUSH_ITLB_VM(vma);
- EXIT_CRITICAL(flags);
+ flush_itlb_vm(vma);
+ local_irq_restore(flags);
}
}
@@ -243,7 +245,7 @@ void local_flush_tlb_one(unsigned long page)
unsigned long flags;
int oldpid, idx;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
oldpid = read_c0_entryhi();
page &= (PAGE_MASK << 1);
write_c0_entryhi(page);
@@ -261,8 +263,8 @@ void local_flush_tlb_one(unsigned long page)
tlbw_use_hazard();
}
write_c0_entryhi(oldpid);
- FLUSH_ITLB;
- EXIT_CRITICAL(flags);
+ flush_itlb();
+ local_irq_restore(flags);
}
/*
@@ -285,7 +287,7 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
if (current->active_mm != vma->vm_mm)
return;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
pid = read_c0_entryhi() & ASID_MASK;
address &= (PAGE_MASK << 1);
@@ -334,8 +336,8 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
tlb_write_indexed();
}
tlbw_use_hazard();
- FLUSH_ITLB_VM(vma);
- EXIT_CRITICAL(flags);
+ flush_itlb_vm(vma);
+ local_irq_restore(flags);
}
void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
@@ -346,7 +348,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
unsigned long old_pagemask;
unsigned long old_ctx;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
/* Save old context and create impossible VPN2 value */
old_ctx = read_c0_entryhi();
old_pagemask = read_c0_pagemask();
@@ -366,7 +368,7 @@ void add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
tlbw_use_hazard(); /* What is the hazard here? */
write_c0_pagemask(old_pagemask);
local_flush_tlb_all();
- EXIT_CRITICAL(flags);
+ local_irq_restore(flags);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -376,20 +378,20 @@ int __init has_transparent_hugepage(void)
unsigned int mask;
unsigned long flags;
- ENTER_CRITICAL(flags);
+ local_irq_save(flags);
write_c0_pagemask(PM_HUGE_MASK);
back_to_back_c0_hazard();
mask = read_c0_pagemask();
write_c0_pagemask(PM_DEFAULT_MASK);
- EXIT_CRITICAL(flags);
+ local_irq_restore(flags);
return mask == PM_HUGE_MASK;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-static int __cpuinitdata ntlb;
+static int ntlb;
static int __init set_ntlb(char *str)
{
get_option(&str, &ntlb);
@@ -398,7 +400,10 @@ static int __init set_ntlb(char *str)
__setup("ntlb=", set_ntlb);
-void __cpuinit tlb_init(void)
+/*
+ * Configure TLB (for init or after a CPU has been powered off).
+ */
+static void r4k_tlb_configure(void)
{
/*
* You should never change this register:
@@ -430,6 +435,11 @@ void __cpuinit tlb_init(void)
local_flush_tlb_all();
/* Did I tell you that ARC SUCKS? */
+}
+
+void tlb_init(void)
+{
+ r4k_tlb_configure();
if (ntlb) {
if (ntlb > 1 && ntlb <= current_cpu_data.tlbsize) {
@@ -443,3 +453,26 @@ void __cpuinit tlb_init(void)
build_tlb_refill_handler();
}
+
+static int r4k_tlb_pm_notifier(struct notifier_block *self, unsigned long cmd,
+ void *v)
+{
+ switch (cmd) {
+ case CPU_PM_ENTER_FAILED:
+ case CPU_PM_EXIT:
+ r4k_tlb_configure();
+ break;
+ }
+
+ return NOTIFY_OK;
+}
+
+static struct notifier_block r4k_tlb_pm_notifier_block = {
+ .notifier_call = r4k_tlb_pm_notifier,
+};
+
+static int __init r4k_tlb_init_pm(void)
+{
+ return cpu_pm_register_notifier(&r4k_tlb_pm_notifier_block);
+}
+arch_initcall(r4k_tlb_init_pm);
diff --git a/arch/mips/mm/tlb-r8k.c b/arch/mips/mm/tlb-r8k.c
index 91c2499f806..138a2ec7cc6 100644
--- a/arch/mips/mm/tlb-r8k.c
+++ b/arch/mips/mm/tlb-r8k.c
@@ -8,7 +8,6 @@
* Carsten Langgaard, carstenl@mips.com
* Copyright (C) 2002 MIPS Technologies, Inc. All rights reserved.
*/
-#include <linux/init.h>
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/mm.h>
@@ -213,14 +212,14 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
local_irq_restore(flags);
}
-static void __cpuinit probe_tlb(unsigned long config)
+static void probe_tlb(unsigned long config)
{
struct cpuinfo_mips *c = &current_cpu_data;
c->tlbsize = 3 * 128; /* 3 sets each 128 entries */
}
-void __cpuinit tlb_init(void)
+void tlb_init(void)
{
unsigned int config = read_c0_config();
unsigned long status;
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 9ab0f907a52..e80e10bafc8 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -26,10 +26,10 @@
#include <linux/types.h>
#include <linux/smp.h>
#include <linux/string.h>
-#include <linux/init.h>
#include <linux/cache.h>
#include <asm/cacheflush.h>
+#include <asm/cpu-type.h>
#include <asm/pgtable.h>
#include <asm/war.h>
#include <asm/uasm.h>
@@ -85,6 +85,7 @@ static int use_bbit_insns(void)
case CPU_CAVIUM_OCTEON:
case CPU_CAVIUM_OCTEON_PLUS:
case CPU_CAVIUM_OCTEON2:
+ case CPU_CAVIUM_OCTEON3:
return 1;
default:
return 0;
@@ -95,6 +96,7 @@ static int use_lwx_insns(void)
{
switch (current_cpu_type()) {
case CPU_CAVIUM_OCTEON2:
+ case CPU_CAVIUM_OCTEON3:
return 1;
default:
return 0;
@@ -136,7 +138,7 @@ static int scratchpad_offset(int i)
* why; it's not an issue caused by the core RTL.
*
*/
-static int __cpuinit m4kc_tlbp_war(void)
+static int m4kc_tlbp_war(void)
{
return (current_cpu_data.processor_id & 0xffff00) ==
(PRID_COMP_MIPS | PRID_IMP_4KC);
@@ -181,11 +183,9 @@ UASM_L_LA(_large_segbits_fault)
UASM_L_LA(_tlb_huge_update)
#endif
-static int __cpuinitdata hazard_instance;
+static int hazard_instance;
-static void __cpuinit uasm_bgezl_hazard(u32 **p,
- struct uasm_reloc **r,
- int instance)
+static void uasm_bgezl_hazard(u32 **p, struct uasm_reloc **r, int instance)
{
switch (instance) {
case 0 ... 7:
@@ -196,9 +196,7 @@ static void __cpuinit uasm_bgezl_hazard(u32 **p,
}
}
-static void __cpuinit uasm_bgezl_label(struct uasm_label **l,
- u32 **p,
- int instance)
+static void uasm_bgezl_label(struct uasm_label **l, u32 **p, int instance)
{
switch (instance) {
case 0 ... 7:
@@ -295,15 +293,15 @@ static inline void dump_handler(const char *symbol, const u32 *handler, int coun
* We deliberately chose a buffer size of 128, so we won't scribble
* over anything important on overflow before we panic.
*/
-static u32 tlb_handler[128] __cpuinitdata;
+static u32 tlb_handler[128];
/* simply assume worst case size for labels and relocs */
-static struct uasm_label labels[128] __cpuinitdata;
-static struct uasm_reloc relocs[128] __cpuinitdata;
+static struct uasm_label labels[128];
+static struct uasm_reloc relocs[128];
-static int check_for_high_segbits __cpuinitdata;
+static int check_for_high_segbits;
-static unsigned int kscratch_used_mask __cpuinitdata;
+static unsigned int kscratch_used_mask;
static inline int __maybe_unused c0_kscratch(void)
{
@@ -316,7 +314,7 @@ static inline int __maybe_unused c0_kscratch(void)
}
}
-static int __cpuinit allocate_kscratch(void)
+static int allocate_kscratch(void)
{
int r;
unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask;
@@ -333,18 +331,14 @@ static int __cpuinit allocate_kscratch(void)
return r;
}
-static int scratch_reg __cpuinitdata;
-static int pgd_reg __cpuinitdata;
+static int scratch_reg;
+static int pgd_reg;
enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch};
-static struct work_registers __cpuinit build_get_work_registers(u32 **p)
+static struct work_registers build_get_work_registers(u32 **p)
{
struct work_registers r;
- int smp_processor_id_reg;
- int smp_processor_id_sel;
- int smp_processor_id_shift;
-
if (scratch_reg >= 0) {
/* Save in CPU local C0_KScratch? */
UASM_i_MTC0(p, 1, c0_kscratch(), scratch_reg);
@@ -355,25 +349,9 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p)
}
if (num_possible_cpus() > 1) {
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
- smp_processor_id_shift = 51;
- smp_processor_id_reg = 20; /* XContext */
- smp_processor_id_sel = 0;
-#else
-# ifdef CONFIG_32BIT
- smp_processor_id_shift = 25;
- smp_processor_id_reg = 4; /* Context */
- smp_processor_id_sel = 0;
-# endif
-# ifdef CONFIG_64BIT
- smp_processor_id_shift = 26;
- smp_processor_id_reg = 4; /* Context */
- smp_processor_id_sel = 0;
-# endif
-#endif
/* Get smp_processor_id */
- UASM_i_MFC0(p, K0, smp_processor_id_reg, smp_processor_id_sel);
- UASM_i_SRL_SAFE(p, K0, K0, smp_processor_id_shift);
+ UASM_i_CPUID_MFC0(p, K0, SMP_CPUID_REG);
+ UASM_i_SRL_SAFE(p, K0, K0, SMP_CPUID_REGSHIFT);
/* handler_reg_save index in K0 */
UASM_i_SLL(p, K0, K0, ilog2(sizeof(struct tlb_reg_save)));
@@ -393,7 +371,7 @@ static struct work_registers __cpuinit build_get_work_registers(u32 **p)
return r;
}
-static void __cpuinit build_restore_work_registers(u32 **p)
+static void build_restore_work_registers(u32 **p)
{
if (scratch_reg >= 0) {
UASM_i_MFC0(p, 1, c0_kscratch(), scratch_reg);
@@ -418,7 +396,7 @@ extern unsigned long pgd_current[];
/*
* The R3000 TLB handler is simple.
*/
-static void __cpuinit build_r3000_tlb_refill_handler(void)
+static void build_r3000_tlb_refill_handler(void)
{
long pgdc = (long)pgd_current;
u32 *p;
@@ -463,7 +441,7 @@ static void __cpuinit build_r3000_tlb_refill_handler(void)
* other one.To keep things simple, we first assume linear space,
* then we relocate it to the final handler layout as needed.
*/
-static u32 final_handler[64] __cpuinitdata;
+static u32 final_handler[64];
/*
* Hazards
@@ -487,7 +465,7 @@ static u32 final_handler[64] __cpuinitdata;
*
* As if we MIPS hackers wouldn't know how to nop pipelines happy ...
*/
-static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
+static void __maybe_unused build_tlb_probe_entry(u32 **p)
{
switch (current_cpu_type()) {
/* Found by experiment: R4600 v2.0/R4700 needs this, too. */
@@ -511,9 +489,9 @@ static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
*/
enum tlb_write_entry { tlb_random, tlb_indexed };
-static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
- struct uasm_reloc **r,
- enum tlb_write_entry wmode)
+static void build_tlb_write_entry(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r,
+ enum tlb_write_entry wmode)
{
void(*tlbw)(u32 **) = NULL;
@@ -531,6 +509,10 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
switch (current_cpu_type()) {
case CPU_M14KC:
case CPU_74K:
+ case CPU_1074K:
+ case CPU_PROAPTIV:
+ case CPU_P5600:
+ case CPU_M5150:
break;
default:
@@ -600,6 +582,7 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
case CPU_BMIPS4380:
case CPU_BMIPS5000:
case CPU_LOONGSON2:
+ case CPU_LOONGSON3:
case CPU_R5500:
if (m4kc_tlbp_war())
uasm_i_nop(p);
@@ -642,13 +625,13 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
default:
panic("No TLB refill handler yet (CPU type: %d)",
- current_cpu_data.cputype);
+ current_cpu_type());
break;
}
}
-static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
- unsigned int reg)
+static __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
+ unsigned int reg)
{
if (cpu_has_rixi) {
UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL));
@@ -663,11 +646,9 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
-static __cpuinit void build_restore_pagemask(u32 **p,
- struct uasm_reloc **r,
- unsigned int tmp,
- enum label_id lid,
- int restore_scratch)
+static void build_restore_pagemask(u32 **p, struct uasm_reloc **r,
+ unsigned int tmp, enum label_id lid,
+ int restore_scratch)
{
if (restore_scratch) {
/* Reset default page size */
@@ -706,12 +687,11 @@ static __cpuinit void build_restore_pagemask(u32 **p,
}
}
-static __cpuinit void build_huge_tlb_write_entry(u32 **p,
- struct uasm_label **l,
- struct uasm_reloc **r,
- unsigned int tmp,
- enum tlb_write_entry wmode,
- int restore_scratch)
+static void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l,
+ struct uasm_reloc **r,
+ unsigned int tmp,
+ enum tlb_write_entry wmode,
+ int restore_scratch)
{
/* Set huge page tlb entry size */
uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
@@ -726,9 +706,9 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p,
/*
* Check if Huge PTE is present, if so then jump to LABEL.
*/
-static void __cpuinit
+static void
build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
- unsigned int pmd, int lid)
+ unsigned int pmd, int lid)
{
UASM_i_LW(p, tmp, 0, pmd);
if (use_bbit_insns()) {
@@ -739,9 +719,8 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
}
}
-static __cpuinit void build_huge_update_entries(u32 **p,
- unsigned int pte,
- unsigned int tmp)
+static void build_huge_update_entries(u32 **p, unsigned int pte,
+ unsigned int tmp)
{
int small_sequence;
@@ -771,11 +750,10 @@ static __cpuinit void build_huge_update_entries(u32 **p,
UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
}
-static __cpuinit void build_huge_handler_tail(u32 **p,
- struct uasm_reloc **r,
- struct uasm_label **l,
- unsigned int pte,
- unsigned int ptr)
+static void build_huge_handler_tail(u32 **p, struct uasm_reloc **r,
+ struct uasm_label **l,
+ unsigned int pte,
+ unsigned int ptr)
{
#ifdef CONFIG_SMP
UASM_i_SC(p, pte, 0, ptr);
@@ -794,7 +772,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p,
* TMP and PTR are scratch.
* TMP will be clobbered, PTR will hold the pmd entry.
*/
-static void __cpuinit
+static void
build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int tmp, unsigned int ptr)
{
@@ -825,11 +803,11 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
}
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg != -1) {
/* pgd is in pgd_reg */
UASM_i_MFC0(p, ptr, c0_kscratch(), pgd_reg);
} else {
+#if defined(CONFIG_MIPS_PGD_C0_CONTEXT)
/*
* &pgd << 11 stored in CONTEXT [23..63].
*/
@@ -841,30 +819,18 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
/* 1 0 1 0 1 << 6 xkphys cached */
uasm_i_ori(p, ptr, ptr, 0x540);
uasm_i_drotr(p, ptr, ptr, 11);
- }
#elif defined(CONFIG_SMP)
-# ifdef CONFIG_MIPS_MT_SMTC
- /*
- * SMTC uses TCBind value as "CPU" index
- */
- uasm_i_mfc0(p, ptr, C0_TCBIND);
- uasm_i_dsrl_safe(p, ptr, ptr, 19);
-# else
- /*
- * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
- * stored in CONTEXT.
- */
- uasm_i_dmfc0(p, ptr, C0_CONTEXT);
- uasm_i_dsrl_safe(p, ptr, ptr, 23);
-# endif
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_daddu(p, ptr, ptr, tmp);
- uasm_i_dmfc0(p, tmp, C0_BADVADDR);
- uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
+ UASM_i_CPUID_MFC0(p, ptr, SMP_CPUID_REG);
+ uasm_i_dsrl_safe(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_daddu(p, ptr, ptr, tmp);
+ uasm_i_dmfc0(p, tmp, C0_BADVADDR);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#else
- UASM_i_LA_mostly(p, ptr, pgdc);
- uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
+ UASM_i_LA_mostly(p, ptr, pgdc);
+ uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
#endif
+ }
uasm_l_vmalloc_done(l, *p);
@@ -886,7 +852,7 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* BVADDR is the faulting address, PTR is scratch.
* PTR will hold the pgd for vmalloc.
*/
-static void __cpuinit
+static void
build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
unsigned int bvaddr, unsigned int ptr,
enum vmalloc64_mode mode)
@@ -956,34 +922,28 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* TMP and PTR are scratch.
* TMP will be clobbered, PTR will hold the pgd entry.
*/
-static void __cpuinit __maybe_unused
+static void __maybe_unused
build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
{
- long pgdc = (long)pgd_current;
+ if (pgd_reg != -1) {
+ /* pgd is in pgd_reg */
+ uasm_i_mfc0(p, ptr, c0_kscratch(), pgd_reg);
+ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ } else {
+ long pgdc = (long)pgd_current;
- /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
+ /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
#ifdef CONFIG_SMP
-#ifdef CONFIG_MIPS_MT_SMTC
- /*
- * SMTC uses TCBind value as "CPU" index
- */
- uasm_i_mfc0(p, ptr, C0_TCBIND);
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_srl(p, ptr, ptr, 19);
-#else
- /*
- * smp_processor_id() << 2 is stored in CONTEXT.
- */
- uasm_i_mfc0(p, ptr, C0_CONTEXT);
- UASM_i_LA_mostly(p, tmp, pgdc);
- uasm_i_srl(p, ptr, ptr, 23);
-#endif
- uasm_i_addu(p, ptr, tmp, ptr);
+ uasm_i_mfc0(p, ptr, SMP_CPUID_REG);
+ UASM_i_LA_mostly(p, tmp, pgdc);
+ uasm_i_srl(p, ptr, ptr, SMP_CPUID_PTRSHIFT);
+ uasm_i_addu(p, ptr, tmp, ptr);
#else
- UASM_i_LA_mostly(p, ptr, pgdc);
+ UASM_i_LA_mostly(p, ptr, pgdc);
#endif
- uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
- uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
+ uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
+ }
uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
@@ -991,7 +951,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
#endif /* !CONFIG_64BIT */
-static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
+static void build_adjust_context(u32 **p, unsigned int ctx)
{
unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
@@ -1017,7 +977,7 @@ static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
uasm_i_andi(p, ctx, ctx, mask);
}
-static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
+static void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
{
/*
* Bug workaround for the Nevada. It seems as if under certain
@@ -1042,8 +1002,7 @@ static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr
UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
}
-static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
- unsigned int ptep)
+static void build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep)
{
/*
* 64bit address support (36bit on a 32bit CPU) in a 32bit
@@ -1104,7 +1063,7 @@ struct mips_huge_tlb_info {
int restore_scratch;
};
-static struct mips_huge_tlb_info __cpuinit
+static struct mips_huge_tlb_info
build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int tmp,
unsigned int ptr, int c0_scratch_reg)
@@ -1282,7 +1241,7 @@ build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l,
*/
#define MIPS64_REFILL_INSNS 32
-static void __cpuinit build_r4000_tlb_refill_handler(void)
+static void build_r4000_tlb_refill_handler(void)
{
u32 *p = tlb_handler;
struct uasm_label *l = labels;
@@ -1297,7 +1256,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
memset(relocs, 0, sizeof(relocs));
memset(final_handler, 0, sizeof(final_handler));
- if ((scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
+ if (IS_ENABLED(CONFIG_64BIT) && (scratch_reg >= 0 || scratchpad_available()) && use_bbit_insns()) {
htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1,
scratch_reg);
vmalloc_mode = refill_scratch;
@@ -1356,95 +1315,100 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
* need three, with the second nop'ed and the third being
* unused.
*/
- /* Loongson2 ebase is different than r4k, we have more space */
-#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
- if ((p - tlb_handler) > 64)
- panic("TLB refill handler space exceeded");
-#else
- if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
- || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
- && uasm_insn_has_bdelay(relocs,
- tlb_handler + MIPS64_REFILL_INSNS - 3)))
- panic("TLB refill handler space exceeded");
-#endif
-
- /*
- * Now fold the handler in the TLB refill handler space.
- */
-#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
- f = final_handler;
- /* Simplest case, just copy the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, p, f);
- final_len = p - tlb_handler;
-#else /* CONFIG_64BIT */
- f = final_handler + MIPS64_REFILL_INSNS;
- if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
- /* Just copy the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, p, f);
- final_len = p - tlb_handler;
- } else {
-#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
- const enum label_id ls = label_tlb_huge_update;
-#else
- const enum label_id ls = label_vmalloc;
-#endif
- u32 *split;
- int ov = 0;
- int i;
-
- for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
- ;
- BUG_ON(i == ARRAY_SIZE(labels));
- split = labels[i].addr;
-
- /*
- * See if we have overflown one way or the other.
- */
- if (split > tlb_handler + MIPS64_REFILL_INSNS ||
- split < p - MIPS64_REFILL_INSNS)
- ov = 1;
-
- if (ov) {
+ switch (boot_cpu_type()) {
+ default:
+ if (sizeof(long) == 4) {
+ case CPU_LOONGSON2:
+ /* Loongson2 ebase is different than r4k, we have more space */
+ if ((p - tlb_handler) > 64)
+ panic("TLB refill handler space exceeded");
/*
- * Split two instructions before the end. One
- * for the branch and one for the instruction
- * in the delay slot.
+ * Now fold the handler in the TLB refill handler space.
*/
- split = tlb_handler + MIPS64_REFILL_INSNS - 2;
-
+ f = final_handler;
+ /* Simplest case, just copy the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
+ final_len = p - tlb_handler;
+ break;
+ } else {
+ if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
+ || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
+ && uasm_insn_has_bdelay(relocs,
+ tlb_handler + MIPS64_REFILL_INSNS - 3)))
+ panic("TLB refill handler space exceeded");
/*
- * If the branch would fall in a delay slot,
- * we must back up an additional instruction
- * so that it is no longer in a delay slot.
+ * Now fold the handler in the TLB refill handler space.
*/
- if (uasm_insn_has_bdelay(relocs, split - 1))
- split--;
- }
- /* Copy first part of the handler. */
- uasm_copy_handler(relocs, labels, tlb_handler, split, f);
- f += split - tlb_handler;
-
- if (ov) {
- /* Insert branch. */
- uasm_l_split(&l, final_handler);
- uasm_il_b(&f, &r, label_split);
- if (uasm_insn_has_bdelay(relocs, split))
- uasm_i_nop(&f);
- else {
- uasm_copy_handler(relocs, labels,
- split, split + 1, f);
- uasm_move_labels(labels, f, f + 1, -1);
- f++;
- split++;
+ f = final_handler + MIPS64_REFILL_INSNS;
+ if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
+ /* Just copy the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, p, f);
+ final_len = p - tlb_handler;
+ } else {
+#ifdef CONFIG_MIPS_HUGE_TLB_SUPPORT
+ const enum label_id ls = label_tlb_huge_update;
+#else
+ const enum label_id ls = label_vmalloc;
+#endif
+ u32 *split;
+ int ov = 0;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
+ ;
+ BUG_ON(i == ARRAY_SIZE(labels));
+ split = labels[i].addr;
+
+ /*
+ * See if we have overflown one way or the other.
+ */
+ if (split > tlb_handler + MIPS64_REFILL_INSNS ||
+ split < p - MIPS64_REFILL_INSNS)
+ ov = 1;
+
+ if (ov) {
+ /*
+ * Split two instructions before the end. One
+ * for the branch and one for the instruction
+ * in the delay slot.
+ */
+ split = tlb_handler + MIPS64_REFILL_INSNS - 2;
+
+ /*
+ * If the branch would fall in a delay slot,
+ * we must back up an additional instruction
+ * so that it is no longer in a delay slot.
+ */
+ if (uasm_insn_has_bdelay(relocs, split - 1))
+ split--;
+ }
+ /* Copy first part of the handler. */
+ uasm_copy_handler(relocs, labels, tlb_handler, split, f);
+ f += split - tlb_handler;
+
+ if (ov) {
+ /* Insert branch. */
+ uasm_l_split(&l, final_handler);
+ uasm_il_b(&f, &r, label_split);
+ if (uasm_insn_has_bdelay(relocs, split))
+ uasm_i_nop(&f);
+ else {
+ uasm_copy_handler(relocs, labels,
+ split, split + 1, f);
+ uasm_move_labels(labels, f, f + 1, -1);
+ f++;
+ split++;
+ }
+ }
+
+ /* Copy the rest of the handler. */
+ uasm_copy_handler(relocs, labels, split, p, final_handler);
+ final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
+ (p - split);
}
}
-
- /* Copy the rest of the handler. */
- uasm_copy_handler(relocs, labels, split, p, final_handler);
- final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
- (p - split);
+ break;
}
-#endif /* CONFIG_64BIT */
uasm_resolve_relocs(relocs, labels);
pr_debug("Wrote TLB refill handler (%u instructions).\n",
@@ -1458,28 +1422,31 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
extern u32 handle_tlbl[], handle_tlbl_end[];
extern u32 handle_tlbs[], handle_tlbs_end[];
extern u32 handle_tlbm[], handle_tlbm_end[];
+extern u32 tlbmiss_handler_setup_pgd_start[], tlbmiss_handler_setup_pgd[];
+extern u32 tlbmiss_handler_setup_pgd_end[];
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
-extern u32 tlbmiss_handler_setup_pgd[], tlbmiss_handler_setup_pgd_end[];
-
-static void __cpuinit build_r4000_setup_pgd(void)
+static void build_setup_pgd(void)
{
const int a0 = 4;
- const int a1 = 5;
- u32 *p = tlbmiss_handler_setup_pgd_array;
+ const int __maybe_unused a1 = 5;
+ const int __maybe_unused a2 = 6;
+ u32 *p = tlbmiss_handler_setup_pgd_start;
const int tlbmiss_handler_setup_pgd_size =
- tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd;
- struct uasm_label *l = labels;
- struct uasm_reloc *r = relocs;
+ tlbmiss_handler_setup_pgd_end - tlbmiss_handler_setup_pgd_start;
+#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
+ long pgdc = (long)pgd_current;
+#endif
memset(tlbmiss_handler_setup_pgd, 0, tlbmiss_handler_setup_pgd_size *
sizeof(tlbmiss_handler_setup_pgd[0]));
memset(labels, 0, sizeof(labels));
memset(relocs, 0, sizeof(relocs));
-
pgd_reg = allocate_kscratch();
-
+#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
if (pgd_reg == -1) {
+ struct uasm_label *l = labels;
+ struct uasm_reloc *r = relocs;
+
/* PGD << 11 in c0_Context */
/*
* If it is a ckseg0 address, convert to a physical
@@ -1501,6 +1468,26 @@ static void __cpuinit build_r4000_setup_pgd(void)
uasm_i_jr(&p, 31);
UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
}
+#else
+#ifdef CONFIG_SMP
+ /* Save PGD to pgd_current[smp_processor_id()] */
+ UASM_i_CPUID_MFC0(&p, a1, SMP_CPUID_REG);
+ UASM_i_SRL_SAFE(&p, a1, a1, SMP_CPUID_PTRSHIFT);
+ UASM_i_LA_mostly(&p, a2, pgdc);
+ UASM_i_ADDU(&p, a2, a2, a1);
+ UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
+#else
+ UASM_i_LA_mostly(&p, a2, pgdc);
+ UASM_i_SW(&p, a0, uasm_rel_lo(pgdc), a2);
+#endif /* SMP */
+ uasm_i_jr(&p, 31);
+
+ /* if pgd_reg is allocated, save PGD also to scratch register */
+ if (pgd_reg != -1)
+ UASM_i_MTC0(&p, a0, c0_kscratch(), pgd_reg);
+ else
+ uasm_i_nop(&p);
+#endif
if (p >= tlbmiss_handler_setup_pgd_end)
panic("tlbmiss_handler_setup_pgd space exceeded");
@@ -1511,9 +1498,8 @@ static void __cpuinit build_r4000_setup_pgd(void)
dump_handler("tlbmiss_handler", tlbmiss_handler_setup_pgd,
tlbmiss_handler_setup_pgd_size);
}
-#endif
-static void __cpuinit
+static void
iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
{
#ifdef CONFIG_SMP
@@ -1533,7 +1519,7 @@ iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
#endif
}
-static void __cpuinit
+static void
iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
unsigned int mode)
{
@@ -1593,7 +1579,7 @@ iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
* the page table where this PTE is located, PTE will be re-loaded
* with it's original value.
*/
-static void __cpuinit
+static void
build_pte_present(u32 **p, struct uasm_reloc **r,
int pte, int ptr, int scratch, enum label_id lid)
{
@@ -1621,7 +1607,7 @@ build_pte_present(u32 **p, struct uasm_reloc **r,
}
/* Make PTE valid, store result in PTR. */
-static void __cpuinit
+static void
build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
@@ -1634,7 +1620,7 @@ build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
* Check if PTE can be written to, if not branch to LABEL. Regardless
* restore PTE with value from PTR when done.
*/
-static void __cpuinit
+static void
build_pte_writable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, int scratch,
enum label_id lid)
@@ -1654,7 +1640,7 @@ build_pte_writable(u32 **p, struct uasm_reloc **r,
/* Make PTE writable, update software status bits as well, then store
* at PTR.
*/
-static void __cpuinit
+static void
build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
unsigned int ptr)
{
@@ -1668,7 +1654,7 @@ build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
* Check if PTE can be modified, if not branch to LABEL. Regardless
* restore PTE with value from PTR when done.
*/
-static void __cpuinit
+static void
build_pte_modifiable(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, int scratch,
enum label_id lid)
@@ -1697,7 +1683,7 @@ build_pte_modifiable(u32 **p, struct uasm_reloc **r,
* This places the pte into ENTRYLO0 and writes it with tlbwi.
* Then it returns.
*/
-static void __cpuinit
+static void
build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
{
uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
@@ -1713,7 +1699,7 @@ build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
* may have the probe fail bit set as a result of a trap on a
* kseg2 access, i.e. without refill. Then it returns.
*/
-static void __cpuinit
+static void
build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int pte,
unsigned int tmp)
@@ -1731,7 +1717,7 @@ build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
uasm_i_rfe(p); /* branch delay */
}
-static void __cpuinit
+static void
build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
unsigned int ptr)
{
@@ -1751,7 +1737,7 @@ build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
uasm_i_tlbp(p); /* load delay */
}
-static void __cpuinit build_r3000_tlb_load_handler(void)
+static void build_r3000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
@@ -1782,7 +1768,7 @@ static void __cpuinit build_r3000_tlb_load_handler(void)
dump_handler("r3000_tlb_load", handle_tlbl, handle_tlbl_size);
}
-static void __cpuinit build_r3000_tlb_store_handler(void)
+static void build_r3000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
@@ -1803,7 +1789,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void)
uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
uasm_i_nop(&p);
- if (p >= handle_tlbs)
+ if (p >= handle_tlbs_end)
panic("TLB store handler fastpath space exceeded");
uasm_resolve_relocs(relocs, labels);
@@ -1813,7 +1799,7 @@ static void __cpuinit build_r3000_tlb_store_handler(void)
dump_handler("r3000_tlb_store", handle_tlbs, handle_tlbs_size);
}
-static void __cpuinit build_r3000_tlb_modify_handler(void)
+static void build_r3000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
@@ -1848,7 +1834,7 @@ static void __cpuinit build_r3000_tlb_modify_handler(void)
/*
* R4000 style TLB load/store/modify handlers.
*/
-static struct work_registers __cpuinit
+static struct work_registers
build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
struct uasm_reloc **r)
{
@@ -1884,7 +1870,7 @@ build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
return wr;
}
-static void __cpuinit
+static void
build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
struct uasm_reloc **r, unsigned int tmp,
unsigned int ptr)
@@ -1902,7 +1888,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
#endif
}
-static void __cpuinit build_r4000_tlb_load_handler(void)
+static void build_r4000_tlb_load_handler(void)
{
u32 *p = handle_tlbl;
const int handle_tlbl_size = handle_tlbl_end - handle_tlbl;
@@ -2085,7 +2071,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
dump_handler("r4000_tlb_load", handle_tlbl, handle_tlbl_size);
}
-static void __cpuinit build_r4000_tlb_store_handler(void)
+static void build_r4000_tlb_store_handler(void)
{
u32 *p = handle_tlbs;
const int handle_tlbs_size = handle_tlbs_end - handle_tlbs;
@@ -2140,7 +2126,7 @@ static void __cpuinit build_r4000_tlb_store_handler(void)
dump_handler("r4000_tlb_store", handle_tlbs, handle_tlbs_size);
}
-static void __cpuinit build_r4000_tlb_modify_handler(void)
+static void build_r4000_tlb_modify_handler(void)
{
u32 *p = handle_tlbm;
const int handle_tlbm_size = handle_tlbm_end - handle_tlbm;
@@ -2196,7 +2182,7 @@ static void __cpuinit build_r4000_tlb_modify_handler(void)
dump_handler("r4000_tlb_modify", handle_tlbm, handle_tlbm_size);
}
-static void __cpuinit flush_tlb_handlers(void)
+static void flush_tlb_handlers(void)
{
local_flush_icache_range((unsigned long)handle_tlbl,
(unsigned long)handle_tlbl_end);
@@ -2204,13 +2190,11 @@ static void __cpuinit flush_tlb_handlers(void)
(unsigned long)handle_tlbs_end);
local_flush_icache_range((unsigned long)handle_tlbm,
(unsigned long)handle_tlbm_end);
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd,
(unsigned long)tlbmiss_handler_setup_pgd_end);
-#endif
}
-void __cpuinit build_tlb_refill_handler(void)
+void build_tlb_refill_handler(void)
{
/*
* The refill handler is generated per-CPU, multi-node systems
@@ -2239,6 +2223,7 @@ void __cpuinit build_tlb_refill_handler(void)
if (!run_once) {
if (!cpu_has_local_ebase)
build_r3000_tlb_refill_handler();
+ build_setup_pgd();
build_r3000_tlb_load_handler();
build_r3000_tlb_store_handler();
build_r3000_tlb_modify_handler();
@@ -2262,9 +2247,7 @@ void __cpuinit build_tlb_refill_handler(void)
default:
if (!run_once) {
scratch_reg = allocate_kscratch();
-#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
- build_r4000_setup_pgd();
-#endif
+ build_setup_pgd();
build_r4000_tlb_load_handler();
build_r4000_tlb_store_handler();
build_r4000_tlb_modify_handler();
diff --git a/arch/mips/mm/uasm-micromips.c b/arch/mips/mm/uasm-micromips.c
index 162ee6d6278..8399ddf03a0 100644
--- a/arch/mips/mm/uasm-micromips.c
+++ b/arch/mips/mm/uasm-micromips.c
@@ -15,7 +15,6 @@
#include <linux/kernel.h>
#include <linux/types.h>
-#include <linux/init.h>
#include <asm/inst.h>
#include <asm/elf.h>
@@ -49,7 +48,7 @@
#include "uasm.c"
-static struct insn insn_table_MM[] __uasminitdata = {
+static struct insn insn_table_MM[] = {
{ insn_addu, M(mm_pool32a_op, 0, 0, 0, 0, mm_addu32_op), RT | RS | RD },
{ insn_addiu, M(mm_addiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
{ insn_and, M(mm_pool32a_op, 0, 0, 0, 0, mm_and_op), RT | RS | RD },
@@ -64,6 +63,7 @@ static struct insn insn_table_MM[] __uasminitdata = {
{ insn_cache, M(mm_pool32b_op, 0, 0, mm_cache_func, 0, 0), RT | RS | SIMM },
{ insn_daddu, 0, 0 },
{ insn_daddiu, 0, 0 },
+ { insn_divu, M(mm_pool32a_op, 0, 0, 0, mm_divu_op, mm_pool32axf_op), RT | RS },
{ insn_dmfc0, 0, 0 },
{ insn_dmtc0, 0, 0 },
{ insn_dsll, 0, 0 },
@@ -79,14 +79,20 @@ static struct insn insn_table_MM[] __uasminitdata = {
{ insn_ext, M(mm_pool32a_op, 0, 0, 0, 0, mm_ext_op), RT | RS | RD | RE },
{ insn_j, M(mm_j32_op, 0, 0, 0, 0, 0), JIMM },
{ insn_jal, M(mm_jal32_op, 0, 0, 0, 0, 0), JIMM },
+ { insn_jalr, M(mm_pool32a_op, 0, 0, 0, mm_jalr_op, mm_pool32axf_op), RT | RS },
{ insn_jr, M(mm_pool32a_op, 0, 0, 0, mm_jalr_op, mm_pool32axf_op), RS },
+ { insn_lb, M(mm_lb32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
{ insn_ld, 0, 0 },
+ { insn_lh, M(mm_lh32_op, 0, 0, 0, 0, 0), RS | RS | SIMM },
{ insn_ll, M(mm_pool32c_op, 0, 0, (mm_ll_func << 1), 0, 0), RS | RT | SIMM },
{ insn_lld, 0, 0 },
{ insn_lui, M(mm_pool32i_op, mm_lui_op, 0, 0, 0, 0), RS | SIMM },
{ insn_lw, M(mm_lw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
{ insn_mfc0, M(mm_pool32a_op, 0, 0, 0, mm_mfc0_op, mm_pool32axf_op), RT | RS | RD },
+ { insn_mfhi, M(mm_pool32a_op, 0, 0, 0, mm_mfhi32_op, mm_pool32axf_op), RS },
+ { insn_mflo, M(mm_pool32a_op, 0, 0, 0, mm_mflo32_op, mm_pool32axf_op), RS },
{ insn_mtc0, M(mm_pool32a_op, 0, 0, 0, mm_mtc0_op, mm_pool32axf_op), RT | RS | RD },
+ { insn_mul, M(mm_pool32a_op, 0, 0, 0, 0, mm_mul_op), RT | RS | RD },
{ insn_or, M(mm_pool32a_op, 0, 0, 0, 0, mm_or32_op), RT | RS | RD },
{ insn_ori, M(mm_ori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM },
{ insn_pref, M(mm_pool32c_op, 0, 0, (mm_pref_func << 1), 0, 0), RT | RS | SIMM },
@@ -95,15 +101,23 @@ static struct insn insn_table_MM[] __uasminitdata = {
{ insn_scd, 0, 0 },
{ insn_sd, 0, 0 },
{ insn_sll, M(mm_pool32a_op, 0, 0, 0, 0, mm_sll32_op), RT | RS | RD },
+ { insn_sllv, M(mm_pool32a_op, 0, 0, 0, 0, mm_sllv32_op), RT | RS | RD },
+ { insn_slt, M(mm_pool32a_op, 0, 0, 0, 0, mm_slt_op), RT | RS | RD },
+ { insn_sltiu, M(mm_sltiu32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
+ { insn_sltu, M(mm_pool32a_op, 0, 0, 0, 0, mm_sltu_op), RT | RS | RD },
{ insn_sra, M(mm_pool32a_op, 0, 0, 0, 0, mm_sra_op), RT | RS | RD },
{ insn_srl, M(mm_pool32a_op, 0, 0, 0, 0, mm_srl32_op), RT | RS | RD },
+ { insn_srlv, M(mm_pool32a_op, 0, 0, 0, 0, mm_srlv32_op), RT | RS | RD },
{ insn_rotr, M(mm_pool32a_op, 0, 0, 0, 0, mm_rotr_op), RT | RS | RD },
{ insn_subu, M(mm_pool32a_op, 0, 0, 0, 0, mm_subu32_op), RT | RS | RD },
{ insn_sw, M(mm_sw32_op, 0, 0, 0, 0, 0), RT | RS | SIMM },
+ { insn_sync, M(mm_pool32a_op, 0, 0, 0, mm_sync_op, mm_pool32axf_op), RS },
{ insn_tlbp, M(mm_pool32a_op, 0, 0, 0, mm_tlbp_op, mm_pool32axf_op), 0 },
{ insn_tlbr, M(mm_pool32a_op, 0, 0, 0, mm_tlbr_op, mm_pool32axf_op), 0 },
{ insn_tlbwi, M(mm_pool32a_op, 0, 0, 0, mm_tlbwi_op, mm_pool32axf_op), 0 },
{ insn_tlbwr, M(mm_pool32a_op, 0, 0, 0, mm_tlbwr_op, mm_pool32axf_op), 0 },
+ { insn_wait, M(mm_pool32a_op, 0, 0, 0, mm_wait_op, mm_pool32axf_op), SCIMM },
+ { insn_wsbh, M(mm_pool32a_op, 0, 0, 0, mm_wsbh_op, mm_pool32axf_op), RT | RS },
{ insn_xor, M(mm_pool32a_op, 0, 0, 0, 0, mm_xor32_op), RT | RS | RD },
{ insn_xori, M(mm_xori32_op, 0, 0, 0, 0, 0), RT | RS | UIMM },
{ insn_dins, 0, 0 },
@@ -118,7 +132,7 @@ static struct insn insn_table_MM[] __uasminitdata = {
#undef M
-static inline __uasminit u32 build_bimm(s32 arg)
+static inline u32 build_bimm(s32 arg)
{
WARN(arg > 0xffff || arg < -0x10000,
KERN_WARNING "Micro-assembler field overflow\n");
@@ -128,7 +142,7 @@ static inline __uasminit u32 build_bimm(s32 arg)
return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 1) & 0x7fff);
}
-static inline __uasminit u32 build_jimm(u32 arg)
+static inline u32 build_jimm(u32 arg)
{
WARN(arg & ~((JIMM_MASK << 2) | 1),
@@ -141,7 +155,7 @@ static inline __uasminit u32 build_jimm(u32 arg)
* The order of opcode arguments is implicitly left to right,
* starting with RS and ending with FUNC or IMM.
*/
-static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
+static void build_insn(u32 **buf, enum opcode opc, ...)
{
struct insn *ip = NULL;
unsigned int i;
@@ -199,7 +213,7 @@ static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
(*buf)++;
}
-static inline void __uasminit
+static inline void
__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
{
long laddr = (long)lab->addr;
diff --git a/arch/mips/mm/uasm-mips.c b/arch/mips/mm/uasm-mips.c
index 5fcdd8fe3e8..6708a2dbf93 100644
--- a/arch/mips/mm/uasm-mips.c
+++ b/arch/mips/mm/uasm-mips.c
@@ -15,7 +15,6 @@
#include <linux/kernel.h>
#include <linux/types.h>
-#include <linux/init.h>
#include <asm/inst.h>
#include <asm/elf.h>
@@ -49,7 +48,7 @@
#include "uasm.c"
-static struct insn insn_table[] __uasminitdata = {
+static struct insn insn_table[] = {
{ insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
{ insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
@@ -68,6 +67,7 @@ static struct insn insn_table[] __uasminitdata = {
{ insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
{ insn_dinsm, M(spec3_op, 0, 0, 0, 0, dinsm_op), RS | RT | RD | RE },
{ insn_dins, M(spec3_op, 0, 0, 0, 0, dins_op), RS | RT | RD | RE },
+ { insn_divu, M(spec_op, 0, 0, 0, 0, divu_op), RS | RT },
{ insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
{ insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
{ insn_drotr32, M(spec_op, 1, 0, 0, 0, dsrl32_op), RT | RD | RE },
@@ -83,17 +83,23 @@ static struct insn insn_table[] __uasminitdata = {
{ insn_ins, M(spec3_op, 0, 0, 0, 0, ins_op), RS | RT | RD | RE },
{ insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
{ insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM },
+ { insn_jalr, M(spec_op, 0, 0, 0, 0, jalr_op), RS | RD },
{ insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
{ insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS },
+ { insn_lb, M(lb_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_ldx, M(spec3_op, 0, 0, 0, ldx_op, lx_op), RS | RT | RD },
+ { insn_lh, M(lh_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM },
{ insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_lwx, M(spec3_op, 0, 0, 0, lwx_op, lx_op), RS | RT | RD },
{ insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_mfhi, M(spec_op, 0, 0, 0, 0, mfhi_op), RD },
+ { insn_mflo, M(spec_op, 0, 0, 0, 0, mflo_op), RD },
{ insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
+ { insn_mul, M(spec2_op, 0, 0, 0, 0, mul_op), RS | RT | RD},
{ insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
{ insn_or, M(spec_op, 0, 0, 0, 0, or_op), RS | RT | RD },
{ insn_pref, M(pref_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
@@ -103,23 +109,32 @@ static struct insn insn_table[] __uasminitdata = {
{ insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
{ insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE },
+ { insn_sllv, M(spec_op, 0, 0, 0, 0, sllv_op), RS | RT | RD },
+ { insn_slt, M(spec_op, 0, 0, 0, 0, slt_op), RS | RT | RD },
+ { insn_sltiu, M(sltiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_sltu, M(spec_op, 0, 0, 0, 0, sltu_op), RS | RT | RD },
{ insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE },
{ insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE },
+ { insn_srlv, M(spec_op, 0, 0, 0, 0, srlv_op), RS | RT | RD },
{ insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD },
{ insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
+ { insn_sync, M(spec_op, 0, 0, 0, 0, sync_op), RE },
{ insn_syscall, M(spec_op, 0, 0, 0, 0, syscall_op), SCIMM},
{ insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 },
{ insn_tlbr, M(cop0_op, cop_op, 0, 0, 0, tlbr_op), 0 },
{ insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 },
{ insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 },
+ { insn_wait, M(cop0_op, cop_op, 0, 0, 0, wait_op), SCIMM },
+ { insn_wsbh, M(spec3_op, 0, 0, 0, wsbh_op, bshfl_op), RT | RD },
{ insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
{ insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD },
+ { insn_yield, M(spec3_op, 0, 0, 0, 0, yield_op), RS | RD },
{ insn_invalid, 0, 0 }
};
#undef M
-static inline __uasminit u32 build_bimm(s32 arg)
+static inline u32 build_bimm(s32 arg)
{
WARN(arg > 0x1ffff || arg < -0x20000,
KERN_WARNING "Micro-assembler field overflow\n");
@@ -129,7 +144,7 @@ static inline __uasminit u32 build_bimm(s32 arg)
return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
}
-static inline __uasminit u32 build_jimm(u32 arg)
+static inline u32 build_jimm(u32 arg)
{
WARN(arg & ~(JIMM_MASK << 2),
KERN_WARNING "Micro-assembler field overflow\n");
@@ -141,7 +156,7 @@ static inline __uasminit u32 build_jimm(u32 arg)
* The order of opcode arguments is implicitly left to right,
* starting with RS and ending with FUNC or IMM.
*/
-static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
+static void build_insn(u32 **buf, enum opcode opc, ...)
{
struct insn *ip = NULL;
unsigned int i;
@@ -187,7 +202,7 @@ static void __uasminit build_insn(u32 **buf, enum opcode opc, ...)
(*buf)++;
}
-static inline void __uasminit
+static inline void
__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab)
{
long laddr = (long)lab->addr;
diff --git a/arch/mips/mm/uasm.c b/arch/mips/mm/uasm.c
index 7eb5e4355d2..a01b0d6cedd 100644
--- a/arch/mips/mm/uasm.c
+++ b/arch/mips/mm/uasm.c
@@ -47,14 +47,16 @@ enum opcode {
insn_addiu, insn_addu, insn_and, insn_andi, insn_bbit0, insn_bbit1,
insn_beq, insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
insn_bne, insn_cache, insn_daddiu, insn_daddu, insn_dins, insn_dinsm,
- insn_dmfc0, insn_dmtc0, insn_drotr, insn_drotr32, insn_dsll,
+ insn_divu, insn_dmfc0, insn_dmtc0, insn_drotr, insn_drotr32, insn_dsll,
insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32, insn_dsubu, insn_eret,
- insn_ext, insn_ins, insn_j, insn_jal, insn_jr, insn_ld, insn_ldx,
- insn_ll, insn_lld, insn_lui, insn_lw, insn_lwx, insn_mfc0, insn_mtc0,
+ insn_ext, insn_ins, insn_j, insn_jal, insn_jalr, insn_jr, insn_lb,
+ insn_ld, insn_ldx, insn_lh, insn_ll, insn_lld, insn_lui, insn_lw,
+ insn_lwx, insn_mfc0, insn_mfhi, insn_mflo, insn_mtc0, insn_mul,
insn_or, insn_ori, insn_pref, insn_rfe, insn_rotr, insn_sc, insn_scd,
- insn_sd, insn_sll, insn_sra, insn_srl, insn_subu, insn_sw,
- insn_syscall, insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_xor,
- insn_xori,
+ insn_sd, insn_sll, insn_sllv, insn_slt, insn_sltiu, insn_sltu, insn_sra,
+ insn_srl, insn_srlv, insn_subu, insn_sw, insn_sync, insn_syscall,
+ insn_tlbp, insn_tlbr, insn_tlbwi, insn_tlbwr, insn_wait, insn_wsbh,
+ insn_xor, insn_xori, insn_yield,
};
struct insn {
@@ -63,35 +65,35 @@ struct insn {
enum fields fields;
};
-static inline __uasminit u32 build_rs(u32 arg)
+static inline u32 build_rs(u32 arg)
{
WARN(arg & ~RS_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return (arg & RS_MASK) << RS_SH;
}
-static inline __uasminit u32 build_rt(u32 arg)
+static inline u32 build_rt(u32 arg)
{
WARN(arg & ~RT_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return (arg & RT_MASK) << RT_SH;
}
-static inline __uasminit u32 build_rd(u32 arg)
+static inline u32 build_rd(u32 arg)
{
WARN(arg & ~RD_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return (arg & RD_MASK) << RD_SH;
}
-static inline __uasminit u32 build_re(u32 arg)
+static inline u32 build_re(u32 arg)
{
WARN(arg & ~RE_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return (arg & RE_MASK) << RE_SH;
}
-static inline __uasminit u32 build_simm(s32 arg)
+static inline u32 build_simm(s32 arg)
{
WARN(arg > 0x7fff || arg < -0x8000,
KERN_WARNING "Micro-assembler field overflow\n");
@@ -99,14 +101,14 @@ static inline __uasminit u32 build_simm(s32 arg)
return arg & 0xffff;
}
-static inline __uasminit u32 build_uimm(u32 arg)
+static inline u32 build_uimm(u32 arg)
{
WARN(arg & ~IMM_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return arg & IMM_MASK;
}
-static inline __uasminit u32 build_scimm(u32 arg)
+static inline u32 build_scimm(u32 arg)
{
WARN(arg & ~SCIMM_MASK,
KERN_WARNING "Micro-assembler field overflow\n");
@@ -114,21 +116,21 @@ static inline __uasminit u32 build_scimm(u32 arg)
return (arg & SCIMM_MASK) << SCIMM_SH;
}
-static inline __uasminit u32 build_func(u32 arg)
+static inline u32 build_func(u32 arg)
{
WARN(arg & ~FUNC_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return arg & FUNC_MASK;
}
-static inline __uasminit u32 build_set(u32 arg)
+static inline u32 build_set(u32 arg)
{
WARN(arg & ~SET_MASK, KERN_WARNING "Micro-assembler field overflow\n");
return arg & SET_MASK;
}
-static void __uasminit build_insn(u32 **buf, enum opcode opc, ...);
+static void build_insn(u32 **buf, enum opcode opc, ...);
#define I_u1u2u3(op) \
Ip_u1u2u3(op) \
@@ -137,6 +139,13 @@ Ip_u1u2u3(op) \
} \
UASM_EXPORT_SYMBOL(uasm_i##op);
+#define I_s3s1s2(op) \
+Ip_s3s1s2(op) \
+{ \
+ build_insn(buf, insn##op, b, c, a); \
+} \
+UASM_EXPORT_SYMBOL(uasm_i##op);
+
#define I_u2u1u3(op) \
Ip_u2u1u3(op) \
{ \
@@ -144,6 +153,13 @@ Ip_u2u1u3(op) \
} \
UASM_EXPORT_SYMBOL(uasm_i##op);
+#define I_u3u2u1(op) \
+Ip_u3u2u1(op) \
+{ \
+ build_insn(buf, insn##op, c, b, a); \
+} \
+UASM_EXPORT_SYMBOL(uasm_i##op);
+
#define I_u3u1u2(op) \
Ip_u3u1u2(op) \
{ \
@@ -200,6 +216,13 @@ Ip_u1u2(op) \
} \
UASM_EXPORT_SYMBOL(uasm_i##op);
+#define I_u2u1(op) \
+Ip_u1u2(op) \
+{ \
+ build_insn(buf, insn##op, b, a); \
+} \
+UASM_EXPORT_SYMBOL(uasm_i##op);
+
#define I_u1s2(op) \
Ip_u1s2(op) \
{ \
@@ -237,6 +260,7 @@ I_u1u2u3(_dmfc0)
I_u1u2u3(_dmtc0)
I_u2u1s3(_daddiu)
I_u3u1u2(_daddu)
+I_u1u2(_divu)
I_u2u1u3(_dsll)
I_u2u1u3(_dsll32)
I_u2u1u3(_dsra)
@@ -250,14 +274,20 @@ I_u2u1msbdu3(_ext)
I_u2u1msbu3(_ins)
I_u1(_j)
I_u1(_jal)
+I_u2u1(_jalr)
I_u1(_jr)
+I_u2s3u1(_lb)
I_u2s3u1(_ld)
+I_u2s3u1(_lh)
I_u2s3u1(_ll)
I_u2s3u1(_lld)
I_u1s2(_lui)
I_u2s3u1(_lw)
I_u1u2u3(_mfc0)
+I_u1(_mfhi)
+I_u1(_mflo)
I_u1u2u3(_mtc0)
+I_u3u1u2(_mul)
I_u2u1u3(_ori)
I_u3u1u2(_or)
I_0(_rfe)
@@ -265,17 +295,26 @@ I_u2s3u1(_sc)
I_u2s3u1(_scd)
I_u2s3u1(_sd)
I_u2u1u3(_sll)
+I_u3u2u1(_sllv)
+I_s3s1s2(_slt)
+I_u2u1s3(_sltiu)
+I_u3u1u2(_sltu)
I_u2u1u3(_sra)
I_u2u1u3(_srl)
+I_u3u2u1(_srlv)
I_u2u1u3(_rotr)
I_u3u1u2(_subu)
I_u2s3u1(_sw)
+I_u1(_sync)
I_0(_tlbp)
I_0(_tlbr)
I_0(_tlbwi)
I_0(_tlbwr)
+I_u1(_wait);
+I_u2u1(_wsbh)
I_u3u1u2(_xor)
I_u2u1u3(_xori)
+I_u2u1(_yield)
I_u2u1msbu3(_dins);
I_u2u1msb32u3(_dinsm);
I_u1(_syscall);
@@ -286,7 +325,7 @@ I_u3u1u2(_ldx)
#ifdef CONFIG_CPU_CAVIUM_OCTEON
#include <asm/octeon/octeon.h>
-void __uasminit ISAFUNC(uasm_i_pref)(u32 **buf, unsigned int a, signed int b,
+void ISAFUNC(uasm_i_pref)(u32 **buf, unsigned int a, signed int b,
unsigned int c)
{
if (OCTEON_IS_MODEL(OCTEON_CN63XX_PASS1_X) && a <= 24 && a != 5)
@@ -304,7 +343,7 @@ I_u2s3u1(_pref)
#endif
/* Handle labels. */
-void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, int lid)
+void ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, int lid)
{
(*lab)->addr = addr;
(*lab)->lab = lid;
@@ -312,7 +351,7 @@ void __uasminit ISAFUNC(uasm_build_label)(struct uasm_label **lab, u32 *addr, in
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_build_label));
-int __uasminit ISAFUNC(uasm_in_compat_space_p)(long addr)
+int ISAFUNC(uasm_in_compat_space_p)(long addr)
{
/* Is this address in 32bit compat space? */
#ifdef CONFIG_64BIT
@@ -323,7 +362,7 @@ int __uasminit ISAFUNC(uasm_in_compat_space_p)(long addr)
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_in_compat_space_p));
-static int __uasminit uasm_rel_highest(long val)
+static int uasm_rel_highest(long val)
{
#ifdef CONFIG_64BIT
return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
@@ -332,7 +371,7 @@ static int __uasminit uasm_rel_highest(long val)
#endif
}
-static int __uasminit uasm_rel_higher(long val)
+static int uasm_rel_higher(long val)
{
#ifdef CONFIG_64BIT
return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
@@ -341,19 +380,19 @@ static int __uasminit uasm_rel_higher(long val)
#endif
}
-int __uasminit ISAFUNC(uasm_rel_hi)(long val)
+int ISAFUNC(uasm_rel_hi)(long val)
{
return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_hi));
-int __uasminit ISAFUNC(uasm_rel_lo)(long val)
+int ISAFUNC(uasm_rel_lo)(long val)
{
return ((val & 0xffff) ^ 0x8000) - 0x8000;
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_rel_lo));
-void __uasminit ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr)
+void ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr)
{
if (!ISAFUNC(uasm_in_compat_space_p)(addr)) {
ISAFUNC(uasm_i_lui)(buf, rs, uasm_rel_highest(addr));
@@ -371,7 +410,7 @@ void __uasminit ISAFUNC(UASM_i_LA_mostly)(u32 **buf, unsigned int rs, long addr)
}
UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA_mostly));
-void __uasminit ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr)
+void ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr)
{
ISAFUNC(UASM_i_LA_mostly)(buf, rs, addr);
if (ISAFUNC(uasm_rel_lo(addr))) {
@@ -386,8 +425,7 @@ void __uasminit ISAFUNC(UASM_i_LA)(u32 **buf, unsigned int rs, long addr)
UASM_EXPORT_SYMBOL(ISAFUNC(UASM_i_LA));
/* Handle relocations. */
-void __uasminit
-ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
+void ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
{
(*rel)->addr = addr;
(*rel)->type = R_MIPS_PC16;
@@ -396,11 +434,11 @@ ISAFUNC(uasm_r_mips_pc16)(struct uasm_reloc **rel, u32 *addr, int lid)
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_r_mips_pc16));
-static inline void __uasminit
-__resolve_relocs(struct uasm_reloc *rel, struct uasm_label *lab);
+static inline void __resolve_relocs(struct uasm_reloc *rel,
+ struct uasm_label *lab);
-void __uasminit
-ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel, struct uasm_label *lab)
+void ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel,
+ struct uasm_label *lab)
{
struct uasm_label *l;
@@ -411,8 +449,8 @@ ISAFUNC(uasm_resolve_relocs)(struct uasm_reloc *rel, struct uasm_label *lab)
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_resolve_relocs));
-void __uasminit
-ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end, long off)
+void ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end,
+ long off)
{
for (; rel->lab != UASM_LABEL_INVALID; rel++)
if (rel->addr >= first && rel->addr < end)
@@ -420,8 +458,8 @@ ISAFUNC(uasm_move_relocs)(struct uasm_reloc *rel, u32 *first, u32 *end, long off
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_relocs));
-void __uasminit
-ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end, long off)
+void ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end,
+ long off)
{
for (; lab->lab != UASM_LABEL_INVALID; lab++)
if (lab->addr >= first && lab->addr < end)
@@ -429,9 +467,8 @@ ISAFUNC(uasm_move_labels)(struct uasm_label *lab, u32 *first, u32 *end, long off
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_move_labels));
-void __uasminit
-ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab, u32 *first,
- u32 *end, u32 *target)
+void ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab,
+ u32 *first, u32 *end, u32 *target)
{
long off = (long)(target - first);
@@ -442,7 +479,7 @@ ISAFUNC(uasm_copy_handler)(struct uasm_reloc *rel, struct uasm_label *lab, u32 *
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_copy_handler));
-int __uasminit ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr)
+int ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr)
{
for (; rel->lab != UASM_LABEL_INVALID; rel++) {
if (rel->addr == addr
@@ -456,83 +493,87 @@ int __uasminit ISAFUNC(uasm_insn_has_bdelay)(struct uasm_reloc *rel, u32 *addr)
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_insn_has_bdelay));
/* Convenience functions for labeled branches. */
-void __uasminit
-ISAFUNC(uasm_il_bltz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_bltz)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bltz)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bltz));
-void __uasminit
-ISAFUNC(uasm_il_b)(u32 **p, struct uasm_reloc **r, int lid)
+void ISAFUNC(uasm_il_b)(u32 **p, struct uasm_reloc **r, int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_b)(p, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_b));
-void __uasminit
-ISAFUNC(uasm_il_beqz)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_beq)(u32 **p, struct uasm_reloc **r, unsigned int r1,
+ unsigned int r2, int lid)
+{
+ uasm_r_mips_pc16(r, *p, lid);
+ ISAFUNC(uasm_i_beq)(p, r1, r2, 0);
+}
+UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beq));
+
+void ISAFUNC(uasm_il_beqz)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_beqz)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqz));
-void __uasminit
-ISAFUNC(uasm_il_beqzl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_beqzl)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_beqzl)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_beqzl));
-void __uasminit
-ISAFUNC(uasm_il_bne)(u32 **p, struct uasm_reloc **r, unsigned int reg1,
- unsigned int reg2, int lid)
+void ISAFUNC(uasm_il_bne)(u32 **p, struct uasm_reloc **r, unsigned int reg1,
+ unsigned int reg2, int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bne)(p, reg1, reg2, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bne));
-void __uasminit
-ISAFUNC(uasm_il_bnez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_bnez)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bnez)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bnez));
-void __uasminit
-ISAFUNC(uasm_il_bgezl)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_bgezl)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bgezl)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgezl));
-void __uasminit
-ISAFUNC(uasm_il_bgez)(u32 **p, struct uasm_reloc **r, unsigned int reg, int lid)
+void ISAFUNC(uasm_il_bgez)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bgez)(p, reg, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bgez));
-void __uasminit
-ISAFUNC(uasm_il_bbit0)(u32 **p, struct uasm_reloc **r, unsigned int reg,
- unsigned int bit, int lid)
+void ISAFUNC(uasm_il_bbit0)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ unsigned int bit, int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bbit0)(p, reg, bit, 0);
}
UASM_EXPORT_SYMBOL(ISAFUNC(uasm_il_bbit0));
-void __uasminit
-ISAFUNC(uasm_il_bbit1)(u32 **p, struct uasm_reloc **r, unsigned int reg,
- unsigned int bit, int lid)
+void ISAFUNC(uasm_il_bbit1)(u32 **p, struct uasm_reloc **r, unsigned int reg,
+ unsigned int bit, int lid)
{
uasm_r_mips_pc16(r, *p, lid);
ISAFUNC(uasm_i_bbit1)(p, reg, bit, 0);
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-10 07:37:14 +0000