aboutsummaryrefslogtreecommitdiff
path: root/arch/sparc/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc/mm')
-rw-r--r--arch/sparc/mm/fault_32.c21
-rw-r--r--arch/sparc/mm/fault_64.c135
-rw-r--r--arch/sparc/mm/gup.c58
-rw-r--r--arch/sparc/mm/hugetlbpage.c3
-rw-r--r--arch/sparc/mm/hypersparc.S8
-rw-r--r--arch/sparc/mm/init_32.c89
-rw-r--r--arch/sparc/mm/init_64.c479
-rw-r--r--arch/sparc/mm/init_64.h8
-rw-r--r--arch/sparc/mm/io-unit.c21
-rw-r--r--arch/sparc/mm/iommu.c27
-rw-r--r--arch/sparc/mm/leon_mm.c4
-rw-r--r--arch/sparc/mm/mm_32.h24
-rw-r--r--arch/sparc/mm/srmmu.c36
-rw-r--r--arch/sparc/mm/srmmu.h4
-rw-r--r--arch/sparc/mm/swift.S8
-rw-r--r--arch/sparc/mm/tlb.c103
-rw-r--r--arch/sparc/mm/tsb.c85
-rw-r--r--arch/sparc/mm/tsunami.S6
-rw-r--r--arch/sparc/mm/ultra.S131
-rw-r--r--arch/sparc/mm/viking.S10
20 files changed, 699 insertions, 561 deletions
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index e98bfda205a..908e8c17c90 100644
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -26,14 +26,14 @@
#include <asm/pgtable.h>
#include <asm/openprom.h>
#include <asm/oplib.h>
+#include <asm/setup.h>
#include <asm/smp.h>
#include <asm/traps.h>
#include <asm/uaccess.h>
-int show_unhandled_signals = 1;
+#include "mm_32.h"
-static void unhandled_fault(unsigned long, struct task_struct *,
- struct pt_regs *) __attribute__ ((noreturn));
+int show_unhandled_signals = 1;
static void __noreturn unhandled_fault(unsigned long address,
struct task_struct *tsk,
@@ -141,9 +141,6 @@ static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs,
force_sig_info (sig, &info, current);
}
-extern unsigned long safe_compute_effective_address(struct pt_regs *,
- unsigned int);
-
static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault)
{
unsigned int insn;
@@ -177,8 +174,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
unsigned long g2;
int from_user = !(regs->psr & PSR_PS);
int fault, code;
- unsigned int flags = (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE |
- (write ? FAULT_FLAG_WRITE : 0));
+ unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
if (text_fault)
address = regs->pc;
@@ -235,6 +231,11 @@ good_area:
goto bad_area;
}
+ if (from_user)
+ flags |= FAULT_FLAG_USER;
+ if (write)
+ flags |= FAULT_FLAG_WRITE;
+
/*
* If for any reason at all we couldn't handle the fault,
* make sure we exit gracefully rather than endlessly redo
@@ -383,6 +384,7 @@ static void force_user_fault(unsigned long address, int write)
struct vm_area_struct *vma;
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
+ unsigned int flags = FAULT_FLAG_USER;
int code;
code = SEGV_MAPERR;
@@ -402,11 +404,12 @@ good_area:
if (write) {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
+ flags |= FAULT_FLAG_WRITE;
} else {
if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
goto bad_area;
}
- switch (handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0)) {
+ switch (handle_mm_fault(mm, vma, address, flags)) {
case VM_FAULT_SIGBUS:
case VM_FAULT_OOM:
goto do_sigbus;
diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 097aee763af..587cd056512 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -21,6 +21,7 @@
#include <linux/kprobes.h>
#include <linux/kdebug.h>
#include <linux/percpu.h>
+#include <linux/context_tracking.h>
#include <asm/page.h>
#include <asm/pgtable.h>
@@ -31,6 +32,7 @@
#include <asm/lsu.h>
#include <asm/sections.h>
#include <asm/mmu_context.h>
+#include <asm/setup.h>
int show_unhandled_signals = 1;
@@ -95,38 +97,51 @@ static unsigned int get_user_insn(unsigned long tpc)
pte_t *ptep, pte;
unsigned long pa;
u32 insn = 0;
- unsigned long pstate;
- if (pgd_none(*pgdp))
- goto outret;
+ if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp)))
+ goto out;
pudp = pud_offset(pgdp, tpc);
- if (pud_none(*pudp))
- goto outret;
- pmdp = pmd_offset(pudp, tpc);
- if (pmd_none(*pmdp))
- goto outret;
-
- /* This disables preemption for us as well. */
- __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
- __asm__ __volatile__("wrpr %0, %1, %%pstate"
- : : "r" (pstate), "i" (PSTATE_IE));
- ptep = pte_offset_map(pmdp, tpc);
- pte = *ptep;
- if (!pte_present(pte))
+ if (pud_none(*pudp) || unlikely(pud_bad(*pudp)))
goto out;
- pa = (pte_pfn(pte) << PAGE_SHIFT);
- pa += (tpc & ~PAGE_MASK);
-
- /* Use phys bypass so we don't pollute dtlb/dcache. */
- __asm__ __volatile__("lduwa [%1] %2, %0"
- : "=r" (insn)
- : "r" (pa), "i" (ASI_PHYS_USE_EC));
+ /* This disables preemption for us as well. */
+ local_irq_disable();
+ pmdp = pmd_offset(pudp, tpc);
+ if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp)))
+ goto out_irq_enable;
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ if (pmd_trans_huge(*pmdp)) {
+ if (pmd_trans_splitting(*pmdp))
+ goto out_irq_enable;
+
+ pa = pmd_pfn(*pmdp) << PAGE_SHIFT;
+ pa += tpc & ~HPAGE_MASK;
+
+ /* Use phys bypass so we don't pollute dtlb/dcache. */
+ __asm__ __volatile__("lduwa [%1] %2, %0"
+ : "=r" (insn)
+ : "r" (pa), "i" (ASI_PHYS_USE_EC));
+ } else
+#endif
+ {
+ ptep = pte_offset_map(pmdp, tpc);
+ pte = *ptep;
+ if (pte_present(pte)) {
+ pa = (pte_pfn(pte) << PAGE_SHIFT);
+ pa += (tpc & ~PAGE_MASK);
+
+ /* Use phys bypass so we don't pollute dtlb/dcache. */
+ __asm__ __volatile__("lduwa [%1] %2, %0"
+ : "=r" (insn)
+ : "r" (pa), "i" (ASI_PHYS_USE_EC));
+ }
+ pte_unmap(ptep);
+ }
+out_irq_enable:
+ local_irq_enable();
out:
- pte_unmap(ptep);
- __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
-outret:
return insn;
}
@@ -152,7 +167,8 @@ show_signal_msg(struct pt_regs *regs, int sig, int code,
}
static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
- unsigned int insn, int fault_code)
+ unsigned long fault_addr, unsigned int insn,
+ int fault_code)
{
unsigned long addr;
siginfo_t info;
@@ -160,10 +176,18 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
info.si_code = code;
info.si_signo = sig;
info.si_errno = 0;
- if (fault_code & FAULT_CODE_ITLB)
+ if (fault_code & FAULT_CODE_ITLB) {
addr = regs->tpc;
- else
- addr = compute_effective_address(regs, insn, 0);
+ } else {
+ /* If we were able to probe the faulting instruction, use it
+ * to compute a precise fault address. Otherwise use the fault
+ * time provided address which may only have page granularity.
+ */
+ if (insn)
+ addr = compute_effective_address(regs, insn, 0);
+ else
+ addr = fault_addr;
+ }
info.si_addr = (void __user *) addr;
info.si_trapno = 0;
@@ -173,9 +197,6 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs,
force_sig_info(sig, &info, current);
}
-extern int handle_ldf_stq(u32, struct pt_regs *);
-extern int handle_ld_nf(u32, struct pt_regs *);
-
static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn)
{
if (!insn) {
@@ -238,7 +259,7 @@ static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code,
/* The si_code was set to make clear whether
* this was a SEGV_MAPERR or SEGV_ACCERR fault.
*/
- do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code);
+ do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code);
return;
}
@@ -258,20 +279,9 @@ static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs)
show_regs(regs);
}
-static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs,
- unsigned long addr)
-{
- static int times;
-
- if (times++ < 10)
- printk(KERN_ERR "FAULT[%s:%d]: 32-bit process "
- "reports 64-bit fault address [%lx]\n",
- current->comm, current->pid, addr);
- show_regs(regs);
-}
-
asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
{
+ enum ctx_state prev_state = exception_enter();
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned int insn = 0;
@@ -282,7 +292,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
fault_code = get_thread_fault_code();
if (notify_page_fault(regs))
- return;
+ goto exit_exception;
si_code = SEGV_MAPERR;
address = current_thread_info()->fault_address;
@@ -298,10 +308,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
goto intr_or_no_mm;
}
}
- if (unlikely((address >> 32) != 0)) {
- bogus_32bit_fault_address(regs, address);
+ if (unlikely((address >> 32) != 0))
goto intr_or_no_mm;
- }
}
if (regs->tstate & TSTATE_PRIV) {
@@ -313,9 +321,10 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
/* Valid, no problems... */
} else {
bad_kernel_pc(regs, address);
- return;
+ goto exit_exception;
}
- }
+ } else
+ flags |= FAULT_FLAG_USER;
/*
* If we're in an interrupt or have no user
@@ -418,17 +427,18 @@ good_area:
vma->vm_file != NULL)
set_thread_fault_code(fault_code |
FAULT_CODE_BLKCOMMIT);
+
+ flags |= FAULT_FLAG_WRITE;
} else {
/* Allow reads even for write-only mappings */
if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
goto bad_area;
}
- flags |= ((fault_code & FAULT_CODE_WRITE) ? FAULT_FLAG_WRITE : 0);
fault = handle_mm_fault(mm, vma, address, flags);
if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
- return;
+ goto exit_exception;
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM)
@@ -472,9 +482,16 @@ good_area:
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
mm_rss = mm->context.huge_pte_count;
if (unlikely(mm_rss >
- mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit))
- tsb_grow(mm, MM_TSB_HUGE, mm_rss);
+ mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) {
+ if (mm->context.tsb_block[MM_TSB_HUGE].tsb)
+ tsb_grow(mm, MM_TSB_HUGE, mm_rss);
+ else
+ hugetlb_setup(regs);
+
+ }
#endif
+exit_exception:
+ exception_exit(prev_state);
return;
/*
@@ -487,7 +504,7 @@ bad_area:
handle_kernel_fault:
do_kernel_fault(regs, si_code, fault_code, insn, address);
- return;
+ goto exit_exception;
/*
* We ran out of memory, or some other thing happened to us that made
@@ -498,7 +515,7 @@ out_of_memory:
up_read(&mm->mmap_sem);
if (!(regs->tstate & TSTATE_PRIV)) {
pagefault_out_of_memory();
- return;
+ goto exit_exception;
}
goto handle_kernel_fault;
@@ -514,7 +531,7 @@ do_sigbus:
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
- do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code);
+ do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code);
/* Kernel mode? Handle exceptions or die */
if (regs->tstate & TSTATE_PRIV)
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index 42c55df3aec..1aed0432c64 100644
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -66,6 +66,55 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
return 1;
}
+static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr,
+ unsigned long end, int write, struct page **pages,
+ int *nr)
+{
+ struct page *head, *page, *tail;
+ int refs;
+
+ if (!(pmd_val(pmd) & _PAGE_VALID))
+ return 0;
+
+ if (write && !pmd_write(pmd))
+ return 0;
+
+ refs = 0;
+ head = pmd_page(pmd);
+ page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
+ tail = page;
+ do {
+ VM_BUG_ON(compound_head(page) != head);
+ pages[*nr] = page;
+ (*nr)++;
+ page++;
+ refs++;
+ } while (addr += PAGE_SIZE, addr != end);
+
+ if (!page_cache_add_speculative(head, refs)) {
+ *nr -= refs;
+ return 0;
+ }
+
+ if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) {
+ *nr -= refs;
+ while (refs--)
+ put_page(head);
+ return 0;
+ }
+
+ /* Any tail page need their mapcount reference taken before we
+ * return.
+ */
+ while (refs--) {
+ if (PageTail(tail))
+ get_huge_page_tail(tail);
+ tail++;
+ }
+
+ return 1;
+}
+
static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
int write, struct page **pages, int *nr)
{
@@ -77,9 +126,14 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
pmd_t pmd = *pmdp;
next = pmd_addr_end(addr, end);
- if (pmd_none(pmd))
+ if (pmd_none(pmd) || pmd_trans_splitting(pmd))
return 0;
- if (!gup_pte_range(pmd, addr, next, write, pages, nr))
+ if (unlikely(pmd_large(pmd))) {
+ if (!gup_huge_pmd(pmdp, pmd, addr, next,
+ write, pages, nr))
+ return 0;
+ } else if (!gup_pte_range(pmd, addr, next, write,
+ pages, nr))
return 0;
} while (pmdp++, addr = next, addr != end);
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index d2b59441ebd..d329537739c 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -4,7 +4,6 @@
* Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net)
*/
-#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/hugetlb.h>
@@ -21,8 +20,6 @@
/* Slightly simplified from the non-hugepage variant because by
* definition we don't have to worry about any page coloring stuff
*/
-#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL))
-#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL))
static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp,
unsigned long addr,
diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S
index 44aad32eeb4..969f96450f6 100644
--- a/arch/sparc/mm/hypersparc.S
+++ b/arch/sparc/mm/hypersparc.S
@@ -74,7 +74,7 @@ hypersparc_flush_cache_mm_out:
/* The things we do for performance... */
hypersparc_flush_cache_range:
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
#ifndef CONFIG_SMP
ld [%o0 + AOFF_mm_context], %g1
cmp %g1, -1
@@ -163,7 +163,7 @@ hypersparc_flush_cache_range_out:
*/
/* Verified, my ass... */
hypersparc_flush_cache_page:
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
ld [%o0 + AOFF_mm_context], %g2
#ifndef CONFIG_SMP
cmp %g2, -1
@@ -284,7 +284,7 @@ hypersparc_flush_tlb_mm_out:
sta %g5, [%g1] ASI_M_MMUREGS
hypersparc_flush_tlb_range:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
lda [%g1] ASI_M_MMUREGS, %g5
@@ -307,7 +307,7 @@ hypersparc_flush_tlb_range_out:
sta %g5, [%g1] ASI_M_MMUREGS
hypersparc_flush_tlb_page:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
andn %o1, (PAGE_SIZE - 1), %o1
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index dde85ef1c56..eb828715527 100644
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -31,10 +31,13 @@
#include <asm/pgtable.h>
#include <asm/vaddrs.h>
#include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */
+#include <asm/setup.h>
#include <asm/tlb.h>
#include <asm/prom.h>
#include <asm/leon.h>
+#include "mm_32.h"
+
unsigned long *sparc_valid_addr_bitmap;
EXPORT_SYMBOL(sparc_valid_addr_bitmap);
@@ -57,13 +60,12 @@ void show_mem(unsigned int filter)
printk("Mem-info:\n");
show_free_areas(filter);
printk("Free swap: %6ldkB\n",
- nr_swap_pages << (PAGE_SHIFT-10));
+ get_nr_swap_pages() << (PAGE_SHIFT-10));
printk("%ld pages of RAM\n", totalram_pages);
printk("%ld free pages\n", nr_free_pages());
}
-extern unsigned long cmdline_memory_size;
unsigned long last_valid_pfn;
unsigned long calc_highpages(void)
@@ -246,9 +248,6 @@ unsigned long __init bootmem_init(unsigned long *pages_avail)
* init routine based upon the Sun model type on the Sparc.
*
*/
-extern void srmmu_paging_init(void);
-extern void device_scan(void);
-
void __init paging_init(void)
{
srmmu_paging_init();
@@ -282,22 +281,12 @@ static void map_high_region(unsigned long start_pfn, unsigned long end_pfn)
printk("mapping high region %08lx - %08lx\n", start_pfn, end_pfn);
#endif
- for (tmp = start_pfn; tmp < end_pfn; tmp++) {
- struct page *page = pfn_to_page(tmp);
-
- ClearPageReserved(page);
- init_page_count(page);
- __free_page(page);
- totalhigh_pages++;
- }
+ for (tmp = start_pfn; tmp < end_pfn; tmp++)
+ free_highmem_page(pfn_to_page(tmp));
}
void __init mem_init(void)
{
- int codepages = 0;
- int datapages = 0;
- int initpages = 0;
- int reservedpages = 0;
int i;
if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
@@ -329,15 +318,12 @@ void __init mem_init(void)
max_mapnr = last_valid_pfn - pfn_base;
high_memory = __va(max_low_pfn << PAGE_SHIFT);
-
- totalram_pages = free_all_bootmem();
+ free_all_bootmem();
for (i = 0; sp_banks[i].num_bytes != 0; i++) {
unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT;
unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT;
- num_physpages += sp_banks[i].num_bytes >> PAGE_SHIFT;
-
if (end_pfn <= highstart_pfn)
continue;
@@ -347,72 +333,19 @@ void __init mem_init(void)
map_high_region(start_pfn, end_pfn);
}
- totalram_pages += totalhigh_pages;
-
- codepages = (((unsigned long) &_etext) - ((unsigned long)&_start));
- codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
- datapages = (((unsigned long) &_edata) - ((unsigned long)&_etext));
- datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
- initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin));
- initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
-
- /* Ignore memory holes for the purpose of counting reserved pages */
- for (i=0; i < max_low_pfn; i++)
- if (test_bit(i >> (20 - PAGE_SHIFT), sparc_valid_addr_bitmap)
- && PageReserved(pfn_to_page(i)))
- reservedpages++;
-
- printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n",
- nr_free_pages() << (PAGE_SHIFT-10),
- num_physpages << (PAGE_SHIFT - 10),
- codepages << (PAGE_SHIFT-10),
- reservedpages << (PAGE_SHIFT - 10),
- datapages << (PAGE_SHIFT-10),
- initpages << (PAGE_SHIFT-10),
- totalhigh_pages << (PAGE_SHIFT-10));
+ mem_init_print_info(NULL);
}
void free_initmem (void)
{
- unsigned long addr;
- unsigned long freed;
-
- addr = (unsigned long)(&__init_begin);
- freed = (unsigned long)(&__init_end) - addr;
- for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) {
- struct page *p;
-
- memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
- p = virt_to_page(addr);
-
- ClearPageReserved(p);
- init_page_count(p);
- __free_page(p);
- totalram_pages++;
- num_physpages++;
- }
- printk(KERN_INFO "Freeing unused kernel memory: %ldk freed\n",
- freed >> 10);
+ free_initmem_default(POISON_FREE_INITMEM);
}
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
- if (start < end)
- printk(KERN_INFO "Freeing initrd memory: %ldk freed\n",
- (end - start) >> 10);
- for (; start < end; start += PAGE_SIZE) {
- struct page *p;
-
- memset((void *)start, POISON_FREE_INITMEM, PAGE_SIZE);
- p = virt_to_page(start);
-
- ClearPageReserved(p);
- init_page_count(p);
- __free_page(p);
- totalram_pages++;
- num_physpages++;
- }
+ free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
+ "initrd");
}
#endif
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index c3b72423c84..16b58ff11e6 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -47,6 +47,7 @@
#include <asm/prom.h>
#include <asm/mdesc.h>
#include <asm/cpudata.h>
+#include <asm/setup.h>
#include <asm/irq.h>
#include "init_64.h"
@@ -314,16 +315,31 @@ static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_inde
struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
unsigned long tag;
+ if (unlikely(!tsb))
+ return;
+
tsb += ((address >> tsb_hash_shift) &
(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
tag = (address >> 22UL);
tsb_insert(tsb, tag, tte);
}
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static inline bool is_hugetlb_pte(pte_t pte)
+{
+ if ((tlb_type == hypervisor &&
+ (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
+ (tlb_type != hypervisor &&
+ (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))
+ return true;
+ return false;
+}
+#endif
+
void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
{
- unsigned long tsb_index, tsb_hash_shift, flags;
struct mm_struct *mm;
+ unsigned long flags;
pte_t pte = *ptep;
if (tlb_type != hypervisor) {
@@ -335,25 +351,16 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
mm = vma->vm_mm;
- tsb_index = MM_TSB_BASE;
- tsb_hash_shift = PAGE_SHIFT;
-
spin_lock_irqsave(&mm->context.lock, flags);
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
- if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
- if ((tlb_type == hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
- (tlb_type != hypervisor &&
- (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) {
- tsb_index = MM_TSB_HUGE;
- tsb_hash_shift = HPAGE_SHIFT;
- }
- }
+ if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
+ __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
+ address, pte_val(pte));
+ else
#endif
-
- __update_mmu_tsb_insert(mm, tsb_index, tsb_hash_shift,
- address, pte_val(pte));
+ __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
+ address, pte_val(pte));
spin_unlock_irqrestore(&mm->context.lock, flags);
}
@@ -582,7 +589,7 @@ static void __init remap_kernel(void)
int i, tlb_ent = sparc64_highest_locked_tlbent();
tte_vaddr = (unsigned long) KERNBASE;
- phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
+ phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
tte_data = kern_large_tte(phys_page);
kern_locked_tte_data = tte_data;
@@ -675,10 +682,9 @@ void get_new_mmu_context(struct mm_struct *mm)
{
unsigned long ctx, new_ctx;
unsigned long orig_pgsz_bits;
- unsigned long flags;
int new_version;
- spin_lock_irqsave(&ctx_alloc_lock, flags);
+ spin_lock(&ctx_alloc_lock);
orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
@@ -714,7 +720,7 @@ void get_new_mmu_context(struct mm_struct *mm)
out:
tlb_context_cache = new_ctx;
mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
- spin_unlock_irqrestore(&ctx_alloc_lock, flags);
+ spin_unlock(&ctx_alloc_lock);
if (unlikely(new_version))
smp_new_mmu_context_version();
@@ -789,11 +795,11 @@ struct node_mem_mask {
static struct node_mem_mask node_masks[MAX_NUMNODES];
static int num_node_masks;
+#ifdef CONFIG_NEED_MULTIPLE_NODES
+
int numa_cpu_lookup_table[NR_CPUS];
cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
-#ifdef CONFIG_NEED_MULTIPLE_NODES
-
struct mdesc_mblock {
u64 base;
u64 size;
@@ -882,17 +888,21 @@ static void __init allocate_node_data(int nid)
static void init_node_masks_nonnuma(void)
{
+#ifdef CONFIG_NEED_MULTIPLE_NODES
int i;
+#endif
numadbg("Initializing tables for non-numa.\n");
node_masks[0].mask = node_masks[0].val = 0;
num_node_masks = 1;
+#ifdef CONFIG_NEED_MULTIPLE_NODES
for (i = 0; i < NR_CPUS; i++)
numa_cpu_lookup_table[i] = 0;
cpumask_setall(&numa_cpumask_lookup_table[0]);
+#endif
}
#ifdef CONFIG_NEED_MULTIPLE_NODES
@@ -1016,7 +1026,8 @@ static void __init add_node_ranges(void)
"start[%lx] end[%lx]\n",
nid, start, this_end);
- memblock_set_node(start, this_end - start, nid);
+ memblock_set_node(start, this_end - start,
+ &memblock.memory, nid);
start = this_end;
}
}
@@ -1093,7 +1104,14 @@ static int __init grab_mblocks(struct mdesc_handle *md)
m->size = *val;
val = mdesc_get_property(md, node,
"address-congruence-offset", NULL);
- m->offset = *val;
+
+ /* The address-congruence-offset property is optional.
+ * Explicity zero it be identifty this.
+ */
+ if (val)
+ m->offset = *val;
+ else
+ m->offset = 0UL;
numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
count - 1, m->base, m->size, m->offset);
@@ -1313,7 +1331,7 @@ static void __init bootmem_init_nonnuma(void)
(top_of_ram - total_ram) >> 20);
init_node_masks_nonnuma();
- memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
+ memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
allocate_node_data(0);
node_set_online(0);
}
@@ -1545,6 +1563,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size)
return ~0UL;
}
+unsigned long PAGE_OFFSET;
+EXPORT_SYMBOL(PAGE_OFFSET);
+
+static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)
+{
+ unsigned long final_shift;
+ unsigned int val = *insn;
+ unsigned int cnt;
+
+ /* We are patching in ilog2(max_supported_phys_address), and
+ * we are doing so in a manner similar to a relocation addend.
+ * That is, we are adding the shift value to whatever value
+ * is in the shift instruction count field already.
+ */
+ cnt = (val & 0x3f);
+ val &= ~0x3f;
+
+ /* If we are trying to shift >= 64 bits, clear the destination
+ * register. This can happen when phys_bits ends up being equal
+ * to MAX_PHYS_ADDRESS_BITS.
+ */
+ final_shift = (cnt + (64 - phys_bits));
+ if (final_shift >= 64) {
+ unsigned int rd = (val >> 25) & 0x1f;
+
+ val = 0x80100000 | (rd << 25);
+ } else {
+ val |= final_shift;
+ }
+ *insn = val;
+
+ __asm__ __volatile__("flush %0"
+ : /* no outputs */
+ : "r" (insn));
+}
+
+static void __init page_offset_shift_patch(unsigned long phys_bits)
+{
+ extern unsigned int __page_offset_shift_patch;
+ extern unsigned int __page_offset_shift_patch_end;
+ unsigned int *p;
+
+ p = &__page_offset_shift_patch;
+ while (p < &__page_offset_shift_patch_end) {
+ unsigned int *insn = (unsigned int *)(unsigned long)*p;
+
+ page_offset_shift_patch_one(insn, phys_bits);
+
+ p++;
+ }
+}
+
+static void __init setup_page_offset(void)
+{
+ unsigned long max_phys_bits = 40;
+
+ if (tlb_type == cheetah || tlb_type == cheetah_plus) {
+ max_phys_bits = 42;
+ } else if (tlb_type == hypervisor) {
+ switch (sun4v_chip_type) {
+ case SUN4V_CHIP_NIAGARA1:
+ case SUN4V_CHIP_NIAGARA2:
+ max_phys_bits = 39;
+ break;
+ case SUN4V_CHIP_NIAGARA3:
+ max_phys_bits = 43;
+ break;
+ case SUN4V_CHIP_NIAGARA4:
+ case SUN4V_CHIP_NIAGARA5:
+ case SUN4V_CHIP_SPARC64X:
+ default:
+ max_phys_bits = 47;
+ break;
+ }
+ }
+
+ if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
+ prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
+ max_phys_bits);
+ prom_halt();
+ }
+
+ PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);
+
+ pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
+ PAGE_OFFSET, max_phys_bits);
+
+ page_offset_shift_patch(max_phys_bits);
+}
+
static void __init tsb_phys_patch(void)
{
struct tsb_ldquad_phys_patch_entry *pquad;
@@ -1682,7 +1790,7 @@ static void __init sun4v_ktsb_init(void)
#endif
}
-void __cpuinit sun4v_ktsb_register(void)
+void sun4v_ktsb_register(void)
{
unsigned long pa, ret;
@@ -1710,7 +1818,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
#ifndef CONFIG_DEBUG_PAGEALLOC
if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
- 0xfffff80000000000UL;
+ PAGE_OFFSET;
kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
_PAGE_P_4V | _PAGE_W_4V);
} else {
@@ -1719,7 +1827,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
- 0xfffff80000000000UL;
+ PAGE_OFFSET;
kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
_PAGE_P_4V | _PAGE_W_4V);
} else {
@@ -1728,7 +1836,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
- 0xfffff80000000000UL;
+ PAGE_OFFSET;
kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |
_PAGE_P_4V | _PAGE_W_4V);
} else {
@@ -1740,7 +1848,7 @@ static void __init sun4v_linear_pte_xor_finalize(void)
/* paging_init() sets up the page tables */
static unsigned long last_valid_pfn;
-pgd_t swapper_pg_dir[2048];
+pgd_t swapper_pg_dir[PTRS_PER_PGD];
static void sun4u_pgprot_init(void);
static void sun4v_pgprot_init(void);
@@ -1751,6 +1859,8 @@ void __init paging_init(void)
unsigned long real_end, i;
int node;
+ setup_page_offset();
+
/* These build time checkes make sure that the dcache_dirty_cpu()
* page->flags usage will work.
*
@@ -1776,7 +1886,7 @@ void __init paging_init(void)
BUILD_BUG_ON(NR_CPUS > 4096);
- kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL;
+ kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
/* Invalidate both kernel TSBs. */
@@ -1832,7 +1942,7 @@ void __init paging_init(void)
shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
real_end = (unsigned long)_end;
- num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22);
+ num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);
printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
num_kernel_image_mappings);
@@ -1989,7 +2099,7 @@ static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
if (new_start <= old_start &&
new_end >= (old_start + PAGE_SIZE)) {
- set_bit(old_start >> 22, bitmap);
+ set_bit(old_start >> ILOG2_4MB, bitmap);
goto do_next_page;
}
}
@@ -2021,15 +2131,24 @@ static void __init patch_tlb_miss_handler_bitmap(void)
flushi(&valid_addr_bitmap_insn[0]);
}
+static void __init register_page_bootmem_info(void)
+{
+#ifdef CONFIG_NEED_MULTIPLE_NODES
+ int i;
+
+ for_each_online_node(i)
+ if (NODE_DATA(i)->node_spanned_pages)
+ register_page_bootmem_info_node(NODE_DATA(i));
+#endif
+}
void __init mem_init(void)
{
- unsigned long codepages, datapages, initpages;
unsigned long addr, last;
addr = PAGE_OFFSET + kern_base;
last = PAGE_ALIGN(kern_size) + addr;
while (addr < last) {
- set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap);
+ set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);
addr += PAGE_SIZE;
}
@@ -2038,26 +2157,8 @@ void __init mem_init(void)
high_memory = __va(last_valid_pfn << PAGE_SHIFT);
-#ifdef CONFIG_NEED_MULTIPLE_NODES
- {
- int i;
- for_each_online_node(i) {
- if (NODE_DATA(i)->node_spanned_pages != 0) {
- totalram_pages +=
- free_all_bootmem_node(NODE_DATA(i));
- }
- }
- totalram_pages += free_low_memory_core_early(MAX_NUMNODES);
- }
-#else
- totalram_pages = free_all_bootmem();
-#endif
-
- /* We subtract one to account for the mem_map_zero page
- * allocated below.
- */
- totalram_pages -= 1;
- num_physpages = totalram_pages;
+ register_page_bootmem_info();
+ free_all_bootmem();
/*
* Set up the zero page, mark it reserved, so that page count
@@ -2068,21 +2169,9 @@ void __init mem_init(void)
prom_printf("paging_init: Cannot alloc zero page.\n");
prom_halt();
}
- SetPageReserved(mem_map_zero);
-
- codepages = (((unsigned long) _etext) - ((unsigned long) _start));
- codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT;
- datapages = (((unsigned long) _edata) - ((unsigned long) _etext));
- datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT;
- initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin));
- initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT;
+ mark_page_reserved(mem_map_zero);
- printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n",
- nr_free_pages() << (PAGE_SHIFT-10),
- codepages << (PAGE_SHIFT-10),
- datapages << (PAGE_SHIFT-10),
- initpages << (PAGE_SHIFT-10),
- PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT));
+ mem_init_print_info(NULL);
if (tlb_type == cheetah || tlb_type == cheetah_plus)
cheetah_ecache_flush_init();
@@ -2108,39 +2197,22 @@ void free_initmem(void)
initend = (unsigned long)(__init_end) & PAGE_MASK;
for (; addr < initend; addr += PAGE_SIZE) {
unsigned long page;
- struct page *p;
page = (addr +
((unsigned long) __va(kern_base)) -
((unsigned long) KERNBASE));
memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
- if (do_free) {
- p = virt_to_page(page);
-
- ClearPageReserved(p);
- init_page_count(p);
- __free_page(p);
- num_physpages++;
- totalram_pages++;
- }
+ if (do_free)
+ free_reserved_page(virt_to_page(page));
}
}
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
- if (start < end)
- printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10);
- for (; start < end; start += PAGE_SIZE) {
- struct page *p = virt_to_page(start);
-
- ClearPageReserved(p);
- init_page_count(p);
- __free_page(p);
- num_physpages++;
- totalram_pages++;
- }
+ free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
+ "initrd");
}
#endif
@@ -2177,10 +2249,9 @@ unsigned long vmemmap_table[VMEMMAP_SIZE];
static long __meminitdata addr_start, addr_end;
static int __meminitdata node_start;
-int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
+int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
+ int node)
{
- unsigned long vstart = (unsigned long) start;
- unsigned long vend = (unsigned long) (start + nr);
unsigned long phys_start = (vstart - VMEMMAP_BASE);
unsigned long phys_end = (vend - VMEMMAP_BASE);
unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
@@ -2201,7 +2272,7 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
void *block;
if (!(*vmem_pp & _PAGE_VALID)) {
- block = vmemmap_alloc_block(1UL << 22, node);
+ block = vmemmap_alloc_block(1UL << ILOG2_4MB, node);
if (!block)
return -ENOMEM;
@@ -2231,6 +2302,11 @@ void __meminit vmemmap_populate_print_last(void)
node_start = 0;
}
}
+
+void vmemmap_free(unsigned long start, unsigned long end)
+{
+}
+
#endif /* CONFIG_SPARSEMEM_VMEMMAP */
static void prot_init_common(unsigned long page_none,
@@ -2283,10 +2359,10 @@ static void __init sun4u_pgprot_init(void)
__ACCESS_BITS_4U | _PAGE_E_4U);
#ifdef CONFIG_DEBUG_PAGEALLOC
- kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
+ kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
#else
kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
- 0xfffff80000000000UL;
+ PAGE_OFFSET;
#endif
kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
_PAGE_P_4U | _PAGE_W_4U);
@@ -2330,10 +2406,10 @@ static void __init sun4v_pgprot_init(void)
_PAGE_CACHE = _PAGE_CACHE_4V;
#ifdef CONFIG_DEBUG_PAGEALLOC
- kern_linear_pte_xor[0] = _PAGE_VALID ^ 0xfffff80000000000UL;
+ kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
#else
kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
- 0xfffff80000000000UL;
+ PAGE_OFFSET;
#endif
kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
_PAGE_P_4V | _PAGE_W_4V);
@@ -2477,53 +2553,13 @@ void __flush_tlb_all(void)
: : "r" (pstate));
}
-static pte_t *get_from_cache(struct mm_struct *mm)
-{
- struct page *page;
- pte_t *ret;
-
- spin_lock(&mm->page_table_lock);
- page = mm->context.pgtable_page;
- ret = NULL;
- if (page) {
- void *p = page_address(page);
-
- mm->context.pgtable_page = NULL;
-
- ret = (pte_t *) (p + (PAGE_SIZE / 2));
- }
- spin_unlock(&mm->page_table_lock);
-
- return ret;
-}
-
-static struct page *__alloc_for_cache(struct mm_struct *mm)
-{
- struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
- __GFP_REPEAT | __GFP_ZERO);
-
- if (page) {
- spin_lock(&mm->page_table_lock);
- if (!mm->context.pgtable_page) {
- atomic_set(&page->_count, 2);
- mm->context.pgtable_page = page;
- }
- spin_unlock(&mm->page_table_lock);
- }
- return page;
-}
-
pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address)
{
- struct page *page;
- pte_t *pte;
-
- pte = get_from_cache(mm);
- if (pte)
- return pte;
+ struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+ __GFP_REPEAT | __GFP_ZERO);
+ pte_t *pte = NULL;
- page = __alloc_for_cache(mm);
if (page)
pte = (pte_t *) page_address(page);
@@ -2533,36 +2569,28 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
pgtable_t pte_alloc_one(struct mm_struct *mm,
unsigned long address)
{
- struct page *page;
- pte_t *pte;
-
- pte = get_from_cache(mm);
- if (pte)
- return pte;
-
- page = __alloc_for_cache(mm);
- if (page) {
- pgtable_page_ctor(page);
- pte = (pte_t *) page_address(page);
+ struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+ __GFP_REPEAT | __GFP_ZERO);
+ if (!page)
+ return NULL;
+ if (!pgtable_page_ctor(page)) {
+ free_hot_cold_page(page, 0);
+ return NULL;
}
-
- return pte;
+ return (pte_t *) page_address(page);
}
void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
- struct page *page = virt_to_page(pte);
- if (put_page_testzero(page))
- free_hot_cold_page(page, 0);
+ free_page((unsigned long)pte);
}
static void __pte_free(pgtable_t pte)
{
struct page *page = virt_to_page(pte);
- if (put_page_testzero(page)) {
- pgtable_page_dtor(page);
- free_hot_cold_page(page, 0);
- }
+
+ pgtable_page_dtor(page);
+ __free_page(page);
}
void pte_free(struct mm_struct *mm, pgtable_t pte)
@@ -2579,124 +2607,27 @@ void pgtable_free(void *table, bool is_page)
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify)
-{
- if (pgprot_val(pgprot) & _PAGE_VALID)
- pmd_val(pmd) |= PMD_HUGE_PRESENT;
- if (tlb_type == hypervisor) {
- if (pgprot_val(pgprot) & _PAGE_WRITE_4V)
- pmd_val(pmd) |= PMD_HUGE_WRITE;
- if (pgprot_val(pgprot) & _PAGE_EXEC_4V)
- pmd_val(pmd) |= PMD_HUGE_EXEC;
-
- if (!for_modify) {
- if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V)
- pmd_val(pmd) |= PMD_HUGE_ACCESSED;
- if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V)
- pmd_val(pmd) |= PMD_HUGE_DIRTY;
- }
- } else {
- if (pgprot_val(pgprot) & _PAGE_WRITE_4U)
- pmd_val(pmd) |= PMD_HUGE_WRITE;
- if (pgprot_val(pgprot) & _PAGE_EXEC_4U)
- pmd_val(pmd) |= PMD_HUGE_EXEC;
-
- if (!for_modify) {
- if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U)
- pmd_val(pmd) |= PMD_HUGE_ACCESSED;
- if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U)
- pmd_val(pmd) |= PMD_HUGE_DIRTY;
- }
- }
-
- return pmd;
-}
-
-pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
-{
- pmd_t pmd;
-
- pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT)));
- pmd_val(pmd) |= PMD_ISHUGE;
- pmd = pmd_set_protbits(pmd, pgprot, false);
- return pmd;
-}
-
-pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
-{
- pmd_val(pmd) &= ~(PMD_HUGE_PRESENT |
- PMD_HUGE_WRITE |
- PMD_HUGE_EXEC);
- pmd = pmd_set_protbits(pmd, newprot, true);
- return pmd;
-}
-
-pgprot_t pmd_pgprot(pmd_t entry)
-{
- unsigned long pte = 0;
-
- if (pmd_val(entry) & PMD_HUGE_PRESENT)
- pte |= _PAGE_VALID;
-
- if (tlb_type == hypervisor) {
- if (pmd_val(entry) & PMD_HUGE_PRESENT)
- pte |= _PAGE_PRESENT_4V;
- if (pmd_val(entry) & PMD_HUGE_EXEC)
- pte |= _PAGE_EXEC_4V;
- if (pmd_val(entry) & PMD_HUGE_WRITE)
- pte |= _PAGE_W_4V;
- if (pmd_val(entry) & PMD_HUGE_ACCESSED)
- pte |= _PAGE_ACCESSED_4V;
- if (pmd_val(entry) & PMD_HUGE_DIRTY)
- pte |= _PAGE_MODIFIED_4V;
- pte |= _PAGE_CP_4V|_PAGE_CV_4V;
- } else {
- if (pmd_val(entry) & PMD_HUGE_PRESENT)
- pte |= _PAGE_PRESENT_4U;
- if (pmd_val(entry) & PMD_HUGE_EXEC)
- pte |= _PAGE_EXEC_4U;
- if (pmd_val(entry) & PMD_HUGE_WRITE)
- pte |= _PAGE_W_4U;
- if (pmd_val(entry) & PMD_HUGE_ACCESSED)
- pte |= _PAGE_ACCESSED_4U;
- if (pmd_val(entry) & PMD_HUGE_DIRTY)
- pte |= _PAGE_MODIFIED_4U;
- pte |= _PAGE_CP_4U|_PAGE_CV_4U;
- }
-
- return __pgprot(pte);
-}
-
void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t *pmd)
{
unsigned long pte, flags;
struct mm_struct *mm;
pmd_t entry = *pmd;
- pgprot_t prot;
if (!pmd_large(entry) || !pmd_young(entry))
return;
- pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS);
- pte <<= PMD_PADDR_SHIFT;
- pte |= _PAGE_VALID;
-
- prot = pmd_pgprot(entry);
-
- if (tlb_type == hypervisor)
- pgprot_val(prot) |= _PAGE_SZHUGE_4V;
- else
- pgprot_val(prot) |= _PAGE_SZHUGE_4U;
+ pte = pmd_val(entry);
- pte |= pgprot_val(prot);
+ /* We are fabricating 8MB pages using 4MB real hw pages. */
+ pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
mm = vma->vm_mm;
spin_lock_irqsave(&mm->context.lock, flags);
if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
- __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT,
+ __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
addr, pte);
spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -2712,14 +2643,28 @@ static void context_reload(void *__data)
load_secondary_context(mm);
}
-void hugetlb_setup(struct mm_struct *mm)
+void hugetlb_setup(struct pt_regs *regs)
{
- struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE];
+ struct mm_struct *mm = current->mm;
+ struct tsb_config *tp;
- if (likely(tp->tsb != NULL))
- return;
+ if (in_atomic() || !mm) {
+ const struct exception_table_entry *entry;
+
+ entry = search_exception_tables(regs->tpc);
+ if (entry) {
+ regs->tpc = entry->fixup;
+ regs->tnpc = regs->tpc + 4;
+ return;
+ }
+ pr_alert("Unexpected HugeTLB setup in atomic context.\n");
+ die_if_kernel("HugeTSB in atomic", regs);
+ }
+
+ tp = &mm->context.tsb_block[MM_TSB_HUGE];
+ if (likely(tp->tsb == NULL))
+ tsb_grow(mm, MM_TSB_HUGE, 0);
- tsb_grow(mm, MM_TSB_HUGE, 0);
tsb_context_switch(mm);
smp_tsb_sync(mm);
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 0661aa606de..0668b364f44 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -1,11 +1,13 @@
#ifndef _SPARC64_MM_INIT_H
#define _SPARC64_MM_INIT_H
+#include <asm/page.h>
+
/* Most of the symbols in this file are defined in init.c and
* marked non-static so that assembler code can get at them.
*/
-#define MAX_PHYS_ADDRESS (1UL << 41UL)
+#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS)
#define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL)
#define KPTE_BITMAP_BYTES \
((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4)
@@ -19,7 +21,7 @@ extern unsigned int sparc64_highest_unlocked_tlb_ent;
extern unsigned long sparc64_kern_pri_context;
extern unsigned long sparc64_kern_pri_nuc_bits;
extern unsigned long sparc64_kern_sec_context;
-extern void mmu_info(struct seq_file *m);
+void mmu_info(struct seq_file *m);
struct linux_prom_translation {
unsigned long virt;
@@ -34,7 +36,7 @@ extern unsigned int prom_trans_ents;
/* Exported for SMP bootup purposes. */
extern unsigned long kern_locked_tte_data;
-extern void prom_world(int enter);
+void prom_world(int enter);
#ifdef CONFIG_SPARSEMEM_VMEMMAP
#define VMEMMAP_CHUNK_SHIFT 22
diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c
index eb99862e965..f311bf21901 100644
--- a/arch/sparc/mm/io-unit.c
+++ b/arch/sparc/mm/io-unit.c
@@ -25,6 +25,8 @@
#include <asm/dma.h>
#include <asm/oplib.h>
+#include "mm_32.h"
+
/* #define IOUNIT_DEBUG */
#ifdef IOUNIT_DEBUG
#define IOD(x) printk(x)
@@ -38,7 +40,8 @@
static void __init iounit_iommu_init(struct platform_device *op)
{
struct iounit_struct *iounit;
- iopte_t *xpt, *xptend;
+ iopte_t __iomem *xpt;
+ iopte_t __iomem *xptend;
iounit = kzalloc(sizeof(struct iounit_struct), GFP_ATOMIC);
if (!iounit) {
@@ -62,10 +65,10 @@ static void __init iounit_iommu_init(struct platform_device *op)
op->dev.archdata.iommu = iounit;
iounit->page_table = xpt;
spin_lock_init(&iounit->lock);
-
- for (xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t);
- xpt < xptend;)
- iopte_val(*xpt++) = 0;
+
+ xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t);
+ for (; xpt < xptend; xpt++)
+ sbus_writel(0, xpt);
}
static int __init iounit_init(void)
@@ -130,7 +133,7 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan);
vaddr = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + (vaddr & ~PAGE_MASK);
for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) {
set_bit(scan, iounit->bmap);
- iounit->page_table[scan] = iopte;
+ sbus_writel(iopte, &iounit->page_table[scan]);
}
IOD(("%08lx\n", vaddr));
return vaddr;
@@ -202,7 +205,7 @@ static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned lon
struct iounit_struct *iounit = dev->archdata.iommu;
unsigned long page, end;
pgprot_t dvma_prot;
- iopte_t *iopte;
+ iopte_t __iomem *iopte;
*pba = addr;
@@ -224,8 +227,8 @@ static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned lon
i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT);
- iopte = (iopte_t *)(iounit->page_table + i);
- *iopte = MKIOPTE(__pa(page));
+ iopte = iounit->page_table + i;
+ sbus_writel(MKIOPTE(__pa(page)), iopte);
}
addr += PAGE_SIZE;
va += PAGE_SIZE;
diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c
index 0f4f7191fbb..491511d37e3 100644
--- a/arch/sparc/mm/iommu.c
+++ b/arch/sparc/mm/iommu.c
@@ -27,6 +27,8 @@
#include <asm/iommu.h>
#include <asm/dma.h>
+#include "mm_32.h"
+
/*
* This can be sized dynamically, but we will do this
* only when we have a guidance about actual I/O pressures.
@@ -34,12 +36,9 @@
#define IOMMU_RNGE IOMMU_RNGE_256MB
#define IOMMU_START 0xF0000000
#define IOMMU_WINSIZE (256*1024*1024U)
-#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */
+#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */
#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
-/* srmmu.c */
-extern int viking_mxcc_present;
-extern int flush_page_for_dma_global;
static int viking_flush;
/* viking.S */
extern void viking_flush_page(unsigned long page);
@@ -59,6 +58,8 @@ static void __init sbus_iommu_init(struct platform_device *op)
struct iommu_struct *iommu;
unsigned int impl, vers;
unsigned long *bitmap;
+ unsigned long control;
+ unsigned long base;
unsigned long tmp;
iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
@@ -73,12 +74,14 @@ static void __init sbus_iommu_init(struct platform_device *op)
prom_printf("Cannot map IOMMU registers\n");
prom_halt();
}
- impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
- vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
- tmp = iommu->regs->control;
- tmp &= ~(IOMMU_CTRL_RNGE);
- tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
- iommu->regs->control = tmp;
+
+ control = sbus_readl(&iommu->regs->control);
+ impl = (control & IOMMU_CTRL_IMPL) >> 28;
+ vers = (control & IOMMU_CTRL_VERS) >> 24;
+ control &= ~(IOMMU_CTRL_RNGE);
+ control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
+ sbus_writel(control, &iommu->regs->control);
+
iommu_invalidate(iommu->regs);
iommu->start = IOMMU_START;
iommu->end = 0xffffffff;
@@ -100,7 +103,9 @@ static void __init sbus_iommu_init(struct platform_device *op)
memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
flush_cache_all();
flush_tlb_all();
- iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
+
+ base = __pa((unsigned long)iommu->page_table) >> 4;
+ sbus_writel(base, &iommu->regs->base);
iommu_invalidate(iommu->regs);
bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c
index 5bed085a2c1..3b17b6f7895 100644
--- a/arch/sparc/mm/leon_mm.c
+++ b/arch/sparc/mm/leon_mm.c
@@ -15,10 +15,10 @@
#include <asm/leon.h>
#include <asm/tlbflush.h>
-#include "srmmu.h"
+#include "mm_32.h"
int leon_flush_during_switch = 1;
-int srmmu_swprobe_trace;
+static int srmmu_swprobe_trace;
static inline unsigned long leon_get_ctable_ptr(void)
{
diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h
new file mode 100644
index 00000000000..a6c27ca9a72
--- /dev/null
+++ b/arch/sparc/mm/mm_32.h
@@ -0,0 +1,24 @@
+/* fault_32.c - visible as they are called from assembler */
+asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc,
+ unsigned long address);
+asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
+ unsigned long address);
+
+void window_overflow_fault(void);
+void window_underflow_fault(unsigned long sp);
+void window_ret_fault(struct pt_regs *regs);
+
+/* srmmu.c */
+extern char *srmmu_name;
+extern int viking_mxcc_present;
+extern int flush_page_for_dma_global;
+
+extern void (*poke_srmmu)(void);
+
+void __init srmmu_paging_init(void);
+
+/* iommu.c */
+void ld_mmu_iommu(void);
+
+/* io-unit.c */
+void ld_mmu_iounit(void);
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index c38bb72e3e8..be65f035d18 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -14,6 +14,7 @@
#include <linux/pagemap.h>
#include <linux/vmalloc.h>
#include <linux/kdebug.h>
+#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/log2.h>
@@ -48,7 +49,7 @@
#include <asm/mxcc.h>
#include <asm/ross.h>
-#include "srmmu.h"
+#include "mm_32.h"
enum mbus_module srmmu_modtype;
static unsigned int hwbug_bitmask;
@@ -62,6 +63,7 @@ extern unsigned long last_valid_pfn;
static pgd_t *srmmu_swapper_pg_dir;
const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops;
+EXPORT_SYMBOL(sparc32_cachetlb_ops);
#ifdef CONFIG_SMP
const struct sparc32_cachetlb_ops *local_ops;
@@ -98,7 +100,6 @@ static unsigned long srmmu_nocache_end;
#define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS)
void *srmmu_nocache_pool;
-void *srmmu_nocache_bitmap;
static struct bit_map srmmu_nocache_map;
static inline int srmmu_pmd_none(pmd_t pmd)
@@ -171,7 +172,7 @@ static void *__srmmu_get_nocache(int size, int align)
printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
size, (int) srmmu_nocache_size,
srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
- return 0;
+ return NULL;
}
addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
@@ -267,6 +268,7 @@ static void __init srmmu_nocache_calcsize(void)
static void __init srmmu_nocache_init(void)
{
+ void *srmmu_nocache_bitmap;
unsigned int bitmap_bits;
pgd_t *pgd;
pmd_t *pmd;
@@ -280,7 +282,9 @@ static void __init srmmu_nocache_init(void)
SRMMU_NOCACHE_ALIGN_MAX, 0UL);
memset(srmmu_nocache_pool, 0, srmmu_nocache_size);
- srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
+ srmmu_nocache_bitmap =
+ __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long),
+ SMP_CACHE_BYTES, 0UL);
bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
@@ -343,7 +347,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
return NULL;
page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
@@ -721,7 +728,7 @@ static inline unsigned long srmmu_probe(unsigned long vaddr)
"=r" (retval) :
"r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE));
} else {
- retval = leon_swprobe(vaddr, 0);
+ retval = leon_swprobe(vaddr, NULL);
}
return retval;
}
@@ -856,9 +863,7 @@ static void __init map_kernel(void)
}
}
-void (*poke_srmmu)(void) __cpuinitdata = NULL;
-
-extern unsigned long bootmem_init(unsigned long *pages_avail);
+void (*poke_srmmu)(void) = NULL;
void __init srmmu_paging_init(void)
{
@@ -1053,7 +1058,7 @@ static void __init init_vac_layout(void)
(int)vac_cache_size, (int)vac_line_size);
}
-static void __cpuinit poke_hypersparc(void)
+static void poke_hypersparc(void)
{
volatile unsigned long clear;
unsigned long mreg = srmmu_get_mmureg();
@@ -1105,7 +1110,7 @@ static void __init init_hypersparc(void)
hypersparc_setup_blockops();
}
-static void __cpuinit poke_swift(void)
+static void poke_swift(void)
{
unsigned long mreg;
@@ -1285,7 +1290,7 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long
}
-static void __cpuinit poke_turbosparc(void)
+static void poke_turbosparc(void)
{
unsigned long mreg = srmmu_get_mmureg();
unsigned long ccreg;
@@ -1348,7 +1353,7 @@ static void __init init_turbosparc(void)
poke_srmmu = poke_turbosparc;
}
-static void __cpuinit poke_tsunami(void)
+static void poke_tsunami(void)
{
unsigned long mreg = srmmu_get_mmureg();
@@ -1389,7 +1394,7 @@ static void __init init_tsunami(void)
tsunami_setup_blockops();
}
-static void __cpuinit poke_viking(void)
+static void poke_viking(void)
{
unsigned long mreg = srmmu_get_mmureg();
static int smp_catch;
@@ -1764,9 +1769,6 @@ static struct sparc32_cachetlb_ops smp_cachetlb_ops = {
/* Load up routines and constants for sun4m and sun4d mmu */
void __init load_mmu(void)
{
- extern void ld_mmu_iommu(void);
- extern void ld_mmu_iounit(void);
-
/* Functions */
get_srmmu_type();
diff --git a/arch/sparc/mm/srmmu.h b/arch/sparc/mm/srmmu.h
deleted file mode 100644
index 5703274ccf8..00000000000
--- a/arch/sparc/mm/srmmu.h
+++ /dev/null
@@ -1,4 +0,0 @@
-/* srmmu.c */
-extern char *srmmu_name;
-
-extern void (*poke_srmmu)(void);
diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S
index c801c3953a0..5d2b88d3942 100644
--- a/arch/sparc/mm/swift.S
+++ b/arch/sparc/mm/swift.S
@@ -105,7 +105,7 @@ swift_flush_cache_mm_out:
.globl swift_flush_cache_range
swift_flush_cache_range:
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
sub %o2, %o1, %o2
sethi %hi(4096), %o3
cmp %o2, %o3
@@ -116,7 +116,7 @@ swift_flush_cache_range:
.globl swift_flush_cache_page
swift_flush_cache_page:
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
70:
ld [%o0 + AOFF_mm_context], %g2
cmp %g2, -1
@@ -219,7 +219,7 @@ swift_flush_sig_insns:
.globl swift_flush_tlb_range
.globl swift_flush_tlb_all
swift_flush_tlb_range:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
swift_flush_tlb_mm:
ld [%o0 + AOFF_mm_context], %g2
cmp %g2, -1
@@ -233,7 +233,7 @@ swift_flush_tlb_all_out:
.globl swift_flush_tlb_page
swift_flush_tlb_page:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
andn %o1, (PAGE_SIZE - 1), %o1
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index 3e8fec391fe..b89aba217e3 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -4,7 +4,6 @@
*/
#include <linux/kernel.h>
-#include <linux/init.h>
#include <linux/percpu.h>
#include <linux/mm.h>
#include <linux/swap.h>
@@ -24,11 +23,17 @@ static DEFINE_PER_CPU(struct tlb_batch, tlb_batch);
void flush_tlb_pending(void)
{
struct tlb_batch *tb = &get_cpu_var(tlb_batch);
+ struct mm_struct *mm = tb->mm;
- if (tb->tlb_nr) {
- flush_tsb_user(tb);
+ if (!tb->tlb_nr)
+ goto out;
- if (CTX_VALID(tb->mm->context)) {
+ flush_tsb_user(tb);
+
+ if (CTX_VALID(mm->context)) {
+ if (tb->tlb_nr == 1) {
+ global_flush_tlb_page(mm, tb->vaddrs[0]);
+ } else {
#ifdef CONFIG_SMP
smp_flush_tlb_pending(tb->mm, tb->tlb_nr,
&tb->vaddrs[0]);
@@ -37,12 +42,30 @@ void flush_tlb_pending(void)
tb->tlb_nr, &tb->vaddrs[0]);
#endif
}
- tb->tlb_nr = 0;
}
+ tb->tlb_nr = 0;
+
+out:
put_cpu_var(tlb_batch);
}
+void arch_enter_lazy_mmu_mode(void)
+{
+ struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
+
+ tb->active = 1;
+}
+
+void arch_leave_lazy_mmu_mode(void)
+{
+ struct tlb_batch *tb = &__get_cpu_var(tlb_batch);
+
+ if (tb->tlb_nr)
+ flush_tlb_pending();
+ tb->active = 0;
+}
+
static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
bool exec)
{
@@ -60,6 +83,12 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
nr = 0;
}
+ if (!tb->active) {
+ flush_tsb_user_page(mm, vaddr);
+ global_flush_tlb_page(mm, vaddr);
+ goto out;
+ }
+
if (nr == 0)
tb->mm = mm;
@@ -68,6 +97,7 @@ static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
if (nr >= TLB_BATCH_NR)
flush_tlb_pending();
+out:
put_cpu_var(tlb_batch);
}
@@ -104,7 +134,7 @@ no_cache_flush:
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
- pmd_t pmd, bool exec)
+ pmd_t pmd)
{
unsigned long end;
pte_t *pte;
@@ -112,8 +142,11 @@ static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
pte = pte_offset_map(&pmd, vaddr);
end = vaddr + HPAGE_SIZE;
while (vaddr < end) {
- if (pte_val(*pte) & _PAGE_VALID)
+ if (pte_val(*pte) & _PAGE_VALID) {
+ bool exec = pte_exec(*pte);
+
tlb_batch_add_one(mm, vaddr, exec);
+ }
pte++;
vaddr += PAGE_SIZE;
}
@@ -130,41 +163,63 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr,
if (mm == &init_mm)
return;
- if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) {
- if (pmd_val(pmd) & PMD_ISHUGE)
+ if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) {
+ if (pmd_val(pmd) & _PAGE_PMD_HUGE)
mm->context.huge_pte_count++;
else
mm->context.huge_pte_count--;
- if (mm->context.huge_pte_count == 1)
- hugetlb_setup(mm);
+
+ /* Do not try to allocate the TSB hash table if we
+ * don't have one already. We have various locks held
+ * and thus we'll end up doing a GFP_KERNEL allocation
+ * in an atomic context.
+ *
+ * Instead, we let the first TLB miss on a hugepage
+ * take care of this.
+ */
}
if (!pmd_none(orig)) {
- bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0);
-
addr &= HPAGE_MASK;
- if (pmd_val(orig) & PMD_ISHUGE)
+ if (pmd_trans_huge(orig)) {
+ pte_t orig_pte = __pte(pmd_val(orig));
+ bool exec = pte_exec(orig_pte);
+
tlb_batch_add_one(mm, addr, exec);
- else
- tlb_batch_pmd_scan(mm, addr, orig, exec);
+ tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec);
+ } else {
+ tlb_batch_pmd_scan(mm, addr, orig);
+ }
}
}
-void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
+void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
+ pmd_t *pmdp)
+{
+ pmd_t entry = *pmdp;
+
+ pmd_val(entry) &= ~_PAGE_VALID;
+
+ set_pmd_at(vma->vm_mm, address, pmdp, entry);
+ flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+}
+
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
+ pgtable_t pgtable)
{
struct list_head *lh = (struct list_head *) pgtable;
assert_spin_locked(&mm->page_table_lock);
/* FIFO */
- if (!mm->pmd_huge_pte)
+ if (!pmd_huge_pte(mm, pmdp))
INIT_LIST_HEAD(lh);
else
- list_add(lh, (struct list_head *) mm->pmd_huge_pte);
- mm->pmd_huge_pte = pgtable;
+ list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
+ pmd_huge_pte(mm, pmdp) = pgtable;
}
-pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
{
struct list_head *lh;
pgtable_t pgtable;
@@ -172,12 +227,12 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
assert_spin_locked(&mm->page_table_lock);
/* FIFO */
- pgtable = mm->pmd_huge_pte;
+ pgtable = pmd_huge_pte(mm, pmdp);
lh = (struct list_head *) pgtable;
if (list_empty(lh))
- mm->pmd_huge_pte = NULL;
+ pmd_huge_pte(mm, pmdp) = NULL;
else {
- mm->pmd_huge_pte = (pgtable_t) lh->next;
+ pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
list_del(lh);
}
pte_val(pgtable[0]) = 0;
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index 7f647434749..a06576683c3 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -7,11 +7,11 @@
#include <linux/preempt.h>
#include <linux/slab.h>
#include <asm/page.h>
-#include <asm/tlbflush.h>
-#include <asm/tlb.h>
-#include <asm/mmu_context.h>
#include <asm/pgtable.h>
+#include <asm/mmu_context.h>
+#include <asm/setup.h>
#include <asm/tsb.h>
+#include <asm/tlb.h>
#include <asm/oplib.h>
extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
@@ -46,28 +46,56 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end)
}
}
+static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v,
+ unsigned long hash_shift,
+ unsigned long nentries)
+{
+ unsigned long tag, ent, hash;
+
+ v &= ~0x1UL;
+ hash = tsb_hash(v, hash_shift, nentries);
+ ent = tsb + (hash * sizeof(struct tsb));
+ tag = (v >> 22UL);
+
+ tsb_flush(ent, tag);
+}
+
static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
unsigned long tsb, unsigned long nentries)
{
unsigned long i;
- for (i = 0; i < tb->tlb_nr; i++) {
- unsigned long v = tb->vaddrs[i];
- unsigned long tag, ent, hash;
+ for (i = 0; i < tb->tlb_nr; i++)
+ __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries);
+}
- v &= ~0x1UL;
+void flush_tsb_user(struct tlb_batch *tb)
+{
+ struct mm_struct *mm = tb->mm;
+ unsigned long nentries, base, flags;
+
+ spin_lock_irqsave(&mm->context.lock, flags);
- hash = tsb_hash(v, hash_shift, nentries);
- ent = tsb + (hash * sizeof(struct tsb));
- tag = (v >> 22UL);
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
+ if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+ base = __pa(base);
+ __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
- tsb_flush(ent, tag);
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
+ base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
+ nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
+ if (tlb_type == cheetah_plus || tlb_type == hypervisor)
+ base = __pa(base);
+ __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries);
}
+#endif
+ spin_unlock_irqrestore(&mm->context.lock, flags);
}
-void flush_tsb_user(struct tlb_batch *tb)
+void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr)
{
- struct mm_struct *mm = tb->mm;
unsigned long nentries, base, flags;
spin_lock_irqsave(&mm->context.lock, flags);
@@ -76,7 +104,7 @@ void flush_tsb_user(struct tlb_batch *tb)
nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
base = __pa(base);
- __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
+ __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries);
#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
@@ -84,7 +112,7 @@ void flush_tsb_user(struct tlb_batch *tb)
nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
if (tlb_type == cheetah_plus || tlb_type == hypervisor)
base = __pa(base);
- __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries);
+ __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries);
}
#endif
spin_unlock_irqrestore(&mm->context.lock, flags);
@@ -106,7 +134,19 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign
mm->context.tsb_block[tsb_idx].tsb_nentries =
tsb_bytes / sizeof(struct tsb);
- base = TSBMAP_BASE;
+ switch (tsb_idx) {
+ case MM_TSB_BASE:
+ base = TSBMAP_8K_BASE;
+ break;
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+ case MM_TSB_HUGE:
+ base = TSBMAP_4M_BASE;
+ break;
+#endif
+ default:
+ BUG();
+ }
+
tte = pgprot_val(PAGE_KERNEL_LOCKED);
tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb);
BUG_ON(tsb_paddr & (tsb_bytes - 1UL));
@@ -246,7 +286,7 @@ void __init pgtable_cache_init(void)
prom_halt();
}
- for (i = 0; i < 8; i++) {
+ for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) {
unsigned long size = 8192 << i;
const char *name = tsb_cache_names[i];
@@ -314,7 +354,7 @@ void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss)
retry_tsb_alloc:
gfp_flags = GFP_KERNEL;
if (new_size > (PAGE_SIZE * 2))
- gfp_flags = __GFP_NOWARN | __GFP_NORETRY;
+ gfp_flags |= __GFP_NOWARN | __GFP_NORETRY;
new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index],
gfp_flags, numa_node_id());
@@ -445,8 +485,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
mm->context.huge_pte_count = 0;
#endif
- mm->context.pgtable_page = NULL;
-
/* copy_mm() copies over the parent's mm_struct before calling
* us, so we need to zero out the TSB pointer or else tsb_grow()
* will be confused and think there is an older TSB to free up.
@@ -485,17 +523,10 @@ static void tsb_destroy_one(struct tsb_config *tp)
void destroy_context(struct mm_struct *mm)
{
unsigned long flags, i;
- struct page *page;
for (i = 0; i < MM_NUM_TSBS; i++)
tsb_destroy_one(&mm->context.tsb_block[i]);
- page = mm->context.pgtable_page;
- if (page && put_page_testzero(page)) {
- pgtable_page_dtor(page);
- free_hot_cold_page(page, 0);
- }
-
spin_lock_irqsave(&ctx_alloc_lock, flags);
if (CTX_VALID(mm->context)) {
diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S
index 4e55e8f7664..bf10a345fa8 100644
--- a/arch/sparc/mm/tsunami.S
+++ b/arch/sparc/mm/tsunami.S
@@ -24,7 +24,7 @@
/* Sliiick... */
tsunami_flush_cache_page:
tsunami_flush_cache_range:
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
tsunami_flush_cache_mm:
ld [%o0 + AOFF_mm_context], %g2
cmp %g2, -1
@@ -46,7 +46,7 @@ tsunami_flush_sig_insns:
/* More slick stuff... */
tsunami_flush_tlb_range:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
tsunami_flush_tlb_mm:
ld [%o0 + AOFF_mm_context], %g2
cmp %g2, -1
@@ -65,7 +65,7 @@ tsunami_flush_tlb_out:
/* This one can be done in a fine grained manner... */
tsunami_flush_tlb_page:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
andn %o1, (PAGE_SIZE - 1), %o1
diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S
index f8e13d421fc..b4f4733abc6 100644
--- a/arch/sparc/mm/ultra.S
+++ b/arch/sparc/mm/ultra.S
@@ -53,6 +53,33 @@ __flush_tlb_mm: /* 18 insns */
nop
.align 32
+ .globl __flush_tlb_page
+__flush_tlb_page: /* 22 insns */
+ /* %o0 = context, %o1 = vaddr */
+ rdpr %pstate, %g7
+ andn %g7, PSTATE_IE, %g2
+ wrpr %g2, %pstate
+ mov SECONDARY_CONTEXT, %o4
+ ldxa [%o4] ASI_DMMU, %g2
+ stxa %o0, [%o4] ASI_DMMU
+ andcc %o1, 1, %g0
+ andn %o1, 1, %o3
+ be,pn %icc, 1f
+ or %o3, 0x10, %o3
+ stxa %g0, [%o3] ASI_IMMU_DEMAP
+1: stxa %g0, [%o3] ASI_DMMU_DEMAP
+ membar #Sync
+ stxa %g2, [%o4] ASI_DMMU
+ sethi %hi(KERNBASE), %o4
+ flush %o4
+ retl
+ wrpr %g7, 0x0, %pstate
+ nop
+ nop
+ nop
+ nop
+
+ .align 32
.globl __flush_tlb_pending
__flush_tlb_pending: /* 26 insns */
/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
@@ -126,10 +153,10 @@ __spitfire_flush_tlb_mm_slow:
.globl __flush_icache_page
__flush_icache_page: /* %o0 = phys_page */
srlx %o0, PAGE_SHIFT, %o0
- sethi %uhi(PAGE_OFFSET), %g1
+ sethi %hi(PAGE_OFFSET), %g1
sllx %o0, PAGE_SHIFT, %o0
sethi %hi(PAGE_SIZE), %g2
- sllx %g1, 32, %g1
+ ldx [%g1 + %lo(PAGE_OFFSET)], %g1
add %o0, %g1, %o0
1: subcc %g2, 32, %g2
bne,pt %icc, 1b
@@ -151,8 +178,8 @@ __flush_icache_page: /* %o0 = phys_page */
.align 64
.globl __flush_dcache_page
__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
- sethi %uhi(PAGE_OFFSET), %g1
- sllx %g1, 32, %g1
+ sethi %hi(PAGE_OFFSET), %g1
+ ldx [%g1 + %lo(PAGE_OFFSET)], %g1
sub %o0, %g1, %o0 ! physical address
srlx %o0, 11, %o0 ! make D-cache TAG
sethi %hi(1 << 14), %o2 ! D-cache size
@@ -203,6 +230,31 @@ __cheetah_flush_tlb_mm: /* 19 insns */
retl
wrpr %g7, 0x0, %pstate
+__cheetah_flush_tlb_page: /* 22 insns */
+ /* %o0 = context, %o1 = vaddr */
+ rdpr %pstate, %g7
+ andn %g7, PSTATE_IE, %g2
+ wrpr %g2, 0x0, %pstate
+ wrpr %g0, 1, %tl
+ mov PRIMARY_CONTEXT, %o4
+ ldxa [%o4] ASI_DMMU, %g2
+ srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
+ sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
+ or %o0, %o3, %o0 /* Preserve nucleus page size fields */
+ stxa %o0, [%o4] ASI_DMMU
+ andcc %o1, 1, %g0
+ be,pn %icc, 1f
+ andn %o1, 1, %o3
+ stxa %g0, [%o3] ASI_IMMU_DEMAP
+1: stxa %g0, [%o3] ASI_DMMU_DEMAP
+ membar #Sync
+ stxa %g2, [%o4] ASI_DMMU
+ sethi %hi(KERNBASE), %o4
+ flush %o4
+ wrpr %g0, 0, %tl
+ retl
+ wrpr %g7, 0x0, %pstate
+
__cheetah_flush_tlb_pending: /* 27 insns */
/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
rdpr %pstate, %g7
@@ -235,8 +287,8 @@ __cheetah_flush_tlb_pending: /* 27 insns */
#ifdef DCACHE_ALIASING_POSSIBLE
__cheetah_flush_dcache_page: /* 11 insns */
- sethi %uhi(PAGE_OFFSET), %g1
- sllx %g1, 32, %g1
+ sethi %hi(PAGE_OFFSET), %g1
+ ldx [%g1 + %lo(PAGE_OFFSET)], %g1
sub %o0, %g1, %o0
sethi %hi(PAGE_SIZE), %o4
1: subcc %o4, (1 << 5), %o4
@@ -269,6 +321,20 @@ __hypervisor_flush_tlb_mm: /* 10 insns */
retl
nop
+__hypervisor_flush_tlb_page: /* 11 insns */
+ /* %o0 = context, %o1 = vaddr */
+ mov %o0, %g2
+ mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */
+ mov %g2, %o1 /* ARG1: mmu context */
+ mov HV_MMU_ALL, %o2 /* ARG2: flags */
+ srlx %o0, PAGE_SHIFT, %o0
+ sllx %o0, PAGE_SHIFT, %o0
+ ta HV_MMU_UNMAP_ADDR_TRAP
+ brnz,pn %o0, __hypervisor_tlb_tl0_error
+ mov HV_MMU_UNMAP_ADDR_TRAP, %o1
+ retl
+ nop
+
__hypervisor_flush_tlb_pending: /* 16 insns */
/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
sllx %o1, 3, %g1
@@ -339,6 +405,13 @@ cheetah_patch_cachetlbops:
call tlb_patch_one
mov 19, %o2
+ sethi %hi(__flush_tlb_page), %o0
+ or %o0, %lo(__flush_tlb_page), %o0
+ sethi %hi(__cheetah_flush_tlb_page), %o1
+ or %o1, %lo(__cheetah_flush_tlb_page), %o1
+ call tlb_patch_one
+ mov 22, %o2
+
sethi %hi(__flush_tlb_pending), %o0
or %o0, %lo(__flush_tlb_pending), %o0
sethi %hi(__cheetah_flush_tlb_pending), %o1
@@ -397,10 +470,9 @@ xcall_flush_tlb_mm: /* 21 insns */
nop
nop
- .globl xcall_flush_tlb_pending
-xcall_flush_tlb_pending: /* 21 insns */
- /* %g5=context, %g1=nr, %g7=vaddrs[] */
- sllx %g1, 3, %g1
+ .globl xcall_flush_tlb_page
+xcall_flush_tlb_page: /* 17 insns */
+ /* %g5=context, %g1=vaddr */
mov PRIMARY_CONTEXT, %g4
ldxa [%g4] ASI_DMMU, %g2
srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
@@ -408,20 +480,16 @@ xcall_flush_tlb_pending: /* 21 insns */
or %g5, %g4, %g5
mov PRIMARY_CONTEXT, %g4
stxa %g5, [%g4] ASI_DMMU
-1: sub %g1, (1 << 3), %g1
- ldx [%g7 + %g1], %g5
- andcc %g5, 0x1, %g0
+ andcc %g1, 0x1, %g0
be,pn %icc, 2f
-
- andn %g5, 0x1, %g5
+ andn %g1, 0x1, %g5
stxa %g0, [%g5] ASI_IMMU_DEMAP
2: stxa %g0, [%g5] ASI_DMMU_DEMAP
membar #Sync
- brnz,pt %g1, 1b
- nop
stxa %g2, [%g4] ASI_DMMU
retry
nop
+ nop
.globl xcall_flush_tlb_kernel_range
xcall_flush_tlb_kernel_range: /* 25 insns */
@@ -656,15 +724,13 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */
membar #Sync
retry
- .globl __hypervisor_xcall_flush_tlb_pending
-__hypervisor_xcall_flush_tlb_pending: /* 21 insns */
- /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */
- sllx %g1, 3, %g1
+ .globl __hypervisor_xcall_flush_tlb_page
+__hypervisor_xcall_flush_tlb_page: /* 17 insns */
+ /* %g5=ctx, %g1=vaddr */
mov %o0, %g2
mov %o1, %g3
mov %o2, %g4
-1: sub %g1, (1 << 3), %g1
- ldx [%g7 + %g1], %o0 /* ARG0: virtual address */
+ mov %g1, %o0 /* ARG0: virtual address */
mov %g5, %o1 /* ARG1: mmu context */
mov HV_MMU_ALL, %o2 /* ARG2: flags */
srlx %o0, PAGE_SHIFT, %o0
@@ -673,8 +739,6 @@ __hypervisor_xcall_flush_tlb_pending: /* 21 insns */
mov HV_MMU_UNMAP_ADDR_TRAP, %g6
brnz,a,pn %o0, __hypervisor_tlb_xcall_error
mov %o0, %g5
- brnz,pt %g1, 1b
- nop
mov %g2, %o0
mov %g3, %o1
mov %g4, %o2
@@ -757,6 +821,13 @@ hypervisor_patch_cachetlbops:
call tlb_patch_one
mov 10, %o2
+ sethi %hi(__flush_tlb_page), %o0
+ or %o0, %lo(__flush_tlb_page), %o0
+ sethi %hi(__hypervisor_flush_tlb_page), %o1
+ or %o1, %lo(__hypervisor_flush_tlb_page), %o1
+ call tlb_patch_one
+ mov 11, %o2
+
sethi %hi(__flush_tlb_pending), %o0
or %o0, %lo(__flush_tlb_pending), %o0
sethi %hi(__hypervisor_flush_tlb_pending), %o1
@@ -788,12 +859,12 @@ hypervisor_patch_cachetlbops:
call tlb_patch_one
mov 21, %o2
- sethi %hi(xcall_flush_tlb_pending), %o0
- or %o0, %lo(xcall_flush_tlb_pending), %o0
- sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1
- or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
+ sethi %hi(xcall_flush_tlb_page), %o0
+ or %o0, %lo(xcall_flush_tlb_page), %o0
+ sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1
+ or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
call tlb_patch_one
- mov 21, %o2
+ mov 17, %o2
sethi %hi(xcall_flush_tlb_kernel_range), %o0
or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S
index bf8ee0613ae..852257fcc82 100644
--- a/arch/sparc/mm/viking.S
+++ b/arch/sparc/mm/viking.S
@@ -108,7 +108,7 @@ viking_mxcc_flush_page:
viking_flush_cache_page:
viking_flush_cache_range:
#ifndef CONFIG_SMP
- ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
#endif
viking_flush_cache_mm:
#ifndef CONFIG_SMP
@@ -148,7 +148,7 @@ viking_flush_tlb_mm:
#endif
viking_flush_tlb_range:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
lda [%g1] ASI_M_MMUREGS, %g5
@@ -173,7 +173,7 @@ viking_flush_tlb_range:
#endif
viking_flush_tlb_page:
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
mov SRMMU_CTX_REG, %g1
ld [%o0 + AOFF_mm_context], %o3
lda [%g1] ASI_M_MMUREGS, %g5
@@ -239,7 +239,7 @@ sun4dsmp_flush_tlb_range:
tst %g5
bne 3f
mov SRMMU_CTX_REG, %g1
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
ld [%o0 + AOFF_mm_context], %o3
lda [%g1] ASI_M_MMUREGS, %g5
sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4
@@ -265,7 +265,7 @@ sun4dsmp_flush_tlb_page:
tst %g5
bne 2f
mov SRMMU_CTX_REG, %g1
- ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */
+ ld [%o0 + VMA_VM_MM], %o0
ld [%o0 + AOFF_mm_context], %o3
lda [%g1] ASI_M_MMUREGS, %g5
and %o1, PAGE_MASK, %o1
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-09 21:16:49 +0000