diff options
Diffstat (limited to 'arch/powerpc/mm/pgtable_32.c')
| -rw-r--r-- | arch/powerpc/mm/pgtable_32.c | 415 |
1 files changed, 204 insertions, 211 deletions
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c index f4e5ac12261..343a87fa78b 100644 --- a/arch/powerpc/mm/pgtable_32.c +++ b/arch/powerpc/mm/pgtable_32.c @@ -8,7 +8,6 @@ * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) * and Cort Dougan (PReP) (cort@cs.nmt.edu) * Copyright (C) 1996 Paul Mackerras - * Amiga/APUS changes by Jesper Skov (jskov@cygnus.co.uk). * * Derived from "arch/i386/mm/init.c" * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds @@ -20,7 +19,6 @@ * */ -#include <linux/config.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/types.h> @@ -28,16 +26,20 @@ #include <linux/vmalloc.h> #include <linux/init.h> #include <linux/highmem.h> +#include <linux/memblock.h> +#include <linux/slab.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> +#include <asm/fixmap.h> #include <asm/io.h> +#include <asm/setup.h> #include "mmu_decl.h" unsigned long ioremap_base; unsigned long ioremap_bot; -int io_bat_index; +EXPORT_SYMBOL(ioremap_bot); /* aka VMALLOC_END */ #if defined(CONFIG_6xx) || defined(CONFIG_POWER3) #define HAVE_BATS 1 @@ -49,14 +51,10 @@ int io_bat_index; extern char etext[], _stext[]; -#ifdef CONFIG_SMP -extern void hash_page_sync(void); -#endif - #ifdef HAVE_BATS -extern unsigned long v_mapped_by_bats(unsigned long va); -extern unsigned long p_mapped_by_bats(unsigned long pa); -void setbat(int index, unsigned long virt, unsigned long phys, +extern phys_addr_t v_mapped_by_bats(unsigned long va); +extern unsigned long p_mapped_by_bats(phys_addr_t pa); +void setbat(int index, unsigned long virt, phys_addr_t phys, unsigned int size, int flags); #else /* !HAVE_BATS */ @@ -66,34 +64,39 @@ void setbat(int index, unsigned long virt, unsigned long phys, #ifdef HAVE_TLBCAM extern unsigned int tlbcam_index; -extern unsigned long v_mapped_by_tlbcam(unsigned long va); -extern unsigned long p_mapped_by_tlbcam(unsigned long pa); +extern phys_addr_t v_mapped_by_tlbcam(unsigned long va); +extern unsigned long p_mapped_by_tlbcam(phys_addr_t pa); #else /* !HAVE_TLBCAM */ #define v_mapped_by_tlbcam(x) (0UL) #define p_mapped_by_tlbcam(x) (0UL) #endif /* HAVE_TLBCAM */ -#ifdef CONFIG_PTE_64BIT -/* 44x uses an 8kB pgdir because it has 8-byte Linux PTEs. */ -#define PGDIR_ORDER 1 -#else -#define PGDIR_ORDER 0 -#endif +#define PGDIR_ORDER (32 + PGD_T_LOG2 - PGDIR_SHIFT) pgd_t *pgd_alloc(struct mm_struct *mm) { pgd_t *ret; - ret = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, PGDIR_ORDER); + /* pgdir take page or two with 4K pages and a page fraction otherwise */ +#ifndef CONFIG_PPC_4K_PAGES + ret = kzalloc(1 << PGDIR_ORDER, GFP_KERNEL); +#else + ret = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, + PGDIR_ORDER - PAGE_SHIFT); +#endif return ret; } -void pgd_free(pgd_t *pgd) +void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - free_pages((unsigned long)pgd, PGDIR_ORDER); +#ifndef CONFIG_PPC_4K_PAGES + kfree((void *)pgd); +#else + free_pages((unsigned long)pgd, PGDIR_ORDER - PAGE_SHIFT); +#endif } -pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) +__init_refok pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { pte_t *pte; extern int mem_init_done; @@ -109,67 +112,82 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) return pte; } -struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { struct page *ptepage; -#ifdef CONFIG_HIGHPTE - gfp_t flags = GFP_KERNEL | __GFP_HIGHMEM | __GFP_REPEAT; -#else - gfp_t flags = GFP_KERNEL | __GFP_REPEAT; -#endif + gfp_t flags = GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO; ptepage = alloc_pages(flags, 0); - if (ptepage) - clear_highpage(ptepage); + if (!ptepage) + return NULL; + if (!pgtable_page_ctor(ptepage)) { + __free_page(ptepage); + return NULL; + } return ptepage; } -void pte_free_kernel(pte_t *pte) -{ -#ifdef CONFIG_SMP - hash_page_sync(); -#endif - free_page((unsigned long)pte); -} - -void pte_free(struct page *ptepage) -{ -#ifdef CONFIG_SMP - hash_page_sync(); -#endif - __free_page(ptepage); -} - -#ifndef CONFIG_PHYS_64BIT void __iomem * ioremap(phys_addr_t addr, unsigned long size) { - return __ioremap(addr, size, _PAGE_NO_CACHE); + return __ioremap_caller(addr, size, _PAGE_NO_CACHE | _PAGE_GUARDED, + __builtin_return_address(0)); } -#else /* CONFIG_PHYS_64BIT */ +EXPORT_SYMBOL(ioremap); + void __iomem * -ioremap64(unsigned long long addr, unsigned long size) +ioremap_wc(phys_addr_t addr, unsigned long size) { - return __ioremap(addr, size, _PAGE_NO_CACHE); + return __ioremap_caller(addr, size, _PAGE_NO_CACHE, + __builtin_return_address(0)); } +EXPORT_SYMBOL(ioremap_wc); void __iomem * -ioremap(phys_addr_t addr, unsigned long size) +ioremap_prot(phys_addr_t addr, unsigned long size, unsigned long flags) { - phys_addr_t addr64 = fixup_bigphys_addr(addr, size); + /* writeable implies dirty for kernel addresses */ + if (flags & _PAGE_RW) + flags |= _PAGE_DIRTY | _PAGE_HWWRITE; - return ioremap64(addr64, size); + /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ + flags &= ~(_PAGE_USER | _PAGE_EXEC); + +#ifdef _PAGE_BAP_SR + /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format + * which means that we just cleared supervisor access... oops ;-) This + * restores it + */ + flags |= _PAGE_BAP_SR; +#endif + + return __ioremap_caller(addr, size, flags, __builtin_return_address(0)); } -#endif /* CONFIG_PHYS_64BIT */ +EXPORT_SYMBOL(ioremap_prot); void __iomem * __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) { + return __ioremap_caller(addr, size, flags, __builtin_return_address(0)); +} + +void __iomem * +__ioremap_caller(phys_addr_t addr, unsigned long size, unsigned long flags, + void *caller) +{ unsigned long v, i; phys_addr_t p; int err; + /* Make sure we have the base flags */ + if ((flags & _PAGE_PRESENT) == 0) + flags |= PAGE_KERNEL; + + /* Non-cacheable page cannot be coherent */ + if (flags & _PAGE_NO_CACHE) + flags &= ~_PAGE_COHERENT; + /* * Choose an address to map it to. * Once the vmalloc system is running, we use it. @@ -186,15 +204,18 @@ __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) if (p < 16*1024*1024) p += _ISA_MEM_BASE; +#ifndef CONFIG_CRASH_DUMP /* * Don't allow anybody to remap normal RAM that we're using. * mem_init() sets high_memory so only do the check after that. */ - if (mem_init_done && (p < virt_to_phys(high_memory))) { - printk("__ioremap(): phys addr "PHYS_FMT" is RAM lr %p\n", p, - __builtin_return_address(0)); + if (mem_init_done && (p < virt_to_phys(high_memory)) && + !(__allow_ioremap_reserved && memblock_is_region_reserved(p, size))) { + printk("__ioremap(): phys addr 0x%llx is RAM lr %pf\n", + (unsigned long long)p, __builtin_return_address(0)); return NULL; } +#endif if (size == 0) return NULL; @@ -218,19 +239,15 @@ __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) if (mem_init_done) { struct vm_struct *area; - area = get_vm_area(size, VM_IOREMAP); + area = get_vm_area_caller(size, VM_IOREMAP, caller); if (area == 0) return NULL; + area->phys_addr = p; v = (unsigned long) area->addr; } else { v = (ioremap_bot -= size); } - if ((flags & _PAGE_PRESENT) == 0) - flags |= _PAGE_KERNEL; - if (flags & _PAGE_NO_CACHE) - flags |= _PAGE_GUARDED; - /* * Should check if it is a candidate for a BAT mapping */ @@ -247,6 +264,7 @@ __ioremap(phys_addr_t addr, unsigned long size, unsigned long flags) out: return (void __iomem *) (v + ((unsigned long)addr & ~PAGE_MASK)); } +EXPORT_SYMBOL(__ioremap); void iounmap(volatile void __iomem *addr) { @@ -259,106 +277,79 @@ void iounmap(volatile void __iomem *addr) if (addr > high_memory && (unsigned long) addr < ioremap_bot) vunmap((void *) (PAGE_MASK & (unsigned long)addr)); } +EXPORT_SYMBOL(iounmap); -void __iomem *ioport_map(unsigned long port, unsigned int len) -{ - return (void __iomem *) (port + _IO_BASE); -} - -void ioport_unmap(void __iomem *addr) -{ - /* Nothing to do */ -} -EXPORT_SYMBOL(ioport_map); -EXPORT_SYMBOL(ioport_unmap); - -int -map_page(unsigned long va, phys_addr_t pa, int flags) +int map_page(unsigned long va, phys_addr_t pa, int flags) { pmd_t *pd; pte_t *pg; int err = -ENOMEM; /* Use upper 10 bits of VA to index the first level map */ - pd = pmd_offset(pgd_offset_k(va), va); + pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va); /* Use middle 10 bits of VA to index the second-level map */ pg = pte_alloc_kernel(pd, va); if (pg != 0) { err = 0; - set_pte_at(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, __pgprot(flags))); - if (mem_init_done) - flush_HPTE(0, va, pmd_val(*pd)); + /* The PTE should never be already set nor present in the + * hash table + */ + BUG_ON((pte_val(*pg) & (_PAGE_PRESENT | _PAGE_HASHPTE)) && + flags); + set_pte_at(&init_mm, va, pg, pfn_pte(pa >> PAGE_SHIFT, + __pgprot(flags))); } + smp_wmb(); return err; } /* - * Map in all of physical memory starting at KERNELBASE. + * Map in a chunk of physical memory starting at start. */ -void __init mapin_ram(void) +void __init __mapin_ram_chunk(unsigned long offset, unsigned long top) { - unsigned long v, p, s, f; - - s = mmu_mapin_ram(); - v = KERNELBASE + s; - p = PPC_MEMSTART + s; - for (; s < total_lowmem; s += PAGE_SIZE) { - if ((char *) v >= _stext && (char *) v < etext) - f = _PAGE_RAM_TEXT; - else - f = _PAGE_RAM; + unsigned long v, s, f; + phys_addr_t p; + int ktext; + + s = offset; + v = PAGE_OFFSET + s; + p = memstart_addr + s; + for (; s < top; s += PAGE_SIZE) { + ktext = ((char *) v >= _stext && (char *) v < etext); + f = ktext ? PAGE_KERNEL_TEXT : PAGE_KERNEL; map_page(v, p, f); +#ifdef CONFIG_PPC_STD_MMU_32 + if (ktext) + hash_preload(&init_mm, v, 0, 0x300); +#endif v += PAGE_SIZE; p += PAGE_SIZE; } } -/* is x a power of 2? */ -#define is_power_of_2(x) ((x) != 0 && (((x) & ((x) - 1)) == 0)) - -/* is x a power of 4? */ -#define is_power_of_4(x) ((x) != 0 && (((x) & (x-1)) == 0) && (ffs(x) & 1)) - -/* - * Set up a mapping for a block of I/O. - * virt, phys, size must all be page-aligned. - * This should only be called before ioremap is called. - */ -void __init io_block_mapping(unsigned long virt, phys_addr_t phys, - unsigned int size, int flags) +void __init mapin_ram(void) { - int i; + unsigned long s, top; - if (virt > KERNELBASE && virt < ioremap_bot) - ioremap_bot = ioremap_base = virt; - -#ifdef HAVE_BATS - /* - * Use a BAT for this if possible... - */ - if (io_bat_index < 2 && is_power_of_2(size) - && (virt & (size - 1)) == 0 && (phys & (size - 1)) == 0) { - setbat(io_bat_index, virt, phys, size, flags); - ++io_bat_index; - return; - } -#endif /* HAVE_BATS */ +#ifndef CONFIG_WII + top = total_lowmem; + s = mmu_mapin_ram(top); + __mapin_ram_chunk(s, top); +#else + if (!wii_hole_size) { + s = mmu_mapin_ram(total_lowmem); + __mapin_ram_chunk(s, total_lowmem); + } else { + top = wii_hole_start; + s = mmu_mapin_ram(top); + __mapin_ram_chunk(s, top); -#ifdef HAVE_TLBCAM - /* - * Use a CAM for this if possible... - */ - if (tlbcam_index < num_tlbcam_entries && is_power_of_4(size) - && (virt & (size - 1)) == 0 && (phys & (size - 1)) == 0) { - settlbcam(tlbcam_index, virt, phys, size, flags, 0); - ++tlbcam_index; - return; + top = memblock_end_of_DRAM(); + s = wii_mmu_mapin_mem2(top); + __mapin_ram_chunk(s, top); } -#endif /* HAVE_TLBCAM */ - - /* No BATs available, put it in the page tables. */ - for (i = 0; i < size; i += PAGE_SIZE) - map_page(virt + i, phys + i, flags); +#endif } /* Scan the real Linux page tables and return a PTE pointer for @@ -367,101 +358,103 @@ void __init io_block_mapping(unsigned long virt, phys_addr_t phys, * the PTE pointer is unmodified if PTE is not found. */ int -get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep) +get_pteptr(struct mm_struct *mm, unsigned long addr, pte_t **ptep, pmd_t **pmdp) { pgd_t *pgd; + pud_t *pud; pmd_t *pmd; pte_t *pte; int retval = 0; pgd = pgd_offset(mm, addr & PAGE_MASK); if (pgd) { - pmd = pmd_offset(pgd, addr & PAGE_MASK); - if (pmd_present(*pmd)) { - pte = pte_offset_map(pmd, addr & PAGE_MASK); - if (pte) { - retval = 1; - *ptep = pte; - /* XXX caller needs to do pte_unmap, yuck */ - } - } + pud = pud_offset(pgd, addr & PAGE_MASK); + if (pud && pud_present(*pud)) { + pmd = pmd_offset(pud, addr & PAGE_MASK); + if (pmd_present(*pmd)) { + pte = pte_offset_map(pmd, addr & PAGE_MASK); + if (pte) { + retval = 1; + *ptep = pte; + if (pmdp) + *pmdp = pmd; + /* XXX caller needs to do pte_unmap, yuck */ + } + } + } } return(retval); } -/* Find physical address for this virtual address. Normally used by - * I/O functions, but anyone can call it. - */ -unsigned long iopa(unsigned long addr) +#ifdef CONFIG_DEBUG_PAGEALLOC + +static int __change_page_attr(struct page *page, pgprot_t prot) { - unsigned long pa; + pte_t *kpte; + pmd_t *kpmd; + unsigned long address; + + BUG_ON(PageHighMem(page)); + address = (unsigned long)page_address(page); + + if (v_mapped_by_bats(address) || v_mapped_by_tlbcam(address)) + return 0; + if (!get_pteptr(&init_mm, address, &kpte, &kpmd)) + return -EINVAL; + __set_pte_at(&init_mm, address, kpte, mk_pte(page, prot), 0); + wmb(); + flush_tlb_page(NULL, address); + pte_unmap(kpte); + + return 0; +} - /* I don't know why this won't work on PMacs or CHRP. It - * appears there is some bug, or there is some implicit - * mapping done not properly represented by BATs or in page - * tables.......I am actively working on resolving this, but - * can't hold up other stuff. -- Dan - */ - pte_t *pte; - struct mm_struct *mm; +/* + * Change the page attributes of an page in the linear mapping. + * + * THIS CONFLICTS WITH BAT MAPPINGS, DEBUG USE ONLY + */ +static int change_page_attr(struct page *page, int numpages, pgprot_t prot) +{ + int i, err = 0; + unsigned long flags; + + local_irq_save(flags); + for (i = 0; i < numpages; i++, page++) { + err = __change_page_attr(page, prot); + if (err) + break; + } + local_irq_restore(flags); + return err; +} - /* Check the BATs */ - pa = v_mapped_by_bats(addr); - if (pa) - return pa; - /* Allow mapping of user addresses (within the thread) - * for DMA if necessary. - */ - if (addr < TASK_SIZE) - mm = current->mm; - else - mm = &init_mm; - - pa = 0; - if (get_pteptr(mm, addr, &pte)) { - pa = (pte_val(*pte) & PAGE_MASK) | (addr & ~PAGE_MASK); - pte_unmap(pte); - } +void kernel_map_pages(struct page *page, int numpages, int enable) +{ + if (PageHighMem(page)) + return; - return(pa); + change_page_attr(page, numpages, enable ? PAGE_KERNEL : __pgprot(0)); } +#endif /* CONFIG_DEBUG_PAGEALLOC */ -/* This is will find the virtual address for a physical one.... - * Swiped from APUS, could be dangerous :-). - * This is only a placeholder until I really find a way to make this - * work. -- Dan - */ -unsigned long -mm_ptov (unsigned long paddr) +static int fixmaps; + +void __set_fixmap (enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags) { - unsigned long ret; -#if 0 - if (paddr < 16*1024*1024) - ret = ZTWO_VADDR(paddr); - else { - int i; - - for (i = 0; i < kmap_chunk_count;){ - unsigned long phys = kmap_chunks[i++]; - unsigned long size = kmap_chunks[i++]; - unsigned long virt = kmap_chunks[i++]; - if (paddr >= phys - && paddr < (phys + size)){ - ret = virt + paddr - phys; - goto exit; - } - } - - ret = (unsigned long) __va(paddr); + unsigned long address = __fix_to_virt(idx); + + if (idx >= __end_of_fixed_addresses) { + BUG(); + return; } -exit: -#ifdef DEBUGPV - printk ("PTOV(%lx)=%lx\n", paddr, ret); -#endif -#else - ret = (unsigned long)paddr + KERNELBASE; -#endif - return ret; + + map_page(address, phys, pgprot_val(flags)); + fixmaps++; } +void __this_fixmap_does_not_exist(void) +{ + WARN_ON(1); +} |