diff options
Diffstat (limited to 'arch/xtensa/mm/init.c')
| -rw-r--r-- | arch/xtensa/mm/init.c | 644 |
1 files changed, 253 insertions, 391 deletions
diff --git a/arch/xtensa/mm/init.c b/arch/xtensa/mm/init.c index 5a91d6c9e66..77ed20209ca 100644 --- a/arch/xtensa/mm/init.c +++ b/arch/xtensa/mm/init.c @@ -8,6 +8,7 @@ * for more details. * * Copyright (C) 2001 - 2005 Tensilica Inc. + * Copyright (C) 2014 Cadence Design Systems Inc. * * Chris Zankel <chris@zankel.net> * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> @@ -15,51 +16,146 @@ * Kevin Chea */ -#include <linux/config.h> -#include <linux/init.h> -#include <linux/signal.h> -#include <linux/sched.h> #include <linux/kernel.h> #include <linux/errno.h> -#include <linux/string.h> -#include <linux/types.h> -#include <linux/ptrace.h> #include <linux/bootmem.h> +#include <linux/gfp.h> +#include <linux/highmem.h> #include <linux/swap.h> +#include <linux/mman.h> +#include <linux/nodemask.h> +#include <linux/mm.h> -#include <asm/pgtable.h> #include <asm/bootparam.h> -#include <asm/mmu_context.h> -#include <asm/tlb.h> -#include <asm/tlbflush.h> #include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> +#include <asm/sections.h> +#include <asm/sysmem.h> +struct sysmem_info sysmem __initdata; -#define DEBUG 0 +static void __init sysmem_dump(void) +{ + unsigned i; + + pr_debug("Sysmem:\n"); + for (i = 0; i < sysmem.nr_banks; ++i) + pr_debug(" 0x%08lx - 0x%08lx (%ldK)\n", + sysmem.bank[i].start, sysmem.bank[i].end, + (sysmem.bank[i].end - sysmem.bank[i].start) >> 10); +} + +/* + * Find bank with maximal .start such that bank.start <= start + */ +static inline struct meminfo * __init find_bank(unsigned long start) +{ + unsigned i; + struct meminfo *it = NULL; + + for (i = 0; i < sysmem.nr_banks; ++i) + if (sysmem.bank[i].start <= start) + it = sysmem.bank + i; + else + break; + return it; +} -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); -//static DEFINE_SPINLOCK(tlb_lock); +/* + * Move all memory banks starting at 'from' to a new place at 'to', + * adjust nr_banks accordingly. + * Both 'from' and 'to' must be inside the sysmem.bank. + * + * Returns: 0 (success), -ENOMEM (not enough space in the sysmem.bank). + */ +static int __init move_banks(struct meminfo *to, struct meminfo *from) +{ + unsigned n = sysmem.nr_banks - (from - sysmem.bank); + + if (to > from && to - from + sysmem.nr_banks > SYSMEM_BANKS_MAX) + return -ENOMEM; + if (to != from) + memmove(to, from, n * sizeof(struct meminfo)); + sysmem.nr_banks += to - from; + return 0; +} /* - * This flag is used to indicate that the page was mapped and modified in - * kernel space, so the cache is probably dirty at that address. - * If cache aliasing is enabled and the page color mismatches, update_mmu_cache - * synchronizes the caches if this bit is set. + * Add new bank to sysmem. Resulting sysmem is the union of bytes of the + * original sysmem and the new bank. + * + * Returns: 0 (success), < 0 (error) */ +int __init add_sysmem_bank(unsigned long start, unsigned long end) +{ + unsigned i; + struct meminfo *it = NULL; + unsigned long sz; + unsigned long bank_sz = 0; + + if (start == end || + (start < end) != (PAGE_ALIGN(start) < (end & PAGE_MASK))) { + pr_warn("Ignoring small memory bank 0x%08lx size: %ld bytes\n", + start, end - start); + return -EINVAL; + } -#define PG_cache_clean PG_arch_1 + start = PAGE_ALIGN(start); + end &= PAGE_MASK; + sz = end - start; -/* References to section boundaries */ + it = find_bank(start); -extern char _ftext, _etext, _fdata, _edata, _rodata_end; -extern char __init_begin, __init_end; + if (it) + bank_sz = it->end - it->start; + + if (it && bank_sz >= start - it->start) { + if (end - it->start > bank_sz) + it->end = end; + else + return 0; + } else { + if (!it) + it = sysmem.bank; + else + ++it; + + if (it - sysmem.bank < sysmem.nr_banks && + it->start - start <= sz) { + it->start = start; + if (it->end - it->start < sz) + it->end = end; + else + return 0; + } else { + if (move_banks(it + 1, it) < 0) { + pr_warn("Ignoring memory bank 0x%08lx size %ld bytes\n", + start, end - start); + return -EINVAL; + } + it->start = start; + it->end = end; + return 0; + } + } + sz = it->end - it->start; + for (i = it + 1 - sysmem.bank; i < sysmem.nr_banks; ++i) + if (sysmem.bank[i].start - it->start <= sz) { + if (sz < sysmem.bank[i].end - it->start) + it->end = sysmem.bank[i].end; + } else { + break; + } + + move_banks(it + 1, sysmem.bank + i); + return 0; +} /* * mem_reserve(start, end, must_exist) * * Reserve some memory from the memory pool. + * If must_exist is set and a part of the region being reserved does not exist + * memory map is not altered. * * Parameters: * start Start of region, @@ -67,58 +163,74 @@ extern char __init_begin, __init_end; * must_exist Must exist in memory pool. * * Returns: - * 0 (memory area couldn't be mapped) - * -1 (success) + * 0 (success) + * < 0 (error) */ int __init mem_reserve(unsigned long start, unsigned long end, int must_exist) { - int i; - - if (start == end) - return 0; + struct meminfo *it; + struct meminfo *rm = NULL; + unsigned long sz; + unsigned long bank_sz = 0; start = start & PAGE_MASK; end = PAGE_ALIGN(end); + sz = end - start; + if (!sz) + return -EINVAL; - for (i = 0; i < sysmem.nr_banks; i++) - if (start < sysmem.bank[i].end - && end >= sysmem.bank[i].start) - break; + it = find_bank(start); + + if (it) + bank_sz = it->end - it->start; - if (i == sysmem.nr_banks) { - if (must_exist) - printk (KERN_WARNING "mem_reserve: [0x%0lx, 0x%0lx) " - "not in any region!\n", start, end); - return 0; + if ((!it || end - it->start > bank_sz) && must_exist) { + pr_warn("mem_reserve: [0x%0lx, 0x%0lx) not in any region!\n", + start, end); + return -EINVAL; } - if (start > sysmem.bank[i].start) { - if (end < sysmem.bank[i].end) { - /* split entry */ - if (sysmem.nr_banks >= SYSMEM_BANKS_MAX) - panic("meminfo overflow\n"); - sysmem.bank[sysmem.nr_banks].start = end; - sysmem.bank[sysmem.nr_banks].end = sysmem.bank[i].end; - sysmem.nr_banks++; + if (it && start - it->start <= bank_sz) { + if (start == it->start) { + if (end - it->start < bank_sz) { + it->start = end; + return 0; + } else { + rm = it; + } + } else { + it->end = start; + if (end - it->start < bank_sz) + return add_sysmem_bank(end, + it->start + bank_sz); + ++it; } - sysmem.bank[i].end = start; - } else { - if (end < sysmem.bank[i].end) - sysmem.bank[i].start = end; - else { - /* remove entry */ - sysmem.nr_banks--; - sysmem.bank[i].start = sysmem.bank[sysmem.nr_banks].start; - sysmem.bank[i].end = sysmem.bank[sysmem.nr_banks].end; + } + + if (!it) + it = sysmem.bank; + + for (; it < sysmem.bank + sysmem.nr_banks; ++it) { + if (it->end - start <= sz) { + if (!rm) + rm = it; + } else { + if (it->start - start < sz) + it->start = end; + break; } } - return -1; + + if (rm) + move_banks(rm, it); + + return 0; } /* - * Initialize the bootmem system and give it all the memory we have available. + * Initialize the bootmem system and give it all low memory we have available. */ void __init bootmem_init(void) @@ -127,6 +239,7 @@ void __init bootmem_init(void) unsigned long bootmap_start, bootmap_size; int i; + sysmem_dump(); max_low_pfn = max_pfn = 0; min_low_pfn = ~0; @@ -142,12 +255,13 @@ void __init bootmem_init(void) if (min_low_pfn > max_pfn) panic("No memory found!\n"); - max_low_pfn = max_pfn < MAX_LOW_MEMORY >> PAGE_SHIFT ? - max_pfn : MAX_LOW_MEMORY >> PAGE_SHIFT; + max_low_pfn = max_pfn < MAX_MEM_PFN >> PAGE_SHIFT ? + max_pfn : MAX_MEM_PFN >> PAGE_SHIFT; /* Find an area to use for the bootmem bitmap. */ - bootmap_size = bootmem_bootmap_pages(max_low_pfn) << PAGE_SHIFT; + bootmap_size = bootmem_bootmap_pages(max_low_pfn - min_low_pfn); + bootmap_size <<= PAGE_SHIFT; bootmap_start = ~0; for (i=0; i<sysmem.nr_banks; i++) @@ -162,68 +276,35 @@ void __init bootmem_init(void) /* Reserve the bootmem bitmap area */ mem_reserve(bootmap_start, bootmap_start + bootmap_size, 1); - bootmap_size = init_bootmem_node(NODE_DATA(0), min_low_pfn, + bootmap_size = init_bootmem_node(NODE_DATA(0), bootmap_start >> PAGE_SHIFT, + min_low_pfn, max_low_pfn); /* Add all remaining memory pieces into the bootmem map */ - for (i=0; i<sysmem.nr_banks; i++) - free_bootmem(sysmem.bank[i].start, - sysmem.bank[i].end - sysmem.bank[i].start); + for (i = 0; i < sysmem.nr_banks; i++) { + if (sysmem.bank[i].start >> PAGE_SHIFT < max_low_pfn) { + unsigned long end = min(max_low_pfn << PAGE_SHIFT, + sysmem.bank[i].end); + free_bootmem(sysmem.bank[i].start, + end - sysmem.bank[i].start); + } + } } -void __init paging_init(void) +void __init zones_init(void) { - unsigned long zones_size[MAX_NR_ZONES]; - int i; - /* All pages are DMA-able, so we put them all in the DMA zone. */ - - zones_size[ZONE_DMA] = max_low_pfn; - for (i = 1; i < MAX_NR_ZONES; i++) - zones_size[i] = 0; - + unsigned long zones_size[MAX_NR_ZONES] = { + [ZONE_DMA] = max_low_pfn - ARCH_PFN_OFFSET, #ifdef CONFIG_HIGHMEM - zones_size[ZONE_HIGHMEM] = max_pfn - max_low_pfn; + [ZONE_HIGHMEM] = max_pfn - max_low_pfn, #endif - - /* Initialize the kernel's page tables. */ - - memset(swapper_pg_dir, 0, PAGE_SIZE); - - free_area_init(zones_size); -} - -/* - * Flush the mmu and reset associated register to default values. - */ - -void __init init_mmu (void) -{ - /* Writing zeros to the <t>TLBCFG special registers ensure - * that valid values exist in the register. For existing - * PGSZID<w> fields, zero selects the first element of the - * page-size array. For nonexistant PGSZID<w> fields, zero is - * the best value to write. Also, when changing PGSZID<w> - * fields, the corresponding TLB must be flushed. - */ - set_itlbcfg_register (0); - set_dtlbcfg_register (0); - flush_tlb_all (); - - /* Set rasid register to a known value. */ - - set_rasid_register (ASID_ALL_RESERVED); - - /* Set PTEVADDR special register to the start of the page - * table, which is in kernel mappable space (ie. not - * statically mapped). This register's value is undefined on - * reset. - */ - set_ptevaddr_register (PGTABLE_START); + }; + free_area_init_node(0, zones_size, ARCH_PFN_OFFSET, NULL); } /* @@ -232,50 +313,38 @@ void __init init_mmu (void) void __init mem_init(void) { - unsigned long codesize, reservedpages, datasize, initsize; - unsigned long highmemsize, tmp, ram; - - max_mapnr = num_physpages = max_low_pfn; - high_memory = (void *) __va(max_mapnr << PAGE_SHIFT); - highmemsize = 0; - #ifdef CONFIG_HIGHMEM -#error HIGHGMEM not implemented in init.c -#endif + unsigned long tmp; - totalram_pages += free_all_bootmem(); + reset_all_zones_managed_pages(); + for (tmp = max_low_pfn; tmp < max_pfn; tmp++) + free_highmem_page(pfn_to_page(tmp)); +#endif - reservedpages = ram = 0; - for (tmp = 0; tmp < max_low_pfn; tmp++) { - ram++; - if (PageReserved(mem_map+tmp)) - reservedpages++; - } + max_mapnr = max_pfn - ARCH_PFN_OFFSET; + high_memory = (void *)__va(max_low_pfn << PAGE_SHIFT); - codesize = (unsigned long) &_etext - (unsigned long) &_ftext; - datasize = (unsigned long) &_edata - (unsigned long) &_fdata; - initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin; - - printk("Memory: %luk/%luk available (%ldk kernel code, %ldk reserved, " - "%ldk data, %ldk init %ldk highmem)\n", - (unsigned long) nr_free_pages() << (PAGE_SHIFT-10), - ram << (PAGE_SHIFT-10), - codesize >> 10, - reservedpages << (PAGE_SHIFT-10), - datasize >> 10, - initsize >> 10, - highmemsize >> 10); -} + free_all_bootmem(); -void -free_reserved_mem(void *start, void *end) -{ - for (; start < end; start += PAGE_SIZE) { - ClearPageReserved(virt_to_page(start)); - set_page_count(virt_to_page(start), 1); - free_page((unsigned long)start); - totalram_pages++; - } + mem_init_print_info(NULL); + pr_info("virtual kernel memory layout:\n" +#ifdef CONFIG_HIGHMEM + " pkmap : 0x%08lx - 0x%08lx (%5lu kB)\n" + " fixmap : 0x%08lx - 0x%08lx (%5lu kB)\n" +#endif + " vmalloc : 0x%08x - 0x%08x (%5u MB)\n" + " lowmem : 0x%08x - 0x%08lx (%5lu MB)\n", +#ifdef CONFIG_HIGHMEM + PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE, + (LAST_PKMAP*PAGE_SIZE) >> 10, + FIXADDR_START, FIXADDR_TOP, + (FIXADDR_TOP - FIXADDR_START) >> 10, +#endif + VMALLOC_START, VMALLOC_END, + (VMALLOC_END - VMALLOC_START) >> 20, + PAGE_OFFSET, PAGE_OFFSET + + (max_low_pfn - min_low_pfn) * PAGE_SIZE, + ((max_low_pfn - min_low_pfn) * PAGE_SIZE) >> 20); } #ifdef CONFIG_BLK_DEV_INITRD @@ -283,269 +352,62 @@ extern int initrd_is_mapped; void free_initrd_mem(unsigned long start, unsigned long end) { - if (initrd_is_mapped) { - free_reserved_mem((void*)start, (void*)end); - printk ("Freeing initrd memory: %ldk freed\n",(end-start)>>10); - } + if (initrd_is_mapped) + free_reserved_area((void *)start, (void *)end, -1, "initrd"); } #endif void free_initmem(void) { - free_reserved_mem(&__init_begin, &__init_end); - printk("Freeing unused kernel memory: %dk freed\n", - (&__init_end - &__init_begin) >> 10); + free_initmem_default(-1); } -void show_mem(void) +static void __init parse_memmap_one(char *p) { - int i, free = 0, total = 0, reserved = 0; - int shared = 0, cached = 0; - - printk("Mem-info:\n"); - show_free_areas(); - printk("Free swap: %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10)); - i = max_mapnr; - while (i-- > 0) { - total++; - if (PageReserved(mem_map+i)) - reserved++; - else if (PageSwapCache(mem_map+i)) - cached++; - else if (!page_count(mem_map + i)) - free++; - else - shared += page_count(mem_map + i) - 1; - } - printk("%d pages of RAM\n", total); - printk("%d reserved pages\n", reserved); - printk("%d pages shared\n", shared); - printk("%d pages swap cached\n",cached); - printk("%d free pages\n", free); -} - -/* ------------------------------------------------------------------------- */ - -#if (DCACHE_WAY_SIZE > PAGE_SIZE) + char *oldp; + unsigned long start_at, mem_size; -/* - * With cache aliasing, the page color of the page in kernel space and user - * space might mismatch. We temporarily map the page to a different virtual - * address with the same color and clear the page there. - */ - -void clear_user_page(void *kaddr, unsigned long vaddr, struct page* page) -{ - - /* There shouldn't be any entries for this page. */ - - __flush_invalidate_dcache_page_phys(__pa(page_address(page))); - - if (!PAGE_COLOR_EQ(vaddr, kaddr)) { - unsigned long v, p; - - /* Temporarily map page to DTLB_WAY_DCACHE_ALIAS0. */ - - spin_lock(&tlb_lock); - - p = (unsigned long)pte_val((mk_pte(page,PAGE_KERNEL))); - kaddr = (void*)PAGE_COLOR_MAP0(vaddr); - v = (unsigned long)kaddr | DTLB_WAY_DCACHE_ALIAS0; - __asm__ __volatile__("wdtlb %0,%1; dsync" : :"a" (p), "a" (v)); - - clear_page(kaddr); - - spin_unlock(&tlb_lock); - } else { - clear_page(kaddr); - } - - /* We need to make sure that i$ and d$ are coherent. */ - - clear_bit(PG_cache_clean, &page->flags); -} - -/* - * With cache aliasing, we have to make sure that the page color of the page - * in kernel space matches that of the virtual user address before we read - * the page. If the page color differ, we create a temporary DTLB entry with - * the corrent page color and use this 'temporary' address as the source. - * We then use the same approach as in clear_user_page and copy the data - * to the kernel space and clear the PG_cache_clean bit to synchronize caches - * later. - * - * Note: - * Instead of using another 'way' for the temporary DTLB entry, we could - * probably use the same entry that points to the kernel address (after - * saving the original value and restoring it when we are done). - */ - -void copy_user_page(void* to, void* from, unsigned long vaddr, - struct page* to_page) -{ - /* There shouldn't be any entries for the new page. */ - - __flush_invalidate_dcache_page_phys(__pa(page_address(to_page))); - - spin_lock(&tlb_lock); - - if (!PAGE_COLOR_EQ(vaddr, from)) { - unsigned long v, p, t; - - __asm__ __volatile__ ("pdtlb %1,%2; rdtlb1 %0,%1" - : "=a"(p), "=a"(t) : "a"(from)); - from = (void*)PAGE_COLOR_MAP0(vaddr); - v = (unsigned long)from | DTLB_WAY_DCACHE_ALIAS0; - __asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v)); - } - - if (!PAGE_COLOR_EQ(vaddr, to)) { - unsigned long v, p; - - p = (unsigned long)pte_val((mk_pte(to_page,PAGE_KERNEL))); - to = (void*)PAGE_COLOR_MAP1(vaddr); - v = (unsigned long)to | DTLB_WAY_DCACHE_ALIAS1; - __asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v)); - } - copy_page(to, from); - - spin_unlock(&tlb_lock); - - /* We need to make sure that i$ and d$ are coherent. */ - - clear_bit(PG_cache_clean, &to_page->flags); -} - - - -/* - * Any time the kernel writes to a user page cache page, or it is about to - * read from a page cache page this routine is called. - * - * Note: - * The kernel currently only provides one architecture bit in the page - * flags that we use for I$/D$ coherency. Maybe, in future, we can - * use a sepearte bit for deferred dcache aliasing: - * If the page is not mapped yet, we only need to set a flag, - * if mapped, we need to invalidate the page. - */ -// FIXME: we probably need this for WB caches not only for Page Coloring.. - -void flush_dcache_page(struct page *page) -{ - unsigned long addr = __pa(page_address(page)); - struct address_space *mapping = page_mapping(page); - - __flush_invalidate_dcache_page_phys(addr); - - if (!test_bit(PG_cache_clean, &page->flags)) + if (!p) return; - /* If this page hasn't been mapped, yet, handle I$/D$ coherency later.*/ -#if 0 - if (mapping && !mapping_mapped(mapping)) - clear_bit(PG_cache_clean, &page->flags); - else -#endif - __invalidate_icache_page_phys(addr); -} - -void flush_cache_range(struct vm_area_struct* vma, unsigned long s, - unsigned long e) -{ - __flush_invalidate_cache_all(); -} - -void flush_cache_page(struct vm_area_struct* vma, unsigned long address, - unsigned long pfn) -{ - struct page *page = pfn_to_page(pfn); - - /* Remove any entry for the old mapping. */ + oldp = p; + mem_size = memparse(p, &p); + if (p == oldp) + return; - if (current->active_mm == vma->vm_mm) { - unsigned long addr = __pa(page_address(page)); - __flush_invalidate_dcache_page_phys(addr); - if ((vma->vm_flags & VM_EXEC) != 0) - __invalidate_icache_page_phys(addr); - } else { - BUG(); - } -} + switch (*p) { + case '@': + start_at = memparse(p + 1, &p); + add_sysmem_bank(start_at, start_at + mem_size); + break; -#endif /* (DCACHE_WAY_SIZE > PAGE_SIZE) */ + case '$': + start_at = memparse(p + 1, &p); + mem_reserve(start_at, start_at + mem_size, 0); + break; + case 0: + mem_reserve(mem_size, 0, 0); + break; -pte_t* pte_alloc_one_kernel (struct mm_struct* mm, unsigned long addr) -{ - pte_t* pte = (pte_t*)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, 0); - if (likely(pte)) { - pte_t* ptep = (pte_t*)(pte_val(*pte) + PAGE_OFFSET); - int i; - for (i = 0; i < 1024; i++, ptep++) - pte_clear(mm, addr, ptep); + default: + pr_warn("Unrecognized memmap syntax: %s\n", p); + break; } - return pte; } -struct page* pte_alloc_one(struct mm_struct *mm, unsigned long addr) +static int __init parse_memmap_opt(char *str) { - struct page *page; + while (str) { + char *k = strchr(str, ','); - page = alloc_pages(GFP_KERNEL | __GFP_REPEAT, 0); + if (k) + *k++ = 0; - if (likely(page)) { - pte_t* ptep = kmap_atomic(page, KM_USER0); - int i; - - for (i = 0; i < 1024; i++, ptep++) - pte_clear(mm, addr, ptep); - - kunmap_atomic(ptep, KM_USER0); + parse_memmap_one(str); + str = k; } - return page; -} - -/* - * Handle D$/I$ coherency. - * - * Note: - * We only have one architecture bit for the page flags, so we cannot handle - * cache aliasing, yet. - */ - -void -update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t pte) -{ - unsigned long pfn = pte_pfn(pte); - struct page *page; - unsigned long vaddr = addr & PAGE_MASK; - - if (!pfn_valid(pfn)) - return; - - page = pfn_to_page(pfn); - - invalidate_itlb_mapping(addr); - invalidate_dtlb_mapping(addr); - - /* We have a new mapping. Use it. */ - - write_dtlb_entry(pte, dtlb_probe(addr)); - - /* If the processor can execute from this page, synchronize D$/I$. */ - - if ((vma->vm_flags & VM_EXEC) != 0) { - - write_itlb_entry(pte, itlb_probe(addr)); - - /* Synchronize caches, if not clean. */ - - if (!test_and_set_bit(PG_cache_clean, &page->flags)) { - __flush_dcache_page(vaddr); - __invalidate_icache_page(vaddr); - } - } + return 0; } - +early_param("memmap", parse_memmap_opt); |
