diff options
Diffstat (limited to 'arch/arm/kernel/setup.c')
| -rw-r--r-- | arch/arm/kernel/setup.c | 1174 |
1 files changed, 626 insertions, 548 deletions
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index b7b0720bc1b..8a16ee5d8a9 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -7,7 +7,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/stddef.h> #include <linux/ioport.h> @@ -18,31 +18,47 @@ #include <linux/bootmem.h> #include <linux/seq_file.h> #include <linux/screen_info.h> +#include <linux/of_platform.h> #include <linux/init.h> -#include <linux/root_dev.h> +#include <linux/kexec.h> +#include <linux/of_fdt.h> #include <linux/cpu.h> #include <linux/interrupt.h> #include <linux/smp.h> -#include <linux/fs.h> - +#include <linux/proc_fs.h> +#include <linux/memblock.h> +#include <linux/bug.h> +#include <linux/compiler.h> +#include <linux/sort.h> + +#include <asm/unified.h> +#include <asm/cp15.h> #include <asm/cpu.h> +#include <asm/cputype.h> #include <asm/elf.h> #include <asm/procinfo.h> +#include <asm/psci.h> +#include <asm/sections.h> #include <asm/setup.h> +#include <asm/smp_plat.h> #include <asm/mach-types.h> #include <asm/cacheflush.h> +#include <asm/cachetype.h> #include <asm/tlbflush.h> +#include <asm/prom.h> #include <asm/mach/arch.h> #include <asm/mach/irq.h> #include <asm/mach/time.h> +#include <asm/system_info.h> +#include <asm/system_misc.h> +#include <asm/traps.h> +#include <asm/unwind.h> +#include <asm/memblock.h> +#include <asm/virt.h> -#include "compat.h" #include "atags.h" -#ifndef MEM_SIZE -#define MEM_SIZE (16*1024*1024) -#endif #if defined(CONFIG_FPE_NWFPE) || defined(CONFIG_FPE_FASTFPE) char fpe_type[8]; @@ -56,15 +72,20 @@ static int __init fpe_setup(char *line) __setup("fpe=", fpe_setup); #endif -extern void paging_init(struct meminfo *, struct machine_desc *desc); -extern void reboot_setup(char *str); -extern int root_mountflags; -extern void _stext, _text, _etext, __data_start, _edata, _end; +extern void init_default_cache_policy(unsigned long); +extern void paging_init(const struct machine_desc *desc); +extern void early_paging_init(const struct machine_desc *, + struct proc_info_list *); +extern void sanity_check_meminfo(void); +extern enum reboot_mode reboot_mode; +extern void setup_dma_zone(const struct machine_desc *desc); unsigned int processor_id; EXPORT_SYMBOL(processor_id); -unsigned int __machine_arch_type; +unsigned int __machine_arch_type __read_mostly; EXPORT_SYMBOL(__machine_arch_type); +unsigned int cacheid __read_mostly; +EXPORT_SYMBOL(cacheid); unsigned int __atags_pointer __initdata; @@ -77,46 +98,55 @@ EXPORT_SYMBOL(system_serial_low); unsigned int system_serial_high; EXPORT_SYMBOL(system_serial_high); -unsigned int elf_hwcap; +unsigned int elf_hwcap __read_mostly; EXPORT_SYMBOL(elf_hwcap); +unsigned int elf_hwcap2 __read_mostly; +EXPORT_SYMBOL(elf_hwcap2); + #ifdef MULTI_CPU -struct processor processor; +struct processor processor __read_mostly; #endif #ifdef MULTI_TLB -struct cpu_tlb_fns cpu_tlb; +struct cpu_tlb_fns cpu_tlb __read_mostly; #endif #ifdef MULTI_USER -struct cpu_user_fns cpu_user; +struct cpu_user_fns cpu_user __read_mostly; #endif #ifdef MULTI_CACHE -struct cpu_cache_fns cpu_cache; +struct cpu_cache_fns cpu_cache __read_mostly; #endif #ifdef CONFIG_OUTER_CACHE -struct outer_cache_fns outer_cache; +struct outer_cache_fns outer_cache __read_mostly; +EXPORT_SYMBOL(outer_cache); #endif +/* + * Cached cpu_architecture() result for use by assembler code. + * C code should use the cpu_architecture() function instead of accessing this + * variable directly. + */ +int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN; + struct stack { u32 irq[3]; u32 abt[3]; u32 und[3]; } ____cacheline_aligned; +#ifndef CONFIG_CPU_V7M static struct stack stacks[NR_CPUS]; +#endif char elf_platform[ELF_PLATFORM_SIZE]; EXPORT_SYMBOL(elf_platform); -unsigned long phys_initrd_start __initdata = 0; -unsigned long phys_initrd_size __initdata = 0; - -static struct meminfo meminfo __initdata = { 0, }; static const char *cpu_name; static const char *machine_name; -static char __initdata command_line[COMMAND_LINE_SIZE]; +static char __initdata cmd_line[COMMAND_LINE_SIZE]; +const struct machine_desc *machine_desc __initdata; -static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; static union { char c[4]; unsigned long l; } endian_test __initdata = { { 'l', '?', '?', 'b' } }; #define ENDIANNESS ((char)endian_test.l) @@ -133,7 +163,7 @@ static struct resource mem_res[] = { .flags = IORESOURCE_MEM }, { - .name = "Kernel text", + .name = "Kernel code", .start = 0, .end = 0, .flags = IORESOURCE_MEM @@ -175,63 +205,6 @@ static struct resource io_res[] = { #define lp1 io_res[1] #define lp2 io_res[2] -static const char *cache_types[16] = { - "write-through", - "write-back", - "write-back", - "undefined 3", - "undefined 4", - "undefined 5", - "write-back", - "write-back", - "undefined 8", - "undefined 9", - "undefined 10", - "undefined 11", - "undefined 12", - "undefined 13", - "write-back", - "undefined 15", -}; - -static const char *cache_clean[16] = { - "not required", - "read-block", - "cp15 c7 ops", - "undefined 3", - "undefined 4", - "undefined 5", - "cp15 c7 ops", - "cp15 c7 ops", - "undefined 8", - "undefined 9", - "undefined 10", - "undefined 11", - "undefined 12", - "undefined 13", - "cp15 c7 ops", - "undefined 15", -}; - -static const char *cache_lockdown[16] = { - "not supported", - "not supported", - "not supported", - "undefined 3", - "undefined 4", - "undefined 5", - "format A", - "format B", - "undefined 8", - "undefined 9", - "undefined 10", - "undefined 11", - "undefined 12", - "undefined 13", - "format C", - "undefined 15", -}; - static const char *proc_arch[] = { "undefined/unknown", "3", @@ -243,7 +216,7 @@ static const char *proc_arch[] = { "5TEJ", "6TEJ", "7", - "?(11)", + "7M", "?(12)", "?(13)", "?(14)", @@ -252,69 +225,33 @@ static const char *proc_arch[] = { "?(17)", }; -#define CACHE_TYPE(x) (((x) >> 25) & 15) -#define CACHE_S(x) ((x) & (1 << 24)) -#define CACHE_DSIZE(x) (((x) >> 12) & 4095) /* only if S=1 */ -#define CACHE_ISIZE(x) ((x) & 4095) - -#define CACHE_SIZE(y) (((y) >> 6) & 7) -#define CACHE_ASSOC(y) (((y) >> 3) & 7) -#define CACHE_M(y) ((y) & (1 << 2)) -#define CACHE_LINE(y) ((y) & 3) - -static inline void dump_cache(const char *prefix, int cpu, unsigned int cache) -{ - unsigned int mult = 2 + (CACHE_M(cache) ? 1 : 0); - - printk("CPU%u: %s: %d bytes, associativity %d, %d byte lines, %d sets\n", - cpu, prefix, - mult << (8 + CACHE_SIZE(cache)), - (mult << CACHE_ASSOC(cache)) >> 1, - 8 << CACHE_LINE(cache), - 1 << (6 + CACHE_SIZE(cache) - CACHE_ASSOC(cache) - - CACHE_LINE(cache))); -} - -static void __init dump_cpu_info(int cpu) +#ifdef CONFIG_CPU_V7M +static int __get_cpu_architecture(void) { - unsigned int info = read_cpuid(CPUID_CACHETYPE); - - if (info != processor_id) { - printk("CPU%u: D %s %s cache\n", cpu, cache_is_vivt() ? "VIVT" : "VIPT", - cache_types[CACHE_TYPE(info)]); - if (CACHE_S(info)) { - dump_cache("I cache", cpu, CACHE_ISIZE(info)); - dump_cache("D cache", cpu, CACHE_DSIZE(info)); - } else { - dump_cache("cache", cpu, CACHE_ISIZE(info)); - } - } - - if (arch_is_coherent()) - printk("Cache coherency enabled\n"); + return CPU_ARCH_ARMv7M; } - -int cpu_architecture(void) +#else +static int __get_cpu_architecture(void) { int cpu_arch; - if ((processor_id & 0x0008f000) == 0) { + if ((read_cpuid_id() & 0x0008f000) == 0) { cpu_arch = CPU_ARCH_UNKNOWN; - } else if ((processor_id & 0x0008f000) == 0x00007000) { - cpu_arch = (processor_id & (1 << 23)) ? CPU_ARCH_ARMv4T : CPU_ARCH_ARMv3; - } else if ((processor_id & 0x00080000) == 0x00000000) { - cpu_arch = (processor_id >> 16) & 7; + } else if ((read_cpuid_id() & 0x0008f000) == 0x00007000) { + cpu_arch = (read_cpuid_id() & (1 << 23)) ? CPU_ARCH_ARMv4T : CPU_ARCH_ARMv3; + } else if ((read_cpuid_id() & 0x00080000) == 0x00000000) { + cpu_arch = (read_cpuid_id() >> 16) & 7; if (cpu_arch) cpu_arch += CPU_ARCH_ARMv3; - } else if ((processor_id & 0x000f0000) == 0x000f0000) { + } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) { unsigned int mmfr0; /* Revised CPUID format. Read the Memory Model Feature * Register 0 and check for VMSAv7 or PMSAv7 */ asm("mrc p15, 0, %0, c0, c1, 4" : "=r" (mmfr0)); - if ((mmfr0 & 0x0000000f) == 0x00000003 || - (mmfr0 & 0x000000f0) == 0x00000030) + if ((mmfr0 & 0x0000000f) >= 0x00000003 || + (mmfr0 & 0x000000f0) >= 0x00000030) cpu_arch = CPU_ARCH_ARMv7; else if ((mmfr0 & 0x0000000f) == 0x00000002 || (mmfr0 & 0x000000f0) == 0x00000020) @@ -326,159 +263,441 @@ int cpu_architecture(void) return cpu_arch; } +#endif + +int __pure cpu_architecture(void) +{ + BUG_ON(__cpu_architecture == CPU_ARCH_UNKNOWN); + + return __cpu_architecture; +} + +static int cpu_has_aliasing_icache(unsigned int arch) +{ + int aliasing_icache; + unsigned int id_reg, num_sets, line_size; + + /* PIPT caches never alias. */ + if (icache_is_pipt()) + return 0; + + /* arch specifies the register format */ + switch (arch) { + case CPU_ARCH_ARMv7: + asm("mcr p15, 2, %0, c0, c0, 0 @ set CSSELR" + : /* No output operands */ + : "r" (1)); + isb(); + asm("mrc p15, 1, %0, c0, c0, 0 @ read CCSIDR" + : "=r" (id_reg)); + line_size = 4 << ((id_reg & 0x7) + 2); + num_sets = ((id_reg >> 13) & 0x7fff) + 1; + aliasing_icache = (line_size * num_sets) > PAGE_SIZE; + break; + case CPU_ARCH_ARMv6: + aliasing_icache = read_cpuid_cachetype() & (1 << 11); + break; + default: + /* I-cache aliases will be handled by D-cache aliasing code */ + aliasing_icache = 0; + } + + return aliasing_icache; +} + +static void __init cacheid_init(void) +{ + unsigned int arch = cpu_architecture(); + + if (arch == CPU_ARCH_ARMv7M) { + cacheid = 0; + } else if (arch >= CPU_ARCH_ARMv6) { + unsigned int cachetype = read_cpuid_cachetype(); + if ((cachetype & (7 << 29)) == 4 << 29) { + /* ARMv7 register format */ + arch = CPU_ARCH_ARMv7; + cacheid = CACHEID_VIPT_NONALIASING; + switch (cachetype & (3 << 14)) { + case (1 << 14): + cacheid |= CACHEID_ASID_TAGGED; + break; + case (3 << 14): + cacheid |= CACHEID_PIPT; + break; + } + } else { + arch = CPU_ARCH_ARMv6; + if (cachetype & (1 << 23)) + cacheid = CACHEID_VIPT_ALIASING; + else + cacheid = CACHEID_VIPT_NONALIASING; + } + if (cpu_has_aliasing_icache(arch)) + cacheid |= CACHEID_VIPT_I_ALIASING; + } else { + cacheid = CACHEID_VIVT; + } + + pr_info("CPU: %s data cache, %s instruction cache\n", + cache_is_vivt() ? "VIVT" : + cache_is_vipt_aliasing() ? "VIPT aliasing" : + cache_is_vipt_nonaliasing() ? "PIPT / VIPT nonaliasing" : "unknown", + cache_is_vivt() ? "VIVT" : + icache_is_vivt_asid_tagged() ? "VIVT ASID tagged" : + icache_is_vipt_aliasing() ? "VIPT aliasing" : + icache_is_pipt() ? "PIPT" : + cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown"); +} /* * These functions re-use the assembly code in head.S, which * already provide the required functionality. */ extern struct proc_info_list *lookup_processor_type(unsigned int); -extern struct machine_desc *lookup_machine_type(unsigned int); -static void __init setup_processor(void) +void __init early_print(const char *str, ...) { - struct proc_info_list *list; - - /* - * locate processor in the list of supported processor - * types. The linker builds this table for us from the - * entries in arch/arm/mm/proc-*.S - */ - list = lookup_processor_type(processor_id); - if (!list) { - printk("CPU configuration botched (ID %08x), unable " - "to continue.\n", processor_id); - while (1); - } + extern void printascii(const char *); + char buf[256]; + va_list ap; - cpu_name = list->cpu_name; + va_start(ap, str); + vsnprintf(buf, sizeof(buf), str, ap); + va_end(ap); -#ifdef MULTI_CPU - processor = *list->proc; -#endif -#ifdef MULTI_TLB - cpu_tlb = *list->tlb; -#endif -#ifdef MULTI_USER - cpu_user = *list->user; -#endif -#ifdef MULTI_CACHE - cpu_cache = *list->cache; +#ifdef CONFIG_DEBUG_LL + printascii(buf); #endif + printk("%s", buf); +} - printk("CPU: %s [%08x] revision %d (ARMv%s), cr=%08lx\n", - cpu_name, processor_id, (int)processor_id & 15, - proc_arch[cpu_architecture()], cr_alignment); +static void __init cpuid_init_hwcaps(void) +{ + unsigned int divide_instrs, vmsa; - sprintf(init_utsname()->machine, "%s%c", list->arch_name, ENDIANNESS); - sprintf(elf_platform, "%s%c", list->elf_name, ENDIANNESS); - elf_hwcap = list->elf_hwcap; -#ifndef CONFIG_ARM_THUMB - elf_hwcap &= ~HWCAP_THUMB; -#endif + if (cpu_architecture() < CPU_ARCH_ARMv7) + return; - cpu_proc_init(); + divide_instrs = (read_cpuid_ext(CPUID_EXT_ISAR0) & 0x0f000000) >> 24; + + switch (divide_instrs) { + case 2: + elf_hwcap |= HWCAP_IDIVA; + case 1: + elf_hwcap |= HWCAP_IDIVT; + } + + /* LPAE implies atomic ldrd/strd instructions */ + vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0; + if (vmsa >= 5) + elf_hwcap |= HWCAP_LPAE; +} + +static void __init feat_v6_fixup(void) +{ + int id = read_cpuid_id(); + + if ((id & 0xff0f0000) != 0x41070000) + return; + + /* + * HWCAP_TLS is available only on 1136 r1p0 and later, + * see also kuser_get_tls_init. + */ + if ((((id >> 4) & 0xfff) == 0xb36) && (((id >> 20) & 3) == 0)) + elf_hwcap &= ~HWCAP_TLS; } /* * cpu_init - initialise one CPU. * - * cpu_init dumps the cache information, initialises SMP specific - * information, and sets up the per-CPU stacks. + * cpu_init sets up the per-CPU stacks. */ -void cpu_init(void) +void notrace cpu_init(void) { +#ifndef CONFIG_CPU_V7M unsigned int cpu = smp_processor_id(); struct stack *stk = &stacks[cpu]; if (cpu >= NR_CPUS) { - printk(KERN_CRIT "CPU%u: bad primary CPU number\n", cpu); + pr_crit("CPU%u: bad primary CPU number\n", cpu); BUG(); } - if (system_state == SYSTEM_BOOTING) - dump_cpu_info(cpu); + /* + * This only works on resume and secondary cores. For booting on the + * boot cpu, smp_prepare_boot_cpu is called after percpu area setup. + */ + set_my_cpu_offset(per_cpu_offset(cpu)); + + cpu_proc_init(); + + /* + * Define the placement constraint for the inline asm directive below. + * In Thumb-2, msr with an immediate value is not allowed. + */ +#ifdef CONFIG_THUMB2_KERNEL +#define PLC "r" +#else +#define PLC "I" +#endif /* * setup stacks for re-entrant exception handlers */ __asm__ ( "msr cpsr_c, %1\n\t" - "add sp, %0, %2\n\t" + "add r14, %0, %2\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %3\n\t" - "add sp, %0, %4\n\t" + "add r14, %0, %4\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %5\n\t" - "add sp, %0, %6\n\t" + "add r14, %0, %6\n\t" + "mov sp, r14\n\t" "msr cpsr_c, %7" : : "r" (stk), - "I" (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), "I" (offsetof(struct stack, irq[0])), - "I" (PSR_F_BIT | PSR_I_BIT | ABT_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | ABT_MODE), "I" (offsetof(struct stack, abt[0])), - "I" (PSR_F_BIT | PSR_I_BIT | UND_MODE), + PLC (PSR_F_BIT | PSR_I_BIT | UND_MODE), "I" (offsetof(struct stack, und[0])), - "I" (PSR_F_BIT | PSR_I_BIT | SVC_MODE) + PLC (PSR_F_BIT | PSR_I_BIT | SVC_MODE) : "r14"); +#endif } -static struct machine_desc * __init setup_machine(unsigned int nr) +u32 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = MPIDR_INVALID }; + +void __init smp_setup_processor_id(void) { - struct machine_desc *list; + int i; + u32 mpidr = is_smp() ? read_cpuid_mpidr() & MPIDR_HWID_BITMASK : 0; + u32 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); + + cpu_logical_map(0) = cpu; + for (i = 1; i < nr_cpu_ids; ++i) + cpu_logical_map(i) = i == cpu ? 0 : i; + + /* + * clear __my_cpu_offset on boot CPU to avoid hang caused by + * using percpu variable early, for example, lockdep will + * access percpu variable inside lock_release + */ + set_my_cpu_offset(0); + + pr_info("Booting Linux on physical CPU 0x%x\n", mpidr); +} +struct mpidr_hash mpidr_hash; +#ifdef CONFIG_SMP +/** + * smp_build_mpidr_hash - Pre-compute shifts required at each affinity + * level in order to build a linear index from an + * MPIDR value. Resulting algorithm is a collision + * free hash carried out through shifting and ORing + */ +static void __init smp_build_mpidr_hash(void) +{ + u32 i, affinity; + u32 fs[3], bits[3], ls, mask = 0; + /* + * Pre-scan the list of MPIDRS and filter out bits that do + * not contribute to affinity levels, ie they never toggle. + */ + for_each_possible_cpu(i) + mask |= (cpu_logical_map(i) ^ cpu_logical_map(0)); + pr_debug("mask of set bits 0x%x\n", mask); /* - * locate machine in the list of supported machines. + * Find and stash the last and first bit set at all affinity levels to + * check how many bits are required to represent them. */ - list = lookup_machine_type(nr); + for (i = 0; i < 3; i++) { + affinity = MPIDR_AFFINITY_LEVEL(mask, i); + /* + * Find the MSB bit and LSB bits position + * to determine how many bits are required + * to express the affinity level. + */ + ls = fls(affinity); + fs[i] = affinity ? ffs(affinity) - 1 : 0; + bits[i] = ls - fs[i]; + } + /* + * An index can be created from the MPIDR by isolating the + * significant bits at each affinity level and by shifting + * them in order to compress the 24 bits values space to a + * compressed set of values. This is equivalent to hashing + * the MPIDR through shifting and ORing. It is a collision free + * hash though not minimal since some levels might contain a number + * of CPUs that is not an exact power of 2 and their bit + * representation might contain holes, eg MPIDR[7:0] = {0x2, 0x80}. + */ + mpidr_hash.shift_aff[0] = fs[0]; + mpidr_hash.shift_aff[1] = MPIDR_LEVEL_BITS + fs[1] - bits[0]; + mpidr_hash.shift_aff[2] = 2*MPIDR_LEVEL_BITS + fs[2] - + (bits[1] + bits[0]); + mpidr_hash.mask = mask; + mpidr_hash.bits = bits[2] + bits[1] + bits[0]; + pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] mask[0x%x] bits[%u]\n", + mpidr_hash.shift_aff[0], + mpidr_hash.shift_aff[1], + mpidr_hash.shift_aff[2], + mpidr_hash.mask, + mpidr_hash.bits); + /* + * 4x is an arbitrary value used to warn on a hash table much bigger + * than expected on most systems. + */ + if (mpidr_hash_size() > 4 * num_possible_cpus()) + pr_warn("Large number of MPIDR hash buckets detected\n"); + sync_cache_w(&mpidr_hash); +} +#endif + +static void __init setup_processor(void) +{ + struct proc_info_list *list; + + /* + * locate processor in the list of supported processor + * types. The linker builds this table for us from the + * entries in arch/arm/mm/proc-*.S + */ + list = lookup_processor_type(read_cpuid_id()); if (!list) { - printk("Machine configuration botched (nr %d), unable " - "to continue.\n", nr); + pr_err("CPU configuration botched (ID %08x), unable to continue.\n", + read_cpuid_id()); while (1); } - printk("Machine: %s\n", list->name); + cpu_name = list->cpu_name; + __cpu_architecture = __get_cpu_architecture(); - return list; +#ifdef MULTI_CPU + processor = *list->proc; +#endif +#ifdef MULTI_TLB + cpu_tlb = *list->tlb; +#endif +#ifdef MULTI_USER + cpu_user = *list->user; +#endif +#ifdef MULTI_CACHE + cpu_cache = *list->cache; +#endif + + pr_info("CPU: %s [%08x] revision %d (ARMv%s), cr=%08lx\n", + cpu_name, read_cpuid_id(), read_cpuid_id() & 15, + proc_arch[cpu_architecture()], get_cr()); + + snprintf(init_utsname()->machine, __NEW_UTS_LEN + 1, "%s%c", + list->arch_name, ENDIANNESS); + snprintf(elf_platform, ELF_PLATFORM_SIZE, "%s%c", + list->elf_name, ENDIANNESS); + elf_hwcap = list->elf_hwcap; + + cpuid_init_hwcaps(); + +#ifndef CONFIG_ARM_THUMB + elf_hwcap &= ~(HWCAP_THUMB | HWCAP_IDIVT); +#endif +#ifdef CONFIG_MMU + init_default_cache_policy(list->__cpu_mm_mmu_flags); +#endif + erratum_a15_798181_init(); + + feat_v6_fixup(); + + cacheid_init(); + cpu_init(); } -static void __init early_initrd(char **p) +void __init dump_machine_table(void) { - unsigned long start, size; + const struct machine_desc *p; - start = memparse(*p, p); - if (**p == ',') { - size = memparse((*p) + 1, p); + early_print("Available machine support:\n\nID (hex)\tNAME\n"); + for_each_machine_desc(p) + early_print("%08x\t%s\n", p->nr, p->name); - phys_initrd_start = start; - phys_initrd_size = size; - } + early_print("\nPlease check your kernel config and/or bootloader.\n"); + + while (true) + /* can't use cpu_relax() here as it may require MMU setup */; } -__early_param("initrd=", early_initrd); -static void __init arm_add_memory(unsigned long start, unsigned long size) +int __init arm_add_memory(u64 start, u64 size) { - struct membank *bank; + u64 aligned_start; /* * Ensure that start/size are aligned to a page boundary. * Size is appropriately rounded down, start is rounded up. */ size -= start & ~PAGE_MASK; + aligned_start = PAGE_ALIGN(start); + +#ifndef CONFIG_ARCH_PHYS_ADDR_T_64BIT + if (aligned_start > ULONG_MAX) { + pr_crit("Ignoring memory at 0x%08llx outside 32-bit physical address space\n", + (long long)start); + return -EINVAL; + } + + if (aligned_start + size > ULONG_MAX) { + pr_crit("Truncating memory at 0x%08llx to fit in 32-bit physical address space\n", + (long long)start); + /* + * To ensure bank->start + bank->size is representable in + * 32 bits, we use ULONG_MAX as the upper limit rather than 4GB. + * This means we lose a page after masking. + */ + size = ULONG_MAX - aligned_start; + } +#endif + + if (aligned_start < PHYS_OFFSET) { + if (aligned_start + size <= PHYS_OFFSET) { + pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n", + aligned_start, aligned_start + size); + return -EINVAL; + } + + pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n", + aligned_start, (u64)PHYS_OFFSET); + + size -= PHYS_OFFSET - aligned_start; + aligned_start = PHYS_OFFSET; + } + + start = aligned_start; + size = size & ~(phys_addr_t)(PAGE_SIZE - 1); - bank = &meminfo.bank[meminfo.nr_banks++]; + /* + * Check whether this memory region has non-zero size or + * invalid node number. + */ + if (size == 0) + return -EINVAL; - bank->start = PAGE_ALIGN(start); - bank->size = size & PAGE_MASK; - bank->node = PHYS_TO_NID(start); + memblock_add(start, size); + return 0; } /* * Pick out the memory size. We look for mem=size@start, * where start and size are "size[KkMm]" */ -static void __init early_mem(char **p) + +static int __init early_mem(char *p) { static int usermem __initdata = 0; - unsigned long size, start; + u64 size; + u64 start; + char *endp; /* * If the user specifies memory size, we @@ -487,96 +706,36 @@ static void __init early_mem(char **p) */ if (usermem == 0) { usermem = 1; - meminfo.nr_banks = 0; + memblock_remove(memblock_start_of_DRAM(), + memblock_end_of_DRAM() - memblock_start_of_DRAM()); } start = PHYS_OFFSET; - size = memparse(*p, p); - if (**p == '@') - start = memparse(*p + 1, p); + size = memparse(p, &endp); + if (*endp == '@') + start = memparse(endp + 1, NULL); arm_add_memory(start, size); -} -__early_param("mem=", early_mem); - -/* - * Initial parsing of the command line. - */ -static void __init parse_cmdline(char **cmdline_p, char *from) -{ - char c = ' ', *to = command_line; - int len = 0; - - for (;;) { - if (c == ' ') { - extern struct early_params __early_begin, __early_end; - struct early_params *p; - - for (p = &__early_begin; p < &__early_end; p++) { - int len = strlen(p->arg); - - if (memcmp(from, p->arg, len) == 0) { - if (to != command_line) - to -= 1; - from += len; - p->fn(&from); - - while (*from != ' ' && *from != '\0') - from++; - break; - } - } - } - c = *from++; - if (!c) - break; - if (COMMAND_LINE_SIZE <= ++len) - break; - *to++ = c; - } - *to = '\0'; - *cmdline_p = command_line; -} - -static void __init -setup_ramdisk(int doload, int prompt, int image_start, unsigned int rd_sz) -{ -#ifdef CONFIG_BLK_DEV_RAM - extern int rd_size, rd_image_start, rd_prompt, rd_doload; - rd_image_start = image_start; - rd_prompt = prompt; - rd_doload = doload; - - if (rd_sz) - rd_size = rd_sz; -#endif + return 0; } +early_param("mem", early_mem); -static void __init -request_standard_resources(struct meminfo *mi, struct machine_desc *mdesc) +static void __init request_standard_resources(const struct machine_desc *mdesc) { + struct memblock_region *region; struct resource *res; - int i; - - kernel_code.start = virt_to_phys(&_text); - kernel_code.end = virt_to_phys(&_etext - 1); - kernel_data.start = virt_to_phys(&__data_start); - kernel_data.end = virt_to_phys(&_end - 1); - for (i = 0; i < mi->nr_banks; i++) { - unsigned long virt_start, virt_end; + kernel_code.start = virt_to_phys(_text); + kernel_code.end = virt_to_phys(_etext - 1); + kernel_data.start = virt_to_phys(_sdata); + kernel_data.end = virt_to_phys(_end - 1); - if (mi->bank[i].size == 0) - continue; - - virt_start = __phys_to_virt(mi->bank[i].start); - virt_end = virt_start + mi->bank[i].size - 1; - - res = alloc_bootmem_low(sizeof(*res)); + for_each_memblock(memory, region) { + res = memblock_virt_alloc(sizeof(*res), 0); res->name = "System RAM"; - res->start = __virt_to_phys(virt_start); - res->end = __virt_to_phys(virt_end); + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; request_resource(&iomem_resource, res); @@ -607,42 +766,6 @@ request_standard_resources(struct meminfo *mi, struct machine_desc *mdesc) request_resource(&ioport_resource, &lp2); } -/* - * Tag parsing. - * - * This is the new way of passing data to the kernel at boot time. Rather - * than passing a fixed inflexible structure to the kernel, we pass a list - * of variable-sized tags to the kernel. The first tag must be a ATAG_CORE - * tag for the list to be recognised (to distinguish the tagged list from - * a param_struct). The list is terminated with a zero-length tag (this tag - * is not parsed in any way). - */ -static int __init parse_tag_core(const struct tag *tag) -{ - if (tag->hdr.size > 2) { - if ((tag->u.core.flags & 1) == 0) - root_mountflags &= ~MS_RDONLY; - ROOT_DEV = old_decode_dev(tag->u.core.rootdev); - } - return 0; -} - -__tagtable(ATAG_CORE, parse_tag_core); - -static int __init parse_tag_mem32(const struct tag *tag) -{ - if (meminfo.nr_banks >= NR_BANKS) { - printk(KERN_WARNING - "Ignoring memory bank 0x%08x size %dKB\n", - tag->u.mem.start, tag->u.mem.size / 1024); - return -EINVAL; - } - arm_add_memory(tag->u.mem.start, tag->u.mem.size); - return 0; -} - -__tagtable(ATAG_MEM, parse_tag_mem32); - #if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) struct screen_info screen_info = { .orig_video_lines = 30, @@ -652,199 +775,161 @@ struct screen_info screen_info = { .orig_video_isVGA = 1, .orig_video_points = 8 }; - -static int __init parse_tag_videotext(const struct tag *tag) -{ - screen_info.orig_x = tag->u.videotext.x; - screen_info.orig_y = tag->u.videotext.y; - screen_info.orig_video_page = tag->u.videotext.video_page; - screen_info.orig_video_mode = tag->u.videotext.video_mode; - screen_info.orig_video_cols = tag->u.videotext.video_cols; - screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; - screen_info.orig_video_lines = tag->u.videotext.video_lines; - screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; - screen_info.orig_video_points = tag->u.videotext.video_points; - return 0; -} - -__tagtable(ATAG_VIDEOTEXT, parse_tag_videotext); #endif -static int __init parse_tag_ramdisk(const struct tag *tag) -{ - setup_ramdisk((tag->u.ramdisk.flags & 1) == 0, - (tag->u.ramdisk.flags & 2) == 0, - tag->u.ramdisk.start, tag->u.ramdisk.size); - return 0; -} - -__tagtable(ATAG_RAMDISK, parse_tag_ramdisk); - -static int __init parse_tag_initrd(const struct tag *tag) -{ - printk(KERN_WARNING "ATAG_INITRD is deprecated; " - "please update your bootloader.\n"); - phys_initrd_start = __virt_to_phys(tag->u.initrd.start); - phys_initrd_size = tag->u.initrd.size; - return 0; -} - -__tagtable(ATAG_INITRD, parse_tag_initrd); - -static int __init parse_tag_initrd2(const struct tag *tag) +static int __init customize_machine(void) { - phys_initrd_start = tag->u.initrd.start; - phys_initrd_size = tag->u.initrd.size; + /* + * customizes platform devices, or adds new ones + * On DT based machines, we fall back to populating the + * machine from the device tree, if no callback is provided, + * otherwise we would always need an init_machine callback. + */ + if (machine_desc->init_machine) + machine_desc->init_machine(); +#ifdef CONFIG_OF + else + of_platform_populate(NULL, of_default_bus_match_table, + NULL, NULL); +#endif return 0; } +arch_initcall(customize_machine); -__tagtable(ATAG_INITRD2, parse_tag_initrd2); - -static int __init parse_tag_serialnr(const struct tag *tag) +static int __init init_machine_late(void) { - system_serial_low = tag->u.serialnr.low; - system_serial_high = tag->u.serialnr.high; + if (machine_desc->init_late) + machine_desc->init_late(); return 0; } +late_initcall(init_machine_late); -__tagtable(ATAG_SERIAL, parse_tag_serialnr); - -static int __init parse_tag_revision(const struct tag *tag) +#ifdef CONFIG_KEXEC +static inline unsigned long long get_total_mem(void) { - system_rev = tag->u.revision.rev; - return 0; -} + unsigned long total; -__tagtable(ATAG_REVISION, parse_tag_revision); - -static int __init parse_tag_cmdline(const struct tag *tag) -{ - strlcpy(default_command_line, tag->u.cmdline.cmdline, COMMAND_LINE_SIZE); - return 0; + total = max_low_pfn - min_low_pfn; + return total << PAGE_SHIFT; } -__tagtable(ATAG_CMDLINE, parse_tag_cmdline); - -/* - * Scan the tag table for this tag, and call its parse function. - * The tag table is built by the linker from all the __tagtable - * declarations. +/** + * reserve_crashkernel() - reserves memory are for crash kernel + * + * This function reserves memory area given in "crashkernel=" kernel command + * line parameter. The memory reserved is used by a dump capture kernel when + * primary kernel is crashing. */ -static int __init parse_tag(const struct tag *tag) +static void __init reserve_crashkernel(void) { - extern struct tagtable __tagtable_begin, __tagtable_end; - struct tagtable *t; + unsigned long long crash_size, crash_base; + unsigned long long total_mem; + int ret; + + total_mem = get_total_mem(); + ret = parse_crashkernel(boot_command_line, total_mem, + &crash_size, &crash_base); + if (ret) + return; + + ret = memblock_reserve(crash_base, crash_size); + if (ret < 0) { + pr_warn("crashkernel reservation failed - memory is in use (0x%lx)\n", + (unsigned long)crash_base); + return; + } - for (t = &__tagtable_begin; t < &__tagtable_end; t++) - if (tag->hdr.tag == t->tag) { - t->parse(tag); - break; - } + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n", + (unsigned long)(crash_size >> 20), + (unsigned long)(crash_base >> 20), + (unsigned long)(total_mem >> 20)); - return t < &__tagtable_end; + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; + insert_resource(&iomem_resource, &crashk_res); } +#else +static inline void reserve_crashkernel(void) {} +#endif /* CONFIG_KEXEC */ -/* - * Parse all tags in the list, checking both the global and architecture - * specific tag tables. - */ -static void __init parse_tags(const struct tag *t) -{ - for (; t->hdr.size; t = tag_next(t)) - if (!parse_tag(t)) - printk(KERN_WARNING - "Ignoring unrecognised tag 0x%08x\n", - t->hdr.tag); -} - -/* - * This holds our defaults. - */ -static struct init_tags { - struct tag_header hdr1; - struct tag_core core; - struct tag_header hdr2; - struct tag_mem32 mem; - struct tag_header hdr3; -} init_tags __initdata = { - { tag_size(tag_core), ATAG_CORE }, - { 1, PAGE_SIZE, 0xff }, - { tag_size(tag_mem32), ATAG_MEM }, - { MEM_SIZE, PHYS_OFFSET }, - { 0, ATAG_NONE } -}; - -static void (*init_machine)(void) __initdata; - -static int __init customize_machine(void) +void __init hyp_mode_check(void) { - /* customizes platform devices, or adds new ones */ - if (init_machine) - init_machine(); - return 0; +#ifdef CONFIG_ARM_VIRT_EXT + sync_boot_mode(); + + if (is_hyp_mode_available()) { + pr_info("CPU: All CPU(s) started in HYP mode.\n"); + pr_info("CPU: Virtualization extensions available.\n"); + } else if (is_hyp_mode_mismatched()) { + pr_warn("CPU: WARNING: CPU(s) started in wrong/inconsistent modes (primary CPU mode 0x%x)\n", + __boot_cpu_mode & MODE_MASK); + pr_warn("CPU: This may indicate a broken bootloader or firmware.\n"); + } else + pr_info("CPU: All CPU(s) started in SVC mode.\n"); +#endif } -arch_initcall(customize_machine); void __init setup_arch(char **cmdline_p) { - struct tag *tags = (struct tag *)&init_tags; - struct machine_desc *mdesc; - char *from = default_command_line; + const struct machine_desc *mdesc; setup_processor(); - mdesc = setup_machine(machine_arch_type); + mdesc = setup_machine_fdt(__atags_pointer); + if (!mdesc) + mdesc = setup_machine_tags(__atags_pointer, __machine_arch_type); + machine_desc = mdesc; machine_name = mdesc->name; - if (mdesc->soft_reboot) - reboot_setup("s"); + if (mdesc->reboot_mode != REBOOT_HARD) + reboot_mode = mdesc->reboot_mode; - if (__atags_pointer) - tags = phys_to_virt(__atags_pointer); - else if (mdesc->boot_params) - tags = phys_to_virt(mdesc->boot_params); + init_mm.start_code = (unsigned long) _text; + init_mm.end_code = (unsigned long) _etext; + init_mm.end_data = (unsigned long) _edata; + init_mm.brk = (unsigned long) _end; - /* - * If we have the old style parameters, convert them to - * a tag list. - */ - if (tags->hdr.tag != ATAG_CORE) - convert_to_tag_list(tags); - if (tags->hdr.tag != ATAG_CORE) - tags = (struct tag *)&init_tags; - - if (mdesc->fixup) - mdesc->fixup(mdesc, tags, &from, &meminfo); - - if (tags->hdr.tag == ATAG_CORE) { - if (meminfo.nr_banks != 0) - squash_mem_tags(tags); - save_atags(tags); - parse_tags(tags); - } + /* populate cmd_line too for later use, preserving boot_command_line */ + strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); + *cmdline_p = cmd_line; + + parse_early_param(); + + early_paging_init(mdesc, lookup_processor_type(read_cpuid_id())); + setup_dma_zone(mdesc); + sanity_check_meminfo(); + arm_memblock_init(mdesc); + + paging_init(mdesc); + request_standard_resources(mdesc); - init_mm.start_code = (unsigned long) &_text; - init_mm.end_code = (unsigned long) &_etext; - init_mm.end_data = (unsigned long) &_edata; - init_mm.brk = (unsigned long) &_end; + if (mdesc->restart) + arm_pm_restart = mdesc->restart; - memcpy(boot_command_line, from, COMMAND_LINE_SIZE); - boot_command_line[COMMAND_LINE_SIZE-1] = '\0'; - parse_cmdline(cmdline_p, from); - paging_init(&meminfo, mdesc); - request_standard_resources(&meminfo, mdesc); + unflatten_device_tree(); + arm_dt_init_cpu_maps(); + psci_init(); #ifdef CONFIG_SMP - smp_init_cpus(); + if (is_smp()) { + if (!mdesc->smp_init || !mdesc->smp_init()) { + if (psci_smp_available()) + smp_set_ops(&psci_smp_ops); + else if (mdesc->smp) + smp_set_ops(mdesc->smp); + } + smp_init_cpus(); + smp_build_mpidr_hash(); + } #endif - cpu_init(); + if (!is_smp()) + hyp_mode_check(); - /* - * Set up various architecture-specific pointers - */ - init_arch_irq = mdesc->init_irq; - system_timer = mdesc->timer; - init_machine = mdesc->init_machine; + reserve_crashkernel(); + +#ifdef CONFIG_MULTI_IRQ_HANDLER + handle_arch_irq = mdesc->handle_irq; +#endif #ifdef CONFIG_VT #if defined(CONFIG_VGA_CONSOLE) @@ -853,6 +938,9 @@ void __init setup_arch(char **cmdline_p) conswitchp = &dummy_con; #endif #endif + + if (mdesc->init_early) + mdesc->init_early(); } @@ -868,9 +956,21 @@ static int __init topology_init(void) return 0; } - subsys_initcall(topology_init); +#ifdef CONFIG_HAVE_PROC_CPU +static int __init proc_cpu_init(void) +{ + struct proc_dir_entry *res; + + res = proc_mkdir("cpu", NULL); + if (!res) + return -ENOMEM; + return 0; +} +fs_initcall(proc_cpu_init); +#endif + static const char *hwcap_str[] = { "swp", "half", @@ -883,33 +983,34 @@ static const char *hwcap_str[] = { "java", "iwmmxt", "crunch", + "thumbee", + "neon", + "vfpv3", + "vfpv3d16", + "tls", + "vfpv4", + "idiva", + "idivt", + "vfpd32", + "lpae", + "evtstrm", NULL }; -static void -c_show_cache(struct seq_file *m, const char *type, unsigned int cache) -{ - unsigned int mult = 2 + (CACHE_M(cache) ? 1 : 0); - - seq_printf(m, "%s size\t\t: %d\n" - "%s assoc\t\t: %d\n" - "%s line length\t: %d\n" - "%s sets\t\t: %d\n", - type, mult << (8 + CACHE_SIZE(cache)), - type, (mult << CACHE_ASSOC(cache)) >> 1, - type, 8 << CACHE_LINE(cache), - type, 1 << (6 + CACHE_SIZE(cache) - CACHE_ASSOC(cache) - - CACHE_LINE(cache))); -} +static const char *hwcap2_str[] = { + "aes", + "pmull", + "sha1", + "sha2", + "crc32", + NULL +}; static int c_show(struct seq_file *m, void *v) { - int i; + int i, j; + u32 cpuid; - seq_printf(m, "Processor\t: %s rev %d (%s)\n", - cpu_name, (int)processor_id & 15, elf_platform); - -#if defined(CONFIG_SMP) for_each_online_cpu(i) { /* * glibc reads /proc/cpuinfo to determine the number of @@ -917,67 +1018,44 @@ static int c_show(struct seq_file *m, void *v) * "processor". Give glibc what it expects. */ seq_printf(m, "processor\t: %d\n", i); - seq_printf(m, "BogoMIPS\t: %lu.%02lu\n\n", - per_cpu(cpu_data, i).loops_per_jiffy / (500000UL/HZ), - (per_cpu(cpu_data, i).loops_per_jiffy / (5000UL/HZ)) % 100); - } -#else /* CONFIG_SMP */ - seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", - loops_per_jiffy / (500000/HZ), - (loops_per_jiffy / (5000/HZ)) % 100); -#endif + cpuid = is_smp() ? per_cpu(cpu_data, i).cpuid : read_cpuid_id(); + seq_printf(m, "model name\t: %s rev %d (%s)\n", + cpu_name, cpuid & 15, elf_platform); - /* dump out the processor features */ - seq_puts(m, "Features\t: "); + /* dump out the processor features */ + seq_puts(m, "Features\t: "); - for (i = 0; hwcap_str[i]; i++) - if (elf_hwcap & (1 << i)) - seq_printf(m, "%s ", hwcap_str[i]); + for (j = 0; hwcap_str[j]; j++) + if (elf_hwcap & (1 << j)) + seq_printf(m, "%s ", hwcap_str[j]); - seq_printf(m, "\nCPU implementer\t: 0x%02x\n", processor_id >> 24); - seq_printf(m, "CPU architecture: %s\n", proc_arch[cpu_architecture()]); + for (j = 0; hwcap2_str[j]; j++) + if (elf_hwcap2 & (1 << j)) + seq_printf(m, "%s ", hwcap2_str[j]); - if ((processor_id & 0x0008f000) == 0x00000000) { - /* pre-ARM7 */ - seq_printf(m, "CPU part\t: %07x\n", processor_id >> 4); - } else { - if ((processor_id & 0x0008f000) == 0x00007000) { - /* ARM7 */ - seq_printf(m, "CPU variant\t: 0x%02x\n", - (processor_id >> 16) & 127); - } else { - /* post-ARM7 */ - seq_printf(m, "CPU variant\t: 0x%x\n", - (processor_id >> 20) & 15); - } - seq_printf(m, "CPU part\t: 0x%03x\n", - (processor_id >> 4) & 0xfff); - } - seq_printf(m, "CPU revision\t: %d\n", processor_id & 15); + seq_printf(m, "\nCPU implementer\t: 0x%02x\n", cpuid >> 24); + seq_printf(m, "CPU architecture: %s\n", + proc_arch[cpu_architecture()]); - { - unsigned int cache_info = read_cpuid(CPUID_CACHETYPE); - if (cache_info != processor_id) { - seq_printf(m, "Cache type\t: %s\n" - "Cache clean\t: %s\n" - "Cache lockdown\t: %s\n" - "Cache format\t: %s\n", - cache_types[CACHE_TYPE(cache_info)], - cache_clean[CACHE_TYPE(cache_info)], - cache_lockdown[CACHE_TYPE(cache_info)], - CACHE_S(cache_info) ? "Harvard" : "Unified"); - - if (CACHE_S(cache_info)) { - c_show_cache(m, "I", CACHE_ISIZE(cache_info)); - c_show_cache(m, "D", CACHE_DSIZE(cache_info)); + if ((cpuid & 0x0008f000) == 0x00000000) { + /* pre-ARM7 */ + seq_printf(m, "CPU part\t: %07x\n", cpuid >> 4); + } else { + if ((cpuid & 0x0008f000) == 0x00007000) { + /* ARM7 */ + seq_printf(m, "CPU variant\t: 0x%02x\n", + (cpuid >> 16) & 127); } else { - c_show_cache(m, "Cache", CACHE_ISIZE(cache_info)); + /* post-ARM7 */ + seq_printf(m, "CPU variant\t: 0x%x\n", + (cpuid >> 20) & 15); } + seq_printf(m, "CPU part\t: 0x%03x\n", + (cpuid >> 4) & 0xfff); } + seq_printf(m, "CPU revision\t: %d\n\n", cpuid & 15); } - seq_puts(m, "\n"); - seq_printf(m, "Hardware\t: %s\n", machine_name); seq_printf(m, "Revision\t: %04x\n", system_rev); seq_printf(m, "Serial\t\t: %08x%08x\n", |
