diff options
Diffstat (limited to 'arch/mips/kernel')
117 files changed, 12933 insertions, 9924 deletions
diff --git a/arch/mips/kernel/8250-platform.c b/arch/mips/kernel/8250-platform.c index cbf3fe20ad1..5c6b2ab1f56 100644 --- a/arch/mips/kernel/8250-platform.c +++ b/arch/mips/kernel/8250-platform.c @@ -5,7 +5,6 @@ * * Copyright (C) 2007 Ralf Baechle (ralf@linux-mips.org) */ -#include <linux/module.h> #include <linux/init.h> #include <linux/serial_8250.h> diff --git a/arch/mips/kernel/Makefile b/arch/mips/kernel/Makefile index 22b2e0e3861..008a2fed058 100644 --- a/arch/mips/kernel/Makefile +++ b/arch/mips/kernel/Makefile @@ -2,75 +2,69 @@ # Makefile for the Linux/MIPS kernel. # -extra-y := head.o init_task.o vmlinux.lds +extra-y := head.o vmlinux.lds -obj-y += cpu-probe.o branch.o entry.o genex.o irq.o process.o \ - ptrace.o reset.o setup.o signal.o syscall.o \ +obj-y += cpu-probe.o branch.o entry.o genex.o idle.o irq.o process.o \ + prom.o ptrace.o reset.o setup.o signal.o syscall.o \ time.o topology.o traps.o unaligned.o watch.o vdso.o ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_ftrace.o = -pg CFLAGS_REMOVE_early_printk.o = -pg +CFLAGS_REMOVE_perf_event.o = -pg +CFLAGS_REMOVE_perf_event_mipsxx.o = -pg endif obj-$(CONFIG_CEVT_BCM1480) += cevt-bcm1480.o -obj-$(CONFIG_CEVT_R4K_LIB) += cevt-r4k.o -obj-$(CONFIG_MIPS_MT_SMTC) += cevt-smtc.o +obj-$(CONFIG_CEVT_R4K) += cevt-r4k.o obj-$(CONFIG_CEVT_DS1287) += cevt-ds1287.o +obj-$(CONFIG_CEVT_GIC) += cevt-gic.o obj-$(CONFIG_CEVT_GT641XX) += cevt-gt641xx.o obj-$(CONFIG_CEVT_SB1250) += cevt-sb1250.o obj-$(CONFIG_CEVT_TXX9) += cevt-txx9.o obj-$(CONFIG_CSRC_BCM1480) += csrc-bcm1480.o +obj-$(CONFIG_CSRC_GIC) += csrc-gic.o obj-$(CONFIG_CSRC_IOASIC) += csrc-ioasic.o -obj-$(CONFIG_CSRC_POWERTV) += csrc-powertv.o -obj-$(CONFIG_CSRC_R4K_LIB) += csrc-r4k.o +obj-$(CONFIG_CSRC_R4K) += csrc-r4k.o obj-$(CONFIG_CSRC_SB1250) += csrc-sb1250.o obj-$(CONFIG_SYNC_R4K) += sync-r4k.o +obj-$(CONFIG_DEBUG_FS) += segment.o obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_MODULES) += mips_ksyms.o module.o +obj-$(CONFIG_MODULES_USE_ELF_RELA) += module-rela.o +obj-$(CONFIG_FTRACE_SYSCALLS) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o ftrace.o -obj-$(CONFIG_CPU_LOONGSON2) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_MIPS32) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_MIPS64) += r4k_fpu.o r4k_switch.o +obj-$(CONFIG_CPU_R4K_FPU) += r4k_fpu.o r4k_switch.o obj-$(CONFIG_CPU_R3000) += r2300_fpu.o r2300_switch.o -obj-$(CONFIG_CPU_R4300) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R4X00) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R5000) += r4k_fpu.o r4k_switch.o obj-$(CONFIG_CPU_R6000) += r6000_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R5432) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R5500) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R8000) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_RM7000) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_RM9000) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_NEVADA) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_R10000) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_SB1) += r4k_fpu.o r4k_switch.o obj-$(CONFIG_CPU_TX39XX) += r2300_fpu.o r2300_switch.o -obj-$(CONFIG_CPU_TX49XX) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_VR41XX) += r4k_fpu.o r4k_switch.o -obj-$(CONFIG_CPU_CAVIUM_OCTEON) += octeon_switch.o +obj-$(CONFIG_CPU_CAVIUM_OCTEON) += r4k_fpu.o octeon_switch.o obj-$(CONFIG_SMP) += smp.o obj-$(CONFIG_SMP_UP) += smp-up.o +obj-$(CONFIG_CPU_BMIPS) += smp-bmips.o bmips_vec.o obj-$(CONFIG_MIPS_MT) += mips-mt.o obj-$(CONFIG_MIPS_MT_FPAFF) += mips-mt-fpaff.o -obj-$(CONFIG_MIPS_MT_SMTC) += smtc.o smtc-asm.o smtc-proc.o obj-$(CONFIG_MIPS_MT_SMP) += smp-mt.o obj-$(CONFIG_MIPS_CMP) += smp-cmp.o +obj-$(CONFIG_MIPS_CPS) += smp-cps.o cps-vec.o +obj-$(CONFIG_MIPS_GIC_IPI) += smp-gic.o obj-$(CONFIG_CPU_MIPSR2) += spram.o obj-$(CONFIG_MIPS_VPE_LOADER) += vpe.o -obj-$(CONFIG_MIPS_VPE_APSP_API) += rtlx.o -obj-$(CONFIG_MIPS_APSP_KSPD) += kspd.o +obj-$(CONFIG_MIPS_VPE_LOADER_CMP) += vpe-cmp.o +obj-$(CONFIG_MIPS_VPE_LOADER_MT) += vpe-mt.o +obj-$(CONFIG_MIPS_VPE_APSP_API) += rtlx.o +obj-$(CONFIG_MIPS_VPE_APSP_API_CMP) += rtlx-cmp.o +obj-$(CONFIG_MIPS_VPE_APSP_API_MT) += rtlx-mt.o obj-$(CONFIG_I8259) += i8259.o obj-$(CONFIG_IRQ_CPU) += irq_cpu.o obj-$(CONFIG_IRQ_CPU_RM7K) += irq-rm7000.o -obj-$(CONFIG_IRQ_CPU_RM9K) += irq-rm9000.o obj-$(CONFIG_MIPS_MSC) += irq-msc01.o obj-$(CONFIG_IRQ_TXX9) += irq_txx9.o obj-$(CONFIG_IRQ_GT641XX) += irq-gt641xx.o @@ -92,18 +86,44 @@ obj-$(CONFIG_I8253) += i8253.o obj-$(CONFIG_GPIO_TXX9) += gpio_txx9.o -obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o crash.o +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +obj-$(CONFIG_EARLY_PRINTK_8250) += early_printk_8250.o obj-$(CONFIG_SPINLOCK_TEST) += spinlock_test.o +obj-$(CONFIG_MIPS_MACHINE) += mips_machine.o -obj-$(CONFIG_OF) += prom.o - -CFLAGS_cpu-bugs64.o = $(shell if $(CC) $(KBUILD_CFLAGS) -Wa,-mdaddi -c -o /dev/null -xc /dev/null >/dev/null 2>&1; then echo "-DHAVE_AS_SET_DADDI"; fi) +CFLAGS_cpu-bugs64.o = $(shell if $(CC) $(KBUILD_CFLAGS) -Wa,-mdaddi -c -o /dev/null -x c /dev/null >/dev/null 2>&1; then echo "-DHAVE_AS_SET_DADDI"; fi) obj-$(CONFIG_HAVE_STD_PC_SERIAL_PORT) += 8250-platform.o -obj-$(CONFIG_MIPS_CPUFREQ) += cpufreq/ +obj-$(CONFIG_PERF_EVENTS) += perf_event.o +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event_mipsxx.o + +obj-$(CONFIG_JUMP_LABEL) += jump_label.o + +obj-$(CONFIG_MIPS_CM) += mips-cm.o +obj-$(CONFIG_MIPS_CPC) += mips-cpc.o + +obj-$(CONFIG_CPU_PM) += pm.o +obj-$(CONFIG_MIPS_CPS_PM) += pm-cps.o -obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +# +# DSP ASE supported for MIPS32 or MIPS64 Release 2 cores only. It is not +# safe to unconditionnaly use the assembler -mdsp / -mdspr2 switches +# here because the compiler may use DSP ASE instructions (such as lwx) in +# code paths where we cannot check that the CPU we are running on supports it. +# Proper abstraction using HAVE_AS_DSP and macros is done in +# arch/mips/include/asm/mipsregs.h. +# +ifeq ($(CONFIG_CPU_MIPSR2), y) +CFLAGS_DSP = -DHAVE_AS_DSP + +CFLAGS_signal.o = $(CFLAGS_DSP) +CFLAGS_signal32.o = $(CFLAGS_DSP) +CFLAGS_process.o = $(CFLAGS_DSP) +CFLAGS_branch.o = $(CFLAGS_DSP) +CFLAGS_ptrace.o = $(CFLAGS_DSP) +endif CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS) diff --git a/arch/mips/kernel/asm-offsets.c b/arch/mips/kernel/asm-offsets.c index 6b30fb2caa6..4bb5107511e 100644 --- a/arch/mips/kernel/asm-offsets.c +++ b/arch/mips/kernel/asm-offsets.c @@ -12,11 +12,14 @@ #include <linux/types.h> #include <linux/sched.h> #include <linux/mm.h> -#include <linux/interrupt.h> #include <linux/kbuild.h> #include <linux/suspend.h> +#include <asm/pm.h> #include <asm/ptrace.h> #include <asm/processor.h> +#include <asm/smp-cps.h> + +#include <linux/kvm_host.h> void output_ptreg_defines(void) { @@ -62,9 +65,6 @@ void output_ptreg_defines(void) OFFSET(PT_BVADDR, pt_regs, cp0_badvaddr); OFFSET(PT_STATUS, pt_regs, cp0_status); OFFSET(PT_CAUSE, pt_regs, cp0_cause); -#ifdef CONFIG_MIPS_MT_SMTC - OFFSET(PT_TCSTATUS, pt_regs, cp0_tcstatus); -#endif /* CONFIG_MIPS_MT_SMTC */ #ifdef CONFIG_CPU_CAVIUM_OCTEON OFFSET(PT_MPL, pt_regs, mpl); OFFSET(PT_MTP, pt_regs, mtp); @@ -81,6 +81,9 @@ void output_task_defines(void) OFFSET(TASK_FLAGS, task_struct, flags); OFFSET(TASK_MM, task_struct, mm); OFFSET(TASK_PID, task_struct, pid); +#if defined(CONFIG_CC_STACKPROTECTOR) + OFFSET(TASK_STACK_CANARY, task_struct, stack_canary); +#endif DEFINE(TASK_STRUCT_SIZE, sizeof(struct task_struct)); BLANK(); } @@ -126,10 +129,6 @@ void output_thread_defines(void) thread.cp0_baduaddr); OFFSET(THREAD_ECODE, task_struct, \ thread.error_code); - OFFSET(THREAD_TRAMP, task_struct, \ - thread.irix_trampoline); - OFFSET(THREAD_OLDCTX, task_struct, \ - thread.irix_oldctx); BLANK(); } @@ -168,6 +167,72 @@ void output_thread_fpu_defines(void) OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]); OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]); + /* the least significant 64 bits of each FP register */ + OFFSET(THREAD_FPR0_LS64, task_struct, + thread.fpu.fpr[0].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR1_LS64, task_struct, + thread.fpu.fpr[1].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR2_LS64, task_struct, + thread.fpu.fpr[2].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR3_LS64, task_struct, + thread.fpu.fpr[3].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR4_LS64, task_struct, + thread.fpu.fpr[4].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR5_LS64, task_struct, + thread.fpu.fpr[5].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR6_LS64, task_struct, + thread.fpu.fpr[6].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR7_LS64, task_struct, + thread.fpu.fpr[7].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR8_LS64, task_struct, + thread.fpu.fpr[8].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR9_LS64, task_struct, + thread.fpu.fpr[9].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR10_LS64, task_struct, + thread.fpu.fpr[10].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR11_LS64, task_struct, + thread.fpu.fpr[11].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR12_LS64, task_struct, + thread.fpu.fpr[12].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR13_LS64, task_struct, + thread.fpu.fpr[13].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR14_LS64, task_struct, + thread.fpu.fpr[14].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR15_LS64, task_struct, + thread.fpu.fpr[15].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR16_LS64, task_struct, + thread.fpu.fpr[16].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR17_LS64, task_struct, + thread.fpu.fpr[17].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR18_LS64, task_struct, + thread.fpu.fpr[18].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR19_LS64, task_struct, + thread.fpu.fpr[19].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR20_LS64, task_struct, + thread.fpu.fpr[20].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR21_LS64, task_struct, + thread.fpu.fpr[21].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR22_LS64, task_struct, + thread.fpu.fpr[22].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR23_LS64, task_struct, + thread.fpu.fpr[23].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR24_LS64, task_struct, + thread.fpu.fpr[24].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR25_LS64, task_struct, + thread.fpu.fpr[25].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR26_LS64, task_struct, + thread.fpu.fpr[26].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR27_LS64, task_struct, + thread.fpu.fpr[27].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR28_LS64, task_struct, + thread.fpu.fpr[28].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR29_LS64, task_struct, + thread.fpu.fpr[29].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR30_LS64, task_struct, + thread.fpu.fpr[30].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FPR31_LS64, task_struct, + thread.fpu.fpr[31].val64[FPR_IDX(64, 0)]); + OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31); BLANK(); } @@ -205,6 +270,9 @@ void output_mm_defines(void) DEFINE(_PTRS_PER_PMD, PTRS_PER_PMD); DEFINE(_PTRS_PER_PTE, PTRS_PER_PTE); BLANK(); + DEFINE(_PAGE_SHIFT, PAGE_SHIFT); + DEFINE(_PAGE_SIZE, PAGE_SIZE); + BLANK(); } #ifdef CONFIG_32BIT @@ -292,15 +360,6 @@ void output_signal_defined(void) BLANK(); } -void output_irq_cpustat_t_defines(void) -{ - COMMENT("Linux irq_cpustat_t offsets."); - DEFINE(IC_SOFTIRQ_PENDING, - offsetof(irq_cpustat_t, __softirq_pending)); - DEFINE(IC_IRQ_CPUSTAT_T, sizeof(irq_cpustat_t)); - BLANK(); -} - #ifdef CONFIG_CPU_CAVIUM_OCTEON void output_octeon_cop2_state_defines(void) { @@ -339,3 +398,97 @@ void output_pbe_defines(void) BLANK(); } #endif + +#ifdef CONFIG_CPU_PM +void output_pm_defines(void) +{ + COMMENT(" PM offsets. "); +#ifdef CONFIG_EVA + OFFSET(SSS_SEGCTL0, mips_static_suspend_state, segctl[0]); + OFFSET(SSS_SEGCTL1, mips_static_suspend_state, segctl[1]); + OFFSET(SSS_SEGCTL2, mips_static_suspend_state, segctl[2]); +#endif + OFFSET(SSS_SP, mips_static_suspend_state, sp); + BLANK(); +} +#endif + +void output_kvm_defines(void) +{ + COMMENT(" KVM/MIPS Specfic offsets. "); + DEFINE(VCPU_ARCH_SIZE, sizeof(struct kvm_vcpu_arch)); + OFFSET(VCPU_RUN, kvm_vcpu, run); + OFFSET(VCPU_HOST_ARCH, kvm_vcpu, arch); + + OFFSET(VCPU_HOST_EBASE, kvm_vcpu_arch, host_ebase); + OFFSET(VCPU_GUEST_EBASE, kvm_vcpu_arch, guest_ebase); + + OFFSET(VCPU_HOST_STACK, kvm_vcpu_arch, host_stack); + OFFSET(VCPU_HOST_GP, kvm_vcpu_arch, host_gp); + + OFFSET(VCPU_HOST_CP0_BADVADDR, kvm_vcpu_arch, host_cp0_badvaddr); + OFFSET(VCPU_HOST_CP0_CAUSE, kvm_vcpu_arch, host_cp0_cause); + OFFSET(VCPU_HOST_EPC, kvm_vcpu_arch, host_cp0_epc); + OFFSET(VCPU_HOST_ENTRYHI, kvm_vcpu_arch, host_cp0_entryhi); + + OFFSET(VCPU_GUEST_INST, kvm_vcpu_arch, guest_inst); + + OFFSET(VCPU_R0, kvm_vcpu_arch, gprs[0]); + OFFSET(VCPU_R1, kvm_vcpu_arch, gprs[1]); + OFFSET(VCPU_R2, kvm_vcpu_arch, gprs[2]); + OFFSET(VCPU_R3, kvm_vcpu_arch, gprs[3]); + OFFSET(VCPU_R4, kvm_vcpu_arch, gprs[4]); + OFFSET(VCPU_R5, kvm_vcpu_arch, gprs[5]); + OFFSET(VCPU_R6, kvm_vcpu_arch, gprs[6]); + OFFSET(VCPU_R7, kvm_vcpu_arch, gprs[7]); + OFFSET(VCPU_R8, kvm_vcpu_arch, gprs[8]); + OFFSET(VCPU_R9, kvm_vcpu_arch, gprs[9]); + OFFSET(VCPU_R10, kvm_vcpu_arch, gprs[10]); + OFFSET(VCPU_R11, kvm_vcpu_arch, gprs[11]); + OFFSET(VCPU_R12, kvm_vcpu_arch, gprs[12]); + OFFSET(VCPU_R13, kvm_vcpu_arch, gprs[13]); + OFFSET(VCPU_R14, kvm_vcpu_arch, gprs[14]); + OFFSET(VCPU_R15, kvm_vcpu_arch, gprs[15]); + OFFSET(VCPU_R16, kvm_vcpu_arch, gprs[16]); + OFFSET(VCPU_R17, kvm_vcpu_arch, gprs[17]); + OFFSET(VCPU_R18, kvm_vcpu_arch, gprs[18]); + OFFSET(VCPU_R19, kvm_vcpu_arch, gprs[19]); + OFFSET(VCPU_R20, kvm_vcpu_arch, gprs[20]); + OFFSET(VCPU_R21, kvm_vcpu_arch, gprs[21]); + OFFSET(VCPU_R22, kvm_vcpu_arch, gprs[22]); + OFFSET(VCPU_R23, kvm_vcpu_arch, gprs[23]); + OFFSET(VCPU_R24, kvm_vcpu_arch, gprs[24]); + OFFSET(VCPU_R25, kvm_vcpu_arch, gprs[25]); + OFFSET(VCPU_R26, kvm_vcpu_arch, gprs[26]); + OFFSET(VCPU_R27, kvm_vcpu_arch, gprs[27]); + OFFSET(VCPU_R28, kvm_vcpu_arch, gprs[28]); + OFFSET(VCPU_R29, kvm_vcpu_arch, gprs[29]); + OFFSET(VCPU_R30, kvm_vcpu_arch, gprs[30]); + OFFSET(VCPU_R31, kvm_vcpu_arch, gprs[31]); + OFFSET(VCPU_LO, kvm_vcpu_arch, lo); + OFFSET(VCPU_HI, kvm_vcpu_arch, hi); + OFFSET(VCPU_PC, kvm_vcpu_arch, pc); + OFFSET(VCPU_COP0, kvm_vcpu_arch, cop0); + OFFSET(VCPU_GUEST_KERNEL_ASID, kvm_vcpu_arch, guest_kernel_asid); + OFFSET(VCPU_GUEST_USER_ASID, kvm_vcpu_arch, guest_user_asid); + + OFFSET(COP0_TLB_HI, mips_coproc, reg[MIPS_CP0_TLB_HI][0]); + OFFSET(COP0_STATUS, mips_coproc, reg[MIPS_CP0_STATUS][0]); + BLANK(); +} + +#ifdef CONFIG_MIPS_CPS +void output_cps_defines(void) +{ + COMMENT(" MIPS CPS offsets. "); + + OFFSET(COREBOOTCFG_VPEMASK, core_boot_config, vpe_mask); + OFFSET(COREBOOTCFG_VPECONFIG, core_boot_config, vpe_config); + DEFINE(COREBOOTCFG_SIZE, sizeof(struct core_boot_config)); + + OFFSET(VPEBOOTCFG_PC, vpe_boot_config, pc); + OFFSET(VPEBOOTCFG_SP, vpe_boot_config, sp); + OFFSET(VPEBOOTCFG_GP, vpe_boot_config, gp); + DEFINE(VPEBOOTCFG_SIZE, sizeof(struct vpe_boot_config)); +} +#endif diff --git a/arch/mips/kernel/binfmt_elfn32.c b/arch/mips/kernel/binfmt_elfn32.c index 9fdd8bcdd21..1188e00bb12 100644 --- a/arch/mips/kernel/binfmt_elfn32.c +++ b/arch/mips/kernel/binfmt_elfn32.c @@ -6,7 +6,7 @@ * * Heavily inspired by the 32-bit Sparc compat code which is * Copyright (C) 1995, 1996, 1997, 1998 David S. Miller (davem@redhat.com) - * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) */ #define ELF_ARCH EM_MIPS @@ -48,7 +48,7 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; #define TASK32_SIZE 0x7fff8000UL #undef ELF_ET_DYN_BASE -#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) +#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) #include <asm/processor.h> #include <linux/module.h> @@ -67,8 +67,8 @@ struct elf_prstatus32 pid_t pr_ppid; pid_t pr_pgrp; pid_t pr_sid; - struct compat_timeval pr_utime; /* User time */ - struct compat_timeval pr_stime; /* System time */ + struct compat_timeval pr_utime; /* User time */ + struct compat_timeval pr_stime; /* System time */ struct compat_timeval pr_cutime;/* Cumulative user time */ struct compat_timeval pr_cstime;/* Cumulative system time */ elf_gregset_t pr_reg; /* GP registers */ @@ -88,7 +88,7 @@ struct elf_prpsinfo32 pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; /* Lots missing */ char pr_fname[16]; /* filename of executable */ - char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ + char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ }; #define elf_caddr_t u32 @@ -119,4 +119,15 @@ MODULE_AUTHOR("Ralf Baechle (ralf@linux-mips.org)"); #undef TASK_SIZE #define TASK_SIZE TASK_SIZE32 +#undef cputime_to_timeval +#define cputime_to_timeval cputime_to_compat_timeval +static __inline__ void +cputime_to_compat_timeval(const cputime_t cputime, struct compat_timeval *value) +{ + unsigned long jiffies = cputime_to_jiffies(cputime); + + value->tv_usec = (jiffies % HZ) * (1000000L / HZ); + value->tv_sec = jiffies / HZ; +} + #include "../../../fs/binfmt_elf.c" diff --git a/arch/mips/kernel/binfmt_elfo32.c b/arch/mips/kernel/binfmt_elfo32.c index ff448233dab..7faf5f2bee2 100644 --- a/arch/mips/kernel/binfmt_elfo32.c +++ b/arch/mips/kernel/binfmt_elfo32.c @@ -6,7 +6,7 @@ * * Heavily inspired by the 32-bit Sparc compat code which is * Copyright (C) 1995, 1996, 1997, 1998 David S. Miller (davem@redhat.com) - * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1995, 1996, 1997, 1998 Jakub Jelinek (jj@ultra.linux.cz) */ #define ELF_ARCH EM_MIPS @@ -28,6 +28,18 @@ typedef double elf_fpreg_t; typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; /* + * In order to be sure that we don't attempt to execute an O32 binary which + * requires 64 bit FP (FR=1) on a system which does not support it we refuse + * to execute any binary which has bits specified by the following macro set + * in its ELF header flags. + */ +#ifdef CONFIG_MIPS_O32_FP64_SUPPORT +# define __MIPS_O32_FP64_MUST_BE_ZERO 0 +#else +# define __MIPS_O32_FP64_MUST_BE_ZERO EF_MIPS_FP64 +#endif + +/* * This is used to ensure we don't load something for the wrong architecture. */ #define elf_check_arch(hdr) \ @@ -44,13 +56,19 @@ typedef elf_fpreg_t elf_fpregset_t[ELF_NFPREG]; if (((__h->e_flags & EF_MIPS_ABI) != 0) && \ ((__h->e_flags & EF_MIPS_ABI) != EF_MIPS_ABI_O32)) \ __res = 0; \ + if (__h->e_flags & __MIPS_O32_FP64_MUST_BE_ZERO) \ + __res = 0; \ \ __res; \ }) +#ifdef CONFIG_KVM_GUEST +#define TASK32_SIZE 0x3fff8000UL +#else #define TASK32_SIZE 0x7fff8000UL +#endif #undef ELF_ET_DYN_BASE -#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) +#define ELF_ET_DYN_BASE (TASK32_SIZE / 3 * 2) #include <asm/processor.h> @@ -86,8 +104,8 @@ struct elf_prstatus32 pid_t pr_ppid; pid_t pr_pgrp; pid_t pr_sid; - struct compat_timeval pr_utime; /* User time */ - struct compat_timeval pr_stime; /* System time */ + struct compat_timeval pr_utime; /* User time */ + struct compat_timeval pr_stime; /* System time */ struct compat_timeval pr_cutime;/* Cumulative user time */ struct compat_timeval pr_cstime;/* Cumulative system time */ elf_gregset_t pr_reg; /* GP registers */ @@ -107,7 +125,7 @@ struct elf_prpsinfo32 pid_t pr_pid, pr_ppid, pr_pgrp, pr_sid; /* Lots missing */ char pr_fname[16]; /* filename of executable */ - char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ + char pr_psargs[ELF_PRARGSZ]; /* initial part of arg list */ }; #define elf_caddr_t u32 @@ -158,4 +176,15 @@ MODULE_AUTHOR("Ralf Baechle (ralf@linux-mips.org)"); #undef TASK_SIZE #define TASK_SIZE TASK_SIZE32 +#undef cputime_to_timeval +#define cputime_to_timeval cputime_to_compat_timeval +static __inline__ void +cputime_to_compat_timeval(const cputime_t cputime, struct compat_timeval *value) +{ + unsigned long jiffies = cputime_to_jiffies(cputime); + + value->tv_usec = (jiffies % HZ) * (1000000L / HZ); + value->tv_sec = jiffies / HZ; +} + #include "../../../fs/binfmt_elf.c" diff --git a/arch/mips/kernel/bmips_vec.S b/arch/mips/kernel/bmips_vec.S new file mode 100644 index 00000000000..290c23b5167 --- /dev/null +++ b/arch/mips/kernel/bmips_vec.S @@ -0,0 +1,285 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2011 by Kevin Cernekee (cernekee@gmail.com) + * + * Reset/NMI/re-entry vectors for BMIPS processors + */ + + +#include <asm/asm.h> +#include <asm/asmmacro.h> +#include <asm/cacheops.h> +#include <asm/cpu.h> +#include <asm/regdef.h> +#include <asm/mipsregs.h> +#include <asm/stackframe.h> +#include <asm/addrspace.h> +#include <asm/hazards.h> +#include <asm/bmips.h> + + .macro BARRIER + .set mips32 + _ssnop + _ssnop + _ssnop + .set mips0 + .endm + +/*********************************************************************** + * Alternate CPU1 startup vector for BMIPS4350 + * + * On some systems the bootloader has already started CPU1 and configured + * it to resume execution at 0x8000_0200 (!BEV IV vector) when it is + * triggered by the SW1 interrupt. If that is the case we try to move + * it to a more convenient place: BMIPS_WARM_RESTART_VEC @ 0x8000_0380. + ***********************************************************************/ + +LEAF(bmips_smp_movevec) + la k0, 1f + li k1, CKSEG1 + or k0, k1 + jr k0 + +1: + /* clear IV, pending IPIs */ + mtc0 zero, CP0_CAUSE + + /* re-enable IRQs to wait for SW1 */ + li k0, ST0_IE | ST0_BEV | STATUSF_IP1 + mtc0 k0, CP0_STATUS + + /* set up CPU1 CBR; move BASE to 0xa000_0000 */ + li k0, 0xff400000 + mtc0 k0, $22, 6 + /* set up relocation vector address based on thread ID */ + mfc0 k1, $22, 3 + srl k1, 16 + andi k1, 0x8000 + or k1, CKSEG1 | BMIPS_RELO_VECTOR_CONTROL_0 + or k0, k1 + li k1, 0xa0080000 + sw k1, 0(k0) + + /* wait here for SW1 interrupt from bmips_boot_secondary() */ + wait + + la k0, bmips_reset_nmi_vec + li k1, CKSEG1 + or k0, k1 + jr k0 +END(bmips_smp_movevec) + +/*********************************************************************** + * Reset/NMI vector + * For BMIPS processors that can relocate their exception vectors, this + * entire function gets copied to 0x8000_0000. + ***********************************************************************/ + +NESTED(bmips_reset_nmi_vec, PT_SIZE, sp) + .set push + .set noat + .align 4 + +#ifdef CONFIG_SMP + /* if the NMI bit is clear, assume this is a CPU1 reset instead */ + li k1, (1 << 19) + mfc0 k0, CP0_STATUS + and k0, k1 + beqz k0, bmips_smp_entry + +#if defined(CONFIG_CPU_BMIPS5000) + mfc0 k0, CP0_PRID + li k1, PRID_IMP_BMIPS5000 + andi k0, 0xff00 + bne k0, k1, 1f + + /* if we're not on core 0, this must be the SMP boot signal */ + li k1, (3 << 25) + mfc0 k0, $22 + and k0, k1 + bnez k0, bmips_smp_entry +1: +#endif /* CONFIG_CPU_BMIPS5000 */ +#endif /* CONFIG_SMP */ + + /* nope, it's just a regular NMI */ + SAVE_ALL + move a0, sp + + /* clear EXL, ERL, BEV so that TLB refills still work */ + mfc0 k0, CP0_STATUS + li k1, ST0_ERL | ST0_EXL | ST0_BEV | ST0_IE + or k0, k1 + xor k0, k1 + mtc0 k0, CP0_STATUS + BARRIER + + /* jump to the NMI handler function */ + la k0, nmi_handler + jr k0 + + RESTORE_ALL + .set arch=r4000 + eret + +/*********************************************************************** + * CPU1 reset vector (used for the initial boot only) + * This is still part of bmips_reset_nmi_vec(). + ***********************************************************************/ + +#ifdef CONFIG_SMP + +bmips_smp_entry: + + /* set up CP0 STATUS; enable FPU */ + li k0, 0x30000000 + mtc0 k0, CP0_STATUS + BARRIER + + /* set local CP0 CONFIG to make kseg0 cacheable, write-back */ + mfc0 k0, CP0_CONFIG + ori k0, 0x07 + xori k0, 0x04 + mtc0 k0, CP0_CONFIG + + mfc0 k0, CP0_PRID + andi k0, 0xff00 +#if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) + li k1, PRID_IMP_BMIPS43XX + bne k0, k1, 2f + + /* initialize CPU1's local I-cache */ + li k0, 0x80000000 + li k1, 0x80010000 + mtc0 zero, $28 + mtc0 zero, $28, 1 + BARRIER + +1: cache Index_Store_Tag_I, 0(k0) + addiu k0, 16 + bne k0, k1, 1b + + b 3f +2: +#endif /* CONFIG_CPU_BMIPS4350 || CONFIG_CPU_BMIPS4380 */ +#if defined(CONFIG_CPU_BMIPS5000) + /* set exception vector base */ + li k1, PRID_IMP_BMIPS5000 + bne k0, k1, 3f + + la k0, ebase + lw k0, 0(k0) + mtc0 k0, $15, 1 + BARRIER +#endif /* CONFIG_CPU_BMIPS5000 */ +3: + /* jump back to kseg0 in case we need to remap the kseg1 area */ + la k0, 1f + jr k0 +1: + la k0, bmips_enable_xks01 + jalr k0 + + /* use temporary stack to set up upper memory TLB */ + li sp, BMIPS_WARM_RESTART_VEC + la k0, plat_wired_tlb_setup + jalr k0 + + /* switch to permanent stack and continue booting */ + + .global bmips_secondary_reentry +bmips_secondary_reentry: + la k0, bmips_smp_boot_sp + lw sp, 0(k0) + la k0, bmips_smp_boot_gp + lw gp, 0(k0) + la k0, start_secondary + jr k0 + +#endif /* CONFIG_SMP */ + + .align 4 + .global bmips_reset_nmi_vec_end +bmips_reset_nmi_vec_end: + +END(bmips_reset_nmi_vec) + + .set pop + .previous + +/*********************************************************************** + * CPU1 warm restart vector (used for second and subsequent boots). + * Also used for S2 standby recovery (PM). + * This entire function gets copied to (BMIPS_WARM_RESTART_VEC) + ***********************************************************************/ + +LEAF(bmips_smp_int_vec) + + .align 4 + mfc0 k0, CP0_STATUS + ori k0, 0x01 + xori k0, 0x01 + mtc0 k0, CP0_STATUS + eret + + .align 4 + .global bmips_smp_int_vec_end +bmips_smp_int_vec_end: + +END(bmips_smp_int_vec) + +/*********************************************************************** + * XKS01 support + * Certain CPUs support extending kseg0 to 1024MB. + ***********************************************************************/ + +LEAF(bmips_enable_xks01) + +#if defined(CONFIG_XKS01) + mfc0 t0, CP0_PRID + andi t2, t0, 0xff00 +#if defined(CONFIG_CPU_BMIPS4380) + li t1, PRID_IMP_BMIPS43XX + bne t2, t1, 1f + + andi t0, 0xff + addiu t1, t0, -PRID_REV_BMIPS4380_HI + bgtz t1, 2f + addiu t0, -PRID_REV_BMIPS4380_LO + bltz t0, 2f + + mfc0 t0, $22, 3 + li t1, 0x1ff0 + li t2, (1 << 12) | (1 << 9) + or t0, t1 + xor t0, t1 + or t0, t2 + mtc0 t0, $22, 3 + BARRIER + b 2f +1: +#endif /* CONFIG_CPU_BMIPS4380 */ +#if defined(CONFIG_CPU_BMIPS5000) + li t1, PRID_IMP_BMIPS5000 + bne t2, t1, 2f + + mfc0 t0, $22, 5 + li t1, 0x01ff + li t2, (1 << 8) | (1 << 5) + or t0, t1 + xor t0, t1 + or t0, t2 + mtc0 t0, $22, 5 + BARRIER +#endif /* CONFIG_CPU_BMIPS5000 */ +2: +#endif /* defined(CONFIG_XKS01) */ + + jr ra + +END(bmips_enable_xks01) + + .previous diff --git a/arch/mips/kernel/branch.c b/arch/mips/kernel/branch.c index 32103cc2a25..7b2df224f04 100644 --- a/arch/mips/kernel/branch.c +++ b/arch/mips/kernel/branch.c @@ -9,37 +9,404 @@ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/signal.h> +#include <linux/module.h> #include <asm/branch.h> #include <asm/cpu.h> #include <asm/cpu-features.h> #include <asm/fpu.h> +#include <asm/fpu_emulator.h> #include <asm/inst.h> #include <asm/ptrace.h> #include <asm/uaccess.h> /* - * Compute the return address and do emulate branch simulation, if required. + * Calculate and return exception PC in case of branch delay slot + * for microMIPS and MIPS16e. It does not clear the ISA mode bit. */ -int __compute_return_epc(struct pt_regs *regs) +int __isa_exception_epc(struct pt_regs *regs) { - unsigned int __user *addr; - unsigned int bit, fcr31, dspcontrol; + unsigned short inst; + long epc = regs->cp0_epc; + + /* Calculate exception PC in branch delay slot. */ + if (__get_user(inst, (u16 __user *) msk_isa16_mode(epc))) { + /* This should never happen because delay slot was checked. */ + force_sig(SIGSEGV, current); + return epc; + } + if (cpu_has_mips16) { + if (((union mips16e_instruction)inst).ri.opcode + == MIPS16e_jal_op) + epc += 4; + else + epc += 2; + } else if (mm_insn_16bit(inst)) + epc += 2; + else + epc += 4; + + return epc; +} + +/* (microMIPS) Convert 16-bit register encoding to 32-bit register encoding. */ +static const unsigned int reg16to32map[8] = {16, 17, 2, 3, 4, 5, 6, 7}; + +int __mm_isBranchInstr(struct pt_regs *regs, struct mm_decoded_insn dec_insn, + unsigned long *contpc) +{ + union mips_instruction insn = (union mips_instruction)dec_insn.insn; + int bc_false = 0; + unsigned int fcr31; + unsigned int bit; + + if (!cpu_has_mmips) + return 0; + + switch (insn.mm_i_format.opcode) { + case mm_pool32a_op: + if ((insn.mm_i_format.simmediate & MM_POOL32A_MINOR_MASK) == + mm_pool32axf_op) { + switch (insn.mm_i_format.simmediate >> + MM_POOL32A_MINOR_SHIFT) { + case mm_jalr_op: + case mm_jalrhb_op: + case mm_jalrs_op: + case mm_jalrshb_op: + if (insn.mm_i_format.rt != 0) /* Not mm_jr */ + regs->regs[insn.mm_i_format.rt] = + regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + *contpc = regs->regs[insn.mm_i_format.rs]; + return 1; + } + } + break; + case mm_pool32i_op: + switch (insn.mm_i_format.rt) { + case mm_bltzals_op: + case mm_bltzal_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + /* Fall through */ + case mm_bltz_op: + if ((long)regs->regs[insn.mm_i_format.rs] < 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bgezals_op: + case mm_bgezal_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + /* Fall through */ + case mm_bgez_op: + if ((long)regs->regs[insn.mm_i_format.rs] >= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_blez_op: + if ((long)regs->regs[insn.mm_i_format.rs] <= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bgtz_op: + if ((long)regs->regs[insn.mm_i_format.rs] <= 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bc2f_op: + case mm_bc1f_op: + bc_false = 1; + /* Fall through */ + case mm_bc2t_op: + case mm_bc1t_op: + preempt_disable(); + if (is_fpu_owner()) + asm volatile("cfc1\t%0,$31" : "=r" (fcr31)); + else + fcr31 = current->thread.fpu.fcr31; + preempt_enable(); + + if (bc_false) + fcr31 = ~fcr31; + + bit = (insn.mm_i_format.rs >> 2); + bit += (bit != 0); + bit += 23; + if (fcr31 & (1 << bit)) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + } + break; + case mm_pool16c_op: + switch (insn.mm_i_format.rt) { + case mm_jalr16_op: + case mm_jalrs16_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + /* Fall through */ + case mm_jr16_op: + *contpc = regs->regs[insn.mm_i_format.rs]; + return 1; + } + break; + case mm_beqz16_op: + if ((long)regs->regs[reg16to32map[insn.mm_b1_format.rs]] == 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_b1_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_bnez16_op: + if ((long)regs->regs[reg16to32map[insn.mm_b1_format.rs]] != 0) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_b1_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_b16_op: + *contpc = regs->cp0_epc + dec_insn.pc_inc + + (insn.mm_b0_format.simmediate << 1); + return 1; + case mm_beq32_op: + if (regs->regs[insn.mm_i_format.rs] == + regs->regs[insn.mm_i_format.rt]) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + dec_insn.next_pc_inc; + return 1; + case mm_bne32_op: + if (regs->regs[insn.mm_i_format.rs] != + regs->regs[insn.mm_i_format.rt]) + *contpc = regs->cp0_epc + + dec_insn.pc_inc + + (insn.mm_i_format.simmediate << 1); + else + *contpc = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + return 1; + case mm_jalx32_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + *contpc = regs->cp0_epc + dec_insn.pc_inc; + *contpc >>= 28; + *contpc <<= 28; + *contpc |= (insn.j_format.target << 2); + return 1; + case mm_jals32_op: + case mm_jal32_op: + regs->regs[31] = regs->cp0_epc + + dec_insn.pc_inc + dec_insn.next_pc_inc; + /* Fall through */ + case mm_j32_op: + *contpc = regs->cp0_epc + dec_insn.pc_inc; + *contpc >>= 27; + *contpc <<= 27; + *contpc |= (insn.j_format.target << 1); + set_isa16_mode(*contpc); + return 1; + } + return 0; +} + +/* + * Compute return address and emulate branch in microMIPS mode after an + * exception only. It does not handle compact branches/jumps and cannot + * be used in interrupt context. (Compact branches/jumps do not cause + * exceptions.) + */ +int __microMIPS_compute_return_epc(struct pt_regs *regs) +{ + u16 __user *pc16; + u16 halfword; + unsigned int word; + unsigned long contpc; + struct mm_decoded_insn mminsn = { 0 }; + + mminsn.micro_mips_mode = 1; + + /* This load never faults. */ + pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc); + __get_user(halfword, pc16); + pc16++; + contpc = regs->cp0_epc + 2; + word = ((unsigned int)halfword << 16); + mminsn.pc_inc = 2; + + if (!mm_insn_16bit(halfword)) { + __get_user(halfword, pc16); + pc16++; + contpc = regs->cp0_epc + 4; + mminsn.pc_inc = 4; + word |= halfword; + } + mminsn.insn = word; + + if (get_user(halfword, pc16)) + goto sigsegv; + mminsn.next_pc_inc = 2; + word = ((unsigned int)halfword << 16); + + if (!mm_insn_16bit(halfword)) { + pc16++; + if (get_user(halfword, pc16)) + goto sigsegv; + mminsn.next_pc_inc = 4; + word |= halfword; + } + mminsn.next_insn = word; + + mm_isBranchInstr(regs, mminsn, &contpc); + + regs->cp0_epc = contpc; + + return 0; + +sigsegv: + force_sig(SIGSEGV, current); + return -EFAULT; +} + +/* + * Compute return address and emulate branch in MIPS16e mode after an + * exception only. It does not handle compact branches/jumps and cannot + * be used in interrupt context. (Compact branches/jumps do not cause + * exceptions.) + */ +int __MIPS16e_compute_return_epc(struct pt_regs *regs) +{ + u16 __user *addr; + union mips16e_instruction inst; + u16 inst2; + u32 fullinst; long epc; - union mips_instruction insn; epc = regs->cp0_epc; - if (epc & 3) - goto unaligned; - /* - * Read the instruction - */ - addr = (unsigned int __user *) epc; - if (__get_user(insn.word, addr)) { + /* Read the instruction. */ + addr = (u16 __user *)msk_isa16_mode(epc); + if (__get_user(inst.full, addr)) { force_sig(SIGSEGV, current); return -EFAULT; } + switch (inst.ri.opcode) { + case MIPS16e_extend_op: + regs->cp0_epc += 4; + return 0; + + /* + * JAL and JALX in MIPS16e mode + */ + case MIPS16e_jal_op: + addr += 1; + if (__get_user(inst2, addr)) { + force_sig(SIGSEGV, current); + return -EFAULT; + } + fullinst = ((unsigned)inst.full << 16) | inst2; + regs->regs[31] = epc + 6; + epc += 4; + epc >>= 28; + epc <<= 28; + /* + * JAL:5 X:1 TARGET[20-16]:5 TARGET[25:21]:5 TARGET[15:0]:16 + * + * ......TARGET[15:0].................TARGET[20:16]........... + * ......TARGET[25:21] + */ + epc |= + ((fullinst & 0xffff) << 2) | ((fullinst & 0x3e00000) >> 3) | + ((fullinst & 0x1f0000) << 7); + if (!inst.jal.x) + set_isa16_mode(epc); /* Set ISA mode bit. */ + regs->cp0_epc = epc; + return 0; + + /* + * J(AL)R(C) + */ + case MIPS16e_rr_op: + if (inst.rr.func == MIPS16e_jr_func) { + + if (inst.rr.ra) + regs->cp0_epc = regs->regs[31]; + else + regs->cp0_epc = + regs->regs[reg16to32[inst.rr.rx]]; + + if (inst.rr.l) { + if (inst.rr.nd) + regs->regs[31] = epc + 2; + else + regs->regs[31] = epc + 4; + } + return 0; + } + break; + } + + /* + * All other cases have no branch delay slot and are 16-bits. + * Branches do not cause an exception. + */ + regs->cp0_epc += 2; + + return 0; +} + +/** + * __compute_return_epc_for_insn - Computes the return address and do emulate + * branch simulation, if required. + * + * @regs: Pointer to pt_regs + * @insn: branch instruction to decode + * @returns: -EFAULT on error and forces SIGBUS, and on success + * returns 0 or BRANCH_LIKELY_TAKEN as appropriate after + * evaluating the branch. + */ +int __compute_return_epc_for_insn(struct pt_regs *regs, + union mips_instruction insn) +{ + unsigned int bit, fcr31, dspcontrol; + long epc = regs->cp0_epc; + int ret = 0; + switch (insn.i_format.opcode) { /* * jr and jalr are in r_format format. @@ -62,20 +429,24 @@ int __compute_return_epc(struct pt_regs *regs) */ case bcond_op: switch (insn.i_format.rt) { - case bltz_op: + case bltz_op: case bltzl_op: - if ((long)regs->regs[insn.i_format.rs] < 0) + if ((long)regs->regs[insn.i_format.rs] < 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == bltzl_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; case bgez_op: case bgezl_op: - if ((long)regs->regs[insn.i_format.rs] >= 0) + if ((long)regs->regs[insn.i_format.rs] >= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == bgezl_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -83,9 +454,11 @@ int __compute_return_epc(struct pt_regs *regs) case bltzal_op: case bltzall_op: regs->regs[31] = epc + 8; - if ((long)regs->regs[insn.i_format.rs] < 0) + if ((long)regs->regs[insn.i_format.rs] < 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == bltzall_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -93,12 +466,15 @@ int __compute_return_epc(struct pt_regs *regs) case bgezal_op: case bgezall_op: regs->regs[31] = epc + 8; - if ((long)regs->regs[insn.i_format.rs] >= 0) + if ((long)regs->regs[insn.i_format.rs] >= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == bgezall_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; + case bposge32_op: if (!cpu_has_dsp) goto sigill; @@ -125,6 +501,8 @@ int __compute_return_epc(struct pt_regs *regs) epc <<= 28; epc |= (insn.j_format.target << 2); regs->cp0_epc = epc; + if (insn.i_format.opcode == jalx_op) + set_isa16_mode(regs->cp0_epc); break; /* @@ -133,9 +511,11 @@ int __compute_return_epc(struct pt_regs *regs) case beq_op: case beql_op: if (regs->regs[insn.i_format.rs] == - regs->regs[insn.i_format.rt]) + regs->regs[insn.i_format.rt]) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.opcode == beql_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -143,9 +523,11 @@ int __compute_return_epc(struct pt_regs *regs) case bne_op: case bnel_op: if (regs->regs[insn.i_format.rs] != - regs->regs[insn.i_format.rt]) + regs->regs[insn.i_format.rt]) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.opcode == bnel_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -153,9 +535,11 @@ int __compute_return_epc(struct pt_regs *regs) case blez_op: /* not really i_format */ case blezl_op: /* rt field assumed to be zero */ - if ((long)regs->regs[insn.i_format.rs] <= 0) + if ((long)regs->regs[insn.i_format.rs] <= 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.opcode == blezl_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -163,9 +547,11 @@ int __compute_return_epc(struct pt_regs *regs) case bgtz_op: case bgtzl_op: /* rt field assumed to be zero */ - if ((long)regs->regs[insn.i_format.rs] > 0) + if ((long)regs->regs[insn.i_format.rs] > 0) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.opcode == bgtzl_op) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -176,7 +562,11 @@ int __compute_return_epc(struct pt_regs *regs) case cop1_op: preempt_disable(); if (is_fpu_owner()) - asm volatile("cfc1\t%0,$31" : "=r" (fcr31)); + asm volatile( + ".set push\n" + "\t.set mips1\n" + "\tcfc1\t%0,$31\n" + "\t.set pop" : "=r" (fcr31)); else fcr31 = current->thread.fpu.fcr31; preempt_enable(); @@ -185,20 +575,24 @@ int __compute_return_epc(struct pt_regs *regs) bit += (bit != 0); bit += 23; switch (insn.i_format.rt & 3) { - case 0: /* bc1f */ - case 2: /* bc1fl */ - if (~fcr31 & (1 << bit)) + case 0: /* bc1f */ + case 2: /* bc1fl */ + if (~fcr31 & (1 << bit)) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == 2) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; - case 1: /* bc1t */ - case 3: /* bc1tl */ - if (fcr31 & (1 << bit)) + case 1: /* bc1t */ + case 3: /* bc1tl */ + if (fcr31 & (1 << bit)) { epc = epc + 4 + (insn.i_format.simmediate << 2); - else + if (insn.i_format.rt == 3) + ret = BRANCH_LIKELY_TAKEN; + } else epc += 8; regs->cp0_epc = epc; break; @@ -239,15 +633,38 @@ int __compute_return_epc(struct pt_regs *regs) #endif } - return 0; + return ret; -unaligned: - printk("%s: unaligned epc - sending SIGBUS.\n", current->comm); +sigill: + printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm); force_sig(SIGBUS, current); return -EFAULT; +} +EXPORT_SYMBOL_GPL(__compute_return_epc_for_insn); -sigill: - printk("%s: DSP branch but not DSP ASE - sending SIGBUS.\n", current->comm); +int __compute_return_epc(struct pt_regs *regs) +{ + unsigned int __user *addr; + long epc; + union mips_instruction insn; + + epc = regs->cp0_epc; + if (epc & 3) + goto unaligned; + + /* + * Read the instruction + */ + addr = (unsigned int __user *) epc; + if (__get_user(insn.word, addr)) { + force_sig(SIGSEGV, current); + return -EFAULT; + } + + return __compute_return_epc_for_insn(regs, insn); + +unaligned: + printk("%s: unaligned epc - sending SIGBUS.\n", current->comm); force_sig(SIGBUS, current); return -EFAULT; } diff --git a/arch/mips/kernel/cevt-bcm1480.c b/arch/mips/kernel/cevt-bcm1480.c index 36c3898b76d..7976457184b 100644 --- a/arch/mips/kernel/cevt-bcm1480.c +++ b/arch/mips/kernel/cevt-bcm1480.c @@ -41,7 +41,7 @@ * the rest of the system */ static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -109,7 +109,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1480_clockevent_init(void) +void sb1480_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_BCM1480_INT_TIMER_0 + cpu; @@ -144,8 +144,8 @@ void __cpuinit sb1480_clockevent_init(void) bcm1480_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER; + action->handler = sibyte_counter_handler; + action->flags = IRQF_PERCPU | IRQF_TIMER; action->name = name; action->dev_id = cd; diff --git a/arch/mips/kernel/cevt-ds1287.c b/arch/mips/kernel/cevt-ds1287.c index 939157e397b..ff1f01b7227 100644 --- a/arch/mips/kernel/cevt-ds1287.c +++ b/arch/mips/kernel/cevt-ds1287.c @@ -1,7 +1,7 @@ /* * DS1287 clockevent driver * - * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> + * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -89,7 +89,7 @@ static void ds1287_event_handler(struct clock_event_device *dev) static struct clock_event_device ds1287_clockevent = { .name = "ds1287", .features = CLOCK_EVT_FEAT_PERIODIC, - .set_next_event = ds1287_set_next_event, + .set_next_event = ds1287_set_next_event, .set_mode = ds1287_set_mode, .event_handler = ds1287_event_handler, }; @@ -108,7 +108,7 @@ static irqreturn_t ds1287_interrupt(int irq, void *dev_id) static struct irqaction ds1287_irqaction = { .handler = ds1287_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "ds1287", }; diff --git a/arch/mips/kernel/cevt-gic.c b/arch/mips/kernel/cevt-gic.c new file mode 100644 index 00000000000..6093716980b --- /dev/null +++ b/arch/mips/kernel/cevt-gic.c @@ -0,0 +1,105 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/clockchips.h> +#include <linux/interrupt.h> +#include <linux/percpu.h> +#include <linux/smp.h> +#include <linux/irq.h> + +#include <asm/time.h> +#include <asm/gic.h> +#include <asm/mips-boards/maltaint.h> + +DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device); +int gic_timer_irq_installed; + + +static int gic_next_event(unsigned long delta, struct clock_event_device *evt) +{ + u64 cnt; + int res; + + cnt = gic_read_count(); + cnt += (u64)delta; + gic_write_cpu_compare(cnt, cpumask_first(evt->cpumask)); + res = ((int)(gic_read_count() - cnt) >= 0) ? -ETIME : 0; + return res; +} + +void gic_set_clock_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ + /* Nothing to do ... */ +} + +irqreturn_t gic_compare_interrupt(int irq, void *dev_id) +{ + struct clock_event_device *cd; + int cpu = smp_processor_id(); + + gic_write_compare(gic_read_compare()); + cd = &per_cpu(gic_clockevent_device, cpu); + cd->event_handler(cd); + return IRQ_HANDLED; +} + +struct irqaction gic_compare_irqaction = { + .handler = gic_compare_interrupt, + .flags = IRQF_PERCPU | IRQF_TIMER, + .name = "timer", +}; + + +void gic_event_handler(struct clock_event_device *dev) +{ +} + +int gic_clockevent_init(void) +{ + unsigned int cpu = smp_processor_id(); + struct clock_event_device *cd; + unsigned int irq; + + if (!cpu_has_counter || !gic_frequency) + return -ENXIO; + + irq = MIPS_GIC_IRQ_BASE; + + cd = &per_cpu(gic_clockevent_device, cpu); + + cd->name = "MIPS GIC"; + cd->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_C3STOP; + + clockevent_set_clock(cd, gic_frequency); + + /* Calculate the min / max delta */ + cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); + cd->min_delta_ns = clockevent_delta2ns(0x300, cd); + + cd->rating = 300; + cd->irq = irq; + cd->cpumask = cpumask_of(cpu); + cd->set_next_event = gic_next_event; + cd->set_mode = gic_set_clock_mode; + cd->event_handler = gic_event_handler; + + clockevents_register_device(cd); + + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_MAP), 0x80000002); + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_SMASK), GIC_VPE_SMASK_CMP_MSK); + + if (gic_timer_irq_installed) + return 0; + + gic_timer_irq_installed = 1; + + setup_irq(irq, &gic_compare_irqaction); + irq_set_handler(irq, handle_percpu_irq); + return 0; +} diff --git a/arch/mips/kernel/cevt-gt641xx.c b/arch/mips/kernel/cevt-gt641xx.c index 339f3639b90..f069460751a 100644 --- a/arch/mips/kernel/cevt-gt641xx.c +++ b/arch/mips/kernel/cevt-gt641xx.c @@ -1,7 +1,7 @@ /* * GT641xx clockevent routines. * - * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> + * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -98,7 +98,7 @@ static struct clock_event_device gt641xx_timer0_clockevent = { .name = "gt641xx-timer0", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, .irq = GT641XX_TIMER0_IRQ, - .set_next_event = gt641xx_timer0_set_next_event, + .set_next_event = gt641xx_timer0_set_next_event, .set_mode = gt641xx_timer0_set_mode, .event_handler = gt641xx_timer0_event_handler, }; @@ -114,7 +114,7 @@ static irqreturn_t gt641xx_timer0_interrupt(int irq, void *dev_id) static struct irqaction gt641xx_timer0_irqaction = { .handler = gt641xx_timer0_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "gt641xx_timer0", }; diff --git a/arch/mips/kernel/cevt-r4k.c b/arch/mips/kernel/cevt-r4k.c index 2f4d7a99bcc..bc127e22fda 100644 --- a/arch/mips/kernel/cevt-r4k.c +++ b/arch/mips/kernel/cevt-r4k.c @@ -12,19 +12,12 @@ #include <linux/smp.h> #include <linux/irq.h> -#include <asm/smtc_ipi.h> #include <asm/time.h> #include <asm/cevt-r4k.h> - -/* - * The SMTC Kernel for the 34K, 1004K, et. al. replaces several - * of these routines with SMTC-specific variants. - */ - -#ifndef CONFIG_MIPS_MT_SMTC +#include <asm/gic.h> static int mips_next_event(unsigned long delta, - struct clock_event_device *evt) + struct clock_event_device *evt) { unsigned int cnt; int res; @@ -32,12 +25,10 @@ static int mips_next_event(unsigned long delta, cnt = read_c0_count(); cnt += delta; write_c0_compare(cnt); - res = ((int)(read_c0_count() - cnt) > 0) ? -ETIME : 0; + res = ((int)(read_c0_count() - cnt) >= 0) ? -ETIME : 0; return res; } -#endif /* CONFIG_MIPS_MT_SMTC */ - void mips_set_clock_mode(enum clock_event_mode mode, struct clock_event_device *evt) { @@ -47,8 +38,6 @@ void mips_set_clock_mode(enum clock_event_mode mode, DEFINE_PER_CPU(struct clock_event_device, mips_clockevent_device); int cp0_timer_irq_installed; -#ifndef CONFIG_MIPS_MT_SMTC - irqreturn_t c0_compare_interrupt(int irq, void *dev_id) { const int r2 = cpu_has_mips_r2; @@ -65,7 +54,7 @@ irqreturn_t c0_compare_interrupt(int irq, void *dev_id) goto out; /* - * The same applies to performance counter interrupts. But with the + * The same applies to performance counter interrupts. But with the * above we now know that the reason we got here must be a timer * interrupt. Being the paranoiacs we are we check anyway. */ @@ -80,11 +69,9 @@ out: return IRQ_HANDLED; } -#endif /* Not CONFIG_MIPS_MT_SMTC */ - struct irqaction c0_compare_irqaction = { .handler = c0_compare_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "timer", }; @@ -98,36 +85,39 @@ void mips_event_handler(struct clock_event_device *dev) */ static int c0_compare_int_pending(void) { +#ifdef CONFIG_IRQ_GIC + if (cpu_has_veic) + return gic_get_timer_pending(); +#endif return (read_c0_cause() >> cp0_compare_irq_shift) & (1ul << CAUSEB_IP); } /* * Compare interrupt can be routed and latched outside the core, - * so a single execution hazard barrier may not be enough to give - * it time to clear as seen in the Cause register. 4 time the - * pipeline depth seems reasonably conservative, and empirically - * works better in configurations with high CPU/bus clock ratios. + * so wait up to worst case number of cycle counter ticks for timer interrupt + * changes to propagate to the cause register. */ - -#define compare_change_hazard() \ - do { \ - irq_disable_hazard(); \ - irq_disable_hazard(); \ - irq_disable_hazard(); \ - irq_disable_hazard(); \ - } while (0) +#define COMPARE_INT_SEEN_TICKS 50 int c0_compare_int_usable(void) { unsigned int delta; unsigned int cnt; +#ifdef CONFIG_KVM_GUEST + return 1; +#endif + /* - * IP7 already pending? Try to clear it by acking the timer. + * IP7 already pending? Try to clear it by acking the timer. */ if (c0_compare_int_pending()) { - write_c0_compare(read_c0_count()); - compare_change_hazard(); + cnt = read_c0_count(); + write_c0_compare(cnt); + back_to_back_c0_hazard(); + while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) + if (!c0_compare_int_pending()) + break; if (c0_compare_int_pending()) return 0; } @@ -136,7 +126,7 @@ int c0_compare_int_usable(void) cnt = read_c0_count(); cnt += delta; write_c0_compare(cnt); - compare_change_hazard(); + back_to_back_c0_hazard(); if ((int)(read_c0_count() - cnt) < 0) break; /* increase delta if the timer was already expired */ @@ -145,12 +135,17 @@ int c0_compare_int_usable(void) while ((int)(read_c0_count() - cnt) <= 0) ; /* Wait for expiry */ - compare_change_hazard(); + while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) + if (c0_compare_int_pending()) + break; if (!c0_compare_int_pending()) return 0; - - write_c0_compare(read_c0_count()); - compare_change_hazard(); + cnt = read_c0_count(); + write_c0_compare(cnt); + back_to_back_c0_hazard(); + while (read_c0_count() < (cnt + COMPARE_INT_SEEN_TICKS)) + if (!c0_compare_int_pending()) + break; if (c0_compare_int_pending()) return 0; @@ -160,9 +155,7 @@ int c0_compare_int_usable(void) return 1; } -#ifndef CONFIG_MIPS_MT_SMTC - -int __cpuinit r4k_clockevent_init(void) +int r4k_clockevent_init(void) { unsigned int cpu = smp_processor_id(); struct clock_event_device *cd; @@ -186,7 +179,9 @@ int __cpuinit r4k_clockevent_init(void) cd = &per_cpu(mips_clockevent_device, cpu); cd->name = "MIPS"; - cd->features = CLOCK_EVT_FEAT_ONESHOT; + cd->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_C3STOP | + CLOCK_EVT_FEAT_PERCPU; clockevent_set_clock(cd, mips_hpt_frequency); @@ -213,4 +208,3 @@ int __cpuinit r4k_clockevent_init(void) return 0; } -#endif /* Not CONFIG_MIPS_MT_SMTC */ diff --git a/arch/mips/kernel/cevt-sb1250.c b/arch/mips/kernel/cevt-sb1250.c index 590c54f28a8..5ea6d6b1de1 100644 --- a/arch/mips/kernel/cevt-sb1250.c +++ b/arch/mips/kernel/cevt-sb1250.c @@ -39,7 +39,7 @@ * the rest of the system */ static void sibyte_set_mode(enum clock_event_mode mode, - struct clock_event_device *evt) + struct clock_event_device *evt) { unsigned int cpu = smp_processor_id(); void __iomem *cfg, *init; @@ -107,7 +107,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sibyte_hpt_clockevent); static DEFINE_PER_CPU(struct irqaction, sibyte_hpt_irqaction); static DEFINE_PER_CPU(char [18], sibyte_hpt_name); -void __cpuinit sb1250_clockevent_init(void) +void sb1250_clockevent_init(void) { unsigned int cpu = smp_processor_id(); unsigned int irq = K_INT_TIMER_0 + cpu; @@ -143,8 +143,8 @@ void __cpuinit sb1250_clockevent_init(void) sb1250_unmask_irq(cpu, irq); - action->handler = sibyte_counter_handler; - action->flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER; + action->handler = sibyte_counter_handler; + action->flags = IRQF_PERCPU | IRQF_TIMER; action->name = name; action->dev_id = cd; diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c deleted file mode 100644 index 2e72d30b2f0..00000000000 --- a/arch/mips/kernel/cevt-smtc.c +++ /dev/null @@ -1,324 +0,0 @@ -/* - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 2007 MIPS Technologies, Inc. - * Copyright (C) 2007 Ralf Baechle <ralf@linux-mips.org> - * Copyright (C) 2008 Kevin D. Kissell, Paralogos sarl - */ -#include <linux/clockchips.h> -#include <linux/interrupt.h> -#include <linux/percpu.h> -#include <linux/smp.h> -#include <linux/irq.h> - -#include <asm/smtc_ipi.h> -#include <asm/time.h> -#include <asm/cevt-r4k.h> - -/* - * Variant clock event timer support for SMTC on MIPS 34K, 1004K - * or other MIPS MT cores. - * - * Notes on SMTC Support: - * - * SMTC has multiple microthread TCs pretending to be Linux CPUs. - * But there's only one Count/Compare pair per VPE, and Compare - * interrupts are taken opportunisitically by available TCs - * bound to the VPE with the Count register. The new timer - * framework provides for global broadcasts, but we really - * want VPE-level multicasts for best behavior. So instead - * of invoking the high-level clock-event broadcast code, - * this version of SMTC support uses the historical SMTC - * multicast mechanisms "under the hood", appearing to the - * generic clock layer as if the interrupts are per-CPU. - * - * The approach taken here is to maintain a set of NR_CPUS - * virtual timers, and track which "CPU" needs to be alerted - * at each event. - * - * It's unlikely that we'll see a MIPS MT core with more than - * 2 VPEs, but we *know* that we won't need to handle more - * VPEs than we have "CPUs". So NCPUs arrays of NCPUs elements - * is always going to be overkill, but always going to be enough. - */ - -unsigned long smtc_nexttime[NR_CPUS][NR_CPUS]; -static int smtc_nextinvpe[NR_CPUS]; - -/* - * Timestamps stored are absolute values to be programmed - * into Count register. Valid timestamps will never be zero. - * If a Zero Count value is actually calculated, it is converted - * to be a 1, which will introduce 1 or two CPU cycles of error - * roughly once every four billion events, which at 1000 HZ means - * about once every 50 days. If that's actually a problem, one - * could alternate squashing 0 to 1 and to -1. - */ - -#define MAKEVALID(x) (((x) == 0L) ? 1L : (x)) -#define ISVALID(x) ((x) != 0L) - -/* - * Time comparison is subtle, as it's really truncated - * modular arithmetic. - */ - -#define IS_SOONER(a, b, reference) \ - (((a) - (unsigned long)(reference)) < ((b) - (unsigned long)(reference))) - -/* - * CATCHUP_INCREMENT, used when the function falls behind the counter. - * Could be an increasing function instead of a constant; - */ - -#define CATCHUP_INCREMENT 64 - -static int mips_next_event(unsigned long delta, - struct clock_event_device *evt) -{ - unsigned long flags; - unsigned int mtflags; - unsigned long timestamp, reference, previous; - unsigned long nextcomp = 0L; - int vpe = current_cpu_data.vpe_id; - int cpu = smp_processor_id(); - local_irq_save(flags); - mtflags = dmt(); - - /* - * Maintain the per-TC virtual timer - * and program the per-VPE shared Count register - * as appropriate here... - */ - reference = (unsigned long)read_c0_count(); - timestamp = MAKEVALID(reference + delta); - /* - * To really model the clock, we have to catch the case - * where the current next-in-VPE timestamp is the old - * timestamp for the calling CPE, but the new value is - * in fact later. In that case, we have to do a full - * scan and discover the new next-in-VPE CPU id and - * timestamp. - */ - previous = smtc_nexttime[vpe][cpu]; - if (cpu == smtc_nextinvpe[vpe] && ISVALID(previous) - && IS_SOONER(previous, timestamp, reference)) { - int i; - int soonest = cpu; - - /* - * Update timestamp array here, so that new - * value gets considered along with those of - * other virtual CPUs on the VPE. - */ - smtc_nexttime[vpe][cpu] = timestamp; - for_each_online_cpu(i) { - if (ISVALID(smtc_nexttime[vpe][i]) - && IS_SOONER(smtc_nexttime[vpe][i], - smtc_nexttime[vpe][soonest], reference)) { - soonest = i; - } - } - smtc_nextinvpe[vpe] = soonest; - nextcomp = smtc_nexttime[vpe][soonest]; - /* - * Otherwise, we don't have to process the whole array rank, - * we just have to see if the event horizon has gotten closer. - */ - } else { - if (!ISVALID(smtc_nexttime[vpe][smtc_nextinvpe[vpe]]) || - IS_SOONER(timestamp, - smtc_nexttime[vpe][smtc_nextinvpe[vpe]], reference)) { - smtc_nextinvpe[vpe] = cpu; - nextcomp = timestamp; - } - /* - * Since next-in-VPE may me the same as the executing - * virtual CPU, we update the array *after* checking - * its value. - */ - smtc_nexttime[vpe][cpu] = timestamp; - } - - /* - * It may be that, in fact, we don't need to update Compare, - * but if we do, we want to make sure we didn't fall into - * a crack just behind Count. - */ - if (ISVALID(nextcomp)) { - write_c0_compare(nextcomp); - ehb(); - /* - * We never return an error, we just make sure - * that we trigger the handlers as quickly as - * we can if we fell behind. - */ - while ((nextcomp - (unsigned long)read_c0_count()) - > (unsigned long)LONG_MAX) { - nextcomp += CATCHUP_INCREMENT; - write_c0_compare(nextcomp); - ehb(); - } - } - emt(mtflags); - local_irq_restore(flags); - return 0; -} - - -void smtc_distribute_timer(int vpe) -{ - unsigned long flags; - unsigned int mtflags; - int cpu; - struct clock_event_device *cd; - unsigned long nextstamp; - unsigned long reference; - - -repeat: - nextstamp = 0L; - for_each_online_cpu(cpu) { - /* - * Find virtual CPUs within the current VPE who have - * unserviced timer requests whose time is now past. - */ - local_irq_save(flags); - mtflags = dmt(); - if (cpu_data[cpu].vpe_id == vpe && - ISVALID(smtc_nexttime[vpe][cpu])) { - reference = (unsigned long)read_c0_count(); - if ((smtc_nexttime[vpe][cpu] - reference) - > (unsigned long)LONG_MAX) { - smtc_nexttime[vpe][cpu] = 0L; - emt(mtflags); - local_irq_restore(flags); - /* - * We don't send IPIs to ourself. - */ - if (cpu != smp_processor_id()) { - smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0); - } else { - cd = &per_cpu(mips_clockevent_device, cpu); - cd->event_handler(cd); - } - } else { - /* Local to VPE but Valid Time not yet reached. */ - if (!ISVALID(nextstamp) || - IS_SOONER(smtc_nexttime[vpe][cpu], nextstamp, - reference)) { - smtc_nextinvpe[vpe] = cpu; - nextstamp = smtc_nexttime[vpe][cpu]; - } - emt(mtflags); - local_irq_restore(flags); - } - } else { - emt(mtflags); - local_irq_restore(flags); - - } - } - /* Reprogram for interrupt at next soonest timestamp for VPE */ - if (ISVALID(nextstamp)) { - write_c0_compare(nextstamp); - ehb(); - if ((nextstamp - (unsigned long)read_c0_count()) - > (unsigned long)LONG_MAX) - goto repeat; - } -} - - -irqreturn_t c0_compare_interrupt(int irq, void *dev_id) -{ - int cpu = smp_processor_id(); - - /* If we're running SMTC, we've got MIPS MT and therefore MIPS32R2 */ - handle_perf_irq(1); - - if (read_c0_cause() & (1 << 30)) { - /* Clear Count/Compare Interrupt */ - write_c0_compare(read_c0_compare()); - smtc_distribute_timer(cpu_data[cpu].vpe_id); - } - return IRQ_HANDLED; -} - - -int __cpuinit smtc_clockevent_init(void) -{ - uint64_t mips_freq = mips_hpt_frequency; - unsigned int cpu = smp_processor_id(); - struct clock_event_device *cd; - unsigned int irq; - int i; - int j; - - if (!cpu_has_counter || !mips_hpt_frequency) - return -ENXIO; - if (cpu == 0) { - for (i = 0; i < num_possible_cpus(); i++) { - smtc_nextinvpe[i] = 0; - for (j = 0; j < num_possible_cpus(); j++) - smtc_nexttime[i][j] = 0L; - } - /* - * SMTC also can't have the usablility test - * run by secondary TCs once Compare is in use. - */ - if (!c0_compare_int_usable()) - return -ENXIO; - } - - /* - * With vectored interrupts things are getting platform specific. - * get_c0_compare_int is a hook to allow a platform to return the - * interrupt number of it's liking. - */ - irq = MIPS_CPU_IRQ_BASE + cp0_compare_irq; - if (get_c0_compare_int) - irq = get_c0_compare_int(); - - cd = &per_cpu(mips_clockevent_device, cpu); - - cd->name = "MIPS"; - cd->features = CLOCK_EVT_FEAT_ONESHOT; - - /* Calculate the min / max delta */ - cd->mult = div_sc((unsigned long) mips_freq, NSEC_PER_SEC, 32); - cd->shift = 32; - cd->max_delta_ns = clockevent_delta2ns(0x7fffffff, cd); - cd->min_delta_ns = clockevent_delta2ns(0x300, cd); - - cd->rating = 300; - cd->irq = irq; - cd->cpumask = cpumask_of(cpu); - cd->set_next_event = mips_next_event; - cd->set_mode = mips_set_clock_mode; - cd->event_handler = mips_event_handler; - - clockevents_register_device(cd); - - /* - * On SMTC we only want to do the data structure - * initialization and IRQ setup once. - */ - if (cpu) - return 0; - /* - * And we need the hwmask associated with the c0_compare - * vector to be initialized. - */ - irq_hwmask[irq] = (0x100 << cp0_compare_irq); - if (cp0_timer_irq_installed) - return 0; - - cp0_timer_irq_installed = 1; - - setup_irq(irq, &c0_compare_irqaction); - - return 0; -} diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c index 0b7377361e2..2ae08462e46 100644 --- a/arch/mips/kernel/cevt-txx9.c +++ b/arch/mips/kernel/cevt-txx9.c @@ -4,7 +4,7 @@ * for more details. * * Based on linux/arch/mips/kernel/cevt-r4k.c, - * linux/arch/mips/jmr3927/rbhma3100/setup.c + * linux/arch/mips/jmr3927/rbhma3100/setup.c * * Copyright 2001 MontaVista Software Inc. * Copyright (C) 2000-2001 Toshiba Corporation @@ -51,8 +51,7 @@ void __init txx9_clocksource_init(unsigned long baseaddr, { struct txx9_tmr_reg __iomem *tmrptr; - clocksource_set_clock(&txx9_clocksource.cs, TIMER_CLK(imbusclk)); - clocksource_register(&txx9_clocksource.cs); + clocksource_register_hz(&txx9_clocksource.cs, TIMER_CLK(imbusclk)); tmrptr = ioremap(baseaddr, sizeof(struct txx9_tmr_reg)); __raw_writel(TCR_BASE, &tmrptr->tcr); @@ -130,7 +129,7 @@ static struct txx9_clock_event_device txx9_clock_event_device = { CLOCK_EVT_FEAT_ONESHOT, .rating = 200, .set_mode = txx9tmr_set_mode, - .set_next_event = txx9tmr_set_next_event, + .set_next_event = txx9tmr_set_next_event, }, }; @@ -140,14 +139,14 @@ static irqreturn_t txx9tmr_interrupt(int irq, void *dev_id) struct clock_event_device *cd = &txx9_cd->cd; struct txx9_tmr_reg __iomem *tmrptr = txx9_cd->tmrptr; - __raw_writel(0, &tmrptr->tisr); /* ack interrupt */ + __raw_writel(0, &tmrptr->tisr); /* ack interrupt */ cd->event_handler(cd); return IRQ_HANDLED; } static struct irqaction txx9tmr_irq = { .handler = txx9tmr_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU | IRQF_TIMER, + .flags = IRQF_PERCPU | IRQF_TIMER, .name = "txx9tmr", .dev_id = &txx9_clock_event_device, }; diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S new file mode 100644 index 00000000000..6f4f739dad9 --- /dev/null +++ b/arch/mips/kernel/cps-vec.S @@ -0,0 +1,487 @@ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <asm/addrspace.h> +#include <asm/asm.h> +#include <asm/asm-offsets.h> +#include <asm/asmmacro.h> +#include <asm/cacheops.h> +#include <asm/mipsregs.h> +#include <asm/mipsmtregs.h> +#include <asm/pm.h> + +#define GCR_CL_COHERENCE_OFS 0x2008 +#define GCR_CL_ID_OFS 0x2028 + +.extern mips_cm_base + +.set noreorder + + /* + * Set dest to non-zero if the core supports the MT ASE, else zero. If + * MT is not supported then branch to nomt. + */ + .macro has_mt dest, nomt + mfc0 \dest, CP0_CONFIG + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 1 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 2 + bgez \dest, \nomt + mfc0 \dest, CP0_CONFIG, 3 + andi \dest, \dest, MIPS_CONF3_MT + beqz \dest, \nomt + .endm + +.section .text.cps-vec +.balign 0x1000 + +LEAF(mips_cps_core_entry) + /* + * These first 12 bytes will be patched by cps_smp_setup to load the + * base address of the CM GCRs into register v1 and the CCA to use into + * register s0. + */ + .quad 0 + .word 0 + + /* Check whether we're here due to an NMI */ + mfc0 k0, CP0_STATUS + and k0, k0, ST0_NMI + beqz k0, not_nmi + nop + + /* This is an NMI */ + la k0, nmi_handler + jr k0 + nop + +not_nmi: + /* Setup Cause */ + li t0, CAUSEF_IV + mtc0 t0, CP0_CAUSE + + /* Setup Status */ + li t0, ST0_CU1 | ST0_CU0 + mtc0 t0, CP0_STATUS + + /* + * Clear the bits used to index the caches. Note that the architecture + * dictates that writing to any of TagLo or TagHi selects 0 or 2 should + * be valid for all MIPS32 CPUs, even those for which said writes are + * unnecessary. + */ + mtc0 zero, CP0_TAGLO, 0 + mtc0 zero, CP0_TAGHI, 0 + mtc0 zero, CP0_TAGLO, 2 + mtc0 zero, CP0_TAGHI, 2 + ehb + + /* Primary cache configuration is indicated by Config1 */ + mfc0 v0, CP0_CONFIG, 1 + + /* Detect I-cache line size */ + _EXT t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ + beqz t0, icache_done + li t1, 2 + sllv t0, t1, t0 + + /* Detect I-cache size */ + _EXT t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ + xori t2, t1, 0x7 + beqz t2, 1f + li t3, 32 + addi t1, t1, 1 + sllv t1, t3, t1 +1: /* At this point t1 == I-cache sets per way */ + _EXT t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ + addi t2, t2, 1 + mul t1, t1, t0 + mul t1, t1, t2 + + li a0, KSEG0 + add a1, a0, t1 +1: cache Index_Store_Tag_I, 0(a0) + add a0, a0, t0 + bne a0, a1, 1b + nop +icache_done: + + /* Detect D-cache line size */ + _EXT t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ + beqz t0, dcache_done + li t1, 2 + sllv t0, t1, t0 + + /* Detect D-cache size */ + _EXT t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ + xori t2, t1, 0x7 + beqz t2, 1f + li t3, 32 + addi t1, t1, 1 + sllv t1, t3, t1 +1: /* At this point t1 == D-cache sets per way */ + _EXT t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ + addi t2, t2, 1 + mul t1, t1, t0 + mul t1, t1, t2 + + li a0, KSEG0 + addu a1, a0, t1 + subu a1, a1, t0 +1: cache Index_Store_Tag_D, 0(a0) + bne a0, a1, 1b + add a0, a0, t0 +dcache_done: + + /* Set Kseg0 CCA to that in s0 */ + mfc0 t0, CP0_CONFIG + ori t0, 0x7 + xori t0, 0x7 + or t0, t0, s0 + mtc0 t0, CP0_CONFIG + ehb + + /* Enter the coherent domain */ + li t0, 0xff + sw t0, GCR_CL_COHERENCE_OFS(v1) + ehb + + /* Jump to kseg0 */ + la t0, 1f + jr t0 + nop + + /* + * We're up, cached & coherent. Perform any further required core-level + * initialisation. + */ +1: jal mips_cps_core_init + nop + + /* + * Boot any other VPEs within this core that should be online, and + * deactivate this VPE if it should be offline. + */ + jal mips_cps_boot_vpes + nop + + /* Off we go! */ + lw t1, VPEBOOTCFG_PC(v0) + lw gp, VPEBOOTCFG_GP(v0) + lw sp, VPEBOOTCFG_SP(v0) + jr t1 + nop + END(mips_cps_core_entry) + +.org 0x200 +LEAF(excep_tlbfill) + b . + nop + END(excep_tlbfill) + +.org 0x280 +LEAF(excep_xtlbfill) + b . + nop + END(excep_xtlbfill) + +.org 0x300 +LEAF(excep_cache) + b . + nop + END(excep_cache) + +.org 0x380 +LEAF(excep_genex) + b . + nop + END(excep_genex) + +.org 0x400 +LEAF(excep_intex) + b . + nop + END(excep_intex) + +.org 0x480 +LEAF(excep_ejtag) + la k0, ejtag_debug_handler + jr k0 + nop + END(excep_ejtag) + +LEAF(mips_cps_core_init) +#ifdef CONFIG_MIPS_MT + /* Check that the core implements the MT ASE */ + has_mt t0, 3f + nop + + .set push + .set mt + + /* Only allow 1 TC per VPE to execute... */ + dmt + + /* ...and for the moment only 1 VPE */ + dvpe + la t1, 1f + jr.hb t1 + nop + + /* Enter VPE configuration state */ +1: mfc0 t0, CP0_MVPCONTROL + ori t0, t0, MVPCONTROL_VPC + mtc0 t0, CP0_MVPCONTROL + + /* Retrieve the number of VPEs within the core */ + mfc0 t0, CP0_MVPCONF0 + srl t0, t0, MVPCONF0_PVPE_SHIFT + andi t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT) + addi t7, t0, 1 + + /* If there's only 1, we're done */ + beqz t0, 2f + nop + + /* Loop through each VPE within this core */ + li t5, 1 + +1: /* Operate on the appropriate TC */ + mtc0 t5, CP0_VPECONTROL + ehb + + /* Bind TC to VPE (1:1 TC:VPE mapping) */ + mttc0 t5, CP0_TCBIND + + /* Set exclusive TC, non-active, master */ + li t0, VPECONF0_MVP + sll t1, t5, VPECONF0_XTC_SHIFT + or t0, t0, t1 + mttc0 t0, CP0_VPECONF0 + + /* Set TC non-active, non-allocatable */ + mttc0 zero, CP0_TCSTATUS + + /* Set TC halted */ + li t0, TCHALT_H + mttc0 t0, CP0_TCHALT + + /* Next VPE */ + addi t5, t5, 1 + slt t0, t5, t7 + bnez t0, 1b + nop + + /* Leave VPE configuration state */ +2: mfc0 t0, CP0_MVPCONTROL + xori t0, t0, MVPCONTROL_VPC + mtc0 t0, CP0_MVPCONTROL + +3: .set pop +#endif + jr ra + nop + END(mips_cps_core_init) + +LEAF(mips_cps_boot_vpes) + /* Retrieve CM base address */ + la t0, mips_cm_base + lw t0, 0(t0) + + /* Calculate a pointer to this cores struct core_boot_config */ + lw t0, GCR_CL_ID_OFS(t0) + li t1, COREBOOTCFG_SIZE + mul t0, t0, t1 + la t1, mips_cps_core_bootcfg + lw t1, 0(t1) + addu t0, t0, t1 + + /* Calculate this VPEs ID. If the core doesn't support MT use 0 */ + has_mt t6, 1f + li t9, 0 + + /* Find the number of VPEs present in the core */ + mfc0 t1, CP0_MVPCONF0 + srl t1, t1, MVPCONF0_PVPE_SHIFT + andi t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT + addi t1, t1, 1 + + /* Calculate a mask for the VPE ID from EBase.CPUNum */ + clz t1, t1 + li t2, 31 + subu t1, t2, t1 + li t2, 1 + sll t1, t2, t1 + addiu t1, t1, -1 + + /* Retrieve the VPE ID from EBase.CPUNum */ + mfc0 t9, $15, 1 + and t9, t9, t1 + +1: /* Calculate a pointer to this VPEs struct vpe_boot_config */ + li t1, VPEBOOTCFG_SIZE + mul v0, t9, t1 + lw t7, COREBOOTCFG_VPECONFIG(t0) + addu v0, v0, t7 + +#ifdef CONFIG_MIPS_MT + + /* If the core doesn't support MT then return */ + bnez t6, 1f + nop + jr ra + nop + + .set push + .set mt + +1: /* Enter VPE configuration state */ + dvpe + la t1, 1f + jr.hb t1 + nop +1: mfc0 t1, CP0_MVPCONTROL + ori t1, t1, MVPCONTROL_VPC + mtc0 t1, CP0_MVPCONTROL + ehb + + /* Loop through each VPE */ + lw t6, COREBOOTCFG_VPEMASK(t0) + move t8, t6 + li t5, 0 + + /* Check whether the VPE should be running. If not, skip it */ +1: andi t0, t6, 1 + beqz t0, 2f + nop + + /* Operate on the appropriate TC */ + mfc0 t0, CP0_VPECONTROL + ori t0, t0, VPECONTROL_TARGTC + xori t0, t0, VPECONTROL_TARGTC + or t0, t0, t5 + mtc0 t0, CP0_VPECONTROL + ehb + + /* Skip the VPE if its TC is not halted */ + mftc0 t0, CP0_TCHALT + beqz t0, 2f + nop + + /* Calculate a pointer to the VPEs struct vpe_boot_config */ + li t0, VPEBOOTCFG_SIZE + mul t0, t0, t5 + addu t0, t0, t7 + + /* Set the TC restart PC */ + lw t1, VPEBOOTCFG_PC(t0) + mttc0 t1, CP0_TCRESTART + + /* Set the TC stack pointer */ + lw t1, VPEBOOTCFG_SP(t0) + mttgpr t1, sp + + /* Set the TC global pointer */ + lw t1, VPEBOOTCFG_GP(t0) + mttgpr t1, gp + + /* Copy config from this VPE */ + mfc0 t0, CP0_CONFIG + mttc0 t0, CP0_CONFIG + + /* Ensure no software interrupts are pending */ + mttc0 zero, CP0_CAUSE + mttc0 zero, CP0_STATUS + + /* Set TC active, not interrupt exempt */ + mftc0 t0, CP0_TCSTATUS + li t1, ~TCSTATUS_IXMT + and t0, t0, t1 + ori t0, t0, TCSTATUS_A + mttc0 t0, CP0_TCSTATUS + + /* Clear the TC halt bit */ + mttc0 zero, CP0_TCHALT + + /* Set VPE active */ + mftc0 t0, CP0_VPECONF0 + ori t0, t0, VPECONF0_VPA + mttc0 t0, CP0_VPECONF0 + + /* Next VPE */ +2: srl t6, t6, 1 + addi t5, t5, 1 + bnez t6, 1b + nop + + /* Leave VPE configuration state */ + mfc0 t1, CP0_MVPCONTROL + xori t1, t1, MVPCONTROL_VPC + mtc0 t1, CP0_MVPCONTROL + ehb + evpe + + /* Check whether this VPE is meant to be running */ + li t0, 1 + sll t0, t0, t9 + and t0, t0, t8 + bnez t0, 2f + nop + + /* This VPE should be offline, halt the TC */ + li t0, TCHALT_H + mtc0 t0, CP0_TCHALT + la t0, 1f +1: jr.hb t0 + nop + +2: .set pop + +#endif /* CONFIG_MIPS_MT */ + + /* Return */ + jr ra + nop + END(mips_cps_boot_vpes) + +#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM) + + /* Calculate a pointer to this CPUs struct mips_static_suspend_state */ + .macro psstate dest + .set push + .set noat + lw $1, TI_CPU(gp) + sll $1, $1, LONGLOG + la \dest, __per_cpu_offset + addu $1, $1, \dest + lw $1, 0($1) + la \dest, cps_cpu_state + addu \dest, \dest, $1 + .set pop + .endm + +LEAF(mips_cps_pm_save) + /* Save CPU state */ + SUSPEND_SAVE_REGS + psstate t1 + SUSPEND_SAVE_STATIC + jr v0 + nop + END(mips_cps_pm_save) + +LEAF(mips_cps_pm_restore) + /* Restore CPU state */ + psstate t1 + RESUME_RESTORE_STATIC + RESUME_RESTORE_REGS_RETURN + END(mips_cps_pm_restore) + +#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */ diff --git a/arch/mips/kernel/cpu-bugs64.c b/arch/mips/kernel/cpu-bugs64.c index b8bb8ba6086..2d80b5f1aea 100644 --- a/arch/mips/kernel/cpu-bugs64.c +++ b/arch/mips/kernel/cpu-bugs64.c @@ -6,6 +6,7 @@ * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ +#include <linux/context_tracking.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/ptrace.h> @@ -16,7 +17,7 @@ #include <asm/cpu.h> #include <asm/fpu.h> #include <asm/mipsregs.h> -#include <asm/system.h> +#include <asm/setup.h> static char bug64hit[] __initdata = "reliable operation impossible!\n%s"; @@ -73,7 +74,7 @@ static inline void mult_sh_align_mod(long *v1, long *v2, long *w, : "0" (5), "1" (8), "2" (5)); align_mod(align, mod); /* - * The trailing nop is needed to fullfill the two-instruction + * The trailing nop is needed to fulfill the two-instruction * requirement between reading hi/lo and staring a mult/div. * Leaving it out may cause gas insert a nop itself breaking * the desired alignment of the next chunk. @@ -84,9 +85,9 @@ static inline void mult_sh_align_mod(long *v1, long *v2, long *w, ".set noreorder\n\t" ".set nomacro\n\t" "mult %2, %3\n\t" - "dsll32 %0, %4, %5\n\t" + "dsll32 %0, %4, %5\n\t" "mflo $0\n\t" - "dsll32 %1, %4, %5\n\t" + "dsll32 %1, %4, %5\n\t" "nop\n\t" ".set pop" : "=&r" (lv1), "=r" (lw) @@ -167,12 +168,16 @@ static inline void check_mult_sh(void) panic(bug64hit, !R4000_WAR ? r4kwar : nowar); } -static volatile int daddi_ov __cpuinitdata; +static volatile int daddi_ov; asmlinkage void __init do_daddi_ov(struct pt_regs *regs) { + enum ctx_state prev_state; + + prev_state = exception_enter(); daddi_ov = 1; regs->cp0_epc += 4; + exception_exit(prev_state); } static inline void check_daddi(void) @@ -239,7 +244,7 @@ static inline void check_daddi(void) panic(bug64hit, !DADDI_WAR ? daddiwar : nowar); } -int daddiu_bug = -1; +int daddiu_bug = -1; static inline void check_daddiu(void) { @@ -273,7 +278,7 @@ static inline void check_daddiu(void) #ifdef HAVE_AS_SET_DADDI ".set daddi\n\t" #endif - "daddiu %0, %2, %4\n\t" + "daddiu %0, %2, %4\n\t" "addiu %1, $0, %4\n\t" "daddu %1, %2\n\t" ".set pop" @@ -292,7 +297,7 @@ static inline void check_daddiu(void) asm volatile( "addiu %2, $0, %3\n\t" "dsrl %2, %2, 1\n\t" - "daddiu %0, %2, %4\n\t" + "daddiu %0, %2, %4\n\t" "addiu %1, $0, %4\n\t" "daddu %1, %2" : "=&r" (v), "=&r" (w), "=&r" (tmp) diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 71620e19827..d74f957c561 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -4,7 +4,7 @@ * Copyright (C) xxxx the Anonymous * Copyright (C) 1994 - 2006 Ralf Baechle * Copyright (C) 2003, 2004 Maciej W. Rozycki - * Copyright (C) 2001, 2004 MIPS Inc. + * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,118 +16,21 @@ #include <linux/ptrace.h> #include <linux/smp.h> #include <linux/stddef.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/bugs.h> #include <asm/cpu.h> +#include <asm/cpu-type.h> #include <asm/fpu.h> #include <asm/mipsregs.h> -#include <asm/system.h> +#include <asm/mipsmtregs.h> +#include <asm/msa.h> #include <asm/watch.h> +#include <asm/elf.h> #include <asm/spram.h> #include <asm/uaccess.h> -/* - * Not all of the MIPS CPUs have the "wait" instruction available. Moreover, - * the implementation of the "wait" feature differs between CPU families. This - * points to the function that implements CPU specific wait. - * The wait instruction stops the pipeline and reduces the power consumption of - * the CPU very much. - */ -void (*cpu_wait)(void); -EXPORT_SYMBOL(cpu_wait); - -static void r3081_wait(void) -{ - unsigned long cfg = read_c0_conf(); - write_c0_conf(cfg | R30XX_CONF_HALT); -} - -static void r39xx_wait(void) -{ - local_irq_disable(); - if (!need_resched()) - write_c0_conf(read_c0_conf() | TX39_CONF_HALT); - local_irq_enable(); -} - -extern void r4k_wait(void); - -/* - * This variant is preferable as it allows testing need_resched and going to - * sleep depending on the outcome atomically. Unfortunately the "It is - * implementation-dependent whether the pipeline restarts when a non-enabled - * interrupt is requested" restriction in the MIPS32/MIPS64 architecture makes - * using this version a gamble. - */ -void r4k_wait_irqoff(void) -{ - local_irq_disable(); - if (!need_resched()) - __asm__(" .set push \n" - " .set mips3 \n" - " wait \n" - " .set pop \n"); - local_irq_enable(); - __asm__(" .globl __pastwait \n" - "__pastwait: \n"); - return; -} - -/* - * The RM7000 variant has to handle erratum 38. The workaround is to not - * have any pending stores when the WAIT instruction is executed. - */ -static void rm7k_wait_irqoff(void) -{ - local_irq_disable(); - if (!need_resched()) - __asm__( - " .set push \n" - " .set mips3 \n" - " .set noat \n" - " mfc0 $1, $12 \n" - " sync \n" - " mtc0 $1, $12 # stalls until W stage \n" - " wait \n" - " mtc0 $1, $12 # stalls until W stage \n" - " .set pop \n"); - local_irq_enable(); -} - -/* - * The Au1xxx wait is available only if using 32khz counter or - * external timer source, but specifically not CP0 Counter. - * alchemy/common/time.c may override cpu_wait! - */ -static void au1k_wait(void) -{ - __asm__(" .set mips3 \n" - " cache 0x14, 0(%0) \n" - " cache 0x14, 32(%0) \n" - " sync \n" - " nop \n" - " wait \n" - " nop \n" - " nop \n" - " nop \n" - " nop \n" - " .set mips0 \n" - : : "r" (au1k_wait)); -} - -static int __initdata nowait; - -static int __init wait_disable(char *s) -{ - nowait = 1; - - return 1; -} - -__setup("nowait", wait_disable); - -static int __cpuinitdata mips_fpu_disabled; +static int mips_fpu_disabled; static int __init fpu_disable(char *s) { @@ -139,11 +42,11 @@ static int __init fpu_disable(char *s) __setup("nofpu", fpu_disable); -int __cpuinitdata mips_dsp_disabled; +int mips_dsp_disabled; static int __init dsp_disable(char *s) { - cpu_data[0].ases &= ~MIPS_ASE_DSP; + cpu_data[0].ases &= ~(MIPS_ASE_DSP | MIPS_ASE_DSP2P); mips_dsp_disabled = 1; return 1; @@ -151,109 +54,15 @@ static int __init dsp_disable(char *s) __setup("nodsp", dsp_disable); -void __init check_wait(void) -{ - struct cpuinfo_mips *c = ¤t_cpu_data; - - if (nowait) { - printk("Wait instruction disabled.\n"); - return; - } - - switch (c->cputype) { - case CPU_R3081: - case CPU_R3081E: - cpu_wait = r3081_wait; - break; - case CPU_TX3927: - cpu_wait = r39xx_wait; - break; - case CPU_R4200: -/* case CPU_R4300: */ - case CPU_R4600: - case CPU_R4640: - case CPU_R4650: - case CPU_R4700: - case CPU_R5000: - case CPU_R5500: - case CPU_NEVADA: - case CPU_4KC: - case CPU_4KEC: - case CPU_4KSC: - case CPU_5KC: - case CPU_25KF: - case CPU_PR4450: - case CPU_BMIPS3300: - case CPU_BMIPS4350: - case CPU_BMIPS4380: - case CPU_BMIPS5000: - case CPU_CAVIUM_OCTEON: - case CPU_CAVIUM_OCTEON_PLUS: - case CPU_CAVIUM_OCTEON2: - case CPU_JZRISC: - cpu_wait = r4k_wait; - break; - - case CPU_RM7000: - cpu_wait = rm7k_wait_irqoff; - break; - - case CPU_24K: - case CPU_34K: - case CPU_1004K: - cpu_wait = r4k_wait; - if (read_c0_config7() & MIPS_CONF7_WII) - cpu_wait = r4k_wait_irqoff; - break; - - case CPU_74K: - cpu_wait = r4k_wait; - if ((c->processor_id & 0xff) >= PRID_REV_ENCODE_332(2, 1, 0)) - cpu_wait = r4k_wait_irqoff; - break; - - case CPU_TX49XX: - cpu_wait = r4k_wait_irqoff; - break; - case CPU_ALCHEMY: - cpu_wait = au1k_wait; - break; - case CPU_20KC: - /* - * WAIT on Rev1.0 has E1, E2, E3 and E16. - * WAIT on Rev2.0 and Rev3.0 has E16. - * Rev3.1 WAIT is nop, why bother - */ - if ((c->processor_id & 0xff) <= 0x64) - break; - - /* - * Another rev is incremeting c0_count at a reduced clock - * rate while in WAIT mode. So we basically have the choice - * between using the cp0 timer as clocksource or avoiding - * the WAIT instruction. Until more details are known, - * disable the use of WAIT for 20Kc entirely. - cpu_wait = r4k_wait; - */ - break; - case CPU_RM9000: - if ((c->processor_id & 0x00ff) >= 0x40) - cpu_wait = r4k_wait; - break; - default: - break; - } -} - static inline void check_errata(void) { struct cpuinfo_mips *c = ¤t_cpu_data; - switch (c->cputype) { + switch (current_cpu_type()) { case CPU_34K: /* * Erratum "RPS May Cause Incorrect Instruction Execution" - * This code only handles VPE0, any SMP/SMTC/RTOS code + * This code only handles VPE0, any SMP/RTOS code * making use of VPE1 will be responsable for that VPE. */ if ((c->processor_id & PRID_REV_MASK) <= PRID_REV_34K_V1_0_2) @@ -291,6 +100,12 @@ static inline int cpu_has_confreg(void) #endif } +static inline void set_elf_platform(int cpu, const char *plat) +{ + if (cpu == 0) + __elf_platform = plat; +} + /* * Get the FPU Implementation/Revision. */ @@ -299,7 +114,7 @@ static inline unsigned long cpu_get_fpu_id(void) unsigned long tmp, fpu_id; tmp = read_c0_status(); - __enable_fpu(); + __enable_fpu(FPU_AS_IS); fpu_id = read_32bit_cp1_register(CP1_REVISION); write_c0_status(tmp); return fpu_id; @@ -310,7 +125,21 @@ static inline unsigned long cpu_get_fpu_id(void) */ static inline int __cpu_has_fpu(void) { - return ((cpu_get_fpu_id() & 0xff00) != FPIR_IMP_NONE); + return ((cpu_get_fpu_id() & FPIR_IMP_MASK) != FPIR_IMP_NONE); +} + +static inline unsigned long cpu_get_msa_id(void) +{ + unsigned long status, conf5, msa_id; + + status = read_c0_status(); + __enable_fpu(FPU_64BIT); + conf5 = read_c0_config5(); + enable_msa(); + msa_id = read_msa_ir(); + write_c0_config5(conf5); + write_c0_status(status); + return msa_id; } static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) @@ -322,24 +151,302 @@ static inline void cpu_probe_vmbits(struct cpuinfo_mips *c) #endif } +static void set_isa(struct cpuinfo_mips *c, unsigned int isa) +{ + switch (isa) { + case MIPS_CPU_ISA_M64R2: + c->isa_level |= MIPS_CPU_ISA_M32R2 | MIPS_CPU_ISA_M64R2; + case MIPS_CPU_ISA_M64R1: + c->isa_level |= MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M64R1; + case MIPS_CPU_ISA_V: + c->isa_level |= MIPS_CPU_ISA_V; + case MIPS_CPU_ISA_IV: + c->isa_level |= MIPS_CPU_ISA_IV; + case MIPS_CPU_ISA_III: + c->isa_level |= MIPS_CPU_ISA_II | MIPS_CPU_ISA_III; + break; + + case MIPS_CPU_ISA_M32R2: + c->isa_level |= MIPS_CPU_ISA_M32R2; + case MIPS_CPU_ISA_M32R1: + c->isa_level |= MIPS_CPU_ISA_M32R1; + case MIPS_CPU_ISA_II: + c->isa_level |= MIPS_CPU_ISA_II; + break; + } +} + +static char unknown_isa[] = KERN_ERR \ + "Unsupported ISA type, c0.config0: %d."; + +static void set_ftlb_enable(struct cpuinfo_mips *c, int enable) +{ + unsigned int config6; + + /* It's implementation dependent how the FTLB can be enabled */ + switch (c->cputype) { + case CPU_PROAPTIV: + case CPU_P5600: + /* proAptiv & related cores use Config6 to enable the FTLB */ + config6 = read_c0_config6(); + if (enable) + /* Enable FTLB */ + write_c0_config6(config6 | MIPS_CONF6_FTLBEN); + else + /* Disable FTLB */ + write_c0_config6(config6 & ~MIPS_CONF6_FTLBEN); + back_to_back_c0_hazard(); + break; + } +} + +static inline unsigned int decode_config0(struct cpuinfo_mips *c) +{ + unsigned int config0; + int isa; + + config0 = read_c0_config(); + + /* + * Look for Standard TLB or Dual VTLB and FTLB + */ + if ((((config0 & MIPS_CONF_MT) >> 7) == 1) || + (((config0 & MIPS_CONF_MT) >> 7) == 4)) + c->options |= MIPS_CPU_TLB; + + isa = (config0 & MIPS_CONF_AT) >> 13; + switch (isa) { + case 0: + switch ((config0 & MIPS_CONF_AR) >> 10) { + case 0: + set_isa(c, MIPS_CPU_ISA_M32R1); + break; + case 1: + set_isa(c, MIPS_CPU_ISA_M32R2); + break; + default: + goto unknown; + } + break; + case 2: + switch ((config0 & MIPS_CONF_AR) >> 10) { + case 0: + set_isa(c, MIPS_CPU_ISA_M64R1); + break; + case 1: + set_isa(c, MIPS_CPU_ISA_M64R2); + break; + default: + goto unknown; + } + break; + default: + goto unknown; + } + + return config0 & MIPS_CONF_M; + +unknown: + panic(unknown_isa, config0); +} + +static inline unsigned int decode_config1(struct cpuinfo_mips *c) +{ + unsigned int config1; + + config1 = read_c0_config1(); + + if (config1 & MIPS_CONF1_MD) + c->ases |= MIPS_ASE_MDMX; + if (config1 & MIPS_CONF1_WR) + c->options |= MIPS_CPU_WATCH; + if (config1 & MIPS_CONF1_CA) + c->ases |= MIPS_ASE_MIPS16; + if (config1 & MIPS_CONF1_EP) + c->options |= MIPS_CPU_EJTAG; + if (config1 & MIPS_CONF1_FP) { + c->options |= MIPS_CPU_FPU; + c->options |= MIPS_CPU_32FPR; + } + if (cpu_has_tlb) { + c->tlbsize = ((config1 & MIPS_CONF1_TLBS) >> 25) + 1; + c->tlbsizevtlb = c->tlbsize; + c->tlbsizeftlbsets = 0; + } + + return config1 & MIPS_CONF_M; +} + +static inline unsigned int decode_config2(struct cpuinfo_mips *c) +{ + unsigned int config2; + + config2 = read_c0_config2(); + + if (config2 & MIPS_CONF2_SL) + c->scache.flags &= ~MIPS_CACHE_NOT_PRESENT; + + return config2 & MIPS_CONF_M; +} + +static inline unsigned int decode_config3(struct cpuinfo_mips *c) +{ + unsigned int config3; + + config3 = read_c0_config3(); + + if (config3 & MIPS_CONF3_SM) { + c->ases |= MIPS_ASE_SMARTMIPS; + c->options |= MIPS_CPU_RIXI; + } + if (config3 & MIPS_CONF3_RXI) + c->options |= MIPS_CPU_RIXI; + if (config3 & MIPS_CONF3_DSP) + c->ases |= MIPS_ASE_DSP; + if (config3 & MIPS_CONF3_DSP2P) + c->ases |= MIPS_ASE_DSP2P; + if (config3 & MIPS_CONF3_VINT) + c->options |= MIPS_CPU_VINT; + if (config3 & MIPS_CONF3_VEIC) + c->options |= MIPS_CPU_VEIC; + if (config3 & MIPS_CONF3_MT) + c->ases |= MIPS_ASE_MIPSMT; + if (config3 & MIPS_CONF3_ULRI) + c->options |= MIPS_CPU_ULRI; + if (config3 & MIPS_CONF3_ISA) + c->options |= MIPS_CPU_MICROMIPS; + if (config3 & MIPS_CONF3_VZ) + c->ases |= MIPS_ASE_VZ; + if (config3 & MIPS_CONF3_SC) + c->options |= MIPS_CPU_SEGMENTS; + if (config3 & MIPS_CONF3_MSA) + c->ases |= MIPS_ASE_MSA; + + return config3 & MIPS_CONF_M; +} + +static inline unsigned int decode_config4(struct cpuinfo_mips *c) +{ + unsigned int config4; + unsigned int newcf4; + unsigned int mmuextdef; + unsigned int ftlb_page = MIPS_CONF4_FTLBPAGESIZE; + + config4 = read_c0_config4(); + + if (cpu_has_tlb) { + if (((config4 & MIPS_CONF4_IE) >> 29) == 2) + c->options |= MIPS_CPU_TLBINV; + mmuextdef = config4 & MIPS_CONF4_MMUEXTDEF; + switch (mmuextdef) { + case MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT: + c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40; + c->tlbsizevtlb = c->tlbsize; + break; + case MIPS_CONF4_MMUEXTDEF_VTLBSIZEEXT: + c->tlbsizevtlb += + ((config4 & MIPS_CONF4_VTLBSIZEEXT) >> + MIPS_CONF4_VTLBSIZEEXT_SHIFT) * 0x40; + c->tlbsize = c->tlbsizevtlb; + ftlb_page = MIPS_CONF4_VFTLBPAGESIZE; + /* fall through */ + case MIPS_CONF4_MMUEXTDEF_FTLBSIZEEXT: + newcf4 = (config4 & ~ftlb_page) | + (page_size_ftlb(mmuextdef) << + MIPS_CONF4_FTLBPAGESIZE_SHIFT); + write_c0_config4(newcf4); + back_to_back_c0_hazard(); + config4 = read_c0_config4(); + if (config4 != newcf4) { + pr_err("PAGE_SIZE 0x%lx is not supported by FTLB (config4=0x%x)\n", + PAGE_SIZE, config4); + /* Switch FTLB off */ + set_ftlb_enable(c, 0); + break; + } + c->tlbsizeftlbsets = 1 << + ((config4 & MIPS_CONF4_FTLBSETS) >> + MIPS_CONF4_FTLBSETS_SHIFT); + c->tlbsizeftlbways = ((config4 & MIPS_CONF4_FTLBWAYS) >> + MIPS_CONF4_FTLBWAYS_SHIFT) + 2; + c->tlbsize += c->tlbsizeftlbways * c->tlbsizeftlbsets; + break; + } + } + + c->kscratch_mask = (config4 >> 16) & 0xff; + + return config4 & MIPS_CONF_M; +} + +static inline unsigned int decode_config5(struct cpuinfo_mips *c) +{ + unsigned int config5; + + config5 = read_c0_config5(); + config5 &= ~MIPS_CONF5_UFR; + write_c0_config5(config5); + + if (config5 & MIPS_CONF5_EVA) + c->options |= MIPS_CPU_EVA; + + return config5 & MIPS_CONF_M; +} + +static void decode_configs(struct cpuinfo_mips *c) +{ + int ok; + + /* MIPS32 or MIPS64 compliant CPU. */ + c->options = MIPS_CPU_4KEX | MIPS_CPU_4K_CACHE | MIPS_CPU_COUNTER | + MIPS_CPU_DIVEC | MIPS_CPU_LLSC | MIPS_CPU_MCHECK; + + c->scache.flags = MIPS_CACHE_NOT_PRESENT; + + /* Enable FTLB if present */ + set_ftlb_enable(c, 1); + + ok = decode_config0(c); /* Read Config registers. */ + BUG_ON(!ok); /* Arch spec violation! */ + if (ok) + ok = decode_config1(c); + if (ok) + ok = decode_config2(c); + if (ok) + ok = decode_config3(c); + if (ok) + ok = decode_config4(c); + if (ok) + ok = decode_config5(c); + + mips_probe_watch_registers(c); + +#ifndef CONFIG_MIPS_CPS + if (cpu_has_mips_r2) { + c->core = get_ebase_cpunum(); + if (cpu_has_mipsmt) + c->core >>= fls(core_nvpes()) - 1; + } +#endif +} + #define R4K_OPTS (MIPS_CPU_TLB | MIPS_CPU_4KEX | MIPS_CPU_4K_CACHE \ | MIPS_CPU_COUNTER) static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) { - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_R2000: c->cputype = CPU_R2000; __cpu_name[cpu] = "R2000"; - c->isa_level = MIPS_CPU_ISA_I; c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | - MIPS_CPU_NOFPUEX; + MIPS_CPU_NOFPUEX; if (__cpu_has_fpu()) c->options |= MIPS_CPU_FPU; c->tlbsize = 64; break; case PRID_IMP_R3000: - if ((c->processor_id & 0xff) == PRID_REV_R3000A) { + if ((c->processor_id & PRID_REV_MASK) == PRID_REV_R3000A) { if (cpu_has_confreg()) { c->cputype = CPU_R3081E; __cpu_name[cpu] = "R3081"; @@ -347,21 +454,20 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_R3000A; __cpu_name[cpu] = "R3000A"; } - break; } else { c->cputype = CPU_R3000; __cpu_name[cpu] = "R3000"; } - c->isa_level = MIPS_CPU_ISA_I; c->options = MIPS_CPU_TLB | MIPS_CPU_3K_CACHE | - MIPS_CPU_NOFPUEX; + MIPS_CPU_NOFPUEX; if (__cpu_has_fpu()) c->options |= MIPS_CPU_FPU; c->tlbsize = 64; break; case PRID_IMP_R4000: if (read_c0_config() & CONF_SC) { - if ((c->processor_id & 0xff) >= PRID_REV_R4400) { + if ((c->processor_id & PRID_REV_MASK) >= + PRID_REV_R4400) { c->cputype = CPU_R4400PC; __cpu_name[cpu] = "R4400PC"; } else { @@ -369,22 +475,46 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "R4000PC"; } } else { - if ((c->processor_id & 0xff) >= PRID_REV_R4400) { - c->cputype = CPU_R4400SC; - __cpu_name[cpu] = "R4400SC"; + int cca = read_c0_config() & CONF_CM_CMASK; + int mc; + + /* + * SC and MC versions can't be reliably told apart, + * but only the latter support coherent caching + * modes so assume the firmware has set the KSEG0 + * coherency attribute reasonably (if uncached, we + * assume SC). + */ + switch (cca) { + case CONF_CM_CACHABLE_CE: + case CONF_CM_CACHABLE_COW: + case CONF_CM_CACHABLE_CUW: + mc = 1; + break; + default: + mc = 0; + break; + } + if ((c->processor_id & PRID_REV_MASK) >= + PRID_REV_R4400) { + c->cputype = mc ? CPU_R4400MC : CPU_R4400SC; + __cpu_name[cpu] = mc ? "R4400MC" : "R4400SC"; } else { - c->cputype = CPU_R4000SC; - __cpu_name[cpu] = "R4000SC"; + c->cputype = mc ? CPU_R4000MC : CPU_R4000SC; + __cpu_name[cpu] = mc ? "R4000MC" : "R4000SC"; } } - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_WATCH | MIPS_CPU_VCE | - MIPS_CPU_LLSC; + MIPS_CPU_WATCH | MIPS_CPU_VCE | + MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_VR41XX: + set_isa(c, MIPS_CPU_ISA_III); + c->options = R4K_OPTS; + c->tlbsize = 32; switch (c->processor_id & 0xf0) { case PRID_REV_VR4111: c->cputype = CPU_VR4111; @@ -409,6 +539,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "NEC VR4131"; } else { c->cputype = CPU_VR4133; + c->options |= MIPS_CPU_LLSC; __cpu_name[cpu] = "NEC VR4133"; } break; @@ -418,28 +549,25 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "NEC Vr41xx"; break; } - c->isa_level = MIPS_CPU_ISA_III; - c->options = R4K_OPTS; - c->tlbsize = 32; break; case PRID_IMP_R4300: c->cputype = CPU_R4300; __cpu_name[cpu] = "R4300"; - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 32; break; case PRID_IMP_R4600: c->cputype = CPU_R4600; __cpu_name[cpu] = "R4600"; - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_LLSC; c->tlbsize = 48; break; #if 0 - case PRID_IMP_R4650: + case PRID_IMP_R4650: /* * This processor doesn't have an MMU, so it's not * "real easy" to run Linux on it. It is left purely @@ -448,13 +576,12 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) */ c->cputype = CPU_R4650; __cpu_name[cpu] = "R4650"; - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_LLSC; - c->tlbsize = 48; + c->tlbsize = 48; break; #endif case PRID_IMP_TX39: - c->isa_level = MIPS_CPU_ISA_I; c->options = MIPS_CPU_TLB | MIPS_CPU_TX39_CACHE; if ((c->processor_id & 0xf0) == (PRID_REV_TX3927 & 0xf0)) { @@ -462,7 +589,7 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) __cpu_name[cpu] = "TX3927"; c->tlbsize = 64; } else { - switch (c->processor_id & 0xff) { + switch (c->processor_id & PRID_REV_MASK) { case PRID_REV_TX3912: c->cputype = CPU_TX3912; __cpu_name[cpu] = "TX3912"; @@ -479,15 +606,15 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) case PRID_IMP_R4700: c->cputype = CPU_R4700; __cpu_name[cpu] = "R4700"; - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_TX49: c->cputype = CPU_TX49XX; __cpu_name[cpu] = "R49XX"; - c->isa_level = MIPS_CPU_ISA_III; + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_LLSC; if (!(c->processor_id & 0x08)) c->options |= MIPS_CPU_FPU | MIPS_CPU_32FPR; @@ -496,283 +623,149 @@ static inline void cpu_probe_legacy(struct cpuinfo_mips *c, unsigned int cpu) case PRID_IMP_R5000: c->cputype = CPU_R5000; __cpu_name[cpu] = "R5000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_R5432: c->cputype = CPU_R5432; __cpu_name[cpu] = "R5432"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_WATCH | MIPS_CPU_LLSC; + MIPS_CPU_WATCH | MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_R5500: c->cputype = CPU_R5500; __cpu_name[cpu] = "R5500"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_WATCH | MIPS_CPU_LLSC; + MIPS_CPU_WATCH | MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_NEVADA: c->cputype = CPU_NEVADA; __cpu_name[cpu] = "Nevada"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_DIVEC | MIPS_CPU_LLSC; + MIPS_CPU_DIVEC | MIPS_CPU_LLSC; c->tlbsize = 48; break; case PRID_IMP_R6000: c->cputype = CPU_R6000; __cpu_name[cpu] = "R6000"; - c->isa_level = MIPS_CPU_ISA_II; + set_isa(c, MIPS_CPU_ISA_II); c->options = MIPS_CPU_TLB | MIPS_CPU_FPU | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 32; break; case PRID_IMP_R6000A: c->cputype = CPU_R6000A; __cpu_name[cpu] = "R6000A"; - c->isa_level = MIPS_CPU_ISA_II; + set_isa(c, MIPS_CPU_ISA_II); c->options = MIPS_CPU_TLB | MIPS_CPU_FPU | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 32; break; case PRID_IMP_RM7000: c->cputype = CPU_RM7000; __cpu_name[cpu] = "RM7000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; /* - * Undocumented RM7000: Bit 29 in the info register of + * Undocumented RM7000: Bit 29 in the info register of * the RM7000 v2.0 indicates if the TLB has 48 or 64 * entries. * - * 29 1 => 64 entry JTLB - * 0 => 48 entry JTLB - */ - c->tlbsize = (read_c0_info() & (1 << 29)) ? 64 : 48; - break; - case PRID_IMP_RM9000: - c->cputype = CPU_RM9000; - __cpu_name[cpu] = "RM9000"; - c->isa_level = MIPS_CPU_ISA_IV; - c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; - /* - * Bit 29 in the info register of the RM9000 - * indicates if the TLB has 48 or 64 entries. - * - * 29 1 => 64 entry JTLB - * 0 => 48 entry JTLB + * 29 1 => 64 entry JTLB + * 0 => 48 entry JTLB */ c->tlbsize = (read_c0_info() & (1 << 29)) ? 64 : 48; break; case PRID_IMP_R8000: c->cputype = CPU_R8000; __cpu_name[cpu] = "RM8000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = MIPS_CPU_TLB | MIPS_CPU_4KEX | - MIPS_CPU_FPU | MIPS_CPU_32FPR | - MIPS_CPU_LLSC; + MIPS_CPU_FPU | MIPS_CPU_32FPR | + MIPS_CPU_LLSC; c->tlbsize = 384; /* has weird TLB: 3-way x 128 */ break; case PRID_IMP_R10000: c->cputype = CPU_R10000; __cpu_name[cpu] = "R10000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = MIPS_CPU_TLB | MIPS_CPU_4K_CACHE | MIPS_CPU_4KEX | - MIPS_CPU_FPU | MIPS_CPU_32FPR | + MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_COUNTER | MIPS_CPU_WATCH | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 64; break; case PRID_IMP_R12000: c->cputype = CPU_R12000; __cpu_name[cpu] = "R12000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = MIPS_CPU_TLB | MIPS_CPU_4K_CACHE | MIPS_CPU_4KEX | - MIPS_CPU_FPU | MIPS_CPU_32FPR | + MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_COUNTER | MIPS_CPU_WATCH | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 64; break; case PRID_IMP_R14000: c->cputype = CPU_R14000; __cpu_name[cpu] = "R14000"; - c->isa_level = MIPS_CPU_ISA_IV; + set_isa(c, MIPS_CPU_ISA_IV); c->options = MIPS_CPU_TLB | MIPS_CPU_4K_CACHE | MIPS_CPU_4KEX | - MIPS_CPU_FPU | MIPS_CPU_32FPR | + MIPS_CPU_FPU | MIPS_CPU_32FPR | MIPS_CPU_COUNTER | MIPS_CPU_WATCH | - MIPS_CPU_LLSC; + MIPS_CPU_LLSC; c->tlbsize = 64; break; - case PRID_IMP_LOONGSON2: - c->cputype = CPU_LOONGSON2; - __cpu_name[cpu] = "ICT Loongson-2"; - c->isa_level = MIPS_CPU_ISA_III; + case PRID_IMP_LOONGSON_64: /* Loongson-2/3 */ + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON2E: + c->cputype = CPU_LOONGSON2; + __cpu_name[cpu] = "ICT Loongson-2"; + set_elf_platform(cpu, "loongson2e"); + break; + case PRID_REV_LOONGSON2F: + c->cputype = CPU_LOONGSON2; + __cpu_name[cpu] = "ICT Loongson-2"; + set_elf_platform(cpu, "loongson2f"); + break; + case PRID_REV_LOONGSON3A: + c->cputype = CPU_LOONGSON3; + __cpu_name[cpu] = "ICT Loongson-3"; + set_elf_platform(cpu, "loongson3a"); + break; + } + + set_isa(c, MIPS_CPU_ISA_III); c->options = R4K_OPTS | MIPS_CPU_FPU | MIPS_CPU_LLSC | MIPS_CPU_32FPR; c->tlbsize = 64; break; - } -} + case PRID_IMP_LOONGSON_32: /* Loongson-1 */ + decode_configs(c); -static char unknown_isa[] __cpuinitdata = KERN_ERR \ - "Unsupported ISA type, c0.config0: %d."; + c->cputype = CPU_LOONGSON1; -static inline unsigned int decode_config0(struct cpuinfo_mips *c) -{ - unsigned int config0; - int isa; - - config0 = read_c0_config(); - - if (((config0 & MIPS_CONF_MT) >> 7) == 1) - c->options |= MIPS_CPU_TLB; - isa = (config0 & MIPS_CONF_AT) >> 13; - switch (isa) { - case 0: - switch ((config0 & MIPS_CONF_AR) >> 10) { - case 0: - c->isa_level = MIPS_CPU_ISA_M32R1; + switch (c->processor_id & PRID_REV_MASK) { + case PRID_REV_LOONGSON1B: + __cpu_name[cpu] = "Loongson 1B"; break; - case 1: - c->isa_level = MIPS_CPU_ISA_M32R2; - break; - default: - goto unknown; } - break; - case 2: - switch ((config0 & MIPS_CONF_AR) >> 10) { - case 0: - c->isa_level = MIPS_CPU_ISA_M64R1; - break; - case 1: - c->isa_level = MIPS_CPU_ISA_M64R2; - break; - default: - goto unknown; - } - break; - default: - goto unknown; - } - - return config0 & MIPS_CONF_M; - -unknown: - panic(unknown_isa, config0); -} - -static inline unsigned int decode_config1(struct cpuinfo_mips *c) -{ - unsigned int config1; - - config1 = read_c0_config1(); - if (config1 & MIPS_CONF1_MD) - c->ases |= MIPS_ASE_MDMX; - if (config1 & MIPS_CONF1_WR) - c->options |= MIPS_CPU_WATCH; - if (config1 & MIPS_CONF1_CA) - c->ases |= MIPS_ASE_MIPS16; - if (config1 & MIPS_CONF1_EP) - c->options |= MIPS_CPU_EJTAG; - if (config1 & MIPS_CONF1_FP) { - c->options |= MIPS_CPU_FPU; - c->options |= MIPS_CPU_32FPR; + break; } - if (cpu_has_tlb) - c->tlbsize = ((config1 & MIPS_CONF1_TLBS) >> 25) + 1; - - return config1 & MIPS_CONF_M; -} - -static inline unsigned int decode_config2(struct cpuinfo_mips *c) -{ - unsigned int config2; - - config2 = read_c0_config2(); - - if (config2 & MIPS_CONF2_SL) - c->scache.flags &= ~MIPS_CACHE_NOT_PRESENT; - - return config2 & MIPS_CONF_M; -} - -static inline unsigned int decode_config3(struct cpuinfo_mips *c) -{ - unsigned int config3; - - config3 = read_c0_config3(); - - if (config3 & MIPS_CONF3_SM) - c->ases |= MIPS_ASE_SMARTMIPS; - if (config3 & MIPS_CONF3_DSP) - c->ases |= MIPS_ASE_DSP; - if (config3 & MIPS_CONF3_VINT) - c->options |= MIPS_CPU_VINT; - if (config3 & MIPS_CONF3_VEIC) - c->options |= MIPS_CPU_VEIC; - if (config3 & MIPS_CONF3_MT) - c->ases |= MIPS_ASE_MIPSMT; - if (config3 & MIPS_CONF3_ULRI) - c->options |= MIPS_CPU_ULRI; - - return config3 & MIPS_CONF_M; -} - -static inline unsigned int decode_config4(struct cpuinfo_mips *c) -{ - unsigned int config4; - - config4 = read_c0_config4(); - - if ((config4 & MIPS_CONF4_MMUEXTDEF) == MIPS_CONF4_MMUEXTDEF_MMUSIZEEXT - && cpu_has_tlb) - c->tlbsize += (config4 & MIPS_CONF4_MMUSIZEEXT) * 0x40; - - return config4 & MIPS_CONF_M; -} - -static void __cpuinit decode_configs(struct cpuinfo_mips *c) -{ - int ok; - - /* MIPS32 or MIPS64 compliant CPU. */ - c->options = MIPS_CPU_4KEX | MIPS_CPU_4K_CACHE | MIPS_CPU_COUNTER | - MIPS_CPU_DIVEC | MIPS_CPU_LLSC | MIPS_CPU_MCHECK; - - c->scache.flags = MIPS_CACHE_NOT_PRESENT; - - ok = decode_config0(c); /* Read Config registers. */ - BUG_ON(!ok); /* Arch spec violation! */ - if (ok) - ok = decode_config1(c); - if (ok) - ok = decode_config2(c); - if (ok) - ok = decode_config3(c); - if (ok) - ok = decode_config4(c); - - mips_probe_watch_registers(c); - - if (cpu_has_mips_r2) - c->core = read_c0_ebase() & 0x3ff; } static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) { - decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_4KC: c->cputype = CPU_4KC; __cpu_name[cpu] = "MIPS 4Kc"; @@ -791,15 +784,22 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_5KC; __cpu_name[cpu] = "MIPS 5Kc"; break; + case PRID_IMP_5KE: + c->cputype = CPU_5KE; + __cpu_name[cpu] = "MIPS 5KE"; + break; case PRID_IMP_20KC: c->cputype = CPU_20KC; __cpu_name[cpu] = "MIPS 20Kc"; break; case PRID_IMP_24K: - case PRID_IMP_24KE: c->cputype = CPU_24K; __cpu_name[cpu] = "MIPS 24Kc"; break; + case PRID_IMP_24KE: + c->cputype = CPU_24K; + __cpu_name[cpu] = "MIPS 24KEc"; + break; case PRID_IMP_25KF: c->cputype = CPU_25KF; __cpu_name[cpu] = "MIPS 25Kc"; @@ -812,19 +812,57 @@ static inline void cpu_probe_mips(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_74K; __cpu_name[cpu] = "MIPS 74Kc"; break; + case PRID_IMP_M14KC: + c->cputype = CPU_M14KC; + __cpu_name[cpu] = "MIPS M14Kc"; + break; + case PRID_IMP_M14KEC: + c->cputype = CPU_M14KEC; + __cpu_name[cpu] = "MIPS M14KEc"; + break; case PRID_IMP_1004K: c->cputype = CPU_1004K; __cpu_name[cpu] = "MIPS 1004Kc"; break; + case PRID_IMP_1074K: + c->cputype = CPU_1074K; + __cpu_name[cpu] = "MIPS 1074Kc"; + break; + case PRID_IMP_INTERAPTIV_UP: + c->cputype = CPU_INTERAPTIV; + __cpu_name[cpu] = "MIPS interAptiv"; + break; + case PRID_IMP_INTERAPTIV_MP: + c->cputype = CPU_INTERAPTIV; + __cpu_name[cpu] = "MIPS interAptiv (multi)"; + break; + case PRID_IMP_PROAPTIV_UP: + c->cputype = CPU_PROAPTIV; + __cpu_name[cpu] = "MIPS proAptiv"; + break; + case PRID_IMP_PROAPTIV_MP: + c->cputype = CPU_PROAPTIV; + __cpu_name[cpu] = "MIPS proAptiv (multi)"; + break; + case PRID_IMP_P5600: + c->cputype = CPU_P5600; + __cpu_name[cpu] = "MIPS P5600"; + break; + case PRID_IMP_M5150: + c->cputype = CPU_M5150; + __cpu_name[cpu] = "MIPS M5150"; + break; } + decode_configs(c); + spram_config(); } static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_AU1_REV1: case PRID_IMP_AU1_REV2: c->cputype = CPU_ALCHEMY; @@ -843,7 +881,7 @@ static inline void cpu_probe_alchemy(struct cpuinfo_mips *c, unsigned int cpu) break; case 4: __cpu_name[cpu] = "Au1200"; - if ((c->processor_id & 0xff) == 2) + if ((c->processor_id & PRID_REV_MASK) == 2) __cpu_name[cpu] = "Au1250"; break; case 5: @@ -861,12 +899,12 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_SB1: c->cputype = CPU_SB1; __cpu_name[cpu] = "SiByte SB1"; /* FPU in pass1 is known to have issues. */ - if ((c->processor_id & 0xff) < 0x02) + if ((c->processor_id & PRID_REV_MASK) < 0x02) c->options &= ~(MIPS_CPU_FPU | MIPS_CPU_32FPR); break; case PRID_IMP_SB1A: @@ -879,7 +917,7 @@ static inline void cpu_probe_sibyte(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_SR71000: c->cputype = CPU_SR71000; __cpu_name[cpu] = "Sandcraft SR71000"; @@ -892,11 +930,11 @@ static inline void cpu_probe_sandcraft(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_PR4450: c->cputype = CPU_PR4450; __cpu_name[cpu] = "Philips PR4450"; - c->isa_level = MIPS_CPU_ISA_M32R1; + set_isa(c, MIPS_CPU_ISA_M32R1); break; } } @@ -904,46 +942,48 @@ static inline void cpu_probe_nxp(struct cpuinfo_mips *c, unsigned int cpu) static inline void cpu_probe_broadcom(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { - case PRID_IMP_BMIPS32: + switch (c->processor_id & PRID_IMP_MASK) { + case PRID_IMP_BMIPS32_REV4: + case PRID_IMP_BMIPS32_REV8: c->cputype = CPU_BMIPS32; __cpu_name[cpu] = "Broadcom BMIPS32"; + set_elf_platform(cpu, "bmips32"); break; case PRID_IMP_BMIPS3300: case PRID_IMP_BMIPS3300_ALT: case PRID_IMP_BMIPS3300_BUG: c->cputype = CPU_BMIPS3300; __cpu_name[cpu] = "Broadcom BMIPS3300"; + set_elf_platform(cpu, "bmips3300"); break; case PRID_IMP_BMIPS43XX: { - int rev = c->processor_id & 0xff; + int rev = c->processor_id & PRID_REV_MASK; if (rev >= PRID_REV_BMIPS4380_LO && rev <= PRID_REV_BMIPS4380_HI) { c->cputype = CPU_BMIPS4380; __cpu_name[cpu] = "Broadcom BMIPS4380"; + set_elf_platform(cpu, "bmips4380"); } else { c->cputype = CPU_BMIPS4350; __cpu_name[cpu] = "Broadcom BMIPS4350"; + set_elf_platform(cpu, "bmips4350"); } break; } case PRID_IMP_BMIPS5000: c->cputype = CPU_BMIPS5000; __cpu_name[cpu] = "Broadcom BMIPS5000"; + set_elf_platform(cpu, "bmips5000"); c->options |= MIPS_CPU_ULRI; break; - case PRID_IMP_BMIPS4KC: - c->cputype = CPU_4KC; - __cpu_name[cpu] = "MIPS 4Kc"; - break; } } static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu) { decode_configs(c); - switch (c->processor_id & 0xff00) { + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_CAVIUM_CN38XX: case PRID_IMP_CAVIUM_CN31XX: case PRID_IMP_CAVIUM_CN30XX: @@ -957,14 +997,22 @@ static inline void cpu_probe_cavium(struct cpuinfo_mips *c, unsigned int cpu) c->cputype = CPU_CAVIUM_OCTEON_PLUS; __cpu_name[cpu] = "Cavium Octeon+"; platform: - if (cpu == 0) - __elf_platform = "octeon"; + set_elf_platform(cpu, "octeon"); break; + case PRID_IMP_CAVIUM_CN61XX: case PRID_IMP_CAVIUM_CN63XX: + case PRID_IMP_CAVIUM_CN66XX: + case PRID_IMP_CAVIUM_CN68XX: + case PRID_IMP_CAVIUM_CNF71XX: c->cputype = CPU_CAVIUM_OCTEON2; __cpu_name[cpu] = "Cavium Octeon II"; - if (cpu == 0) - __elf_platform = "octeon2"; + set_elf_platform(cpu, "octeon2"); + break; + case PRID_IMP_CAVIUM_CN70XX: + case PRID_IMP_CAVIUM_CN78XX: + c->cputype = CPU_CAVIUM_OCTEON3; + __cpu_name[cpu] = "Cavium Octeon III"; + set_elf_platform(cpu, "octeon3"); break; default: printk(KERN_INFO "Unknown Octeon chip!\n"); @@ -978,7 +1026,8 @@ static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) decode_configs(c); /* JZRISC does not implement the CP0 counter. */ c->options &= ~MIPS_CPU_COUNTER; - switch (c->processor_id & 0xff00) { + BUG_ON(!__builtin_constant_p(cpu_has_counter) || cpu_has_counter); + switch (c->processor_id & PRID_IMP_MASK) { case PRID_IMP_JZRISC: c->cputype = CPU_JZRISC; __cpu_name[cpu] = "Ingenic JZRISC"; @@ -989,6 +1038,87 @@ static inline void cpu_probe_ingenic(struct cpuinfo_mips *c, unsigned int cpu) } } +static inline void cpu_probe_netlogic(struct cpuinfo_mips *c, int cpu) +{ + decode_configs(c); + + if ((c->processor_id & PRID_IMP_MASK) == PRID_IMP_NETLOGIC_AU13XX) { + c->cputype = CPU_ALCHEMY; + __cpu_name[cpu] = "Au1300"; + /* following stuff is not for Alchemy */ + return; + } + + c->options = (MIPS_CPU_TLB | + MIPS_CPU_4KEX | + MIPS_CPU_COUNTER | + MIPS_CPU_DIVEC | + MIPS_CPU_WATCH | + MIPS_CPU_EJTAG | + MIPS_CPU_LLSC); + + switch (c->processor_id & PRID_IMP_MASK) { + case PRID_IMP_NETLOGIC_XLP2XX: + case PRID_IMP_NETLOGIC_XLP9XX: + case PRID_IMP_NETLOGIC_XLP5XX: + c->cputype = CPU_XLP; + __cpu_name[cpu] = "Broadcom XLPII"; + break; + + case PRID_IMP_NETLOGIC_XLP8XX: + case PRID_IMP_NETLOGIC_XLP3XX: + c->cputype = CPU_XLP; + __cpu_name[cpu] = "Netlogic XLP"; + break; + + case PRID_IMP_NETLOGIC_XLR732: + case PRID_IMP_NETLOGIC_XLR716: + case PRID_IMP_NETLOGIC_XLR532: + case PRID_IMP_NETLOGIC_XLR308: + case PRID_IMP_NETLOGIC_XLR532C: + case PRID_IMP_NETLOGIC_XLR516C: + case PRID_IMP_NETLOGIC_XLR508C: + case PRID_IMP_NETLOGIC_XLR308C: + c->cputype = CPU_XLR; + __cpu_name[cpu] = "Netlogic XLR"; + break; + + case PRID_IMP_NETLOGIC_XLS608: + case PRID_IMP_NETLOGIC_XLS408: + case PRID_IMP_NETLOGIC_XLS404: + case PRID_IMP_NETLOGIC_XLS208: + case PRID_IMP_NETLOGIC_XLS204: + case PRID_IMP_NETLOGIC_XLS108: + case PRID_IMP_NETLOGIC_XLS104: + case PRID_IMP_NETLOGIC_XLS616B: + case PRID_IMP_NETLOGIC_XLS608B: + case PRID_IMP_NETLOGIC_XLS416B: + case PRID_IMP_NETLOGIC_XLS412B: + case PRID_IMP_NETLOGIC_XLS408B: + case PRID_IMP_NETLOGIC_XLS404B: + c->cputype = CPU_XLR; + __cpu_name[cpu] = "Netlogic XLS"; + break; + + default: + pr_info("Unknown Netlogic chip id [%02x]!\n", + c->processor_id); + c->cputype = CPU_XLR; + break; + } + + if (c->cputype == CPU_XLP) { + set_isa(c, MIPS_CPU_ISA_M64R2); + c->options |= (MIPS_CPU_FPU | MIPS_CPU_ULRI | MIPS_CPU_MCHECK); + /* This will be updated again after all threads are woken up */ + c->tlbsize = ((read_c0_config6() >> 16) & 0xffff) + 1; + } else { + set_isa(c, MIPS_CPU_ISA_M64R1); + c->tlbsize = ((read_c0_config1() >> 25) & 0x3f) + 1; + } + c->kscratch_mask = 0xf; +} + #ifdef CONFIG_64BIT /* For use by uaccess.h */ u64 __ua_limit; @@ -998,17 +1128,17 @@ EXPORT_SYMBOL(__ua_limit); const char *__cpu_name[NR_CPUS]; const char *__elf_platform; -__cpuinit void cpu_probe(void) +void cpu_probe(void) { struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int cpu = smp_processor_id(); - c->processor_id = PRID_IMP_UNKNOWN; + c->processor_id = PRID_IMP_UNKNOWN; c->fpu_id = FPIR_IMP_NONE; c->cputype = CPU_UNKNOWN; c->processor_id = read_c0_prid(); - switch (c->processor_id & 0xff0000) { + switch (c->processor_id & PRID_COMP_MASK) { case PRID_COMP_LEGACY: cpu_probe_legacy(c, cpu); break; @@ -1036,6 +1166,9 @@ __cpuinit void cpu_probe(void) case PRID_COMP_INGENIC: cpu_probe_ingenic(c, cpu); break; + case PRID_COMP_NETLOGIC: + cpu_probe_netlogic(c, cpu); + break; } BUG_ON(!__cpu_name[cpu]); @@ -1052,25 +1185,32 @@ __cpuinit void cpu_probe(void) c->options &= ~MIPS_CPU_FPU; if (mips_dsp_disabled) - c->ases &= ~MIPS_ASE_DSP; + c->ases &= ~(MIPS_ASE_DSP | MIPS_ASE_DSP2P); if (c->options & MIPS_CPU_FPU) { c->fpu_id = cpu_get_fpu_id(); - if (c->isa_level == MIPS_CPU_ISA_M32R1 || - c->isa_level == MIPS_CPU_ISA_M32R2 || - c->isa_level == MIPS_CPU_ISA_M64R1 || - c->isa_level == MIPS_CPU_ISA_M64R2) { + if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | + MIPS_CPU_ISA_M64R1 | MIPS_CPU_ISA_M64R2)) { if (c->fpu_id & MIPS_FPIR_3D) c->ases |= MIPS_ASE_MIPS3D; } } - if (cpu_has_mips_r2) + if (cpu_has_mips_r2) { c->srsets = ((read_c0_srsctl() >> 26) & 0x0f) + 1; + /* R2 has Performance Counter Interrupt indicator */ + c->options |= MIPS_CPU_PCI; + } else c->srsets = 1; + if (cpu_has_msa) { + c->msa_id = cpu_get_msa_id(); + WARN(c->msa_id & MSA_IR_WRPF, + "Vector register partitioning unimplemented!"); + } + cpu_probe_vmbits(c); #ifdef CONFIG_64BIT @@ -1079,12 +1219,14 @@ __cpuinit void cpu_probe(void) #endif } -__cpuinit void cpu_report(void) +void cpu_report(void) { struct cpuinfo_mips *c = ¤t_cpu_data; - printk(KERN_INFO "CPU revision is: %08x (%s)\n", - c->processor_id, cpu_name_string()); + pr_info("CPU%d revision is: %08x (%s)\n", + smp_processor_id(), c->processor_id, cpu_name_string()); if (c->options & MIPS_CPU_FPU) printk(KERN_INFO "FPU revision is: %08x\n", c->fpu_id); + if (cpu_has_msa) + pr_info("MSA revision is: %08x\n", c->msa_id); } diff --git a/arch/mips/kernel/cpufreq/Kconfig b/arch/mips/kernel/cpufreq/Kconfig deleted file mode 100644 index 58c601eee6f..00000000000 --- a/arch/mips/kernel/cpufreq/Kconfig +++ /dev/null @@ -1,41 +0,0 @@ -# -# CPU Frequency scaling -# - -config MIPS_EXTERNAL_TIMER - bool - -config MIPS_CPUFREQ - bool - default y - depends on CPU_SUPPORTS_CPUFREQ && MIPS_EXTERNAL_TIMER - -if MIPS_CPUFREQ - -menu "CPU Frequency scaling" - -source "drivers/cpufreq/Kconfig" - -if CPU_FREQ - -comment "CPUFreq processor drivers" - -config LOONGSON2_CPUFREQ - tristate "Loongson2 CPUFreq Driver" - select CPU_FREQ_TABLE - depends on MIPS_CPUFREQ - help - This option adds a CPUFreq driver for loongson processors which - support software configurable cpu frequency. - - Loongson2F and it's successors support this feature. - - For details, take a look at <file:Documentation/cpu-freq/>. - - If in doubt, say N. - -endif # CPU_FREQ - -endmenu - -endif # MIPS_CPUFREQ diff --git a/arch/mips/kernel/cpufreq/Makefile b/arch/mips/kernel/cpufreq/Makefile deleted file mode 100644 index c3479a432ef..00000000000 --- a/arch/mips/kernel/cpufreq/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -# -# Makefile for the Linux/MIPS cpufreq. -# - -obj-$(CONFIG_LOONGSON2_CPUFREQ) += loongson2_cpufreq.o loongson2_clock.o diff --git a/arch/mips/kernel/cpufreq/loongson2_clock.c b/arch/mips/kernel/cpufreq/loongson2_clock.c deleted file mode 100644 index cefc6e259ba..00000000000 --- a/arch/mips/kernel/cpufreq/loongson2_clock.c +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (C) 2006 - 2008 Lemote Inc. & Insititute of Computing Technology - * Author: Yanhua, yanh@lemote.com - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ - -#include <linux/cpufreq.h> -#include <linux/platform_device.h> - -#include <asm/clock.h> - -#include <loongson.h> - -static LIST_HEAD(clock_list); -static DEFINE_SPINLOCK(clock_lock); -static DEFINE_MUTEX(clock_list_sem); - -/* Minimum CLK support */ -enum { - DC_ZERO, DC_25PT = 2, DC_37PT, DC_50PT, DC_62PT, DC_75PT, - DC_87PT, DC_DISABLE, DC_RESV -}; - -struct cpufreq_frequency_table loongson2_clockmod_table[] = { - {DC_RESV, CPUFREQ_ENTRY_INVALID}, - {DC_ZERO, CPUFREQ_ENTRY_INVALID}, - {DC_25PT, 0}, - {DC_37PT, 0}, - {DC_50PT, 0}, - {DC_62PT, 0}, - {DC_75PT, 0}, - {DC_87PT, 0}, - {DC_DISABLE, 0}, - {DC_RESV, CPUFREQ_TABLE_END}, -}; -EXPORT_SYMBOL_GPL(loongson2_clockmod_table); - -static struct clk cpu_clk = { - .name = "cpu_clk", - .flags = CLK_ALWAYS_ENABLED | CLK_RATE_PROPAGATES, - .rate = 800000000, -}; - -struct clk *clk_get(struct device *dev, const char *id) -{ - return &cpu_clk; -} -EXPORT_SYMBOL(clk_get); - -static void propagate_rate(struct clk *clk) -{ - struct clk *clkp; - - list_for_each_entry(clkp, &clock_list, node) { - if (likely(clkp->parent != clk)) - continue; - if (likely(clkp->ops && clkp->ops->recalc)) - clkp->ops->recalc(clkp); - if (unlikely(clkp->flags & CLK_RATE_PROPAGATES)) - propagate_rate(clkp); - } -} - -int clk_enable(struct clk *clk) -{ - return 0; -} -EXPORT_SYMBOL(clk_enable); - -void clk_disable(struct clk *clk) -{ -} -EXPORT_SYMBOL(clk_disable); - -unsigned long clk_get_rate(struct clk *clk) -{ - return (unsigned long)clk->rate; -} -EXPORT_SYMBOL(clk_get_rate); - -void clk_put(struct clk *clk) -{ -} -EXPORT_SYMBOL(clk_put); - -int clk_set_rate(struct clk *clk, unsigned long rate) -{ - return clk_set_rate_ex(clk, rate, 0); -} -EXPORT_SYMBOL_GPL(clk_set_rate); - -int clk_set_rate_ex(struct clk *clk, unsigned long rate, int algo_id) -{ - int ret = 0; - int regval; - int i; - - if (likely(clk->ops && clk->ops->set_rate)) { - unsigned long flags; - - spin_lock_irqsave(&clock_lock, flags); - ret = clk->ops->set_rate(clk, rate, algo_id); - spin_unlock_irqrestore(&clock_lock, flags); - } - - if (unlikely(clk->flags & CLK_RATE_PROPAGATES)) - propagate_rate(clk); - - for (i = 0; loongson2_clockmod_table[i].frequency != CPUFREQ_TABLE_END; - i++) { - if (loongson2_clockmod_table[i].frequency == - CPUFREQ_ENTRY_INVALID) - continue; - if (rate == loongson2_clockmod_table[i].frequency) - break; - } - if (rate != loongson2_clockmod_table[i].frequency) - return -ENOTSUPP; - - clk->rate = rate; - - regval = LOONGSON_CHIPCFG0; - regval = (regval & ~0x7) | (loongson2_clockmod_table[i].index - 1); - LOONGSON_CHIPCFG0 = regval; - - return ret; -} -EXPORT_SYMBOL_GPL(clk_set_rate_ex); - -long clk_round_rate(struct clk *clk, unsigned long rate) -{ - if (likely(clk->ops && clk->ops->round_rate)) { - unsigned long flags, rounded; - - spin_lock_irqsave(&clock_lock, flags); - rounded = clk->ops->round_rate(clk, rate); - spin_unlock_irqrestore(&clock_lock, flags); - - return rounded; - } - - return rate; -} -EXPORT_SYMBOL_GPL(clk_round_rate); - -/* - * This is the simple version of Loongson-2 wait, Maybe we need do this in - * interrupt disabled content - */ - -DEFINE_SPINLOCK(loongson2_wait_lock); -void loongson2_cpu_wait(void) -{ - u32 cpu_freq; - unsigned long flags; - - spin_lock_irqsave(&loongson2_wait_lock, flags); - cpu_freq = LOONGSON_CHIPCFG0; - LOONGSON_CHIPCFG0 &= ~0x7; /* Put CPU into wait mode */ - LOONGSON_CHIPCFG0 = cpu_freq; /* Restore CPU state */ - spin_unlock_irqrestore(&loongson2_wait_lock, flags); -} -EXPORT_SYMBOL_GPL(loongson2_cpu_wait); - -MODULE_AUTHOR("Yanhua <yanh@lemote.com>"); -MODULE_DESCRIPTION("cpufreq driver for Loongson 2F"); -MODULE_LICENSE("GPL"); diff --git a/arch/mips/kernel/cpufreq/loongson2_cpufreq.c b/arch/mips/kernel/cpufreq/loongson2_cpufreq.c deleted file mode 100644 index ae5db206347..00000000000 --- a/arch/mips/kernel/cpufreq/loongson2_cpufreq.c +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Cpufreq driver for the loongson-2 processors - * - * The 2E revision of loongson processor not support this feature. - * - * Copyright (C) 2006 - 2008 Lemote Inc. & Insititute of Computing Technology - * Author: Yanhua, yanh@lemote.com - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - */ -#include <linux/cpufreq.h> -#include <linux/module.h> -#include <linux/err.h> -#include <linux/sched.h> /* set_cpus_allowed() */ -#include <linux/delay.h> -#include <linux/platform_device.h> - -#include <asm/clock.h> - -#include <loongson.h> - -static uint nowait; - -static struct clk *cpuclk; - -static void (*saved_cpu_wait) (void); - -static int loongson2_cpu_freq_notifier(struct notifier_block *nb, - unsigned long val, void *data); - -static struct notifier_block loongson2_cpufreq_notifier_block = { - .notifier_call = loongson2_cpu_freq_notifier -}; - -static int loongson2_cpu_freq_notifier(struct notifier_block *nb, - unsigned long val, void *data) -{ - if (val == CPUFREQ_POSTCHANGE) - current_cpu_data.udelay_val = loops_per_jiffy; - - return 0; -} - -static unsigned int loongson2_cpufreq_get(unsigned int cpu) -{ - return clk_get_rate(cpuclk); -} - -/* - * Here we notify other drivers of the proposed change and the final change. - */ -static int loongson2_cpufreq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int cpu = policy->cpu; - unsigned int newstate = 0; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; - unsigned int freq; - - if (!cpu_online(cpu)) - return -ENODEV; - - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - if (cpufreq_frequency_table_target - (policy, &loongson2_clockmod_table[0], target_freq, relation, - &newstate)) - return -EINVAL; - - freq = - ((cpu_clock_freq / 1000) * - loongson2_clockmod_table[newstate].index) / 8; - if (freq < policy->min || freq > policy->max) - return -EINVAL; - - pr_debug("cpufreq: requested frequency %u Hz\n", target_freq * 1000); - - freqs.cpu = cpu; - freqs.old = loongson2_cpufreq_get(cpu); - freqs.new = freq; - freqs.flags = 0; - - if (freqs.new == freqs.old) - return 0; - - /* notifiers */ - cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); - - /* setting the cpu frequency */ - clk_set_rate(cpuclk, freq); - - /* notifiers */ - cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); - - pr_debug("cpufreq: set frequency %u kHz\n", freq); - - return 0; -} - -static int loongson2_cpufreq_cpu_init(struct cpufreq_policy *policy) -{ - int i; - - if (!cpu_online(policy->cpu)) - return -ENODEV; - - cpuclk = clk_get(NULL, "cpu_clk"); - if (IS_ERR(cpuclk)) { - printk(KERN_ERR "cpufreq: couldn't get CPU clk\n"); - return PTR_ERR(cpuclk); - } - - cpuclk->rate = cpu_clock_freq / 1000; - if (!cpuclk->rate) - return -EINVAL; - - /* clock table init */ - for (i = 2; - (loongson2_clockmod_table[i].frequency != CPUFREQ_TABLE_END); - i++) - loongson2_clockmod_table[i].frequency = (cpuclk->rate * i) / 8; - - policy->cur = loongson2_cpufreq_get(policy->cpu); - - cpufreq_frequency_table_get_attr(&loongson2_clockmod_table[0], - policy->cpu); - - return cpufreq_frequency_table_cpuinfo(policy, - &loongson2_clockmod_table[0]); -} - -static int loongson2_cpufreq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &loongson2_clockmod_table[0]); -} - -static int loongson2_cpufreq_exit(struct cpufreq_policy *policy) -{ - clk_put(cpuclk); - return 0; -} - -static struct freq_attr *loongson2_table_attr[] = { - &cpufreq_freq_attr_scaling_available_freqs, - NULL, -}; - -static struct cpufreq_driver loongson2_cpufreq_driver = { - .owner = THIS_MODULE, - .name = "loongson2", - .init = loongson2_cpufreq_cpu_init, - .verify = loongson2_cpufreq_verify, - .target = loongson2_cpufreq_target, - .get = loongson2_cpufreq_get, - .exit = loongson2_cpufreq_exit, - .attr = loongson2_table_attr, -}; - -static struct platform_device_id platform_device_ids[] = { - { - .name = "loongson2_cpufreq", - }, - {} -}; - -MODULE_DEVICE_TABLE(platform, platform_device_ids); - -static struct platform_driver platform_driver = { - .driver = { - .name = "loongson2_cpufreq", - .owner = THIS_MODULE, - }, - .id_table = platform_device_ids, -}; - -static int __init cpufreq_init(void) -{ - int ret; - - /* Register platform stuff */ - ret = platform_driver_register(&platform_driver); - if (ret) - return ret; - - pr_info("cpufreq: Loongson-2F CPU frequency driver.\n"); - - cpufreq_register_notifier(&loongson2_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - - ret = cpufreq_register_driver(&loongson2_cpufreq_driver); - - if (!ret && !nowait) { - saved_cpu_wait = cpu_wait; - cpu_wait = loongson2_cpu_wait; - } - - return ret; -} - -static void __exit cpufreq_exit(void) -{ - if (!nowait && saved_cpu_wait) - cpu_wait = saved_cpu_wait; - cpufreq_unregister_driver(&loongson2_cpufreq_driver); - cpufreq_unregister_notifier(&loongson2_cpufreq_notifier_block, - CPUFREQ_TRANSITION_NOTIFIER); - - platform_driver_unregister(&platform_driver); -} - -module_init(cpufreq_init); -module_exit(cpufreq_exit); - -module_param(nowait, uint, 0644); -MODULE_PARM_DESC(nowait, "Disable Loongson-2F specific wait"); - -MODULE_AUTHOR("Yanhua <yanh@lemote.com>"); -MODULE_DESCRIPTION("cpufreq driver for Loongson2F"); -MODULE_LICENSE("GPL"); diff --git a/arch/mips/kernel/crash.c b/arch/mips/kernel/crash.c new file mode 100644 index 00000000000..d21264681e9 --- /dev/null +++ b/arch/mips/kernel/crash.c @@ -0,0 +1,70 @@ +#include <linux/kernel.h> +#include <linux/smp.h> +#include <linux/reboot.h> +#include <linux/kexec.h> +#include <linux/bootmem.h> +#include <linux/crash_dump.h> +#include <linux/delay.h> +#include <linux/irq.h> +#include <linux/types.h> +#include <linux/sched.h> + +/* This keeps a track of which one is crashing cpu. */ +static int crashing_cpu = -1; +static cpumask_t cpus_in_crash = CPU_MASK_NONE; + +#ifdef CONFIG_SMP +static void crash_shutdown_secondary(void *ignore) +{ + struct pt_regs *regs; + int cpu = smp_processor_id(); + + regs = task_pt_regs(current); + + if (!cpu_online(cpu)) + return; + + local_irq_disable(); + if (!cpu_isset(cpu, cpus_in_crash)) + crash_save_cpu(regs, cpu); + cpu_set(cpu, cpus_in_crash); + + while (!atomic_read(&kexec_ready_to_reboot)) + cpu_relax(); + relocated_kexec_smp_wait(NULL); + /* NOTREACHED */ +} + +static void crash_kexec_prepare_cpus(void) +{ + unsigned int msecs; + + unsigned int ncpus = num_online_cpus() - 1;/* Excluding the panic cpu */ + + dump_send_ipi(crash_shutdown_secondary); + smp_wmb(); + + /* + * The crash CPU sends an IPI and wait for other CPUs to + * respond. Delay of at least 10 seconds. + */ + pr_emerg("Sending IPI to other cpus...\n"); + msecs = 10000; + while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) { + cpu_relax(); + mdelay(1); + } +} + +#else /* !defined(CONFIG_SMP) */ +static void crash_kexec_prepare_cpus(void) {} +#endif /* !defined(CONFIG_SMP) */ + +void default_machine_crash_shutdown(struct pt_regs *regs) +{ + local_irq_disable(); + crashing_cpu = smp_processor_id(); + crash_save_cpu(regs, crashing_cpu); + crash_kexec_prepare_cpus(); + cpu_set(crashing_cpu, cpus_in_crash); +} diff --git a/arch/mips/kernel/crash_dump.c b/arch/mips/kernel/crash_dump.c new file mode 100644 index 00000000000..f291cf99b03 --- /dev/null +++ b/arch/mips/kernel/crash_dump.c @@ -0,0 +1,66 @@ +#include <linux/highmem.h> +#include <linux/bootmem.h> +#include <linux/crash_dump.h> +#include <asm/uaccess.h> +#include <linux/slab.h> + +static void *kdump_buf_page; + +/** + * copy_oldmem_page - copy one page from "oldmem" + * @pfn: page frame number to be copied + * @buf: target memory address for the copy; this can be in kernel address + * space or user address space (see @userbuf) + * @csize: number of bytes to copy + * @offset: offset in bytes into the page (based on pfn) to begin the copy + * @userbuf: if set, @buf is in user address space, use copy_to_user(), + * otherwise @buf is in kernel address space, use memcpy(). + * + * Copy a page from "oldmem". For this page, there is no pte mapped + * in the current kernel. + * + * Calling copy_to_user() in atomic context is not desirable. Hence first + * copying the data to a pre-allocated kernel page and then copying to user + * space in non-atomic context. + */ +ssize_t copy_oldmem_page(unsigned long pfn, char *buf, + size_t csize, unsigned long offset, int userbuf) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = kmap_atomic_pfn(pfn); + + if (!userbuf) { + memcpy(buf, (vaddr + offset), csize); + kunmap_atomic(vaddr); + } else { + if (!kdump_buf_page) { + pr_warning("Kdump: Kdump buffer page not allocated\n"); + + return -EFAULT; + } + copy_page(kdump_buf_page, vaddr); + kunmap_atomic(vaddr); + if (copy_to_user(buf, (kdump_buf_page + offset), csize)) + return -EFAULT; + } + + return csize; +} + +static int __init kdump_buf_page_init(void) +{ + int ret = 0; + + kdump_buf_page = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!kdump_buf_page) { + pr_warning("Kdump: Failed to allocate kdump buffer page\n"); + ret = -ENOMEM; + } + + return ret; +} +arch_initcall(kdump_buf_page_init); diff --git a/arch/mips/kernel/csrc-bcm1480.c b/arch/mips/kernel/csrc-bcm1480.c index 51489f8a825..468f3eba413 100644 --- a/arch/mips/kernel/csrc-bcm1480.c +++ b/arch/mips/kernel/csrc-bcm1480.c @@ -35,7 +35,7 @@ static cycle_t bcm1480_hpt_read(struct clocksource *cs) struct clocksource bcm1480_clocksource = { .name = "zbbus-cycles", - .rating = 200, + .rating = 200, .read = bcm1480_hpt_read, .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS, @@ -49,6 +49,5 @@ void __init sb1480_clocksource_init(void) plldiv = G_BCM1480_SYS_PLL_DIV(__raw_readq(IOADDR(A_SCD_SYSTEM_CFG))); zbbus = ((plldiv >> 1) * 50000000) + ((plldiv & 1) * 25000000); - clocksource_set_clock(cs, zbbus); - clocksource_register(cs); + clocksource_register_hz(cs, zbbus); } diff --git a/arch/mips/kernel/csrc-gic.c b/arch/mips/kernel/csrc-gic.c new file mode 100644 index 00000000000..e0262090111 --- /dev/null +++ b/arch/mips/kernel/csrc-gic.c @@ -0,0 +1,40 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. + */ +#include <linux/init.h> +#include <linux/time.h> + +#include <asm/gic.h> + +static cycle_t gic_hpt_read(struct clocksource *cs) +{ + return gic_read_count(); +} + +static struct clocksource gic_clocksource = { + .name = "GIC", + .read = gic_hpt_read, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +void __init gic_clocksource_init(unsigned int frequency) +{ + unsigned int config, bits; + + /* Calculate the clocksource mask. */ + GICREAD(GIC_REG(SHARED, GIC_SH_CONFIG), config); + bits = 32 + ((config & GIC_SH_CONFIG_COUNTBITS_MSK) >> + (GIC_SH_CONFIG_COUNTBITS_SHF - 2)); + + /* Set clocksource mask. */ + gic_clocksource.mask = CLOCKSOURCE_MASK(bits); + + /* Calculate a somewhat reasonable rating value. */ + gic_clocksource.rating = 200 + frequency / 10000000; + + clocksource_register_hz(&gic_clocksource, frequency); +} diff --git a/arch/mips/kernel/csrc-ioasic.c b/arch/mips/kernel/csrc-ioasic.c index 23da108506b..6cbbf6e106b 100644 --- a/arch/mips/kernel/csrc-ioasic.c +++ b/arch/mips/kernel/csrc-ioasic.c @@ -1,7 +1,7 @@ /* * DEC I/O ASIC's counter clocksource * - * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> + * Copyright (C) 2008 Yoichi Yuasa <yuasa@linux-mips.org> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -37,13 +37,13 @@ static struct clocksource clocksource_dec = { .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; -void __init dec_ioasic_clocksource_init(void) +int __init dec_ioasic_clocksource_init(void) { unsigned int freq; u32 start, end; - int i = HZ / 10; - + int i = HZ / 8; + ds1287_timer_state(); while (!ds1287_timer_state()) ; @@ -55,11 +55,15 @@ void __init dec_ioasic_clocksource_init(void) end = dec_ioasic_hpt_read(&clocksource_dec); - freq = (end - start) * 10; + freq = (end - start) * 8; + + /* An early revision of the I/O ASIC didn't have the counter. */ + if (!freq) + return -ENXIO; + printk(KERN_INFO "I/O ASIC clock frequency %dHz\n", freq); clocksource_dec.rating = 200 + freq / 10000000; - clocksource_set_clock(&clocksource_dec, freq); - - clocksource_register(&clocksource_dec); + clocksource_register_hz(&clocksource_dec, freq); + return 0; } diff --git a/arch/mips/kernel/csrc-powertv.c b/arch/mips/kernel/csrc-powertv.c deleted file mode 100644 index a27c16c8690..00000000000 --- a/arch/mips/kernel/csrc-powertv.c +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (C) 2008 Scientific-Atlanta, Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -/* - * The file comes from kernel/csrc-r4k.c - */ -#include <linux/clocksource.h> -#include <linux/init.h> - -#include <asm/time.h> /* Not included in linux/time.h */ - -#include <asm/mach-powertv/asic_regs.h> -#include "powertv-clock.h" - -/* MIPS PLL Register Definitions */ -#define PLL_GET_M(x) (((x) >> 8) & 0x000000FF) -#define PLL_GET_N(x) (((x) >> 16) & 0x000000FF) -#define PLL_GET_P(x) (((x) >> 24) & 0x00000007) - -/* - * returns: Clock frequency in kHz - */ -unsigned int __init mips_get_pll_freq(void) -{ - unsigned int pll_reg, m, n, p; - unsigned int fin = 54000; /* Base frequency in kHz */ - unsigned int fout; - - /* Read PLL register setting */ - pll_reg = asic_read(mips_pll_setup); - m = PLL_GET_M(pll_reg); - n = PLL_GET_N(pll_reg); - p = PLL_GET_P(pll_reg); - pr_info("MIPS PLL Register:0x%x M=%d N=%d P=%d\n", pll_reg, m, n, p); - - /* Calculate clock frequency = (2 * N * 54MHz) / (M * (2**P)) */ - fout = ((2 * n * fin) / (m * (0x01 << p))); - - pr_info("MIPS Clock Freq=%d kHz\n", fout); - - return fout; -} - -static cycle_t c0_hpt_read(struct clocksource *cs) -{ - return read_c0_count(); -} - -static struct clocksource clocksource_mips = { - .name = "powertv-counter", - .read = c0_hpt_read, - .mask = CLOCKSOURCE_MASK(32), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -static void __init powertv_c0_hpt_clocksource_init(void) -{ - unsigned int pll_freq = mips_get_pll_freq(); - - pr_info("CPU frequency %d.%02d MHz\n", pll_freq / 1000, - (pll_freq % 1000) * 100 / 1000); - - mips_hpt_frequency = pll_freq / 2 * 1000; - - clocksource_mips.rating = 200 + mips_hpt_frequency / 10000000; - - clocksource_set_clock(&clocksource_mips, mips_hpt_frequency); - - clocksource_register(&clocksource_mips); -} - -/** - * struct tim_c - free running counter - * @hi: High 16 bits of the counter - * @lo: Low 32 bits of the counter - * - * Lays out the structure of the free running counter in memory. This counter - * increments at a rate of 27 MHz/8 on all platforms. - */ -struct tim_c { - unsigned int hi; - unsigned int lo; -}; - -static struct tim_c *tim_c; - -static cycle_t tim_c_read(struct clocksource *cs) -{ - unsigned int hi; - unsigned int next_hi; - unsigned int lo; - - hi = readl(&tim_c->hi); - - for (;;) { - lo = readl(&tim_c->lo); - next_hi = readl(&tim_c->hi); - if (next_hi == hi) - break; - hi = next_hi; - } - -pr_crit("%s: read %llx\n", __func__, ((u64) hi << 32) | lo); - return ((u64) hi << 32) | lo; -} - -#define TIM_C_SIZE 48 /* # bits in the timer */ - -static struct clocksource clocksource_tim_c = { - .name = "powertv-tim_c", - .read = tim_c_read, - .mask = CLOCKSOURCE_MASK(TIM_C_SIZE), - .flags = CLOCK_SOURCE_IS_CONTINUOUS, -}; - -/** - * powertv_tim_c_clocksource_init - set up a clock source for the TIM_C clock - * - * The hard part here is coming up with a constant k and shift s such that - * the 48-bit TIM_C value multiplied by k doesn't overflow and that value, - * when shifted right by s, yields the corresponding number of nanoseconds. - * We know that TIM_C counts at 27 MHz/8, so each cycle corresponds to - * 1 / (27,000,000/8) seconds. Multiply that by a billion and you get the - * number of nanoseconds. Since the TIM_C value has 48 bits and the math is - * done in 64 bits, avoiding an overflow means that k must be less than - * 64 - 48 = 16 bits. - */ -static void __init powertv_tim_c_clocksource_init(void) -{ - int prescale; - unsigned long dividend; - unsigned long k; - int s; - const int max_k_bits = (64 - 48) - 1; - const unsigned long billion = 1000000000; - const unsigned long counts_per_second = 27000000 / 8; - - prescale = BITS_PER_LONG - ilog2(billion) - 1; - dividend = billion << prescale; - k = dividend / counts_per_second; - s = ilog2(k) - max_k_bits; - - if (s < 0) - s = prescale; - - else { - k >>= s; - s += prescale; - } - - clocksource_tim_c.mult = k; - clocksource_tim_c.shift = s; - clocksource_tim_c.rating = 200; - - clocksource_register(&clocksource_tim_c); - tim_c = (struct tim_c *) asic_reg_addr(tim_ch); -} - -/** - powertv_clocksource_init - initialize all clocksources - */ -void __init powertv_clocksource_init(void) -{ - powertv_c0_hpt_clocksource_init(); - powertv_tim_c_clocksource_init(); -} diff --git a/arch/mips/kernel/csrc-r4k.c b/arch/mips/kernel/csrc-r4k.c index e95a3cd48ee..decd1fa38d5 100644 --- a/arch/mips/kernel/csrc-r4k.c +++ b/arch/mips/kernel/csrc-r4k.c @@ -30,9 +30,7 @@ int __init init_r4k_clocksource(void) /* Calculate a somewhat reasonable rating value */ clocksource_mips.rating = 200 + mips_hpt_frequency / 10000000; - clocksource_set_clock(&clocksource_mips, mips_hpt_frequency); - - clocksource_register(&clocksource_mips); + clocksource_register_hz(&clocksource_mips, mips_hpt_frequency); return 0; } diff --git a/arch/mips/kernel/csrc-sb1250.c b/arch/mips/kernel/csrc-sb1250.c index d14d3d1907f..6ecb77d8206 100644 --- a/arch/mips/kernel/csrc-sb1250.c +++ b/arch/mips/kernel/csrc-sb1250.c @@ -44,7 +44,7 @@ static cycle_t sb1250_hpt_read(struct clocksource *cs) struct clocksource bcm1250_clocksource = { .name = "bcm1250-counter-3", - .rating = 200, + .rating = 200, .read = sb1250_hpt_read, .mask = CLOCKSOURCE_MASK(23), .flags = CLOCK_SOURCE_IS_CONTINUOUS, @@ -65,6 +65,5 @@ void __init sb1250_clocksource_init(void) IOADDR(A_SCD_TIMER_REGISTER(SB1250_HPT_NUM, R_SCD_TIMER_CFG))); - clocksource_set_clock(cs, V_SCD_TIMER_FREQ); - clocksource_register(cs); + clocksource_register_hz(cs, V_SCD_TIMER_FREQ); } diff --git a/arch/mips/kernel/early_printk.c b/arch/mips/kernel/early_printk.c index 9ae813eb782..505cb77d128 100644 --- a/arch/mips/kernel/early_printk.c +++ b/arch/mips/kernel/early_printk.c @@ -7,15 +7,16 @@ * Copyright (C) 2007 MIPS Technologies, Inc. * written by Ralf Baechle (ralf@linux-mips.org) */ +#include <linux/kernel.h> #include <linux/console.h> +#include <linux/printk.h> #include <linux/init.h> #include <asm/setup.h> extern void prom_putchar(char); -static void __init -early_console_write(struct console *con, const char *s, unsigned n) +static void early_console_write(struct console *con, const char *s, unsigned n) { while (n-- && *s) { if (*s == '\n') @@ -25,20 +26,18 @@ early_console_write(struct console *con, const char *s, unsigned n) } } -static struct console early_console __initdata = { +static struct console early_console_prom = { .name = "early", .write = early_console_write, .flags = CON_PRINTBUFFER | CON_BOOT, .index = -1 }; -static int early_console_initialized __initdata; - void __init setup_early_printk(void) { - if (early_console_initialized) + if (early_console) return; - early_console_initialized = 1; + early_console = &early_console_prom; - register_console(&early_console); + register_console(&early_console_prom); } diff --git a/arch/mips/kernel/early_printk_8250.c b/arch/mips/kernel/early_printk_8250.c new file mode 100644 index 00000000000..83cea376755 --- /dev/null +++ b/arch/mips/kernel/early_printk_8250.c @@ -0,0 +1,66 @@ +/* + * 8250/16550-type serial ports prom_putchar() + * + * Copyright (C) 2010 Yoichi Yuasa <yuasa@linux-mips.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include <linux/io.h> +#include <linux/serial_core.h> +#include <linux/serial_reg.h> + +static void __iomem *serial8250_base; +static unsigned int serial8250_reg_shift; +static unsigned int serial8250_tx_timeout; + +void setup_8250_early_printk_port(unsigned long base, unsigned int reg_shift, + unsigned int timeout) +{ + serial8250_base = (void __iomem *)base; + serial8250_reg_shift = reg_shift; + serial8250_tx_timeout = timeout; +} + +static inline u8 serial_in(int offset) +{ + return readb(serial8250_base + (offset << serial8250_reg_shift)); +} + +static inline void serial_out(int offset, char value) +{ + writeb(value, serial8250_base + (offset << serial8250_reg_shift)); +} + +void prom_putchar(char c) +{ + unsigned int timeout; + int status, bits; + + if (!serial8250_base) + return; + + timeout = serial8250_tx_timeout; + bits = UART_LSR_TEMT | UART_LSR_THRE; + + do { + status = serial_in(UART_LSR); + + if (--timeout == 0) + break; + } while ((status & bits) != bits); + + if (timeout) + serial_out(UART_TX, c); +} diff --git a/arch/mips/kernel/entry.S b/arch/mips/kernel/entry.S index ffa331029e0..4353d323f01 100644 --- a/arch/mips/kernel/entry.S +++ b/arch/mips/kernel/entry.S @@ -16,9 +16,6 @@ #include <asm/isadep.h> #include <asm/thread_info.h> #include <asm/war.h> -#ifdef CONFIG_MIPS_MT_SMTC -#include <asm/mipsmtregs.h> -#endif #ifndef CONFIG_PREEMPT #define resume_kernel restore_all @@ -36,6 +33,11 @@ FEXPORT(ret_from_exception) FEXPORT(ret_from_irq) LONG_S s0, TI_REGS($28) FEXPORT(__ret_from_irq) +/* + * We can be coming here from a syscall done in the kernel space, + * e.g. a failed kernel_execve(). + */ +resume_userspace_check: LONG_L t0, PT_STATUS(sp) # returning to kernel mode? andi t0, t0, KU_USER beqz t0, resume_kernel @@ -65,6 +67,12 @@ need_resched: b need_resched #endif +FEXPORT(ret_from_kernel_thread) + jal schedule_tail # a0 = struct task_struct *prev + move a0, s1 + jal s0 + j syscall_exit + FEXPORT(ret_from_fork) jal schedule_tail # a0 = struct task_struct *prev @@ -77,47 +85,12 @@ FEXPORT(syscall_exit) and t0, a2, t0 bnez t0, syscall_exit_work -FEXPORT(restore_all) # restore full frame -#ifdef CONFIG_MIPS_MT_SMTC -#ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP -/* Re-arm any temporarily masked interrupts not explicitly "acked" */ - mfc0 v0, CP0_TCSTATUS - ori v1, v0, TCSTATUS_IXMT - mtc0 v1, CP0_TCSTATUS - andi v0, TCSTATUS_IXMT - _ehb - mfc0 t0, CP0_TCCONTEXT - DMT 9 # dmt t1 - jal mips_ihb - mfc0 t2, CP0_STATUS - andi t3, t0, 0xff00 - or t2, t2, t3 - mtc0 t2, CP0_STATUS - _ehb - andi t1, t1, VPECONTROL_TE - beqz t1, 1f - EMT -1: - mfc0 v1, CP0_TCSTATUS - /* We set IXMT above, XOR should clear it here */ - xori v1, v1, TCSTATUS_IXMT - or v1, v0, v1 - mtc0 v1, CP0_TCSTATUS - _ehb - xor t0, t0, t3 - mtc0 t0, CP0_TCCONTEXT -#endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */ -/* Detect and execute deferred IPI "interrupts" */ - LONG_L s0, TI_REGS($28) - LONG_S sp, TI_REGS($28) - jal deferred_smtc_ipi - LONG_S s0, TI_REGS($28) -#endif /* CONFIG_MIPS_MT_SMTC */ +restore_all: # restore full frame .set noat RESTORE_TEMP RESTORE_AT RESTORE_STATIC -FEXPORT(restore_partial) # restore partial frame +restore_partial: # restore partial frame #ifdef CONFIG_TRACE_IRQFLAGS SAVE_STATIC SAVE_AT @@ -162,19 +135,27 @@ work_notifysig: # deal with pending signals and move a0, sp li a1, 0 jal do_notify_resume # a2 already loaded - j resume_userspace + j resume_userspace_check -FEXPORT(syscall_exit_work_partial) +FEXPORT(syscall_exit_partial) + local_irq_disable # make sure need_resched doesn't + # change between and return + LONG_L a2, TI_FLAGS($28) # current->work + li t0, _TIF_ALLWORK_MASK + and t0, a2 + beqz t0, restore_partial SAVE_STATIC syscall_exit_work: - li t0, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + LONG_L t0, PT_STATUS(sp) # returning to kernel mode? + andi t0, t0, KU_USER + beqz t0, resume_kernel + li t0, _TIF_WORK_SYSCALL_EXIT and t0, a2 # a2 is preloaded with TI_FLAGS beqz t0, work_pending # trace bit set? - local_irq_enable # could let do_syscall_trace() + local_irq_enable # could let syscall_trace_leave() # call schedule() instead move a0, sp - li a1, 1 - jal do_syscall_trace + jal syscall_trace_leave b resume_userspace #if defined(CONFIG_CPU_MIPSR2) || defined(CONFIG_MIPS_MT) diff --git a/arch/mips/kernel/ftrace.c b/arch/mips/kernel/ftrace.c index 5a84a1f1123..60e7e5e45af 100644 --- a/arch/mips/kernel/ftrace.c +++ b/arch/mips/kernel/ftrace.c @@ -11,35 +11,53 @@ #include <linux/uaccess.h> #include <linux/init.h> #include <linux/ftrace.h> +#include <linux/syscalls.h> #include <asm/asm.h> #include <asm/asm-offsets.h> #include <asm/cacheflush.h> +#include <asm/syscall.h> #include <asm/uasm.h> +#include <asm/unistd.h> + +#include <asm-generic/sections.h> + +#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) +#define MCOUNT_OFFSET_INSNS 5 +#else +#define MCOUNT_OFFSET_INSNS 4 +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE + +/* Arch override because MIPS doesn't need to run this from stop_machine() */ +void arch_ftrace_update_code(int command) +{ + ftrace_modify_all_code(command); +} + +#endif /* - * If the Instruction Pointer is in module space (0xc0000000), return true; - * otherwise, it is in kernel space (0x80000000), return false. + * Check if the address is in kernel space * - * FIXME: This will not work when the kernel space and module space are the - * same. If they are the same, we need to modify scripts/recordmcount.pl, - * ftrace_make_nop/call() and the other related parts to ensure the - * enabling/disabling of the calling site to _mcount is right for both kernel - * and module. + * Clone core_kernel_text() from kernel/extable.c, but doesn't call + * init_kernel_text() for Ftrace doesn't trace functions in init sections. */ - -static inline int in_module(unsigned long ip) +static inline int in_kernel_space(unsigned long ip) { - return ip & 0x40000000; + if (ip >= (unsigned long)_stext && + ip <= (unsigned long)_etext) + return 1; + return 0; } #ifdef CONFIG_DYNAMIC_FTRACE #define JAL 0x0c000000 /* jump & link: ip --> ra, jump to target */ #define ADDR_MASK 0x03ffffff /* op_code|addr : 31...26|25 ....0 */ +#define JUMP_RANGE_MASK ((1UL << 28) - 1) -#define INSN_B_1F_4 0x10000004 /* b 1f; offset = 4 */ -#define INSN_B_1F_5 0x10000005 /* b 1f; offset = 5 */ #define INSN_NOP 0x00000000 /* nop */ #define INSN_JAL(addr) \ ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK))) @@ -60,18 +78,19 @@ static inline void ftrace_dyn_arch_init_insns(void) /* jal (ftrace_caller + 8), jump over the first two instruction */ buf = (u32 *)&insn_jal_ftrace_caller; - uasm_i_jal(&buf, (FTRACE_ADDR + 8)); + uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK); #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* j ftrace_graph_caller */ buf = (u32 *)&insn_j_ftrace_graph_caller; - uasm_i_j(&buf, (unsigned long)ftrace_graph_caller); + uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK); #endif } static int ftrace_modify_code(unsigned long ip, unsigned int new_code) { int faulted; + mm_segment_t old_fs; /* *(unsigned int *)ip = new_code; */ safe_store_code(new_code, ip, faulted); @@ -79,11 +98,62 @@ static int ftrace_modify_code(unsigned long ip, unsigned int new_code) if (unlikely(faulted)) return -EFAULT; + old_fs = get_fs(); + set_fs(get_ds()); flush_icache_range(ip, ip + 8); + set_fs(old_fs); return 0; } +#ifndef CONFIG_64BIT +static int ftrace_modify_code_2(unsigned long ip, unsigned int new_code1, + unsigned int new_code2) +{ + int faulted; + + safe_store_code(new_code1, ip, faulted); + if (unlikely(faulted)) + return -EFAULT; + safe_store_code(new_code2, ip + 4, faulted); + if (unlikely(faulted)) + return -EFAULT; + flush_icache_range(ip, ip + 8); + return 0; +} +#endif + +/* + * The details about the calling site of mcount on MIPS + * + * 1. For kernel: + * + * move at, ra + * jal _mcount --> nop + * + * 2. For modules: + * + * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT + * + * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) + * addiu v1, v1, low_16bit_of_mcount + * move at, ra + * move $12, ra_address + * jalr v1 + * sub sp, sp, 8 + * 1: offset = 5 instructions + * 2.2 For the Other situations + * + * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) + * addiu v1, v1, low_16bit_of_mcount + * move at, ra + * jalr v1 + * nop | move $12, ra_address | sub sp, sp, 8 + * 1: offset = 4 instructions + */ + +#define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS) + int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) { @@ -91,40 +161,22 @@ int ftrace_make_nop(struct module *mod, unsigned long ip = rec->ip; /* - * We have compiled module with -mlong-calls, but compiled the kernel - * without it, we need to cope with them respectively. + * If ip is in kernel space, no long call, otherwise, long call is + * needed. */ - if (in_module(ip)) { -#if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT) - /* - * lui v1, hi_16bit_of_mcount --> b 1f (0x10000005) - * addiu v1, v1, low_16bit_of_mcount - * move at, ra - * move $12, ra_address - * jalr v1 - * sub sp, sp, 8 - * 1: offset = 5 instructions - */ - new = INSN_B_1F_5; + new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F; +#ifdef CONFIG_64BIT + return ftrace_modify_code(ip, new); #else - /* - * lui v1, hi_16bit_of_mcount --> b 1f (0x10000004) - * addiu v1, v1, low_16bit_of_mcount - * move at, ra - * jalr v1 - * nop | move $12, ra_address | sub sp, sp, 8 - * 1: offset = 4 instructions - */ - new = INSN_B_1F_4; + /* + * On 32 bit MIPS platforms, gcc adds a stack adjust + * instruction in the delay slot after the branch to + * mcount and expects mcount to restore the sp on return. + * This is based on a legacy API and does nothing but + * waste instructions so it's being removed at runtime. + */ + return ftrace_modify_code_2(ip, new, INSN_NOP); #endif - } else { - /* - * move at, ra - * jal _mcount --> nop - */ - new = INSN_NOP; - } - return ftrace_modify_code(ip, new); } int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) @@ -132,8 +184,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) unsigned int new; unsigned long ip = rec->ip; - /* ip, module: 0xc0000000, kernel: 0x80000000 */ - new = in_module(ip) ? insn_lui_v1_hi16_mcount : insn_jal_ftrace_caller; + new = in_kernel_space(ip) ? insn_jal_ftrace_caller : + insn_lui_v1_hi16_mcount; return ftrace_modify_code(ip, new); } @@ -149,7 +201,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func) return ftrace_modify_code(FTRACE_CALL_IP, new); } -int __init ftrace_dyn_arch_init(void *data) +int __init ftrace_dyn_arch_init(void) { /* Encode the instructions when booting */ ftrace_dyn_arch_init_insns(); @@ -157,9 +209,6 @@ int __init ftrace_dyn_arch_init(void *data) /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */ ftrace_modify_code(MCOUNT_ADDR, INSN_NOP); - /* The return code is retured via data */ - *(unsigned long *)data = 0; - return 0; } #endif /* CONFIG_DYNAMIC_FTRACE */ @@ -186,33 +235,29 @@ int ftrace_disable_ftrace_graph_caller(void) #ifndef KBUILD_MCOUNT_RA_ADDRESS -#define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ -#define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ +#define S_RA_SP (0xafbf << 16) /* s{d,w} ra, offset(sp) */ +#define S_R_SP (0xafb0 << 16) /* s{d,w} R, offset(sp) */ #define OFFSET_MASK 0xffff /* stack offset range: 0 ~ PT_SIZE */ -unsigned long ftrace_get_parent_addr(unsigned long self_addr, - unsigned long parent, - unsigned long parent_addr, - unsigned long fp) +unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long + old_parent_ra, unsigned long parent_ra_addr, unsigned long fp) { - unsigned long sp, ip, ra; + unsigned long sp, ip, tmp; unsigned int code; int faulted; /* - * For module, move the ip from calling site of mcount to the - * instruction "lui v1, hi_16bit_of_mcount"(offset is 20), but for - * kernel, move to the instruction "move ra, at"(offset is 12) + * For module, move the ip from the return address after the + * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for + * kernel, move after the instruction "move ra, at"(offset is 16) */ - ip = self_addr - (in_module(self_addr) ? 20 : 12); + ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24); /* * search the text until finding the non-store instruction or "s{d,w} * ra, offset(sp)" instruction */ do { - ip -= 4; - /* get the code at "ip": code = *(unsigned int *)ip; */ safe_load_code(code, ip, faulted); @@ -224,18 +269,20 @@ unsigned long ftrace_get_parent_addr(unsigned long self_addr, * store the ra on the stack */ if ((code & S_R_SP) != S_R_SP) - return parent_addr; + return parent_ra_addr; - } while (((code & S_RA_SP) != S_RA_SP)); + /* Move to the next instruction */ + ip -= 4; + } while ((code & S_RA_SP) != S_RA_SP); sp = fp + (code & OFFSET_MASK); - /* ra = *(unsigned long *)sp; */ - safe_load_stack(ra, sp, faulted); + /* tmp = *(unsigned long *)sp; */ + safe_load_stack(tmp, sp, faulted); if (unlikely(faulted)) return 0; - if (ra == parent) + if (tmp == old_parent_ra) return sp; return 0; } @@ -246,21 +293,21 @@ unsigned long ftrace_get_parent_addr(unsigned long self_addr, * Hook the return address and push it in the stack of return addrs * in current thread info. */ -void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, +void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra, unsigned long fp) { - unsigned long old; + unsigned long old_parent_ra; struct ftrace_graph_ent trace; unsigned long return_hooker = (unsigned long) &return_to_handler; - int faulted; + int faulted, insns; if (unlikely(atomic_read(¤t->tracing_graph_pause))) return; /* - * "parent" is the stack address saved the return address of the caller - * of _mcount. + * "parent_ra_addr" is the stack address saved the return address of + * the caller of _mcount. * * if the gcc < 4.5, a leaf function does not save the return address * in the stack address, so, we "emulate" one in _mcount's stack space, @@ -275,37 +322,44 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, * do it in ftrace_graph_caller of mcount.S. */ - /* old = *parent; */ - safe_load_stack(old, parent, faulted); + /* old_parent_ra = *parent_ra_addr; */ + safe_load_stack(old_parent_ra, parent_ra_addr, faulted); if (unlikely(faulted)) goto out; #ifndef KBUILD_MCOUNT_RA_ADDRESS - parent = (unsigned long *)ftrace_get_parent_addr(self_addr, old, - (unsigned long)parent, fp); + parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra, + old_parent_ra, (unsigned long)parent_ra_addr, fp); /* * If fails when getting the stack address of the non-leaf function's * ra, stop function graph tracer and return */ - if (parent == 0) + if (parent_ra_addr == 0) goto out; #endif - /* *parent = return_hooker; */ - safe_store_stack(return_hooker, parent, faulted); + /* *parent_ra_addr = return_hooker; */ + safe_store_stack(return_hooker, parent_ra_addr, faulted); if (unlikely(faulted)) goto out; - if (ftrace_push_return_trace(old, self_addr, &trace.depth, fp) == - -EBUSY) { - *parent = old; + if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp) + == -EBUSY) { + *parent_ra_addr = old_parent_ra; return; } - trace.func = self_addr; + /* + * Get the recorded ip of the current mcount calling site in the + * __mcount_loc section, which will be used to filter the function + * entries configured through the tracing/set_graph_function interface. + */ + + insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1; + trace.func = self_ra - (MCOUNT_INSN_SIZE * insns); /* Only trace if the calling function expects to */ if (!ftrace_graph_entry(&trace)) { current->curr_ret_stack--; - *parent = old; + *parent_ra_addr = old_parent_ra; } return; out: @@ -313,3 +367,33 @@ out: WARN_ON(1); } #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ + +#ifdef CONFIG_FTRACE_SYSCALLS + +#ifdef CONFIG_32BIT +unsigned long __init arch_syscall_addr(int nr) +{ + return (unsigned long)sys_call_table[nr - __NR_O32_Linux]; +} +#endif + +#ifdef CONFIG_64BIT + +unsigned long __init arch_syscall_addr(int nr) +{ +#ifdef CONFIG_MIPS32_N32 + if (nr >= __NR_N32_Linux && nr <= __NR_N32_Linux + __NR_N32_Linux_syscalls) + return (unsigned long)sysn32_call_table[nr - __NR_N32_Linux]; +#endif + if (nr >= __NR_64_Linux && nr <= __NR_64_Linux + __NR_64_Linux_syscalls) + return (unsigned long)sys_call_table[nr - __NR_64_Linux]; +#ifdef CONFIG_MIPS32_O32 + if (nr >= __NR_O32_Linux && nr <= __NR_O32_Linux + __NR_O32_Linux_syscalls) + return (unsigned long)sys32_call_table[nr - __NR_O32_Linux]; +#endif + + return (unsigned long) &sys_ni_syscall; +} +#endif + +#endif /* CONFIG_FTRACE_SYSCALLS */ diff --git a/arch/mips/kernel/genex.S b/arch/mips/kernel/genex.S index 8882e5766f2..ac35e12cb1f 100644 --- a/arch/mips/kernel/genex.S +++ b/arch/mips/kernel/genex.S @@ -5,8 +5,8 @@ * * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. - * Copyright (C) 2001 MIPS Technologies, Inc. * Copyright (C) 2002, 2007 Maciej W. Rozycki + * Copyright (C) 2001, 2012 MIPS Technologies, Inc. All rights reserved. */ #include <linux/init.h> @@ -19,30 +19,10 @@ #include <asm/mipsregs.h> #include <asm/stackframe.h> #include <asm/war.h> -#include <asm/page.h> #include <asm/thread_info.h> -#define PANIC_PIC(msg) \ - .set push; \ - .set reorder; \ - PTR_LA a0,8f; \ - .set noat; \ - PTR_LA AT, panic; \ - jr AT; \ -9: b 9b; \ - .set pop; \ - TEXT(msg) - __INIT -NESTED(except_vec0_generic, 0, sp) - PANIC_PIC("Exception vector 0 called") - END(except_vec0_generic) - -NESTED(except_vec1_generic, 0, sp) - PANIC_PIC("Exception vector 1 called") - END(except_vec1_generic) - /* * General exception vector for all other CPUs. * @@ -73,7 +53,7 @@ NESTED(except_vec3_generic, 0, sp) */ NESTED(except_vec3_r4000, 0, sp) .set push - .set mips3 + .set arch=r4000 .set noat mfc0 k1, CP0_CAUSE li k0, 31<<2 @@ -128,7 +108,7 @@ handle_vcei: __FINIT .align 5 /* 32 byte rollback region */ -LEAF(r4k_wait) +LEAF(__r4k_wait) .set push .set noreorder /* start of rollback region */ @@ -139,20 +119,27 @@ LEAF(r4k_wait) nop nop nop - .set mips3 +#ifdef CONFIG_CPU_MICROMIPS + nop + nop + nop + nop +#endif + .set arch=r4000 wait /* end of rollback region (the region size must be power of two) */ - .set pop 1: jr ra - END(r4k_wait) + nop + .set pop + END(__r4k_wait) .macro BUILD_ROLLBACK_PROLOGUE handler FEXPORT(rollback_\handler) .set push .set noat MFC0 k0, CP0_EPC - PTR_LA k1, r4k_wait + PTR_LA k1, __r4k_wait ori k0, 0x1f /* 32 byte rollback region */ xori k0, 0x1f bne k0, k1, 9f @@ -161,7 +148,7 @@ LEAF(r4k_wait) .set pop .endm - .align 5 + .align 5 BUILD_ROLLBACK_PROLOGUE handle_int NESTED(handle_int, PT_SIZE, sp) #ifdef CONFIG_TRACE_IRQFLAGS @@ -202,7 +189,11 @@ NESTED(handle_int, PT_SIZE, sp) LONG_L s0, TI_REGS($28) LONG_S sp, TI_REGS($28) PTR_LA ra, ret_from_irq - j plat_irq_dispatch + PTR_LA v0, plat_irq_dispatch + jr v0 +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(handle_int) __INIT @@ -223,11 +214,14 @@ NESTED(except_vec4, 0, sp) /* * EJTAG debug exception handler. * The EJTAG debug exception entry point is 0xbfc00480, which - * normally is in the boot PROM, so the boot PROM must do a + * normally is in the boot PROM, so the boot PROM must do an * unconditional jump to this vector. */ NESTED(except_vec_ejtag_debug, 0, sp) j ejtag_debug_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(except_vec_ejtag_debug) __FINIT @@ -243,18 +237,10 @@ NESTED(except_vec_vi, 0, sp) SAVE_AT .set push .set noreorder -#ifdef CONFIG_MIPS_MT_SMTC - /* - * To keep from blindly blocking *all* interrupts - * during service by SMTC kernel, we also want to - * pass the IM value to be cleared. - */ -FEXPORT(except_vec_vi_mori) - ori a0, $0, 0 -#endif /* CONFIG_MIPS_MT_SMTC */ + PTR_LA v1, except_vec_vi_handler FEXPORT(except_vec_vi_lui) lui v0, 0 /* Patched */ - j except_vec_vi_handler + jr v1 FEXPORT(except_vec_vi_ori) ori v0, 0 /* Patched */ .set pop @@ -268,37 +254,10 @@ EXPORT(except_vec_vi_end) NESTED(except_vec_vi_handler, 0, sp) SAVE_TEMP SAVE_STATIC -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC has an interesting problem that interrupts are level-triggered, - * and the CLI macro will clear EXL, potentially causing a duplicate - * interrupt service invocation. So we need to clear the associated - * IM bit of Status prior to doing CLI, and restore it after the - * service routine has been invoked - we must assume that the - * service routine will have cleared the state, and any active - * level represents a new or otherwised unserviced event... - */ - mfc0 t1, CP0_STATUS - and t0, a0, t1 -#ifdef CONFIG_MIPS_MT_SMTC_IM_BACKSTOP - mfc0 t2, CP0_TCCONTEXT - or t2, t0, t2 - mtc0 t2, CP0_TCCONTEXT -#endif /* CONFIG_MIPS_MT_SMTC_IM_BACKSTOP */ - xor t1, t1, t0 - mtc0 t1, CP0_STATUS - _ehb -#endif /* CONFIG_MIPS_MT_SMTC */ CLI #ifdef CONFIG_TRACE_IRQFLAGS move s0, v0 -#ifdef CONFIG_MIPS_MT_SMTC - move s1, a0 -#endif TRACE_IRQS_OFF -#ifdef CONFIG_MIPS_MT_SMTC - move a0, s1 -#endif move v0, s0 #endif @@ -355,6 +314,9 @@ EXPORT(ejtag_debug_buffer) */ NESTED(except_vec_nmi, 0, sp) j nmi_handler +#ifdef CONFIG_CPU_MICROMIPS + nop +#endif END(except_vec_nmi) __FINIT @@ -362,12 +324,20 @@ NESTED(except_vec_nmi, 0, sp) NESTED(nmi_handler, PT_SIZE, sp) .set push .set noat + /* + * Clear ERL - restore segment mapping + * Clear BEV - required for page fault exception handler to work + */ + mfc0 k0, CP0_STATUS + ori k0, k0, ST0_EXL + li k1, ~(ST0_BEV | ST0_ERL) + and k0, k0, k1 + mtc0 k0, CP0_STATUS + _ehb SAVE_ALL - move a0, sp + move a0, sp jal nmi_exception_handler - RESTORE_ALL - .set mips3 - eret + /* nmi_exception_handler never returns */ .set pop END(nmi_handler) @@ -410,7 +380,7 @@ NESTED(nmi_handler, PT_SIZE, sp) string escapes and emits bogus warnings if it believes to recognize an unknown escape code. So make the arguments start with an n and gas will believe \n is ok ... */ - .macro __BUILD_verbose nexception + .macro __BUILD_verbose nexception LONG_L a1, PT_EPC(sp) #ifdef CONFIG_32BIT PRINT("Got \nexception at %08lx\012") @@ -443,7 +413,7 @@ NESTED(nmi_handler, PT_SIZE, sp) .endm .macro BUILD_HANDLER exception handler clear verbose - __BUILD_HANDLER \exception \handler \clear \verbose _int + __BUILD_HANDLER \exception \handler \clear \verbose _int .endm BUILD_HANDLER adel ade ade silent /* #4 */ @@ -455,9 +425,12 @@ NESTED(nmi_handler, PT_SIZE, sp) BUILD_HANDLER cpu cpu sti silent /* #11 */ BUILD_HANDLER ov ov sti silent /* #12 */ BUILD_HANDLER tr tr sti silent /* #13 */ + BUILD_HANDLER msa_fpe msa_fpe sti silent /* #14 */ BUILD_HANDLER fpe fpe fpe silent /* #15 */ + BUILD_HANDLER ftlb ftlb none silent /* #16 */ + BUILD_HANDLER msa msa sti silent /* #21 */ BUILD_HANDLER mdmx mdmx sti silent /* #22 */ -#ifdef CONFIG_HARDWARE_WATCHPOINTS +#ifdef CONFIG_HARDWARE_WATCHPOINTS /* * For watch, interrupts will be enabled after the watch * registers are read. @@ -473,9 +446,6 @@ NESTED(nmi_handler, PT_SIZE, sp) .align 5 LEAF(handle_ri_rdhwr_vivt) -#ifdef CONFIG_MIPS_MT_SMTC - PANIC_PIC("handle_ri_rdhwr_vivt called") -#else .set push .set noat .set noreorder @@ -483,8 +453,8 @@ NESTED(nmi_handler, PT_SIZE, sp) MFC0 k1, CP0_ENTRYHI andi k1, 0xff /* ASID_MASK */ MFC0 k0, CP0_EPC - PTR_SRL k0, PAGE_SHIFT + 1 - PTR_SLL k0, PAGE_SHIFT + 1 + PTR_SRL k0, _PAGE_SHIFT + 1 + PTR_SLL k0, _PAGE_SHIFT + 1 or k1, k0 MTC0 k1, CP0_ENTRYHI mtc0_tlbw_hazard @@ -494,20 +464,41 @@ NESTED(nmi_handler, PT_SIZE, sp) .set pop bltz k1, handle_ri /* slow path */ /* fall thru */ -#endif END(handle_ri_rdhwr_vivt) LEAF(handle_ri_rdhwr) .set push .set noat .set noreorder - /* 0x7c03e83b: rdhwr v1,$29 */ + /* MIPS32: 0x7c03e83b: rdhwr v1,$29 */ + /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */ MFC0 k1, CP0_EPC - lui k0, 0x7c03 - lw k1, (k1) - ori k0, 0xe83b - .set reorder +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2) + and k0, k1, 1 + beqz k0, 1f + xor k1, k0 + lhu k0, (k1) + lhu k1, 2(k1) + ins k1, k0, 16, 16 + lui k0, 0x007d + b docheck + ori k0, 0x6b3c +1: + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#else + andi k0, k1, 1 + bnez k0, handle_ri + lui k0, 0x7c03 + lw k1, (k1) + ori k0, 0xe83b +#endif + .set reorder +docheck: bne k0, k1, handle_ri /* if not ours */ + +isrdhwr: /* The insn is rdhwr. No need to check CAUSE.BD here. */ get_saved_sp /* k1 := current_thread_info */ .set noreorder @@ -532,7 +523,7 @@ NESTED(nmi_handler, PT_SIZE, sp) ori k1, _THREAD_MASK xori k1, _THREAD_MASK LONG_L v1, TI_TP_VALUE(k1) - .set mips3 + .set arch=r4000 eret .set mips0 #endif diff --git a/arch/mips/kernel/head.S b/arch/mips/kernel/head.S index ea695d9605e..95afd663cd4 100644 --- a/arch/mips/kernel/head.S +++ b/arch/mips/kernel/head.S @@ -21,7 +21,6 @@ #include <asm/asmmacro.h> #include <asm/irqflags.h> #include <asm/regdef.h> -#include <asm/page.h> #include <asm/pgtable-bits.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> @@ -29,45 +28,6 @@ #include <kernel-entry-init.h> /* - * inputs are the text nasid in t1, data nasid in t2. - */ - .macro MAPPED_KERNEL_SETUP_TLB -#ifdef CONFIG_MAPPED_KERNEL - /* - * This needs to read the nasid - assume 0 for now. - * Drop in 0xffffffffc0000000 in tlbhi, 0+VG in tlblo_0, - * 0+DVG in tlblo_1. - */ - dli t0, 0xffffffffc0000000 - dmtc0 t0, CP0_ENTRYHI - li t0, 0x1c000 # Offset of text into node memory - dsll t1, NASID_SHFT # Shift text nasid into place - dsll t2, NASID_SHFT # Same for data nasid - or t1, t1, t0 # Physical load address of kernel text - or t2, t2, t0 # Physical load address of kernel data - dsrl t1, 12 # 4K pfn - dsrl t2, 12 # 4K pfn - dsll t1, 6 # Get pfn into place - dsll t2, 6 # Get pfn into place - li t0, ((_PAGE_GLOBAL|_PAGE_VALID| _CACHE_CACHABLE_COW) >> 6) - or t0, t0, t1 - mtc0 t0, CP0_ENTRYLO0 # physaddr, VG, cach exlwr - li t0, ((_PAGE_GLOBAL|_PAGE_VALID| _PAGE_DIRTY|_CACHE_CACHABLE_COW) >> 6) - or t0, t0, t2 - mtc0 t0, CP0_ENTRYLO1 # physaddr, DVG, cach exlwr - li t0, 0x1ffe000 # MAPPED_KERN_TLBMASK, TLBPGMASK_16M - mtc0 t0, CP0_PAGEMASK - li t0, 0 # KMAP_INX - mtc0 t0, CP0_INDEX - li t0, 1 - mtc0 t0, CP0_WIRED - tlbwi -#else - mtc0 zero, CP0_WIRED -#endif - .endm - - /* * For the moment disable interrupts, mark the kernel mode and * set ST0_KX so that the CPU does not spit fire when using * 64-bit addresses. A full initialization of the CPU's status @@ -75,33 +35,12 @@ */ .macro setup_c0_status set clr .set push -#ifdef CONFIG_MIPS_MT_SMTC - /* - * For SMTC, we need to set privilege and disable interrupts only for - * the current TC, using the TCStatus register. - */ - mfc0 t0, CP0_TCSTATUS - /* Fortunately CU 0 is in the same place in both registers */ - /* Set TCU0, TMX, TKSU (for later inversion) and IXMT */ - li t1, ST0_CU0 | 0x08001c00 - or t0, t1 - /* Clear TKSU, leave IXMT */ - xori t0, 0x00001800 - mtc0 t0, CP0_TCSTATUS - _ehb - /* We need to leave the global IE bit set, but clear EXL...*/ - mfc0 t0, CP0_STATUS - or t0, ST0_CU0 | ST0_EXL | ST0_ERL | \set | \clr - xor t0, ST0_EXL | ST0_ERL | \clr - mtc0 t0, CP0_STATUS -#else mfc0 t0, CP0_STATUS or t0, ST0_CU0|\set|0x1f|\clr xor t0, 0x1f|\clr mtc0 t0, CP0_STATUS .set noreorder sll zero,3 # ehb -#endif .set pop .endm @@ -134,7 +73,7 @@ EXPORT(_stext) #ifdef CONFIG_BOOT_RAW /* * Give us a fighting chance of running if execution beings at the - * kernel load address. This is needed because this platform does + * kernel load address. This is needed because this platform does * not have a ELF loader yet. */ FEXPORT(__kernel_entry) @@ -155,24 +94,6 @@ NESTED(kernel_entry, 16, sp) # kernel entry point jr t0 0: -#ifdef CONFIG_MIPS_MT_SMTC - /* - * In SMTC kernel, "CLI" is thread-specific, in TCStatus. - * We still need to enable interrupts globally in Status, - * and clear EXL/ERL. - * - * TCContext is used to track interrupt levels under - * service in SMTC kernel. Clear for boot TC before - * allowing any interrupts. - */ - mtc0 zero, CP0_TCCONTEXT - - mfc0 t0, CP0_STATUS - ori t0, t0, 0xff1f - xori t0, t0, 0x001e - mtc0 t0, CP0_STATUS -#endif /* CONFIG_MIPS_MT_SMTC */ - PTR_LA t0, __bss_start # clear .bss LONG_S zero, (t0) PTR_LA t1, __bss_stop - LONGSIZE @@ -198,35 +119,14 @@ NESTED(kernel_entry, 16, sp) # kernel entry point j start_kernel END(kernel_entry) - __CPUINIT - #ifdef CONFIG_SMP /* - * SMP slave cpus entry point. Board specific code for bootstrap calls this + * SMP slave cpus entry point. Board specific code for bootstrap calls this * function after setting up the stack and gp registers. */ NESTED(smp_bootstrap, 16, sp) -#ifdef CONFIG_MIPS_MT_SMTC - /* - * Read-modify-writes of Status must be atomic, and this - * is one case where CLI is invoked without EXL being - * necessarily set. The CLI and setup_c0_status will - * in fact be redundant for all but the first TC of - * each VPE being booted. - */ - DMT 10 # dmt t2 /* t0, t1 are used by CLI and setup_c0_status() */ - jal mips_ihb -#endif /* CONFIG_MIPS_MT_SMTC */ - setup_c0_status_sec smp_slave_setup -#ifdef CONFIG_MIPS_MT_SMTC - andi t2, t2, VPECONTROL_TE - beqz t2, 2f - EMT # emt -2: -#endif /* CONFIG_MIPS_MT_SMTC */ + setup_c0_status_sec j start_secondary END(smp_bootstrap) #endif /* CONFIG_SMP */ - - __FINIT diff --git a/arch/mips/kernel/i8253.c b/arch/mips/kernel/i8253.c index 2392a7a296d..c5bc344fc74 100644 --- a/arch/mips/kernel/i8253.c +++ b/arch/mips/kernel/i8253.c @@ -3,209 +3,37 @@ * */ #include <linux/clockchips.h> -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/jiffies.h> -#include <linux/module.h> +#include <linux/i8253.h> +#include <linux/export.h> #include <linux/smp.h> -#include <linux/spinlock.h> #include <linux/irq.h> -#include <asm/delay.h> -#include <asm/i8253.h> -#include <asm/io.h> #include <asm/time.h> -DEFINE_RAW_SPINLOCK(i8253_lock); -EXPORT_SYMBOL(i8253_lock); - -/* - * Initialize the PIT timer. - * - * This is also called after resume to bring the PIT into operation again. - */ -static void init_pit_timer(enum clock_event_mode mode, - struct clock_event_device *evt) -{ - raw_spin_lock(&i8253_lock); - - switch(mode) { - case CLOCK_EVT_MODE_PERIODIC: - /* binary, mode 2, LSB/MSB, ch 0 */ - outb_p(0x34, PIT_MODE); - outb_p(LATCH & 0xff , PIT_CH0); /* LSB */ - outb(LATCH >> 8 , PIT_CH0); /* MSB */ - break; - - case CLOCK_EVT_MODE_SHUTDOWN: - case CLOCK_EVT_MODE_UNUSED: - if (evt->mode == CLOCK_EVT_MODE_PERIODIC || - evt->mode == CLOCK_EVT_MODE_ONESHOT) { - outb_p(0x30, PIT_MODE); - outb_p(0, PIT_CH0); - outb_p(0, PIT_CH0); - } - break; - - case CLOCK_EVT_MODE_ONESHOT: - /* One shot setup */ - outb_p(0x38, PIT_MODE); - break; - - case CLOCK_EVT_MODE_RESUME: - /* Nothing to do here */ - break; - } - raw_spin_unlock(&i8253_lock); -} - -/* - * Program the next event in oneshot mode - * - * Delta is given in PIT ticks - */ -static int pit_next_event(unsigned long delta, struct clock_event_device *evt) -{ - raw_spin_lock(&i8253_lock); - outb_p(delta & 0xff , PIT_CH0); /* LSB */ - outb(delta >> 8 , PIT_CH0); /* MSB */ - raw_spin_unlock(&i8253_lock); - - return 0; -} - -/* - * On UP the PIT can serve all of the possible timer functions. On SMP systems - * it can be solely used for the global tick. - * - * The profiling and update capabilites are switched off once the local apic is - * registered. This mechanism replaces the previous #ifdef LOCAL_APIC - - * !using_apic_timer decisions in do_timer_interrupt_hook() - */ -static struct clock_event_device pit_clockevent = { - .name = "pit", - .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, - .set_mode = init_pit_timer, - .set_next_event = pit_next_event, - .irq = 0, -}; - static irqreturn_t timer_interrupt(int irq, void *dev_id) { - pit_clockevent.event_handler(&pit_clockevent); + i8253_clockevent.event_handler(&i8253_clockevent); return IRQ_HANDLED; } static struct irqaction irq0 = { .handler = timer_interrupt, - .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER, + .flags = IRQF_NOBALANCING | IRQF_TIMER, .name = "timer" }; -/* - * Initialize the conversion factor and the min/max deltas of the clock event - * structure and register the clock event source with the framework. - */ void __init setup_pit_timer(void) { - struct clock_event_device *cd = &pit_clockevent; - unsigned int cpu = smp_processor_id(); - - /* - * Start pit with the boot cpu mask and make it global after the - * IO_APIC has been initialized. - */ - cd->cpumask = cpumask_of(cpu); - clockevent_set_clock(cd, CLOCK_TICK_RATE); - cd->max_delta_ns = clockevent_delta2ns(0x7FFF, cd); - cd->min_delta_ns = clockevent_delta2ns(0xF, cd); - clockevents_register_device(cd); - + clockevent_i8253_init(true); setup_irq(0, &irq0); } -/* - * Since the PIT overflows every tick, its not very useful - * to just read by itself. So use jiffies to emulate a free - * running counter: - */ -static cycle_t pit_read(struct clocksource *cs) -{ - unsigned long flags; - int count; - u32 jifs; - static int old_count; - static u32 old_jifs; - - raw_spin_lock_irqsave(&i8253_lock, flags); - /* - * Although our caller may have the read side of xtime_lock, - * this is now a seqlock, and we are cheating in this routine - * by having side effects on state that we cannot undo if - * there is a collision on the seqlock and our caller has to - * retry. (Namely, old_jifs and old_count.) So we must treat - * jiffies as volatile despite the lock. We read jiffies - * before latching the timer count to guarantee that although - * the jiffies value might be older than the count (that is, - * the counter may underflow between the last point where - * jiffies was incremented and the point where we latch the - * count), it cannot be newer. - */ - jifs = jiffies; - outb_p(0x00, PIT_MODE); /* latch the count ASAP */ - count = inb_p(PIT_CH0); /* read the latched count */ - count |= inb_p(PIT_CH0) << 8; - - /* VIA686a test code... reset the latch if count > max + 1 */ - if (count > LATCH) { - outb_p(0x34, PIT_MODE); - outb_p(LATCH & 0xff, PIT_CH0); - outb(LATCH >> 8, PIT_CH0); - count = LATCH - 1; - } - - /* - * It's possible for count to appear to go the wrong way for a - * couple of reasons: - * - * 1. The timer counter underflows, but we haven't handled the - * resulting interrupt and incremented jiffies yet. - * 2. Hardware problem with the timer, not giving us continuous time, - * the counter does small "jumps" upwards on some Pentium systems, - * (see c't 95/10 page 335 for Neptun bug.) - * - * Previous attempts to handle these cases intelligently were - * buggy, so we just do the simple thing now. - */ - if (count > old_count && jifs == old_jifs) { - count = old_count; - } - old_count = count; - old_jifs = jifs; - - raw_spin_unlock_irqrestore(&i8253_lock, flags); - - count = (LATCH - 1) - count; - - return (cycle_t)(jifs * LATCH) + count; -} - -static struct clocksource clocksource_pit = { - .name = "pit", - .rating = 110, - .read = pit_read, - .mask = CLOCKSOURCE_MASK(32), - .mult = 0, - .shift = 20, -}; - static int __init init_pit_clocksource(void) { if (num_possible_cpus() > 1) /* PIT does not scale! */ return 0; - clocksource_pit.mult = clocksource_hz2mult(CLOCK_TICK_RATE, 20); - return clocksource_register(&clocksource_pit); + return clocksource_i8253_init(); } arch_initcall(init_pit_clocksource); diff --git a/arch/mips/kernel/i8259.c b/arch/mips/kernel/i8259.c index c58176cc796..50b364897dd 100644 --- a/arch/mips/kernel/i8259.c +++ b/arch/mips/kernel/i8259.c @@ -14,7 +14,7 @@ #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/spinlock.h> -#include <linux/sysdev.h> +#include <linux/syscore_ops.h> #include <linux/irq.h> #include <asm/i8259.h> @@ -31,20 +31,17 @@ static int i8259A_auto_eoi = -1; DEFINE_RAW_SPINLOCK(i8259A_lock); -static void disable_8259A_irq(unsigned int irq); -static void enable_8259A_irq(unsigned int irq); -static void mask_and_ack_8259A(unsigned int irq); +static void disable_8259A_irq(struct irq_data *d); +static void enable_8259A_irq(struct irq_data *d); +static void mask_and_ack_8259A(struct irq_data *d); static void init_8259A(int auto_eoi); static struct irq_chip i8259A_chip = { - .name = "XT-PIC", - .mask = disable_8259A_irq, - .disable = disable_8259A_irq, - .unmask = enable_8259A_irq, - .mask_ack = mask_and_ack_8259A, -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF - .set_affinity = plat_set_irq_affinity, -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ + .name = "XT-PIC", + .irq_mask = disable_8259A_irq, + .irq_disable = disable_8259A_irq, + .irq_unmask = enable_8259A_irq, + .irq_mask_ack = mask_and_ack_8259A, }; /* @@ -59,12 +56,11 @@ static unsigned int cached_irq_mask = 0xffff; #define cached_master_mask (cached_irq_mask) #define cached_slave_mask (cached_irq_mask >> 8) -static void disable_8259A_irq(unsigned int irq) +static void disable_8259A_irq(struct irq_data *d) { - unsigned int mask; + unsigned int mask, irq = d->irq - I8259A_IRQ_BASE; unsigned long flags; - irq -= I8259A_IRQ_BASE; mask = 1 << irq; raw_spin_lock_irqsave(&i8259A_lock, flags); cached_irq_mask |= mask; @@ -75,12 +71,11 @@ static void disable_8259A_irq(unsigned int irq) raw_spin_unlock_irqrestore(&i8259A_lock, flags); } -static void enable_8259A_irq(unsigned int irq) +static void enable_8259A_irq(struct irq_data *d) { - unsigned int mask; + unsigned int mask, irq = d->irq - I8259A_IRQ_BASE; unsigned long flags; - irq -= I8259A_IRQ_BASE; mask = ~(1 << irq); raw_spin_lock_irqsave(&i8259A_lock, flags); cached_irq_mask &= mask; @@ -112,7 +107,7 @@ int i8259A_irq_pending(unsigned int irq) void make_8259A_irq(unsigned int irq) { disable_irq_nosync(irq); - set_irq_chip_and_handler(irq, &i8259A_chip, handle_level_irq); + irq_set_chip_and_handler(irq, &i8259A_chip, handle_level_irq); enable_irq(irq); } @@ -145,12 +140,11 @@ static inline int i8259A_irq_real(unsigned int irq) * first, _then_ send the EOI, and the order of EOI * to the two 8259s is important! */ -static void mask_and_ack_8259A(unsigned int irq) +static void mask_and_ack_8259A(struct irq_data *d) { - unsigned int irqmask; + unsigned int irqmask, irq = d->irq - I8259A_IRQ_BASE; unsigned long flags; - irq -= I8259A_IRQ_BASE; irqmask = 1 << irq; raw_spin_lock_irqsave(&i8259A_lock, flags); /* @@ -181,9 +175,8 @@ handle_real_irq: } else { inb(PIC_MASTER_IMR); /* DUMMY - (do we need this?) */ outb(cached_master_mask, PIC_MASTER_IMR); - outb(0x60+irq, PIC_MASTER_CMD); /* 'Specific EOI to master */ + outb(0x60+irq, PIC_MASTER_CMD); /* 'Specific EOI to master */ } - smtc_im_ack_irq(irq); raw_spin_unlock_irqrestore(&i8259A_lock, flags); return; @@ -218,14 +211,13 @@ spurious_8259A_irq: } } -static int i8259A_resume(struct sys_device *dev) +static void i8259A_resume(void) { if (i8259A_auto_eoi >= 0) init_8259A(i8259A_auto_eoi); - return 0; } -static int i8259A_shutdown(struct sys_device *dev) +static void i8259A_shutdown(void) { /* Put the i8259A into a quiescent state that * the kernel initialization code can get it @@ -233,28 +225,19 @@ static int i8259A_shutdown(struct sys_device *dev) */ if (i8259A_auto_eoi >= 0) { outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ - outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-1 */ + outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ } - return 0; } -static struct sysdev_class i8259_sysdev_class = { - .name = "i8259", +static struct syscore_ops i8259_syscore_ops = { .resume = i8259A_resume, .shutdown = i8259A_shutdown, }; -static struct sys_device device_i8259A = { - .id = 0, - .cls = &i8259_sysdev_class, -}; - static int __init i8259A_init_sysfs(void) { - int error = sysdev_class_register(&i8259_sysdev_class); - if (!error) - error = sysdev_register(&device_i8259A); - return error; + register_syscore_ops(&i8259_syscore_ops); + return 0; } device_initcall(i8259A_init_sysfs); @@ -290,9 +273,9 @@ static void init_8259A(int auto_eoi) * In AEOI mode we just have to mask the interrupt * when acking. */ - i8259A_chip.mask_ack = disable_8259A_irq; + i8259A_chip.irq_mask_ack = disable_8259A_irq; else - i8259A_chip.mask_ack = mask_and_ack_8259A; + i8259A_chip.irq_mask_ack = mask_and_ack_8259A; udelay(100); /* wait for 8259A to initialize */ @@ -308,6 +291,7 @@ static void init_8259A(int auto_eoi) static struct irqaction irq2 = { .handler = no_action, .name = "cascade", + .flags = IRQF_NO_THREAD, }; static struct resource pic1_io_resource = { @@ -339,8 +323,8 @@ void __init init_i8259_irqs(void) init_8259A(0); for (i = I8259A_IRQ_BASE; i < I8259A_IRQ_BASE + 16; i++) { - set_irq_chip_and_handler(i, &i8259A_chip, handle_level_irq); - set_irq_probe(i); + irq_set_chip_and_handler(i, &i8259A_chip, handle_level_irq); + irq_set_probe(i); } setup_irq(I8259A_IRQ_BASE + PIC_CASCADE_IR, &irq2); diff --git a/arch/mips/kernel/idle.c b/arch/mips/kernel/idle.c new file mode 100644 index 00000000000..09ce4598075 --- /dev/null +++ b/arch/mips/kernel/idle.c @@ -0,0 +1,249 @@ +/* + * MIPS idle loop and WAIT instruction support. + * + * Copyright (C) xxxx the Anonymous + * Copyright (C) 1994 - 2006 Ralf Baechle + * Copyright (C) 2003, 2004 Maciej W. Rozycki + * Copyright (C) 2001, 2004, 2011, 2012 MIPS Technologies, Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ +#include <linux/export.h> +#include <linux/init.h> +#include <linux/irqflags.h> +#include <linux/printk.h> +#include <linux/sched.h> +#include <asm/cpu.h> +#include <asm/cpu-info.h> +#include <asm/cpu-type.h> +#include <asm/idle.h> +#include <asm/mipsregs.h> + +/* + * Not all of the MIPS CPUs have the "wait" instruction available. Moreover, + * the implementation of the "wait" feature differs between CPU families. This + * points to the function that implements CPU specific wait. + * The wait instruction stops the pipeline and reduces the power consumption of + * the CPU very much. + */ +void (*cpu_wait)(void); +EXPORT_SYMBOL(cpu_wait); + +static void r3081_wait(void) +{ + unsigned long cfg = read_c0_conf(); + write_c0_conf(cfg | R30XX_CONF_HALT); + local_irq_enable(); +} + +static void r39xx_wait(void) +{ + if (!need_resched()) + write_c0_conf(read_c0_conf() | TX39_CONF_HALT); + local_irq_enable(); +} + +void r4k_wait(void) +{ + local_irq_enable(); + __r4k_wait(); +} + +/* + * This variant is preferable as it allows testing need_resched and going to + * sleep depending on the outcome atomically. Unfortunately the "It is + * implementation-dependent whether the pipeline restarts when a non-enabled + * interrupt is requested" restriction in the MIPS32/MIPS64 architecture makes + * using this version a gamble. + */ +void r4k_wait_irqoff(void) +{ + if (!need_resched()) + __asm__( + " .set push \n" + " .set arch=r4000 \n" + " wait \n" + " .set pop \n"); + local_irq_enable(); + __asm__( + " .globl __pastwait \n" + "__pastwait: \n"); +} + +/* + * The RM7000 variant has to handle erratum 38. The workaround is to not + * have any pending stores when the WAIT instruction is executed. + */ +static void rm7k_wait_irqoff(void) +{ + if (!need_resched()) + __asm__( + " .set push \n" + " .set arch=r4000 \n" + " .set noat \n" + " mfc0 $1, $12 \n" + " sync \n" + " mtc0 $1, $12 # stalls until W stage \n" + " wait \n" + " mtc0 $1, $12 # stalls until W stage \n" + " .set pop \n"); + local_irq_enable(); +} + +/* + * Au1 'wait' is only useful when the 32kHz counter is used as timer, + * since coreclock (and the cp0 counter) stops upon executing it. Only an + * interrupt can wake it, so they must be enabled before entering idle modes. + */ +static void au1k_wait(void) +{ + unsigned long c0status = read_c0_status() | 1; /* irqs on */ + + __asm__( + " .set arch=r4000 \n" + " cache 0x14, 0(%0) \n" + " cache 0x14, 32(%0) \n" + " sync \n" + " mtc0 %1, $12 \n" /* wr c0status */ + " wait \n" + " nop \n" + " nop \n" + " nop \n" + " nop \n" + " .set mips0 \n" + : : "r" (au1k_wait), "r" (c0status)); +} + +static int __initdata nowait; + +static int __init wait_disable(char *s) +{ + nowait = 1; + + return 1; +} + +__setup("nowait", wait_disable); + +void __init check_wait(void) +{ + struct cpuinfo_mips *c = ¤t_cpu_data; + + if (nowait) { + printk("Wait instruction disabled.\n"); + return; + } + + switch (current_cpu_type()) { + case CPU_R3081: + case CPU_R3081E: + cpu_wait = r3081_wait; + break; + case CPU_TX3927: + cpu_wait = r39xx_wait; + break; + case CPU_R4200: +/* case CPU_R4300: */ + case CPU_R4600: + case CPU_R4640: + case CPU_R4650: + case CPU_R4700: + case CPU_R5000: + case CPU_R5500: + case CPU_NEVADA: + case CPU_4KC: + case CPU_4KEC: + case CPU_4KSC: + case CPU_5KC: + case CPU_25KF: + case CPU_PR4450: + case CPU_BMIPS3300: + case CPU_BMIPS4350: + case CPU_BMIPS4380: + case CPU_BMIPS5000: + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + case CPU_CAVIUM_OCTEON3: + case CPU_JZRISC: + case CPU_LOONGSON1: + case CPU_XLR: + case CPU_XLP: + cpu_wait = r4k_wait; + break; + + case CPU_RM7000: + cpu_wait = rm7k_wait_irqoff; + break; + + case CPU_M14KC: + case CPU_M14KEC: + case CPU_24K: + case CPU_34K: + case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_P5600: + case CPU_M5150: + cpu_wait = r4k_wait; + if (read_c0_config7() & MIPS_CONF7_WII) + cpu_wait = r4k_wait_irqoff; + break; + + case CPU_74K: + cpu_wait = r4k_wait; + if ((c->processor_id & 0xff) >= PRID_REV_ENCODE_332(2, 1, 0)) + cpu_wait = r4k_wait_irqoff; + break; + + case CPU_TX49XX: + cpu_wait = r4k_wait_irqoff; + break; + case CPU_ALCHEMY: + cpu_wait = au1k_wait; + break; + case CPU_20KC: + /* + * WAIT on Rev1.0 has E1, E2, E3 and E16. + * WAIT on Rev2.0 and Rev3.0 has E16. + * Rev3.1 WAIT is nop, why bother + */ + if ((c->processor_id & 0xff) <= 0x64) + break; + + /* + * Another rev is incremeting c0_count at a reduced clock + * rate while in WAIT mode. So we basically have the choice + * between using the cp0 timer as clocksource or avoiding + * the WAIT instruction. Until more details are known, + * disable the use of WAIT for 20Kc entirely. + cpu_wait = r4k_wait; + */ + break; + default: + break; + } +} + +void arch_cpu_idle(void) +{ + if (cpu_wait) + cpu_wait(); + else + local_irq_enable(); +} + +#ifdef CONFIG_CPU_IDLE + +int mips_cpuidle_wait_enter(struct cpuidle_device *dev, + struct cpuidle_driver *drv, int index) +{ + arch_cpu_idle(); + return index; +} + +#endif diff --git a/arch/mips/kernel/init_task.c b/arch/mips/kernel/init_task.c deleted file mode 100644 index 6d6ca530589..00000000000 --- a/arch/mips/kernel/init_task.c +++ /dev/null @@ -1,35 +0,0 @@ -#include <linux/mm.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/init_task.h> -#include <linux/fs.h> -#include <linux/mqueue.h> - -#include <asm/thread_info.h> -#include <asm/uaccess.h> -#include <asm/pgtable.h> - -static struct signal_struct init_signals = INIT_SIGNALS(init_signals); -static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand); -/* - * Initial thread structure. - * - * We need to make sure that this is 8192-byte aligned due to the - * way process stacks are handled. This is done by making sure - * the linker maps this in the .text segment right after head.S, - * and making head.S ensure the proper alignment. - * - * The things we do for performance.. - */ -union thread_union init_thread_union __init_task_data - __attribute__((__aligned__(THREAD_SIZE))) = - { INIT_THREAD_INFO(init_task) }; - -/* - * Initial task structure. - * - * All other task structs will be allocated on slabs in fork.c - */ -struct task_struct init_task = INIT_TASK(init_task); - -EXPORT_SYMBOL(init_task); diff --git a/arch/mips/kernel/irq-gic.c b/arch/mips/kernel/irq-gic.c index 1774271af84..88e4c323382 100644 --- a/arch/mips/kernel/irq-gic.c +++ b/arch/mips/kernel/irq-gic.c @@ -1,39 +1,142 @@ -#undef DEBUG - +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2008 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. + */ #include <linux/bitmap.h> #include <linux/init.h> #include <linux/smp.h> #include <linux/irq.h> +#include <linux/clocksource.h> #include <asm/io.h> #include <asm/gic.h> -#include <asm/gcmpregs.h> +#include <asm/setup.h> +#include <asm/traps.h> #include <linux/hardirq.h> #include <asm-generic/bitops/find.h> +unsigned int gic_frequency; +unsigned int gic_present; +unsigned long _gic_base; +unsigned int gic_irq_base; +unsigned int gic_irq_flags[GIC_NUM_INTRS]; -static unsigned long _gic_base; -static unsigned int _irqbase; -static unsigned int gic_irq_flags[GIC_NUM_INTRS]; -#define GIC_IRQ_FLAG_EDGE 0x0001 +/* The index into this array is the vector # of the interrupt. */ +struct gic_shared_intr_map gic_shared_intr_map[GIC_NUM_INTRS]; -struct gic_pcpu_mask pcpu_masks[NR_CPUS]; +static struct gic_pcpu_mask pcpu_masks[NR_CPUS]; static struct gic_pending_regs pending_regs[NR_CPUS]; static struct gic_intrmask_regs intrmask_regs[NR_CPUS]; +#if defined(CONFIG_CSRC_GIC) || defined(CONFIG_CEVT_GIC) +cycle_t gic_read_count(void) +{ + unsigned int hi, hi2, lo; + + do { + GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_63_32), hi); + GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_31_00), lo); + GICREAD(GIC_REG(SHARED, GIC_SH_COUNTER_63_32), hi2); + } while (hi2 != hi); + + return (((cycle_t) hi) << 32) + lo; +} + +void gic_write_compare(cycle_t cnt) +{ + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI), + (int)(cnt >> 32)); + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO), + (int)(cnt & 0xffffffff)); +} + +void gic_write_cpu_compare(cycle_t cnt, int cpu) +{ + unsigned long flags; + + local_irq_save(flags); + + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), cpu); + GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_HI), + (int)(cnt >> 32)); + GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_COMPARE_LO), + (int)(cnt & 0xffffffff)); + + local_irq_restore(flags); +} + +cycle_t gic_read_compare(void) +{ + unsigned int hi, lo; + + GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_HI), hi); + GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_COMPARE_LO), lo); + + return (((cycle_t) hi) << 32) + lo; +} +#endif + +unsigned int gic_get_timer_pending(void) +{ + unsigned int vpe_pending; + + GICWRITE(GIC_REG(VPE_LOCAL, GIC_VPE_OTHER_ADDR), 0); + GICREAD(GIC_REG(VPE_OTHER, GIC_VPE_PEND), vpe_pending); + return (vpe_pending & GIC_VPE_PEND_TIMER_MSK); +} + +void gic_bind_eic_interrupt(int irq, int set) +{ + /* Convert irq vector # to hw int # */ + irq -= GIC_PIN_TO_VEC_OFFSET; + + /* Set irq to use shadow set */ + GICWRITE(GIC_REG_ADDR(VPE_LOCAL, GIC_VPE_EIC_SS(irq)), set); +} + void gic_send_ipi(unsigned int intr) { - pr_debug("CPU%d: %s status %08x\n", smp_processor_id(), __func__, - read_c0_status()); GICWRITE(GIC_REG(SHARED, GIC_SH_WEDGE), 0x80000000 | intr); } -/* This is Malta specific and needs to be exported */ +static void gic_eic_irq_dispatch(void) +{ + unsigned int cause = read_c0_cause(); + int irq; + + irq = (cause & ST0_IM) >> STATUSB_IP2; + if (irq == 0) + irq = -1; + + if (irq >= 0) + do_IRQ(gic_irq_base + irq); + else + spurious_interrupt(); +} + static void __init vpe_local_setup(unsigned int numvpes) { - int i; - unsigned long timer_interrupt = 5, perf_interrupt = 5; + unsigned long timer_intr = GIC_INT_TMR; + unsigned long perf_intr = GIC_INT_PERFCTR; unsigned int vpe_ctl; + int i; + + if (cpu_has_veic) { + /* + * GIC timer interrupt -> CPU HW Int X (vector X+2) -> + * map to pin X+2-1 (since GIC adds 1) + */ + timer_intr += (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); + /* + * GIC perfcnt interrupt -> CPU HW Int X (vector X+2) -> + * map to pin X+2-1 (since GIC adds 1) + */ + perf_intr += (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); + } /* * Setup the default performance counter timer interrupts @@ -46,14 +149,34 @@ static void __init vpe_local_setup(unsigned int numvpes) GICREAD(GIC_REG(VPE_OTHER, GIC_VPE_CTL), vpe_ctl); if (vpe_ctl & GIC_VPE_CTL_TIMER_RTBL_MSK) GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_TIMER_MAP), - GIC_MAP_TO_PIN_MSK | timer_interrupt); + GIC_MAP_TO_PIN_MSK | timer_intr); + if (cpu_has_veic) { + set_vi_handler(timer_intr + GIC_PIN_TO_VEC_OFFSET, + gic_eic_irq_dispatch); + gic_shared_intr_map[timer_intr + GIC_PIN_TO_VEC_OFFSET].local_intr_mask |= GIC_VPE_RMASK_TIMER_MSK; + } if (vpe_ctl & GIC_VPE_CTL_PERFCNT_RTBL_MSK) GICWRITE(GIC_REG(VPE_OTHER, GIC_VPE_PERFCTR_MAP), - GIC_MAP_TO_PIN_MSK | perf_interrupt); + GIC_MAP_TO_PIN_MSK | perf_intr); + if (cpu_has_veic) { + set_vi_handler(perf_intr + GIC_PIN_TO_VEC_OFFSET, gic_eic_irq_dispatch); + gic_shared_intr_map[perf_intr + GIC_PIN_TO_VEC_OFFSET].local_intr_mask |= GIC_VPE_RMASK_PERFCNT_MSK; + } } } +unsigned int gic_compare_int(void) +{ + unsigned int pending; + + GICREAD(GIC_REG(VPE_LOCAL, GIC_VPE_PEND), pending); + if (pending & GIC_VPE_PEND_CMP_MSK) + return 1; + else + return 0; +} + unsigned int gic_get_int(void) { unsigned int i; @@ -80,90 +203,61 @@ unsigned int gic_get_int(void) bitmap_and(pending, pending, intrmask, GIC_NUM_INTRS); bitmap_and(pending, pending, pcpu_mask, GIC_NUM_INTRS); - i = find_first_bit(pending, GIC_NUM_INTRS); - - pr_debug("CPU%d: %s pend=%d\n", smp_processor_id(), __func__, i); - - return i; + return find_first_bit(pending, GIC_NUM_INTRS); } -static unsigned int gic_irq_startup(unsigned int irq) +static void gic_mask_irq(struct irq_data *d) { - irq -= _irqbase; - pr_debug("CPU%d: %s: irq%d\n", smp_processor_id(), __func__, irq); - GIC_SET_INTR_MASK(irq); - return 0; -} - -static void gic_irq_ack(unsigned int irq) -{ - irq -= _irqbase; - pr_debug("CPU%d: %s: irq%d\n", smp_processor_id(), __func__, irq); - GIC_CLR_INTR_MASK(irq); - - if (gic_irq_flags[irq] & GIC_IRQ_FLAG_EDGE) - GICWRITE(GIC_REG(SHARED, GIC_SH_WEDGE), irq); + GIC_CLR_INTR_MASK(d->irq - gic_irq_base); } -static void gic_mask_irq(unsigned int irq) +static void gic_unmask_irq(struct irq_data *d) { - irq -= _irqbase; - pr_debug("CPU%d: %s: irq%d\n", smp_processor_id(), __func__, irq); - GIC_CLR_INTR_MASK(irq); -} - -static void gic_unmask_irq(unsigned int irq) -{ - irq -= _irqbase; - pr_debug("CPU%d: %s: irq%d\n", smp_processor_id(), __func__, irq); - GIC_SET_INTR_MASK(irq); + GIC_SET_INTR_MASK(d->irq - gic_irq_base); } #ifdef CONFIG_SMP - static DEFINE_SPINLOCK(gic_lock); -static int gic_set_affinity(unsigned int irq, const struct cpumask *cpumask) +static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask, + bool force) { + unsigned int irq = (d->irq - gic_irq_base); cpumask_t tmp = CPU_MASK_NONE; unsigned long flags; int i; - irq -= _irqbase; - pr_debug("%s(%d) called\n", __func__, irq); cpumask_and(&tmp, cpumask, cpu_online_mask); if (cpus_empty(tmp)) return -1; /* Assumption : cpumask refers to a single CPU */ spin_lock_irqsave(&gic_lock, flags); - for (;;) { - /* Re-route this IRQ */ - GIC_SH_MAP_TO_VPE_SMASK(irq, first_cpu(tmp)); - /* Update the pcpu_masks */ - for (i = 0; i < NR_CPUS; i++) - clear_bit(irq, pcpu_masks[i].pcpu_mask); - set_bit(irq, pcpu_masks[first_cpu(tmp)].pcpu_mask); + /* Re-route this IRQ */ + GIC_SH_MAP_TO_VPE_SMASK(irq, first_cpu(tmp)); - } - cpumask_copy(irq_desc[irq].affinity, cpumask); + /* Update the pcpu_masks */ + for (i = 0; i < NR_CPUS; i++) + clear_bit(irq, pcpu_masks[i].pcpu_mask); + set_bit(irq, pcpu_masks[first_cpu(tmp)].pcpu_mask); + + cpumask_copy(d->affinity, cpumask); spin_unlock_irqrestore(&gic_lock, flags); - return 0; + return IRQ_SET_MASK_OK_NOCOPY; } #endif static struct irq_chip gic_irq_controller = { - .name = "MIPS GIC", - .startup = gic_irq_startup, - .ack = gic_irq_ack, - .mask = gic_mask_irq, - .mask_ack = gic_mask_irq, - .unmask = gic_unmask_irq, - .eoi = gic_unmask_irq, + .name = "MIPS GIC", + .irq_ack = gic_irq_ack, + .irq_mask = gic_mask_irq, + .irq_mask_ack = gic_mask_irq, + .irq_unmask = gic_unmask_irq, + .irq_eoi = gic_finish_irq, #ifdef CONFIG_SMP - .set_affinity = gic_set_affinity, + .irq_set_affinity = gic_set_affinity, #endif }; @@ -171,6 +265,8 @@ static void __init gic_setup_intr(unsigned int intr, unsigned int cpu, unsigned int pin, unsigned int polarity, unsigned int trigtype, unsigned int flags) { + struct gic_shared_intr_map *map_ptr; + /* Setup Intr to Pin mapping */ if (pin & GIC_MAP_TO_NMI_MSK) { GICWRITE(GIC_REG_ADDR(SHARED, GIC_SH_MAP_TO_PIN(intr)), pin); @@ -185,6 +281,14 @@ static void __init gic_setup_intr(unsigned int intr, unsigned int cpu, GIC_MAP_TO_PIN_MSK | pin); /* Setup Intr to CPU mapping */ GIC_SH_MAP_TO_VPE_SMASK(intr, cpu); + if (cpu_has_veic) { + set_vi_handler(pin + GIC_PIN_TO_VEC_OFFSET, + gic_eic_irq_dispatch); + map_ptr = &gic_shared_intr_map[pin + GIC_PIN_TO_VEC_OFFSET]; + if (map_ptr->num_shared_intr >= GIC_MAX_SHARED_INTR) + BUG(); + map_ptr->intr_list[map_ptr->num_shared_intr++] = intr; + } } /* Setup Intr Polarity */ @@ -198,26 +302,39 @@ static void __init gic_setup_intr(unsigned int intr, unsigned int cpu, /* Initialise per-cpu Interrupt software masks */ if (flags & GIC_FLAG_IPI) set_bit(intr, pcpu_masks[cpu].pcpu_mask); - if (flags & GIC_FLAG_TRANSPARENT) + if ((flags & GIC_FLAG_TRANSPARENT) && (cpu_has_veic == 0)) GIC_SET_INTR_MASK(intr); if (trigtype == GIC_TRIG_EDGE) - gic_irq_flags[intr] |= GIC_IRQ_FLAG_EDGE; + gic_irq_flags[intr] |= GIC_TRIG_EDGE; } static void __init gic_basic_init(int numintrs, int numvpes, struct gic_intr_map *intrmap, int mapsize) { unsigned int i, cpu; + unsigned int pin_offset = 0; + + board_bind_eic_interrupt = &gic_bind_eic_interrupt; /* Setup defaults */ for (i = 0; i < numintrs; i++) { GIC_SET_POLARITY(i, GIC_POL_POS); GIC_SET_TRIGGER(i, GIC_TRIG_LEVEL); GIC_CLR_INTR_MASK(i); - if (i < GIC_NUM_INTRS) + if (i < GIC_NUM_INTRS) { gic_irq_flags[i] = 0; + gic_shared_intr_map[i].num_shared_intr = 0; + gic_shared_intr_map[i].local_intr_mask = 0; + } } + /* + * In EIC mode, the HW_INT# is offset by (2-1). Need to subtract + * one because the GIC will add one (since 0=no intr). + */ + if (cpu_has_veic) + pin_offset = (GIC_CPU_TO_VEC_OFFSET - GIC_PIN_TO_VEC_OFFSET); + /* Setup specifics */ for (i = 0; i < mapsize; i++) { cpu = intrmap[i].cpunum; @@ -227,16 +344,13 @@ static void __init gic_basic_init(int numintrs, int numvpes, continue; gic_setup_intr(i, intrmap[i].cpunum, - intrmap[i].pin, + intrmap[i].pin + pin_offset, intrmap[i].polarity, intrmap[i].trigtype, intrmap[i].flags); } vpe_local_setup(numvpes); - - for (i = _irqbase; i < (_irqbase + numintrs); i++) - set_irq_chip(i, &gic_irq_controller); } void __init gic_init(unsigned long gic_base_addr, @@ -249,7 +363,7 @@ void __init gic_init(unsigned long gic_base_addr, _gic_base = (unsigned long) ioremap_nocache(gic_base_addr, gic_addrspace_size); - _irqbase = irqbase; + gic_irq_base = irqbase; GICREAD(GIC_REG(SHARED, GIC_SH_CONFIG), gicconfig); numintrs = (gicconfig & GIC_SH_CONFIG_NUMINTRS_MSK) >> @@ -258,8 +372,9 @@ void __init gic_init(unsigned long gic_base_addr, numvpes = (gicconfig & GIC_SH_CONFIG_NUMVPES_MSK) >> GIC_SH_CONFIG_NUMVPES_SHF; - - pr_debug("%s called\n", __func__); + numvpes = numvpes + 1; gic_basic_init(numintrs, numvpes, intr_map, intr_map_size); + + gic_platform_init(numintrs, &gic_irq_controller); } diff --git a/arch/mips/kernel/irq-gt641xx.c b/arch/mips/kernel/irq-gt641xx.c index 42ef81461bf..44a1f792e39 100644 --- a/arch/mips/kernel/irq-gt641xx.c +++ b/arch/mips/kernel/irq-gt641xx.c @@ -1,7 +1,7 @@ /* * GT641xx IRQ routines. * - * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> + * Copyright (C) 2007 Yoichi Yuasa <yuasa@linux-mips.org> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,68 +25,68 @@ #include <asm/gt64120.h> -#define GT641XX_IRQ_TO_BIT(irq) (1U << (irq - GT641XX_IRQ_BASE)) +#define GT641XX_IRQ_TO_BIT(irq) (1U << (irq - GT641XX_IRQ_BASE)) static DEFINE_RAW_SPINLOCK(gt641xx_irq_lock); -static void ack_gt641xx_irq(unsigned int irq) +static void ack_gt641xx_irq(struct irq_data *d) { unsigned long flags; u32 cause; raw_spin_lock_irqsave(>641xx_irq_lock, flags); cause = GT_READ(GT_INTRCAUSE_OFS); - cause &= ~GT641XX_IRQ_TO_BIT(irq); + cause &= ~GT641XX_IRQ_TO_BIT(d->irq); GT_WRITE(GT_INTRCAUSE_OFS, cause); raw_spin_unlock_irqrestore(>641xx_irq_lock, flags); } -static void mask_gt641xx_irq(unsigned int irq) +static void mask_gt641xx_irq(struct irq_data *d) { unsigned long flags; u32 mask; raw_spin_lock_irqsave(>641xx_irq_lock, flags); mask = GT_READ(GT_INTRMASK_OFS); - mask &= ~GT641XX_IRQ_TO_BIT(irq); + mask &= ~GT641XX_IRQ_TO_BIT(d->irq); GT_WRITE(GT_INTRMASK_OFS, mask); raw_spin_unlock_irqrestore(>641xx_irq_lock, flags); } -static void mask_ack_gt641xx_irq(unsigned int irq) +static void mask_ack_gt641xx_irq(struct irq_data *d) { unsigned long flags; u32 cause, mask; raw_spin_lock_irqsave(>641xx_irq_lock, flags); mask = GT_READ(GT_INTRMASK_OFS); - mask &= ~GT641XX_IRQ_TO_BIT(irq); + mask &= ~GT641XX_IRQ_TO_BIT(d->irq); GT_WRITE(GT_INTRMASK_OFS, mask); cause = GT_READ(GT_INTRCAUSE_OFS); - cause &= ~GT641XX_IRQ_TO_BIT(irq); + cause &= ~GT641XX_IRQ_TO_BIT(d->irq); GT_WRITE(GT_INTRCAUSE_OFS, cause); raw_spin_unlock_irqrestore(>641xx_irq_lock, flags); } -static void unmask_gt641xx_irq(unsigned int irq) +static void unmask_gt641xx_irq(struct irq_data *d) { unsigned long flags; u32 mask; raw_spin_lock_irqsave(>641xx_irq_lock, flags); mask = GT_READ(GT_INTRMASK_OFS); - mask |= GT641XX_IRQ_TO_BIT(irq); + mask |= GT641XX_IRQ_TO_BIT(d->irq); GT_WRITE(GT_INTRMASK_OFS, mask); raw_spin_unlock_irqrestore(>641xx_irq_lock, flags); } static struct irq_chip gt641xx_irq_chip = { .name = "GT641xx", - .ack = ack_gt641xx_irq, - .mask = mask_gt641xx_irq, - .mask_ack = mask_ack_gt641xx_irq, - .unmask = unmask_gt641xx_irq, + .irq_ack = ack_gt641xx_irq, + .irq_mask = mask_gt641xx_irq, + .irq_mask_ack = mask_ack_gt641xx_irq, + .irq_unmask = unmask_gt641xx_irq, }; void gt641xx_irq_dispatch(void) @@ -126,6 +126,6 @@ void __init gt641xx_irq_init(void) * bit31: logical or of bits[25:1]. */ for (i = 1; i < 30; i++) - set_irq_chip_and_handler(GT641XX_IRQ_BASE + i, - >641xx_irq_chip, handle_level_irq); + irq_set_chip_and_handler(GT641XX_IRQ_BASE + i, + >641xx_irq_chip, handle_level_irq); } diff --git a/arch/mips/kernel/irq-msc01.c b/arch/mips/kernel/irq-msc01.c index 6a8cd28133d..a734b2c2f9e 100644 --- a/arch/mips/kernel/irq-msc01.c +++ b/arch/mips/kernel/irq-msc01.c @@ -1,6 +1,6 @@ /* - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * @@ -9,7 +9,6 @@ * * Copyright (C) 2004, 06 Ralf Baechle <ralf@linux-mips.org> */ -#include <linux/module.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/sched.h> @@ -28,8 +27,10 @@ static unsigned long _icctrl_msc; static unsigned int irq_base; /* mask off an interrupt */ -static inline void mask_msc_irq(unsigned int irq) +static inline void mask_msc_irq(struct irq_data *d) { + unsigned int irq = d->irq; + if (irq < (irq_base + 32)) MSCIC_WRITE(MSC01_IC_DISL, 1<<(irq - irq_base)); else @@ -37,8 +38,10 @@ static inline void mask_msc_irq(unsigned int irq) } /* unmask an interrupt */ -static inline void unmask_msc_irq(unsigned int irq) +static inline void unmask_msc_irq(struct irq_data *d) { + unsigned int irq = d->irq; + if (irq < (irq_base + 32)) MSCIC_WRITE(MSC01_IC_ENAL, 1<<(irq - irq_base)); else @@ -48,21 +51,21 @@ static inline void unmask_msc_irq(unsigned int irq) /* * Masks and ACKs an IRQ */ -static void level_mask_and_ack_msc_irq(unsigned int irq) +static void level_mask_and_ack_msc_irq(struct irq_data *d) { - mask_msc_irq(irq); + mask_msc_irq(d); if (!cpu_has_veic) MSCIC_WRITE(MSC01_IC_EOI, 0); - /* This actually needs to be a call into platform code */ - smtc_im_ack_irq(irq); } /* * Masks and ACKs an IRQ */ -static void edge_mask_and_ack_msc_irq(unsigned int irq) +static void edge_mask_and_ack_msc_irq(struct irq_data *d) { - mask_msc_irq(irq); + unsigned int irq = d->irq; + + mask_msc_irq(d); if (!cpu_has_veic) MSCIC_WRITE(MSC01_IC_EOI, 0); else { @@ -71,16 +74,6 @@ static void edge_mask_and_ack_msc_irq(unsigned int irq) MSCIC_WRITE(MSC01_IC_SUP+irq*8, r | ~MSC01_IC_SUP_EDGE_BIT); MSCIC_WRITE(MSC01_IC_SUP+irq*8, r); } - smtc_im_ack_irq(irq); -} - -/* - * End IRQ processing - */ -static void end_msc_irq(unsigned int irq) -{ - if (!(irq_desc[irq].status & (IRQ_DISABLED|IRQ_INPROGRESS))) - unmask_msc_irq(irq); } /* @@ -88,7 +81,7 @@ static void end_msc_irq(unsigned int irq) */ void ll_msc_irq(void) { - unsigned int irq; + unsigned int irq; /* read the interrupt vector register */ MSCIC_READ(MSC01_IC_VEC, irq); @@ -107,22 +100,20 @@ static void msc_bind_eic_interrupt(int irq, int set) static struct irq_chip msc_levelirq_type = { .name = "SOC-it-Level", - .ack = level_mask_and_ack_msc_irq, - .mask = mask_msc_irq, - .mask_ack = level_mask_and_ack_msc_irq, - .unmask = unmask_msc_irq, - .eoi = unmask_msc_irq, - .end = end_msc_irq, + .irq_ack = level_mask_and_ack_msc_irq, + .irq_mask = mask_msc_irq, + .irq_mask_ack = level_mask_and_ack_msc_irq, + .irq_unmask = unmask_msc_irq, + .irq_eoi = unmask_msc_irq, }; static struct irq_chip msc_edgeirq_type = { .name = "SOC-it-Edge", - .ack = edge_mask_and_ack_msc_irq, - .mask = mask_msc_irq, - .mask_ack = edge_mask_and_ack_msc_irq, - .unmask = unmask_msc_irq, - .eoi = unmask_msc_irq, - .end = end_msc_irq, + .irq_ack = edge_mask_and_ack_msc_irq, + .irq_mask = mask_msc_irq, + .irq_mask_ack = edge_mask_and_ack_msc_irq, + .irq_unmask = unmask_msc_irq, + .irq_eoi = unmask_msc_irq, }; @@ -135,21 +126,25 @@ void __init init_msc_irqs(unsigned long icubase, unsigned int irqbase, msc_irqma board_bind_eic_interrupt = &msc_bind_eic_interrupt; - for (; nirq >= 0; nirq--, imp++) { + for (; nirq > 0; nirq--, imp++) { int n = imp->im_irq; switch (imp->im_type) { case MSC01_IRQ_EDGE: - set_irq_chip_and_handler_name(irqbase + n, - &msc_edgeirq_type, handle_edge_irq, "edge"); + irq_set_chip_and_handler_name(irqbase + n, + &msc_edgeirq_type, + handle_edge_irq, + "edge"); if (cpu_has_veic) MSCIC_WRITE(MSC01_IC_SUP+n*8, MSC01_IC_SUP_EDGE_BIT); else MSCIC_WRITE(MSC01_IC_SUP+n*8, MSC01_IC_SUP_EDGE_BIT | imp->im_lvl); break; case MSC01_IRQ_LEVEL: - set_irq_chip_and_handler_name(irqbase+n, - &msc_levelirq_type, handle_level_irq, "level"); + irq_set_chip_and_handler_name(irqbase + n, + &msc_levelirq_type, + handle_level_irq, + "level"); if (cpu_has_veic) MSCIC_WRITE(MSC01_IC_SUP+n*8, 0); else diff --git a/arch/mips/kernel/irq-rm7000.c b/arch/mips/kernel/irq-rm7000.c index 9731e8b4786..26f4e4c9db1 100644 --- a/arch/mips/kernel/irq-rm7000.c +++ b/arch/mips/kernel/irq-rm7000.c @@ -1,8 +1,8 @@ /* * Copyright (C) 2003 Ralf Baechle * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * @@ -16,25 +16,24 @@ #include <asm/irq_cpu.h> #include <asm/mipsregs.h> -#include <asm/system.h> -static inline void unmask_rm7k_irq(unsigned int irq) +static inline void unmask_rm7k_irq(struct irq_data *d) { - set_c0_intcontrol(0x100 << (irq - RM7K_CPU_IRQ_BASE)); + set_c0_intcontrol(0x100 << (d->irq - RM7K_CPU_IRQ_BASE)); } -static inline void mask_rm7k_irq(unsigned int irq) +static inline void mask_rm7k_irq(struct irq_data *d) { - clear_c0_intcontrol(0x100 << (irq - RM7K_CPU_IRQ_BASE)); + clear_c0_intcontrol(0x100 << (d->irq - RM7K_CPU_IRQ_BASE)); } static struct irq_chip rm7k_irq_controller = { .name = "RM7000", - .ack = mask_rm7k_irq, - .mask = mask_rm7k_irq, - .mask_ack = mask_rm7k_irq, - .unmask = unmask_rm7k_irq, - .eoi = unmask_rm7k_irq + .irq_ack = mask_rm7k_irq, + .irq_mask = mask_rm7k_irq, + .irq_mask_ack = mask_rm7k_irq, + .irq_unmask = unmask_rm7k_irq, + .irq_eoi = unmask_rm7k_irq }; void __init rm7k_cpu_irq_init(void) @@ -45,6 +44,6 @@ void __init rm7k_cpu_irq_init(void) clear_c0_intcontrol(0x00000f00); /* Mask all */ for (i = base; i < base + 4; i++) - set_irq_chip_and_handler(i, &rm7k_irq_controller, + irq_set_chip_and_handler(i, &rm7k_irq_controller, handle_percpu_irq); } diff --git a/arch/mips/kernel/irq-rm9000.c b/arch/mips/kernel/irq-rm9000.c deleted file mode 100644 index b7e4025b58a..00000000000 --- a/arch/mips/kernel/irq-rm9000.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (C) 2003 Ralf Baechle - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - * - * Handler for RM9000 extended interrupts. These are a non-standard - * feature so we handle them separately from standard interrupts. - */ -#include <linux/init.h> -#include <linux/interrupt.h> -#include <linux/irq.h> -#include <linux/kernel.h> -#include <linux/module.h> - -#include <asm/irq_cpu.h> -#include <asm/mipsregs.h> -#include <asm/system.h> - -static inline void unmask_rm9k_irq(unsigned int irq) -{ - set_c0_intcontrol(0x1000 << (irq - RM9K_CPU_IRQ_BASE)); -} - -static inline void mask_rm9k_irq(unsigned int irq) -{ - clear_c0_intcontrol(0x1000 << (irq - RM9K_CPU_IRQ_BASE)); -} - -static inline void rm9k_cpu_irq_enable(unsigned int irq) -{ - unsigned long flags; - - local_irq_save(flags); - unmask_rm9k_irq(irq); - local_irq_restore(flags); -} - -/* - * Performance counter interrupts are global on all processors. - */ -static void local_rm9k_perfcounter_irq_startup(void *args) -{ - unsigned int irq = (unsigned int) args; - - rm9k_cpu_irq_enable(irq); -} - -static unsigned int rm9k_perfcounter_irq_startup(unsigned int irq) -{ - on_each_cpu(local_rm9k_perfcounter_irq_startup, (void *) irq, 1); - - return 0; -} - -static void local_rm9k_perfcounter_irq_shutdown(void *args) -{ - unsigned int irq = (unsigned int) args; - unsigned long flags; - - local_irq_save(flags); - mask_rm9k_irq(irq); - local_irq_restore(flags); -} - -static void rm9k_perfcounter_irq_shutdown(unsigned int irq) -{ - on_each_cpu(local_rm9k_perfcounter_irq_shutdown, (void *) irq, 1); -} - -static struct irq_chip rm9k_irq_controller = { - .name = "RM9000", - .ack = mask_rm9k_irq, - .mask = mask_rm9k_irq, - .mask_ack = mask_rm9k_irq, - .unmask = unmask_rm9k_irq, - .eoi = unmask_rm9k_irq -}; - -static struct irq_chip rm9k_perfcounter_irq = { - .name = "RM9000", - .startup = rm9k_perfcounter_irq_startup, - .shutdown = rm9k_perfcounter_irq_shutdown, - .ack = mask_rm9k_irq, - .mask = mask_rm9k_irq, - .mask_ack = mask_rm9k_irq, - .unmask = unmask_rm9k_irq, -}; - -unsigned int rm9000_perfcount_irq; - -EXPORT_SYMBOL(rm9000_perfcount_irq); - -void __init rm9k_cpu_irq_init(void) -{ - int base = RM9K_CPU_IRQ_BASE; - int i; - - clear_c0_intcontrol(0x0000f000); /* Mask all */ - - for (i = base; i < base + 4; i++) - set_irq_chip_and_handler(i, &rm9k_irq_controller, - handle_level_irq); - - rm9000_perfcount_irq = base + 1; - set_irq_chip_and_handler(rm9000_perfcount_irq, &rm9k_perfcounter_irq, - handle_percpu_irq); -} diff --git a/arch/mips/kernel/irq.c b/arch/mips/kernel/irq.c index 4f93db58a79..d2bfbc2e899 100644 --- a/arch/mips/kernel/irq.c +++ b/arch/mips/kernel/irq.c @@ -13,7 +13,6 @@ #include <linux/init.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> -#include <linux/module.h> #include <linux/proc_fs.h> #include <linux/mm.h> #include <linux/random.h> @@ -23,8 +22,7 @@ #include <linux/kgdb.h> #include <linux/ftrace.h> -#include <asm/atomic.h> -#include <asm/system.h> +#include <linux/atomic.h> #include <asm/uaccess.h> #ifdef CONFIG_KGDB @@ -50,7 +48,7 @@ again: } /* - * Allocate the 16 legacy interrupts for i8259 devices. This happens early + * Allocate the 16 legacy interrupts for i8259 devices. This happens early * in the kernel initialization so treating allocation failure as BUG() is * ok. */ @@ -64,9 +62,9 @@ void __init alloc_legacy_irqno(void) void free_irqno(unsigned int irq) { - smp_mb__before_clear_bit(); + smp_mb__before_atomic(); clear_bit(irq, irq_map); - smp_mb__after_clear_bit(); + smp_mb__after_atomic(); } /* @@ -75,54 +73,14 @@ void free_irqno(unsigned int irq) */ void ack_bad_irq(unsigned int irq) { - smtc_im_ack_irq(irq); printk("unexpected IRQ # %d\n", irq); } atomic_t irq_err_count; -/* - * Generic, controller-independent functions: - */ - -int show_interrupts(struct seq_file *p, void *v) +int arch_show_interrupts(struct seq_file *p, int prec) { - int i = *(loff_t *) v, j; - struct irqaction * action; - unsigned long flags; - - if (i == 0) { - seq_printf(p, " "); - for_each_online_cpu(j) - seq_printf(p, "CPU%d ", j); - seq_putc(p, '\n'); - } - - if (i < NR_IRQS) { - raw_spin_lock_irqsave(&irq_desc[i].lock, flags); - action = irq_desc[i].action; - if (!action) - goto skip; - seq_printf(p, "%3d: ", i); -#ifndef CONFIG_SMP - seq_printf(p, "%10u ", kstat_irqs(i)); -#else - for_each_online_cpu(j) - seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); -#endif - seq_printf(p, " %14s", irq_desc[i].chip->name); - seq_printf(p, " %s", action->name); - - for (action=action->next; action; action = action->next) - seq_printf(p, ", %s", action->name); - - seq_putc(p, '\n'); -skip: - raw_spin_unlock_irqrestore(&irq_desc[i].lock, flags); - } else if (i == NR_IRQS) { - seq_putc(p, '\n'); - seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count)); - } + seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); return 0; } @@ -141,7 +99,7 @@ void __init init_IRQ(void) #endif for (i = 0; i < NR_IRQS; i++) - set_irq_noprobe(i); + irq_set_noprobe(i); arch_init_irq(); @@ -183,23 +141,7 @@ void __irq_entry do_IRQ(unsigned int irq) { irq_enter(); check_stack_overflow(); - __DO_IRQ_SMTC_HOOK(irq); - generic_handle_irq(irq); - irq_exit(); -} - -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF -/* - * To avoid inefficient and in some cases pathological re-checking of - * IRQ affinity, we have this variant that skips the affinity check. - */ - -void __irq_entry do_IRQ_no_affinity(unsigned int irq) -{ - irq_enter(); - __NO_AFFINITY_IRQ_SMTC_HOOK(irq); generic_handle_irq(irq); irq_exit(); } -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ diff --git a/arch/mips/kernel/irq_cpu.c b/arch/mips/kernel/irq_cpu.c index 0262abe0912..e498f2b3646 100644 --- a/arch/mips/kernel/irq_cpu.c +++ b/arch/mips/kernel/irq_cpu.c @@ -3,13 +3,13 @@ * Author: Jun Sun, jsun@mvista.com or jsun@junsun.net * * Copyright (C) 2001 Ralf Baechle - * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. - * Author: Maciej W. Rozycki <macro@mips.com> + * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. + * Author: Maciej W. Rozycki <macro@mips.com> * * This file define the irq handler for MIPS CPU interrupts. * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. */ @@ -31,48 +31,44 @@ #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/irq.h> +#include <linux/irqdomain.h> #include <asm/irq_cpu.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> -#include <asm/system.h> -static inline void unmask_mips_irq(unsigned int irq) +static inline void unmask_mips_irq(struct irq_data *d) { - set_c0_status(0x100 << (irq - MIPS_CPU_IRQ_BASE)); + set_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); irq_enable_hazard(); } -static inline void mask_mips_irq(unsigned int irq) +static inline void mask_mips_irq(struct irq_data *d) { - clear_c0_status(0x100 << (irq - MIPS_CPU_IRQ_BASE)); + clear_c0_status(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); irq_disable_hazard(); } static struct irq_chip mips_cpu_irq_controller = { .name = "MIPS", - .ack = mask_mips_irq, - .mask = mask_mips_irq, - .mask_ack = mask_mips_irq, - .unmask = unmask_mips_irq, - .eoi = unmask_mips_irq, + .irq_ack = mask_mips_irq, + .irq_mask = mask_mips_irq, + .irq_mask_ack = mask_mips_irq, + .irq_unmask = unmask_mips_irq, + .irq_eoi = unmask_mips_irq, }; /* * Basically the same as above but taking care of all the MT stuff */ -#define unmask_mips_mt_irq unmask_mips_irq -#define mask_mips_mt_irq mask_mips_irq - -static unsigned int mips_mt_cpu_irq_startup(unsigned int irq) +static unsigned int mips_mt_cpu_irq_startup(struct irq_data *d) { unsigned int vpflags = dvpe(); - clear_c0_cause(0x100 << (irq - MIPS_CPU_IRQ_BASE)); + clear_c0_cause(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); evpe(vpflags); - unmask_mips_mt_irq(irq); - + unmask_mips_irq(d); return 0; } @@ -80,22 +76,22 @@ static unsigned int mips_mt_cpu_irq_startup(unsigned int irq) * While we ack the interrupt interrupts are disabled and thus we don't need * to deal with concurrency issues. Same for mips_cpu_irq_end. */ -static void mips_mt_cpu_irq_ack(unsigned int irq) +static void mips_mt_cpu_irq_ack(struct irq_data *d) { unsigned int vpflags = dvpe(); - clear_c0_cause(0x100 << (irq - MIPS_CPU_IRQ_BASE)); + clear_c0_cause(0x100 << (d->irq - MIPS_CPU_IRQ_BASE)); evpe(vpflags); - mask_mips_mt_irq(irq); + mask_mips_irq(d); } static struct irq_chip mips_mt_cpu_irq_controller = { .name = "MIPS", - .startup = mips_mt_cpu_irq_startup, - .ack = mips_mt_cpu_irq_ack, - .mask = mask_mips_mt_irq, - .mask_ack = mips_mt_cpu_irq_ack, - .unmask = unmask_mips_mt_irq, - .eoi = unmask_mips_mt_irq, + .irq_startup = mips_mt_cpu_irq_startup, + .irq_ack = mips_mt_cpu_irq_ack, + .irq_mask = mask_mips_irq, + .irq_mask_ack = mips_mt_cpu_irq_ack, + .irq_unmask = unmask_mips_irq, + .irq_eoi = unmask_mips_irq, }; void __init mips_cpu_irq_init(void) @@ -107,16 +103,55 @@ void __init mips_cpu_irq_init(void) clear_c0_status(ST0_IM); clear_c0_cause(CAUSEF_IP); - /* - * Only MT is using the software interrupts currently, so we just - * leave them uninitialized for other processors. - */ - if (cpu_has_mipsmt) - for (i = irq_base; i < irq_base + 2; i++) - set_irq_chip_and_handler(i, &mips_mt_cpu_irq_controller, - handle_percpu_irq); + /* Software interrupts are used for MT/CMT IPI */ + for (i = irq_base; i < irq_base + 2; i++) + irq_set_chip_and_handler(i, cpu_has_mipsmt ? + &mips_mt_cpu_irq_controller : + &mips_cpu_irq_controller, + handle_percpu_irq); for (i = irq_base + 2; i < irq_base + 8; i++) - set_irq_chip_and_handler(i, &mips_cpu_irq_controller, + irq_set_chip_and_handler(i, &mips_cpu_irq_controller, handle_percpu_irq); } + +#ifdef CONFIG_IRQ_DOMAIN +static int mips_cpu_intc_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + static struct irq_chip *chip; + + if (hw < 2 && cpu_has_mipsmt) { + /* Software interrupts are used for MT/CMT IPI */ + chip = &mips_mt_cpu_irq_controller; + } else { + chip = &mips_cpu_irq_controller; + } + + irq_set_chip_and_handler(irq, chip, handle_percpu_irq); + + return 0; +} + +static const struct irq_domain_ops mips_cpu_intc_irq_domain_ops = { + .map = mips_cpu_intc_map, + .xlate = irq_domain_xlate_onecell, +}; + +int __init mips_cpu_intc_init(struct device_node *of_node, + struct device_node *parent) +{ + struct irq_domain *domain; + + /* Mask interrupts. */ + clear_c0_status(ST0_IM); + clear_c0_cause(CAUSEF_IP); + + domain = irq_domain_add_legacy(of_node, 8, MIPS_CPU_IRQ_BASE, 0, + &mips_cpu_intc_irq_domain_ops, NULL); + if (!domain) + panic("Failed to add irqdomain for MIPS CPU"); + + return 0; +} +#endif /* CONFIG_IRQ_DOMAIN */ diff --git a/arch/mips/kernel/irq_txx9.c b/arch/mips/kernel/irq_txx9.c index 95a96f69172..ab00e490482 100644 --- a/arch/mips/kernel/irq_txx9.c +++ b/arch/mips/kernel/irq_txx9.c @@ -1,12 +1,12 @@ /* * Based on linux/arch/mips/jmr3927/rbhma3100/irq.c, - * linux/arch/mips/tx4927/common/tx4927_irq.c, - * linux/arch/mips/tx4938/common/irq.c + * linux/arch/mips/tx4927/common/tx4927_irq.c, + * linux/arch/mips/tx4938/common/irq.c * * Copyright 2001, 2003-2005 MontaVista Software Inc. * Author: MontaVista Software, Inc. - * ahennessy@mvista.com - * source@mvista.com + * ahennessy@mvista.com + * source@mvista.com * Copyright (C) 2000-2001 Toshiba Corporation * * This file is subject to the terms and conditions of the GNU General Public @@ -63,9 +63,9 @@ static struct { unsigned char mode; } txx9irq[TXx9_MAX_IR] __read_mostly; -static void txx9_irq_unmask(unsigned int irq) +static void txx9_irq_unmask(struct irq_data *d) { - unsigned int irq_nr = irq - TXX9_IRQ_BASE; + unsigned int irq_nr = d->irq - TXX9_IRQ_BASE; u32 __iomem *ilrp = &txx9_ircptr->ilr[(irq_nr % 16 ) / 2]; int ofs = irq_nr / 16 * 16 + (irq_nr & 1) * 8; @@ -79,9 +79,9 @@ static void txx9_irq_unmask(unsigned int irq) #endif } -static inline void txx9_irq_mask(unsigned int irq) +static inline void txx9_irq_mask(struct irq_data *d) { - unsigned int irq_nr = irq - TXX9_IRQ_BASE; + unsigned int irq_nr = d->irq - TXX9_IRQ_BASE; u32 __iomem *ilrp = &txx9_ircptr->ilr[(irq_nr % 16) / 2]; int ofs = irq_nr / 16 * 16 + (irq_nr & 1) * 8; @@ -99,19 +99,19 @@ static inline void txx9_irq_mask(unsigned int irq) #endif } -static void txx9_irq_mask_ack(unsigned int irq) +static void txx9_irq_mask_ack(struct irq_data *d) { - unsigned int irq_nr = irq - TXX9_IRQ_BASE; + unsigned int irq_nr = d->irq - TXX9_IRQ_BASE; - txx9_irq_mask(irq); + txx9_irq_mask(d); /* clear edge detection */ if (unlikely(TXx9_IRCR_EDGE(txx9irq[irq_nr].mode))) __raw_writel(TXx9_IRSCR_EIClrE | irq_nr, &txx9_ircptr->scr); } -static int txx9_irq_set_type(unsigned int irq, unsigned int flow_type) +static int txx9_irq_set_type(struct irq_data *d, unsigned int flow_type) { - unsigned int irq_nr = irq - TXX9_IRQ_BASE; + unsigned int irq_nr = d->irq - TXX9_IRQ_BASE; u32 cr; u32 __iomem *crp; int ofs; @@ -122,7 +122,7 @@ static int txx9_irq_set_type(unsigned int irq, unsigned int flow_type) switch (flow_type & IRQF_TRIGGER_MASK) { case IRQF_TRIGGER_RISING: mode = TXx9_IRCR_UP; break; case IRQF_TRIGGER_FALLING: mode = TXx9_IRCR_DOWN; break; - case IRQF_TRIGGER_HIGH: mode = TXx9_IRCR_HIGH; break; + case IRQF_TRIGGER_HIGH: mode = TXx9_IRCR_HIGH; break; case IRQF_TRIGGER_LOW: mode = TXx9_IRCR_LOW; break; default: return -EINVAL; @@ -139,11 +139,11 @@ static int txx9_irq_set_type(unsigned int irq, unsigned int flow_type) static struct irq_chip txx9_irq_chip = { .name = "TXX9", - .ack = txx9_irq_mask_ack, - .mask = txx9_irq_mask, - .mask_ack = txx9_irq_mask_ack, - .unmask = txx9_irq_unmask, - .set_type = txx9_irq_set_type, + .irq_ack = txx9_irq_mask_ack, + .irq_mask = txx9_irq_mask, + .irq_mask_ack = txx9_irq_mask_ack, + .irq_unmask = txx9_irq_unmask, + .irq_set_type = txx9_irq_set_type, }; void __init txx9_irq_init(unsigned long baseaddr) @@ -154,8 +154,8 @@ void __init txx9_irq_init(unsigned long baseaddr) for (i = 0; i < TXx9_MAX_IR; i++) { txx9irq[i].level = 4; /* middle level */ txx9irq[i].mode = TXx9_IRCR_LOW; - set_irq_chip_and_handler(TXX9_IRQ_BASE + i, - &txx9_irq_chip, handle_level_irq); + irq_set_chip_and_handler(TXX9_IRQ_BASE + i, &txx9_irq_chip, + handle_level_irq); } /* mask all IRC interrupts */ diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c new file mode 100644 index 00000000000..6001610cfe5 --- /dev/null +++ b/arch/mips/kernel/jump_label.c @@ -0,0 +1,54 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (c) 2010 Cavium Networks, Inc. + */ + +#include <linux/jump_label.h> +#include <linux/kernel.h> +#include <linux/memory.h> +#include <linux/mutex.h> +#include <linux/types.h> +#include <linux/cpu.h> + +#include <asm/cacheflush.h> +#include <asm/inst.h> + +#ifdef HAVE_JUMP_LABEL + +#define J_RANGE_MASK ((1ul << 28) - 1) + +void arch_jump_label_transform(struct jump_entry *e, + enum jump_label_type type) +{ + union mips_instruction insn; + union mips_instruction *insn_p = + (union mips_instruction *)(unsigned long)e->code; + + /* Jump only works within a 256MB aligned region. */ + BUG_ON((e->target & ~J_RANGE_MASK) != (e->code & ~J_RANGE_MASK)); + + /* Target must have 4 byte alignment. */ + BUG_ON((e->target & 3) != 0); + + if (type == JUMP_LABEL_ENABLE) { + insn.j_format.opcode = j_op; + insn.j_format.target = (e->target & J_RANGE_MASK) >> 2; + } else { + insn.word = 0; /* nop */ + } + + get_online_cpus(); + mutex_lock(&text_mutex); + *insn_p = insn; + + flush_icache_range((unsigned long)insn_p, + (unsigned long)insn_p + sizeof(*insn_p)); + + mutex_unlock(&text_mutex); + put_online_cpus(); +} + +#endif /* HAVE_JUMP_LABEL */ diff --git a/arch/mips/kernel/kgdb.c b/arch/mips/kernel/kgdb.c index f4546e97c60..7afcc2f22c0 100644 --- a/arch/mips/kernel/kgdb.c +++ b/arch/mips/kernel/kgdb.c @@ -32,6 +32,7 @@ #include <asm/cacheflush.h> #include <asm/processor.h> #include <asm/sigcontext.h> +#include <asm/uaccess.h> static struct hard_trap_info { unsigned char tt; /* Trap type code for MIPS R3xxx and R4xxx */ @@ -40,7 +41,7 @@ static struct hard_trap_info { { 6, SIGBUS }, /* instruction bus error */ { 7, SIGBUS }, /* data bus error */ { 9, SIGTRAP }, /* break */ -/* { 11, SIGILL }, */ /* CPU unusable */ +/* { 11, SIGILL }, */ /* CPU unusable */ { 12, SIGFPE }, /* overflow */ { 13, SIGTRAP }, /* trap */ { 14, SIGSEGV }, /* virtual instruction cache coherency */ @@ -208,7 +209,14 @@ void arch_kgdb_breakpoint(void) static void kgdb_call_nmi_hook(void *ignored) { + mm_segment_t old_fs; + + old_fs = get_fs(); + set_fs(get_ds()); + kgdb_nmicallback(raw_smp_processor_id(), NULL); + + set_fs(old_fs); } void kgdb_roundup_cpus(unsigned long flags) @@ -282,16 +290,32 @@ static int kgdb_mips_notify(struct notifier_block *self, unsigned long cmd, struct die_args *args = (struct die_args *)ptr; struct pt_regs *regs = args->regs; int trap = (regs->cp0_cause & 0x7c) >> 2; + mm_segment_t old_fs; + +#ifdef CONFIG_KPROBES + /* + * Return immediately if the kprobes fault notifier has set + * DIE_PAGE_FAULT. + */ + if (cmd == DIE_PAGE_FAULT) + return NOTIFY_DONE; +#endif /* CONFIG_KPROBES */ /* Userspace events, ignore. */ if (user_mode(regs)) return NOTIFY_DONE; + /* Kernel mode. Set correct address limit */ + old_fs = get_fs(); + set_fs(get_ds()); + if (atomic_read(&kgdb_active) != -1) kgdb_nmicallback(smp_processor_id(), regs); - if (kgdb_handle_exception(trap, compute_signal(trap), cmd, regs)) + if (kgdb_handle_exception(trap, compute_signal(trap), cmd, regs)) { + set_fs(old_fs); return NOTIFY_DONE; + } if (atomic_read(&kgdb_setting_breakpoint)) if ((trap == 9) && (regs->cp0_epc == (unsigned long)breakinst)) @@ -301,6 +325,7 @@ static int kgdb_mips_notify(struct notifier_block *self, unsigned long cmd, local_irq_enable(); __flush_cache_all(); + set_fs(old_fs); return NOTIFY_STOP; } @@ -312,7 +337,7 @@ int kgdb_ll_trap(int cmd, const char *str, .regs = regs, .str = str, .err = err, - .trapnr = trap, + .trapnr = trap, .signr = sig, }; @@ -362,7 +387,7 @@ int kgdb_arch_init(void) union mips_instruction insn = { .r_format = { .opcode = spec_op, - .func = break_op, + .func = break_op, } }; memcpy(arch_kgdb_ops.gdb_bpt_instr, insn.byte, BREAK_INSTR_SIZE); diff --git a/arch/mips/kernel/kprobes.c b/arch/mips/kernel/kprobes.c index ee28683fc2a..1f8187ab099 100644 --- a/arch/mips/kernel/kprobes.c +++ b/arch/mips/kernel/kprobes.c @@ -25,10 +25,12 @@ #include <linux/kprobes.h> #include <linux/preempt.h> +#include <linux/uaccess.h> #include <linux/kdebug.h> #include <linux/slab.h> #include <asm/ptrace.h> +#include <asm/branch.h> #include <asm/break.h> #include <asm/inst.h> @@ -112,17 +114,49 @@ insn_ok: return 0; } +/* + * insn_has_ll_or_sc function checks whether instruction is ll or sc + * one; putting breakpoint on top of atomic ll/sc pair is bad idea; + * so we need to prevent it and refuse kprobes insertion for such + * instructions; cannot do much about breakpoint in the middle of + * ll/sc pair; it is upto user to avoid those places + */ +static int __kprobes insn_has_ll_or_sc(union mips_instruction insn) +{ + int ret = 0; + + switch (insn.i_format.opcode) { + case ll_op: + case lld_op: + case sc_op: + case scd_op: + ret = 1; + break; + default: + break; + } + return ret; +} + int __kprobes arch_prepare_kprobe(struct kprobe *p) { union mips_instruction insn; union mips_instruction prev_insn; int ret = 0; - prev_insn = p->addr[-1]; insn = p->addr[0]; - if (insn_has_delayslot(insn) || insn_has_delayslot(prev_insn)) { - pr_notice("Kprobes for branch and jump instructions are not supported\n"); + if (insn_has_ll_or_sc(insn)) { + pr_notice("Kprobes for ll and sc instructions are not" + "supported\n"); + ret = -EINVAL; + goto out; + } + + if ((probe_kernel_read(&prev_insn, p->addr - 1, + sizeof(mips_instruction)) == 0) && + insn_has_delayslot(prev_insn)) { + pr_notice("Kprobes for branch delayslot are not supported\n"); ret = -EINVAL; goto out; } @@ -138,9 +172,20 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p) * In the kprobe->ainsn.insn[] array we store the original * instruction at index zero and a break trap instruction at * index one. + * + * On MIPS arch if the instruction at probed address is a + * branch instruction, we need to execute the instruction at + * Branch Delayslot (BD) at the time of probe hit. As MIPS also + * doesn't have single stepping support, the BD instruction can + * not be executed in-line and it would be executed on SSOL slot + * using a normal breakpoint instruction in the next slot. + * So, read the instruction and save it for later execution. */ + if (insn_has_delayslot(insn)) + memcpy(&p->ainsn.insn[0], p->addr + 1, sizeof(kprobe_opcode_t)); + else + memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t)); - memcpy(&p->ainsn.insn[0], p->addr, sizeof(kprobe_opcode_t)); p->ainsn.insn[1] = breakpoint2_insn; p->opcode = *p->addr; @@ -162,7 +207,10 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - free_insn_slot(p->ainsn.insn, 0); + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } } static void save_previous_kprobe(struct kprobe_ctlblk *kcb) @@ -191,16 +239,96 @@ static void set_current_kprobe(struct kprobe *p, struct pt_regs *regs, kcb->kprobe_saved_epc = regs->cp0_epc; } -static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs) +/** + * evaluate_branch_instrucion - + * + * Evaluate the branch instruction at probed address during probe hit. The + * result of evaluation would be the updated epc. The insturction in delayslot + * would actually be single stepped using a normal breakpoint) on SSOL slot. + * + * The result is also saved in the kprobe control block for later use, + * in case we need to execute the delayslot instruction. The latter will be + * false for NOP instruction in dealyslot and the branch-likely instructions + * when the branch is taken. And for those cases we set a flag as + * SKIP_DELAYSLOT in the kprobe control block + */ +static int evaluate_branch_instruction(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) { + union mips_instruction insn = p->opcode; + long epc; + int ret = 0; + + epc = regs->cp0_epc; + if (epc & 3) + goto unaligned; + + if (p->ainsn.insn->word == 0) + kcb->flags |= SKIP_DELAYSLOT; + else + kcb->flags &= ~SKIP_DELAYSLOT; + + ret = __compute_return_epc_for_insn(regs, insn); + if (ret < 0) + return ret; + + if (ret == BRANCH_LIKELY_TAKEN) + kcb->flags |= SKIP_DELAYSLOT; + + kcb->target_epc = regs->cp0_epc; + + return 0; + +unaligned: + pr_notice("%s: unaligned epc - sending SIGBUS.\n", current->comm); + force_sig(SIGBUS, current); + return -EFAULT; + +} + +static void prepare_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + int ret = 0; + regs->cp0_status &= ~ST0_IE; /* single step inline if the instruction is a break */ if (p->opcode.word == breakpoint_insn.word || p->opcode.word == breakpoint2_insn.word) regs->cp0_epc = (unsigned long)p->addr; - else - regs->cp0_epc = (unsigned long)&p->ainsn.insn[0]; + else if (insn_has_delayslot(p->opcode)) { + ret = evaluate_branch_instruction(p, regs, kcb); + if (ret < 0) { + pr_notice("Kprobes: Error in evaluating branch\n"); + return; + } + } + regs->cp0_epc = (unsigned long)&p->ainsn.insn[0]; +} + +/* + * Called after single-stepping. p->addr is the address of the + * instruction whose first byte has been replaced by the "break 0" + * instruction. To avoid the SMP problems that can occur when we + * temporarily put back the original opcode to single-step, we + * single-stepped a copy of the instruction. The address of this + * copy is p->ainsn.insn. + * + * This function prepares to return from the post-single-step + * breakpoint trap. In case of branch instructions, the target + * epc to be restored. + */ +static void __kprobes resume_execution(struct kprobe *p, + struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + if (insn_has_delayslot(p->opcode)) + regs->cp0_epc = kcb->target_epc; + else { + unsigned long orig_epc = kcb->kprobe_saved_epc; + regs->cp0_epc = orig_epc + 4; + } } static int __kprobes kprobe_handler(struct pt_regs *regs) @@ -239,8 +367,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) save_previous_kprobe(kcb); set_current_kprobe(p, regs, kcb); kprobes_inc_nmissed_count(p); - prepare_singlestep(p, regs); + prepare_singlestep(p, regs, kcb); kcb->kprobe_status = KPROBE_REENTER; + if (kcb->flags & SKIP_DELAYSLOT) { + resume_execution(p, regs, kcb); + restore_previous_kprobe(kcb); + preempt_enable_no_resched(); + } return 1; } else { if (addr->word != breakpoint_insn.word) { @@ -284,8 +417,16 @@ static int __kprobes kprobe_handler(struct pt_regs *regs) } ss_probe: - prepare_singlestep(p, regs); - kcb->kprobe_status = KPROBE_HIT_SS; + prepare_singlestep(p, regs, kcb); + if (kcb->flags & SKIP_DELAYSLOT) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + if (p->post_handler) + p->post_handler(p, regs, 0); + resume_execution(p, regs, kcb); + preempt_enable_no_resched(); + } else + kcb->kprobe_status = KPROBE_HIT_SS; + return 1; no_kprobe: @@ -294,25 +435,6 @@ no_kprobe: } -/* - * Called after single-stepping. p->addr is the address of the - * instruction whose first byte has been replaced by the "break 0" - * instruction. To avoid the SMP problems that can occur when we - * temporarily put back the original opcode to single-step, we - * single-stepped a copy of the instruction. The address of this - * copy is p->ainsn.insn. - * - * This function prepares to return from the post-single-step - * breakpoint trap. - */ -static void __kprobes resume_execution(struct kprobe *p, - struct pt_regs *regs, - struct kprobe_ctlblk *kcb) -{ - unsigned long orig_epc = kcb->kprobe_saved_epc; - regs->cp0_epc = orig_epc + 4; -} - static inline int post_kprobe_handler(struct pt_regs *regs) { struct kprobe *cur = kprobe_running(); @@ -416,7 +538,7 @@ void jprobe_return_end(void); void __kprobes jprobe_return(void) { - /* Assembler quirk necessitates this '0,code' business. */ + /* Assembler quirk necessitates this '0,code' business. */ asm volatile( "break 0,%0\n\t" ".globl jprobe_return_end\n" @@ -479,7 +601,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, { struct kretprobe_instance *ri = NULL; struct hlist_head *head, empty_rp; - struct hlist_node *node, *tmp; + struct hlist_node *tmp; unsigned long flags, orig_ret_address = 0; unsigned long trampoline_address = (unsigned long)kretprobe_trampoline; @@ -495,11 +617,11 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, * We can handle this because: * - instances are always inserted at the head of the list * - when multiple return probes are registered for the same - * function, the first instance's ret_addr will point to the - * real return address, and all the rest will point to - * kretprobe_trampoline + * function, the first instance's ret_addr will point to the + * real return address, and all the rest will point to + * kretprobe_trampoline */ - hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { + hlist_for_each_entry_safe(ri, tmp, head, hlist) { if (ri->task != current) /* another task is sharing our hash bucket */ continue; @@ -526,7 +648,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p, kretprobe_hash_unlock(current, &flags); preempt_enable_no_resched(); - hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { + hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); kfree(ri); } diff --git a/arch/mips/kernel/kspd.c b/arch/mips/kernel/kspd.c deleted file mode 100644 index 29811f04339..00000000000 --- a/arch/mips/kernel/kspd.c +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. - * - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - * - */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/unistd.h> -#include <linux/file.h> -#include <linux/fdtable.h> -#include <linux/fs.h> -#include <linux/syscalls.h> -#include <linux/workqueue.h> -#include <linux/errno.h> -#include <linux/list.h> - -#include <asm/vpe.h> -#include <asm/rtlx.h> -#include <asm/kspd.h> - -static struct workqueue_struct *workqueue; -static struct work_struct work; - -extern unsigned long cpu_khz; - -struct mtsp_syscall { - int cmd; - unsigned char abi; - unsigned char size; -}; - -struct mtsp_syscall_ret { - int retval; - int errno; -}; - -struct mtsp_syscall_generic { - int arg0; - int arg1; - int arg2; - int arg3; - int arg4; - int arg5; - int arg6; -}; - -static struct list_head kspd_notifylist; -static int sp_stopping; - -/* these should match with those in the SDE kit */ -#define MTSP_SYSCALL_BASE 0 -#define MTSP_SYSCALL_EXIT (MTSP_SYSCALL_BASE + 0) -#define MTSP_SYSCALL_OPEN (MTSP_SYSCALL_BASE + 1) -#define MTSP_SYSCALL_READ (MTSP_SYSCALL_BASE + 2) -#define MTSP_SYSCALL_WRITE (MTSP_SYSCALL_BASE + 3) -#define MTSP_SYSCALL_CLOSE (MTSP_SYSCALL_BASE + 4) -#define MTSP_SYSCALL_LSEEK32 (MTSP_SYSCALL_BASE + 5) -#define MTSP_SYSCALL_ISATTY (MTSP_SYSCALL_BASE + 6) -#define MTSP_SYSCALL_GETTIME (MTSP_SYSCALL_BASE + 7) -#define MTSP_SYSCALL_PIPEFREQ (MTSP_SYSCALL_BASE + 8) -#define MTSP_SYSCALL_GETTOD (MTSP_SYSCALL_BASE + 9) -#define MTSP_SYSCALL_IOCTL (MTSP_SYSCALL_BASE + 10) - -#define MTSP_O_RDONLY 0x0000 -#define MTSP_O_WRONLY 0x0001 -#define MTSP_O_RDWR 0x0002 -#define MTSP_O_NONBLOCK 0x0004 -#define MTSP_O_APPEND 0x0008 -#define MTSP_O_SHLOCK 0x0010 -#define MTSP_O_EXLOCK 0x0020 -#define MTSP_O_ASYNC 0x0040 -/* XXX: check which of these is actually O_SYNC vs O_DSYNC */ -#define MTSP_O_FSYNC O_SYNC -#define MTSP_O_NOFOLLOW 0x0100 -#define MTSP_O_SYNC 0x0080 -#define MTSP_O_CREAT 0x0200 -#define MTSP_O_TRUNC 0x0400 -#define MTSP_O_EXCL 0x0800 -#define MTSP_O_BINARY 0x8000 - -extern int tclimit; - -struct apsp_table { - int sp; - int ap; -}; - -/* we might want to do the mode flags too */ -struct apsp_table open_flags_table[] = { - { MTSP_O_RDWR, O_RDWR }, - { MTSP_O_WRONLY, O_WRONLY }, - { MTSP_O_CREAT, O_CREAT }, - { MTSP_O_TRUNC, O_TRUNC }, - { MTSP_O_NONBLOCK, O_NONBLOCK }, - { MTSP_O_APPEND, O_APPEND }, - { MTSP_O_NOFOLLOW, O_NOFOLLOW } -}; - -struct apsp_table syscall_command_table[] = { - { MTSP_SYSCALL_OPEN, __NR_open }, - { MTSP_SYSCALL_CLOSE, __NR_close }, - { MTSP_SYSCALL_READ, __NR_read }, - { MTSP_SYSCALL_WRITE, __NR_write }, - { MTSP_SYSCALL_LSEEK32, __NR_lseek }, - { MTSP_SYSCALL_IOCTL, __NR_ioctl } -}; - -static int sp_syscall(int num, int arg0, int arg1, int arg2, int arg3) -{ - register long int _num __asm__("$2") = num; - register long int _arg0 __asm__("$4") = arg0; - register long int _arg1 __asm__("$5") = arg1; - register long int _arg2 __asm__("$6") = arg2; - register long int _arg3 __asm__("$7") = arg3; - - mm_segment_t old_fs; - - old_fs = get_fs(); - set_fs(KERNEL_DS); - - __asm__ __volatile__ ( - " syscall \n" - : "=r" (_num), "=r" (_arg3) - : "r" (_num), "r" (_arg0), "r" (_arg1), "r" (_arg2), "r" (_arg3)); - - set_fs(old_fs); - - /* $a3 is error flag */ - if (_arg3) - return -_num; - - return _num; -} - -static int translate_syscall_command(int cmd) -{ - int i; - int ret = -1; - - for (i = 0; i < ARRAY_SIZE(syscall_command_table); i++) { - if ((cmd == syscall_command_table[i].sp)) - return syscall_command_table[i].ap; - } - - return ret; -} - -static unsigned int translate_open_flags(int flags) -{ - int i; - unsigned int ret = 0; - - for (i = 0; i < ARRAY_SIZE(open_flags_table); i++) { - if( (flags & open_flags_table[i].sp) ) { - ret |= open_flags_table[i].ap; - } - } - - return ret; -} - - -static int sp_setfsuidgid(uid_t uid, gid_t gid) -{ - struct cred *new; - - new = prepare_creds(); - if (!new) - return -ENOMEM; - - new->fsuid = uid; - new->fsgid = gid; - - commit_creds(new); - - return 0; -} - -/* - * Expects a request to be on the sysio channel. Reads it. Decides whether - * its a linux syscall and runs it, or whatever. Puts the return code back - * into the request and sends the whole thing back. - */ -void sp_work_handle_request(void) -{ - struct mtsp_syscall sc; - struct mtsp_syscall_generic generic; - struct mtsp_syscall_ret ret; - struct kspd_notifications *n; - unsigned long written; - mm_segment_t old_fs; - struct timeval tv; - struct timezone tz; - int err, cmd; - - char *vcwd; - int size; - - ret.retval = -1; - - old_fs = get_fs(); - set_fs(KERNEL_DS); - - if (!rtlx_read(RTLX_CHANNEL_SYSIO, &sc, sizeof(struct mtsp_syscall))) { - set_fs(old_fs); - printk(KERN_ERR "Expected request but nothing to read\n"); - return; - } - - size = sc.size; - - if (size) { - if (!rtlx_read(RTLX_CHANNEL_SYSIO, &generic, size)) { - set_fs(old_fs); - printk(KERN_ERR "Expected request but nothing to read\n"); - return; - } - } - - /* Run the syscall at the privilege of the user who loaded the - SP program */ - - if (vpe_getuid(tclimit)) { - err = sp_setfsuidgid(vpe_getuid(tclimit), vpe_getgid(tclimit)); - if (!err) - pr_err("Change of creds failed\n"); - } - - switch (sc.cmd) { - /* needs the flags argument translating from SDE kit to - linux */ - case MTSP_SYSCALL_PIPEFREQ: - ret.retval = cpu_khz * 1000; - ret.errno = 0; - break; - - case MTSP_SYSCALL_GETTOD: - memset(&tz, 0, sizeof(tz)); - if ((ret.retval = sp_syscall(__NR_gettimeofday, (int)&tv, - (int)&tz, 0, 0)) == 0) - ret.retval = tv.tv_sec; - break; - - case MTSP_SYSCALL_EXIT: - list_for_each_entry(n, &kspd_notifylist, list) - n->kspd_sp_exit(tclimit); - sp_stopping = 1; - - printk(KERN_DEBUG "KSPD got exit syscall from SP exitcode %d\n", - generic.arg0); - break; - - case MTSP_SYSCALL_OPEN: - generic.arg1 = translate_open_flags(generic.arg1); - - vcwd = vpe_getcwd(tclimit); - - /* change to cwd of the process that loaded the SP program */ - old_fs = get_fs(); - set_fs(KERNEL_DS); - sys_chdir(vcwd); - set_fs(old_fs); - - sc.cmd = __NR_open; - - /* fall through */ - - default: - if ((sc.cmd >= __NR_Linux) && - (sc.cmd <= (__NR_Linux + __NR_Linux_syscalls)) ) - cmd = sc.cmd; - else - cmd = translate_syscall_command(sc.cmd); - - if (cmd >= 0) { - ret.retval = sp_syscall(cmd, generic.arg0, generic.arg1, - generic.arg2, generic.arg3); - } else - printk(KERN_WARNING - "KSPD: Unknown SP syscall number %d\n", sc.cmd); - break; - } /* switch */ - - if (vpe_getuid(tclimit)) { - err = sp_setfsuidgid(0, 0); - if (!err) - pr_err("restoring old creds failed\n"); - } - - old_fs = get_fs(); - set_fs(KERNEL_DS); - written = rtlx_write(RTLX_CHANNEL_SYSIO, &ret, sizeof(ret)); - set_fs(old_fs); - if (written < sizeof(ret)) - printk("KSPD: sp_work_handle_request failed to send to SP\n"); -} - -static void sp_cleanup(void) -{ - struct files_struct *files = current->files; - int i, j; - struct fdtable *fdt; - - j = 0; - - /* - * It is safe to dereference the fd table without RCU or - * ->file_lock - */ - fdt = files_fdtable(files); - for (;;) { - unsigned long set; - i = j * __NFDBITS; - if (i >= fdt->max_fds) - break; - set = fdt->open_fds->fds_bits[j++]; - while (set) { - if (set & 1) { - struct file * file = xchg(&fdt->fd[i], NULL); - if (file) - filp_close(file, files); - } - i++; - set >>= 1; - } - } - - /* Put daemon cwd back to root to avoid umount problems */ - sys_chdir("/"); -} - -static int channel_open; - -/* the work handler */ -static void sp_work(struct work_struct *unused) -{ - if (!channel_open) { - if( rtlx_open(RTLX_CHANNEL_SYSIO, 1) != 0) { - printk("KSPD: unable to open sp channel\n"); - sp_stopping = 1; - } else { - channel_open++; - printk(KERN_DEBUG "KSPD: SP channel opened\n"); - } - } else { - /* wait for some data, allow it to sleep */ - rtlx_read_poll(RTLX_CHANNEL_SYSIO, 1); - - /* Check we haven't been woken because we are stopping */ - if (!sp_stopping) - sp_work_handle_request(); - } - - if (!sp_stopping) - queue_work(workqueue, &work); - else - sp_cleanup(); -} - -static void startwork(int vpe) -{ - sp_stopping = channel_open = 0; - - if (workqueue == NULL) { - if ((workqueue = create_singlethread_workqueue("kspd")) == NULL) { - printk(KERN_ERR "unable to start kspd\n"); - return; - } - - INIT_WORK(&work, sp_work); - } - - queue_work(workqueue, &work); -} - -static void stopwork(int vpe) -{ - sp_stopping = 1; - - printk(KERN_DEBUG "KSPD: SP stopping\n"); -} - -void kspd_notify(struct kspd_notifications *notify) -{ - list_add(¬ify->list, &kspd_notifylist); -} - -static struct vpe_notifications notify; -static int kspd_module_init(void) -{ - INIT_LIST_HEAD(&kspd_notifylist); - - notify.start = startwork; - notify.stop = stopwork; - vpe_notify(tclimit, ¬ify); - - return 0; -} - -static void kspd_module_exit(void) -{ - -} - -module_init(kspd_module_init); -module_exit(kspd_module_exit); - -MODULE_DESCRIPTION("MIPS KSPD"); -MODULE_AUTHOR("Elizabeth Oldham, MIPS Technologies, Inc."); -MODULE_LICENSE("GPL"); diff --git a/arch/mips/kernel/linux32.c b/arch/mips/kernel/linux32.c index 6343b4a5b83..0b29646bcee 100644 --- a/arch/mips/kernel/linux32.c +++ b/arch/mips/kernel/linux32.c @@ -3,7 +3,6 @@ * * Copyright (C) 2000 Silicon Graphics, Inc. * Written by Ulf Carlsson (ulfc@engr.sgi.com) - * sys32_execve from ia64/ia32 code, Feb 2000, Kanoj Sarcar (kanoj@sgi.com) */ #include <linux/compiler.h> #include <linux/mm.h> @@ -77,27 +76,7 @@ out: return error; } -/* - * sys_execve() executes a new program. - */ -asmlinkage int sys32_execve(nabi_no_regargs struct pt_regs regs) -{ - int error; - char * filename; - - filename = getname(compat_ptr(regs.regs[4])); - error = PTR_ERR(filename); - if (IS_ERR(filename)) - goto out; - error = compat_do_execve(filename, compat_ptr(regs.regs[5]), - compat_ptr(regs.regs[6]), ®s); - putname(filename); - -out: - return error; -} - -#define RLIM_INFINITY32 0x7fffffff +#define RLIM_INFINITY32 0x7fffffff #define RESOURCE32(x) ((x > RLIM_INFINITY32) ? RLIM_INFINITY32 : x) struct rlimit32 { @@ -126,7 +105,7 @@ SYSCALL_DEFINE5(32_llseek, unsigned int, fd, unsigned int, offset_high, /* From the Single Unix Spec: pread & pwrite act like lseek to pos + op + lseek back to original location. They fail just like lseek does on - non-seekable files. */ + non-seekable files. */ SYSCALL_DEFINE6(32_pread, unsigned long, fd, char __user *, buf, size_t, count, unsigned long, unused, unsigned long, a4, unsigned long, a5) @@ -140,150 +119,22 @@ SYSCALL_DEFINE6(32_pwrite, unsigned int, fd, const char __user *, buf, return sys_pwrite64(fd, buf, count, merge_64(a4, a5)); } -SYSCALL_DEFINE2(32_sched_rr_get_interval, compat_pid_t, pid, - struct compat_timespec __user *, interval) -{ - struct timespec t; - int ret; - mm_segment_t old_fs = get_fs(); - - set_fs(KERNEL_DS); - ret = sys_sched_rr_get_interval(pid, (struct timespec __user *)&t); - set_fs(old_fs); - if (put_user (t.tv_sec, &interval->tv_sec) || - __put_user(t.tv_nsec, &interval->tv_nsec)) - return -EFAULT; - return ret; -} - -#ifdef CONFIG_SYSVIPC - -SYSCALL_DEFINE6(32_ipc, u32, call, long, first, long, second, long, third, - unsigned long, ptr, unsigned long, fifth) -{ - int version, err; - - version = call >> 16; /* hack for backward compatibility */ - call &= 0xffff; - - switch (call) { - case SEMOP: - /* struct sembuf is the same on 32 and 64bit :)) */ - err = sys_semtimedop(first, compat_ptr(ptr), second, NULL); - break; - case SEMTIMEDOP: - err = compat_sys_semtimedop(first, compat_ptr(ptr), second, - compat_ptr(fifth)); - break; - case SEMGET: - err = sys_semget(first, second, third); - break; - case SEMCTL: - err = compat_sys_semctl(first, second, third, compat_ptr(ptr)); - break; - case MSGSND: - err = compat_sys_msgsnd(first, second, third, compat_ptr(ptr)); - break; - case MSGRCV: - err = compat_sys_msgrcv(first, second, fifth, third, - version, compat_ptr(ptr)); - break; - case MSGGET: - err = sys_msgget((key_t) first, second); - break; - case MSGCTL: - err = compat_sys_msgctl(first, second, compat_ptr(ptr)); - break; - case SHMAT: - err = compat_sys_shmat(first, second, third, version, - compat_ptr(ptr)); - break; - case SHMDT: - err = sys_shmdt(compat_ptr(ptr)); - break; - case SHMGET: - err = sys_shmget(first, (unsigned)second, third); - break; - case SHMCTL: - err = compat_sys_shmctl(first, second, compat_ptr(ptr)); - break; - default: - err = -EINVAL; - break; - } - - return err; -} - -#else - -SYSCALL_DEFINE6(32_ipc, u32, call, int, first, int, second, int, third, - u32, ptr, u32, fifth) -{ - return -ENOSYS; -} - -#endif /* CONFIG_SYSVIPC */ - -#ifdef CONFIG_MIPS32_N32 -SYSCALL_DEFINE4(n32_semctl, int, semid, int, semnum, int, cmd, u32, arg) -{ - /* compat_sys_semctl expects a pointer to union semun */ - u32 __user *uptr = compat_alloc_user_space(sizeof(u32)); - if (put_user(arg, uptr)) - return -EFAULT; - return compat_sys_semctl(semid, semnum, cmd, uptr); -} - -SYSCALL_DEFINE4(n32_msgsnd, int, msqid, u32, msgp, unsigned int, msgsz, - int, msgflg) -{ - return compat_sys_msgsnd(msqid, msgsz, msgflg, compat_ptr(msgp)); -} - -SYSCALL_DEFINE5(n32_msgrcv, int, msqid, u32, msgp, size_t, msgsz, - int, msgtyp, int, msgflg) -{ - return compat_sys_msgrcv(msqid, msgsz, msgtyp, msgflg, IPC_64, - compat_ptr(msgp)); -} -#endif - SYSCALL_DEFINE1(32_personality, unsigned long, personality) { + unsigned int p = personality & 0xffffffff; int ret; - personality &= 0xffffffff; - if (personality(current->personality) == PER_LINUX32 && - personality == PER_LINUX) - personality = PER_LINUX32; - ret = sys_personality(personality); - if (ret == PER_LINUX32) - ret = PER_LINUX; - return ret; -} - -SYSCALL_DEFINE4(32_sendfile, long, out_fd, long, in_fd, - compat_off_t __user *, offset, s32, count) -{ - mm_segment_t old_fs = get_fs(); - int ret; - off_t of; - - if (offset && get_user(of, offset)) - return -EFAULT; - - set_fs(KERNEL_DS); - ret = sys_sendfile(out_fd, in_fd, offset ? (off_t __user *)&of : NULL, count); - set_fs(old_fs); - - if (offset && put_user(of, offset)) - return -EFAULT; + if (personality(current->personality) == PER_LINUX32 && + personality(p) == PER_LINUX) + p = (p & ~PER_MASK) | PER_LINUX32; + ret = sys_personality(p); + if (ret != -1 && personality(ret) == PER_LINUX32) + ret = (ret & ~PER_MASK) | PER_LINUX; return ret; } asmlinkage ssize_t sys32_readahead(int fd, u32 pad0, u64 a2, u64 a3, - size_t count) + size_t count) { return sys_readahead(fd, merge_64(a2, a3), count); } @@ -312,39 +163,5 @@ asmlinkage long sys32_fallocate(int fd, int mode, unsigned offset_a2, unsigned offset_a3, unsigned len_a4, unsigned len_a5) { return sys_fallocate(fd, mode, merge_64(offset_a2, offset_a3), - merge_64(len_a4, len_a5)); -} - -save_static_function(sys32_clone); -static int noinline __used -_sys32_clone(nabi_no_regargs struct pt_regs regs) -{ - unsigned long clone_flags; - unsigned long newsp; - int __user *parent_tidptr, *child_tidptr; - - clone_flags = regs.regs[4]; - newsp = regs.regs[5]; - if (!newsp) - newsp = regs.regs[29]; - parent_tidptr = (int __user *) regs.regs[6]; - - /* Use __dummy4 instead of getting it off the stack, so that - syscall() works. */ - child_tidptr = (int __user *) __dummy4; - return do_fork(clone_flags, newsp, ®s, 0, - parent_tidptr, child_tidptr); -} - -asmlinkage long sys32_lookup_dcookie(u32 a0, u32 a1, char __user *buf, - size_t len) -{ - return sys_lookup_dcookie(merge_64(a0, a1), buf, len); -} - -SYSCALL_DEFINE6(32_fanotify_mark, int, fanotify_fd, unsigned int, flags, - u64, a3, u64, a4, int, dfd, const char __user *, pathname) -{ - return sys_fanotify_mark(fanotify_fd, flags, merge_64(a3, a4), - dfd, pathname); + merge_64(len_a4, len_a5)); } diff --git a/arch/mips/kernel/machine_kexec.c b/arch/mips/kernel/machine_kexec.c index 85beb9b0b2d..992e18474da 100644 --- a/arch/mips/kernel/machine_kexec.c +++ b/arch/mips/kernel/machine_kexec.c @@ -5,7 +5,7 @@ * This source code is licensed under the GNU General Public License, * Version 2. See the file COPYING for more details. */ - +#include <linux/compiler.h> #include <linux/kexec.h> #include <linux/mm.h> #include <linux/delay.h> @@ -19,9 +19,19 @@ extern const size_t relocate_new_kernel_size; extern unsigned long kexec_start_address; extern unsigned long kexec_indirection_page; +int (*_machine_kexec_prepare)(struct kimage *) = NULL; +void (*_machine_kexec_shutdown)(void) = NULL; +void (*_machine_crash_shutdown)(struct pt_regs *regs) = NULL; +#ifdef CONFIG_SMP +void (*relocated_kexec_smp_wait) (void *); +atomic_t kexec_ready_to_reboot = ATOMIC_INIT(0); +#endif + int machine_kexec_prepare(struct kimage *kimage) { + if (_machine_kexec_prepare) + return _machine_kexec_prepare(kimage); return 0; } @@ -33,14 +43,20 @@ machine_kexec_cleanup(struct kimage *kimage) void machine_shutdown(void) { + if (_machine_kexec_shutdown) + _machine_kexec_shutdown(); } void machine_crash_shutdown(struct pt_regs *regs) { + if (_machine_crash_shutdown) + _machine_crash_shutdown(regs); + else + default_machine_crash_shutdown(regs); } -typedef void (*noretfun_t)(void) __attribute__((noreturn)); +typedef void (*noretfun_t)(void) __noreturn; void machine_kexec(struct kimage *image) @@ -52,7 +68,9 @@ machine_kexec(struct kimage *image) reboot_code_buffer = (unsigned long)page_address(image->control_code_page); - kexec_start_address = image->start; + kexec_start_address = + (unsigned long) phys_to_virt(image->start); + kexec_indirection_page = (unsigned long) phys_to_virt(image->head & PAGE_MASK); @@ -63,7 +81,7 @@ machine_kexec(struct kimage *image) * The generic kexec code builds a page list with physical * addresses. they are directly accessible through KSEG0 (or * CKSEG0 or XPHYS if on 64bit system), hence the - * pys_to_virt() call. + * phys_to_virt() call. */ for (ptr = &image->head; (entry = *ptr) && !(entry &IND_DONE); ptr = (entry & IND_INDIRECTION) ? @@ -81,5 +99,12 @@ machine_kexec(struct kimage *image) printk("Will call new kernel at %08lx\n", image->start); printk("Bye ...\n"); __flush_cache_all(); +#ifdef CONFIG_SMP + /* All secondary cpus now may jump to kexec_wait cycle */ + relocated_kexec_smp_wait = reboot_code_buffer + + (void *)(kexec_smp_wait - relocate_new_kernel); + smp_wmb(); + atomic_set(&kexec_ready_to_reboot, 1); +#endif ((noretfun_t) reboot_code_buffer)(); } diff --git a/arch/mips/kernel/mcount.S b/arch/mips/kernel/mcount.S index 4c968e7efb7..539b6294b61 100644 --- a/arch/mips/kernel/mcount.S +++ b/arch/mips/kernel/mcount.S @@ -46,11 +46,9 @@ PTR_L a5, PT_R9(sp) PTR_L a6, PT_R10(sp) PTR_L a7, PT_R11(sp) - PTR_ADDIU sp, PT_SIZE -#else - PTR_ADDIU sp, (PT_SIZE + 8) #endif -.endm + PTR_ADDIU sp, PT_SIZE + .endm .macro RETURN_BACK jr ra @@ -69,7 +67,13 @@ NESTED(ftrace_caller, PT_SIZE, ra) .globl _mcount _mcount: b ftrace_stub +#ifdef CONFIG_32BIT + addiu sp,sp,8 +#else nop +#endif + + /* When tracing is activated, it calls ftrace_caller+8 (aka here) */ lw t1, function_trace_stop bnez t1, ftrace_stub nop @@ -79,7 +83,7 @@ _mcount: PTR_S MCOUNT_RA_ADDRESS_REG, PT_R12(sp) #endif - move a0, ra /* arg1: self return address */ + PTR_SUBU a0, ra, 8 /* arg1: self address */ .globl ftrace_call ftrace_call: nop /* a placeholder for the call to a real tracing function */ @@ -164,15 +168,11 @@ NESTED(ftrace_graph_caller, PT_SIZE, ra) #endif /* arg3: Get frame pointer of current stack */ -#ifdef CONFIG_FRAME_POINTER - move a2, fp -#else /* ! CONFIG_FRAME_POINTER */ #ifdef CONFIG_64BIT PTR_LA a2, PT_SIZE(sp) #else PTR_LA a2, (PT_SIZE+8)(sp) #endif -#endif jal prepare_ftrace_return nop diff --git a/arch/mips/kernel/mips-cm.c b/arch/mips/kernel/mips-cm.c new file mode 100644 index 00000000000..f76f7a08412 --- /dev/null +++ b/arch/mips/kernel/mips-cm.c @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/errno.h> + +#include <asm/mips-cm.h> +#include <asm/mipsregs.h> + +void __iomem *mips_cm_base; +void __iomem *mips_cm_l2sync_base; + +phys_t __mips_cm_phys_base(void) +{ + u32 config3 = read_c0_config3(); + u32 cmgcr; + + /* Check the CMGCRBase register is implemented */ + if (!(config3 & MIPS_CONF3_CMGCR)) + return 0; + + /* Read the address from CMGCRBase */ + cmgcr = read_c0_cmgcrbase(); + return (cmgcr & MIPS_CMGCRF_BASE) << (36 - 32); +} + +phys_t mips_cm_phys_base(void) + __attribute__((weak, alias("__mips_cm_phys_base"))); + +phys_t __mips_cm_l2sync_phys_base(void) +{ + u32 base_reg; + + /* + * If the L2-only sync region is already enabled then leave it at it's + * current location. + */ + base_reg = read_gcr_l2_only_sync_base(); + if (base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN_MSK) + return base_reg & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE_MSK; + + /* Default to following the CM */ + return mips_cm_phys_base() + MIPS_CM_GCR_SIZE; +} + +phys_t mips_cm_l2sync_phys_base(void) + __attribute__((weak, alias("__mips_cm_l2sync_phys_base"))); + +static void mips_cm_probe_l2sync(void) +{ + unsigned major_rev; + phys_t addr; + + /* L2-only sync was introduced with CM major revision 6 */ + major_rev = (read_gcr_rev() & CM_GCR_REV_MAJOR_MSK) >> + CM_GCR_REV_MAJOR_SHF; + if (major_rev < 6) + return; + + /* Find a location for the L2 sync region */ + addr = mips_cm_l2sync_phys_base(); + BUG_ON((addr & CM_GCR_L2_ONLY_SYNC_BASE_SYNCBASE_MSK) != addr); + if (!addr) + return; + + /* Set the region base address & enable it */ + write_gcr_l2_only_sync_base(addr | CM_GCR_L2_ONLY_SYNC_BASE_SYNCEN_MSK); + + /* Map the region */ + mips_cm_l2sync_base = ioremap_nocache(addr, MIPS_CM_L2SYNC_SIZE); +} + +int mips_cm_probe(void) +{ + phys_t addr; + u32 base_reg; + + addr = mips_cm_phys_base(); + BUG_ON((addr & CM_GCR_BASE_GCRBASE_MSK) != addr); + if (!addr) + return -ENODEV; + + mips_cm_base = ioremap_nocache(addr, MIPS_CM_GCR_SIZE); + if (!mips_cm_base) + return -ENXIO; + + /* sanity check that we're looking at a CM */ + base_reg = read_gcr_base(); + if ((base_reg & CM_GCR_BASE_GCRBASE_MSK) != addr) { + pr_err("GCRs appear to have been moved (expected them at 0x%08lx)!\n", + (unsigned long)addr); + mips_cm_base = NULL; + return -ENODEV; + } + + /* set default target to memory */ + base_reg &= ~CM_GCR_BASE_CMDEFTGT_MSK; + base_reg |= CM_GCR_BASE_CMDEFTGT_MEM; + write_gcr_base(base_reg); + + /* disable CM regions */ + write_gcr_reg0_base(CM_GCR_REGn_BASE_BASEADDR_MSK); + write_gcr_reg0_mask(CM_GCR_REGn_MASK_ADDRMASK_MSK); + write_gcr_reg1_base(CM_GCR_REGn_BASE_BASEADDR_MSK); + write_gcr_reg1_mask(CM_GCR_REGn_MASK_ADDRMASK_MSK); + write_gcr_reg2_base(CM_GCR_REGn_BASE_BASEADDR_MSK); + write_gcr_reg2_mask(CM_GCR_REGn_MASK_ADDRMASK_MSK); + write_gcr_reg3_base(CM_GCR_REGn_BASE_BASEADDR_MSK); + write_gcr_reg3_mask(CM_GCR_REGn_MASK_ADDRMASK_MSK); + + /* probe for an L2-only sync region */ + mips_cm_probe_l2sync(); + + return 0; +} diff --git a/arch/mips/kernel/mips-cpc.c b/arch/mips/kernel/mips-cpc.c new file mode 100644 index 00000000000..ba473608a34 --- /dev/null +++ b/arch/mips/kernel/mips-cpc.c @@ -0,0 +1,80 @@ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/errno.h> +#include <linux/percpu.h> +#include <linux/spinlock.h> + +#include <asm/mips-cm.h> +#include <asm/mips-cpc.h> + +void __iomem *mips_cpc_base; + +static DEFINE_PER_CPU_ALIGNED(spinlock_t, cpc_core_lock); + +static DEFINE_PER_CPU_ALIGNED(unsigned long, cpc_core_lock_flags); + +phys_t __weak mips_cpc_phys_base(void) +{ + u32 cpc_base; + + if (!mips_cm_present()) + return 0; + + if (!(read_gcr_cpc_status() & CM_GCR_CPC_STATUS_EX_MSK)) + return 0; + + /* If the CPC is already enabled, leave it so */ + cpc_base = read_gcr_cpc_base(); + if (cpc_base & CM_GCR_CPC_BASE_CPCEN_MSK) + return cpc_base & CM_GCR_CPC_BASE_CPCBASE_MSK; + + /* Otherwise, give it the default address & enable it */ + cpc_base = mips_cpc_default_phys_base(); + write_gcr_cpc_base(cpc_base | CM_GCR_CPC_BASE_CPCEN_MSK); + return cpc_base; +} + +int mips_cpc_probe(void) +{ + phys_t addr; + unsigned cpu; + + for_each_possible_cpu(cpu) + spin_lock_init(&per_cpu(cpc_core_lock, cpu)); + + addr = mips_cpc_phys_base(); + if (!addr) + return -ENODEV; + + mips_cpc_base = ioremap_nocache(addr, 0x8000); + if (!mips_cpc_base) + return -ENXIO; + + return 0; +} + +void mips_cpc_lock_other(unsigned int core) +{ + unsigned curr_core; + preempt_disable(); + curr_core = current_cpu_data.core; + spin_lock_irqsave(&per_cpu(cpc_core_lock, curr_core), + per_cpu(cpc_core_lock_flags, curr_core)); + write_cpc_cl_other(core << CPC_Cx_OTHER_CORENUM_SHF); +} + +void mips_cpc_unlock_other(void) +{ + unsigned curr_core = current_cpu_data.core; + spin_unlock_irqrestore(&per_cpu(cpc_core_lock, curr_core), + per_cpu(cpc_core_lock_flags, curr_core)); + preempt_enable(); +} diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c index 802e6160f37..362bb3707e6 100644 --- a/arch/mips/kernel/mips-mt-fpaff.c +++ b/arch/mips/kernel/mips-mt-fpaff.c @@ -1,5 +1,5 @@ /* - * General MIPS MT support routines, usable in AP/SP, SMVP, or SMTC kernels + * General MIPS MT support routines, usable in AP/SP and SMVP. * Copyright (C) 2005 Mips Technologies, Inc */ #include <linux/cpu.h> @@ -27,12 +27,12 @@ unsigned long mt_fpemul_threshold; * FPU affinity with the user's requested processor affinity. * This code is 98% identical with the sys_sched_setaffinity() * and sys_sched_getaffinity() system calls, and should be - * updated when kernel/sched.c changes. + * updated when kernel/sched/core.c changes. */ /* * find_process_by_pid - find a process with a matching PID value. - * used in sys_sched_set/getaffinity() in kernel/sched.c, so + * used in sys_sched_set/getaffinity() in kernel/sched/core.c, so * cloned here. */ static inline struct task_struct *find_process_by_pid(pid_t pid) @@ -50,8 +50,8 @@ static bool check_same_owner(struct task_struct *p) rcu_read_lock(); pcred = __task_cred(p); - match = (cred->euid == pcred->euid || - cred->euid == pcred->uid); + match = (uid_eq(cred->euid, pcred->euid) || + uid_eq(cred->euid, pcred->uid)); rcu_read_unlock(); return match; } @@ -173,7 +173,7 @@ asmlinkage long mipsmt_sys_sched_getaffinity(pid_t pid, unsigned int len, if (retval) goto out_unlock; - cpus_and(mask, p->thread.user_cpus_allowed, cpu_possible_map); + cpumask_and(&mask, &p->thread.user_cpus_allowed, cpu_possible_mask); out_unlock: read_unlock(&tasklist_lock); diff --git a/arch/mips/kernel/mips-mt.c b/arch/mips/kernel/mips-mt.c index b2259e7cd82..88b1ef5f868 100644 --- a/arch/mips/kernel/mips-mt.c +++ b/arch/mips/kernel/mips-mt.c @@ -1,19 +1,18 @@ /* - * General MIPS MT support routines, usable in AP/SP, SMVP, or SMTC kernels + * General MIPS MT support routines, usable in AP/SP and SMVP. * Copyright (C) 2005 Mips Technologies, Inc */ #include <linux/device.h> #include <linux/kernel.h> #include <linux/sched.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/interrupt.h> #include <linux/security.h> #include <asm/cpu.h> #include <asm/processor.h> -#include <asm/atomic.h> -#include <asm/system.h> +#include <linux/atomic.h> #include <asm/hardirq.h> #include <asm/mmu_context.h> #include <asm/mipsmtregs.h> @@ -58,9 +57,6 @@ void mips_mt_regdump(unsigned long mvpctl) int tc; unsigned long haltval; unsigned long tcstatval; -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_soft_dump(void); -#endif /* CONFIG_MIPT_MT_SMTC */ local_irq_save(flags); vpflags = dvpe(); @@ -117,9 +113,6 @@ void mips_mt_regdump(unsigned long mvpctl) if (!haltval) write_tc_c0_tchalt(0); } -#ifdef CONFIG_MIPS_MT_SMTC - smtc_soft_dump(); -#endif /* CONFIG_MIPT_MT_SMTC */ printk("===========================\n"); evpe(vpflags); local_irq_restore(flags); @@ -210,7 +203,7 @@ void mips_mt_set_cpuoptions(void) unsigned int nconfig7 = oconfig7; if (mt_opt_norps) { - printk("\"norps\" option deprectated: use \"rpsctl=\"\n"); + printk("\"norps\" option deprecated: use \"rpsctl=\"\n"); } if (mt_opt_rpsctl >= 0) { printk("34K return prediction stack override set to %d.\n", @@ -296,21 +289,11 @@ void mips_mt_set_cpuoptions(void) void mt_cflush_lockdown(void) { -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_cflush_lockdown(void); - - smtc_cflush_lockdown(); -#endif /* CONFIG_MIPS_MT_SMTC */ /* FILL IN VSMP and AP/SP VERSIONS HERE */ } void mt_cflush_release(void) { -#ifdef CONFIG_MIPS_MT_SMTC - void smtc_cflush_release(void); - - smtc_cflush_release(); -#endif /* CONFIG_MIPS_MT_SMTC */ /* FILL IN VSMP and AP/SP VERSIONS HERE */ } diff --git a/arch/mips/kernel/mips_ksyms.c b/arch/mips/kernel/mips_ksyms.c index 1d04807874d..2607c3a4ff7 100644 --- a/arch/mips/kernel/mips_ksyms.c +++ b/arch/mips/kernel/mips_ksyms.c @@ -5,23 +5,31 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (C) 1996, 97, 98, 99, 2000, 01, 03, 04, 05 by Ralf Baechle + * Copyright (C) 1996, 97, 98, 99, 2000, 01, 03, 04, 05, 12 by Ralf Baechle * Copyright (C) 1999, 2000, 01 Silicon Graphics, Inc. */ #include <linux/interrupt.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/checksum.h> -#include <asm/pgtable.h> +#include <linux/mm.h> #include <asm/uaccess.h> #include <asm/ftrace.h> extern void *__bzero(void *__s, size_t __count); +extern long __strncpy_from_kernel_nocheck_asm(char *__to, + const char *__from, long __len); +extern long __strncpy_from_kernel_asm(char *__to, const char *__from, + long __len); extern long __strncpy_from_user_nocheck_asm(char *__to, - const char *__from, long __len); + const char *__from, long __len); extern long __strncpy_from_user_asm(char *__to, const char *__from, - long __len); + long __len); +extern long __strlen_kernel_nocheck_asm(const char *s); +extern long __strlen_kernel_asm(const char *s); extern long __strlen_user_nocheck_asm(const char *s); extern long __strlen_user_asm(const char *s); +extern long __strnlen_kernel_nocheck_asm(const char *s); +extern long __strnlen_kernel_asm(const char *s); extern long __strnlen_user_nocheck_asm(const char *s); extern long __strnlen_user_asm(const char *s); @@ -32,24 +40,42 @@ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memcpy); EXPORT_SYMBOL(memmove); -EXPORT_SYMBOL(kernel_thread); +/* + * Functions that operate on entire pages. Mostly used by memory management. + */ +EXPORT_SYMBOL(clear_page); +EXPORT_SYMBOL(copy_page); /* * Userspace access stuff. */ EXPORT_SYMBOL(__copy_user); EXPORT_SYMBOL(__copy_user_inatomic); +#ifdef CONFIG_EVA +EXPORT_SYMBOL(__copy_from_user_eva); +EXPORT_SYMBOL(__copy_in_user_eva); +EXPORT_SYMBOL(__copy_to_user_eva); +EXPORT_SYMBOL(__copy_user_inatomic_eva); +#endif EXPORT_SYMBOL(__bzero); +EXPORT_SYMBOL(__strncpy_from_kernel_nocheck_asm); +EXPORT_SYMBOL(__strncpy_from_kernel_asm); EXPORT_SYMBOL(__strncpy_from_user_nocheck_asm); EXPORT_SYMBOL(__strncpy_from_user_asm); +EXPORT_SYMBOL(__strlen_kernel_nocheck_asm); +EXPORT_SYMBOL(__strlen_kernel_asm); EXPORT_SYMBOL(__strlen_user_nocheck_asm); EXPORT_SYMBOL(__strlen_user_asm); +EXPORT_SYMBOL(__strnlen_kernel_nocheck_asm); +EXPORT_SYMBOL(__strnlen_kernel_asm); EXPORT_SYMBOL(__strnlen_user_nocheck_asm); EXPORT_SYMBOL(__strnlen_user_asm); EXPORT_SYMBOL(csum_partial); EXPORT_SYMBOL(csum_partial_copy_nocheck); -EXPORT_SYMBOL(__csum_partial_copy_user); +EXPORT_SYMBOL(__csum_partial_copy_kernel); +EXPORT_SYMBOL(__csum_partial_copy_to_user); +EXPORT_SYMBOL(__csum_partial_copy_from_user); EXPORT_SYMBOL(invalid_pte_table); #ifdef CONFIG_FUNCTION_TRACER diff --git a/arch/mips/kernel/mips_machine.c b/arch/mips/kernel/mips_machine.c new file mode 100644 index 00000000000..87609752969 --- /dev/null +++ b/arch/mips/kernel/mips_machine.c @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2008-2010 Gabor Juhos <juhosg@openwrt.org> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + */ +#include <linux/mm.h> +#include <linux/string.h> +#include <linux/slab.h> + +#include <asm/mips_machine.h> +#include <asm/prom.h> + +static struct mips_machine *mips_machine __initdata; + +#define for_each_machine(mach) \ + for ((mach) = (struct mips_machine *)&__mips_machines_start; \ + (mach) && \ + (unsigned long)(mach) < (unsigned long)&__mips_machines_end; \ + (mach)++) + +__init int mips_machtype_setup(char *id) +{ + struct mips_machine *mach; + + for_each_machine(mach) { + if (mach->mach_id == NULL) + continue; + + if (strcmp(mach->mach_id, id) == 0) { + mips_machtype = mach->mach_type; + return 0; + } + } + + pr_err("MIPS: no machine found for id '%s', supported machines:\n", id); + pr_err("%-24s %s\n", "id", "name"); + for_each_machine(mach) + pr_err("%-24s %s\n", mach->mach_id, mach->mach_name); + + return 1; +} + +__setup("machtype=", mips_machtype_setup); + +__init void mips_machine_setup(void) +{ + struct mips_machine *mach; + + for_each_machine(mach) { + if (mips_machtype == mach->mach_type) { + mips_machine = mach; + break; + } + } + + if (!mips_machine) + return; + + mips_set_machine_name(mips_machine->mach_name); + + if (mips_machine->mach_setup) + mips_machine->mach_setup(); +} diff --git a/arch/mips/kernel/module-rela.c b/arch/mips/kernel/module-rela.c new file mode 100644 index 00000000000..2b70723071c --- /dev/null +++ b/arch/mips/kernel/module-rela.c @@ -0,0 +1,145 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Copyright (C) 2001 Rusty Russell. + * Copyright (C) 2003, 2004 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2005 Thiemo Seufer + */ + +#include <linux/elf.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/moduleloader.h> + +extern int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v); + +static int apply_r_mips_32_rela(struct module *me, u32 *location, Elf_Addr v) +{ + *location = v; + + return 0; +} + +static int apply_r_mips_26_rela(struct module *me, u32 *location, Elf_Addr v) +{ + if (v % 4) { + pr_err("module %s: dangerous R_MIPS_26 RELArelocation\n", + me->name); + return -ENOEXEC; + } + + if ((v & 0xf0000000) != (((unsigned long)location + 4) & 0xf0000000)) { + printk(KERN_ERR + "module %s: relocation overflow\n", + me->name); + return -ENOEXEC; + } + + *location = (*location & ~0x03ffffff) | ((v >> 2) & 0x03ffffff); + + return 0; +} + +static int apply_r_mips_hi16_rela(struct module *me, u32 *location, Elf_Addr v) +{ + *location = (*location & 0xffff0000) | + ((((long long) v + 0x8000LL) >> 16) & 0xffff); + + return 0; +} + +static int apply_r_mips_lo16_rela(struct module *me, u32 *location, Elf_Addr v) +{ + *location = (*location & 0xffff0000) | (v & 0xffff); + + return 0; +} + +static int apply_r_mips_64_rela(struct module *me, u32 *location, Elf_Addr v) +{ + *(Elf_Addr *)location = v; + + return 0; +} + +static int apply_r_mips_higher_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *location = (*location & 0xffff0000) | + ((((long long) v + 0x80008000LL) >> 32) & 0xffff); + + return 0; +} + +static int apply_r_mips_highest_rela(struct module *me, u32 *location, + Elf_Addr v) +{ + *location = (*location & 0xffff0000) | + ((((long long) v + 0x800080008000LL) >> 48) & 0xffff); + + return 0; +} + +static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, + Elf_Addr v) = { + [R_MIPS_NONE] = apply_r_mips_none, + [R_MIPS_32] = apply_r_mips_32_rela, + [R_MIPS_26] = apply_r_mips_26_rela, + [R_MIPS_HI16] = apply_r_mips_hi16_rela, + [R_MIPS_LO16] = apply_r_mips_lo16_rela, + [R_MIPS_64] = apply_r_mips_64_rela, + [R_MIPS_HIGHER] = apply_r_mips_higher_rela, + [R_MIPS_HIGHEST] = apply_r_mips_highest_rela +}; + +int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, + struct module *me) +{ + Elf_Mips_Rela *rel = (void *) sechdrs[relsec].sh_addr; + Elf_Sym *sym; + u32 *location; + unsigned int i; + Elf_Addr v; + int res; + + pr_debug("Applying relocate section %u to %u\n", relsec, + sechdrs[relsec].sh_info); + + for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { + /* This is where to make the change */ + location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + + rel[i].r_offset; + /* This is the symbol it is referring to */ + sym = (Elf_Sym *)sechdrs[symindex].sh_addr + + ELF_MIPS_R_SYM(rel[i]); + if (IS_ERR_VALUE(sym->st_value)) { + /* Ignore unresolved weak symbol */ + if (ELF_ST_BIND(sym->st_info) == STB_WEAK) + continue; + printk(KERN_WARNING "%s: Unknown symbol %s\n", + me->name, strtab + sym->st_name); + return -ENOENT; + } + + v = sym->st_value + rel[i].r_addend; + + res = reloc_handlers_rela[ELF_MIPS_R_TYPE(rel[i])](me, location, v); + if (res) + return res; + } + + return 0; +} diff --git a/arch/mips/kernel/module.c b/arch/mips/kernel/module.c index 6f51dda87fc..2a52568dbcd 100644 --- a/arch/mips/kernel/module.c +++ b/arch/mips/kernel/module.c @@ -23,13 +23,15 @@ #include <linux/moduleloader.h> #include <linux/elf.h> #include <linux/mm.h> +#include <linux/numa.h> #include <linux/vmalloc.h> #include <linux/slab.h> #include <linux/fs.h> #include <linux/string.h> #include <linux/kernel.h> -#include <linux/module.h> #include <linux/spinlock.h> +#include <linux/jump_label.h> + #include <asm/pgtable.h> /* MODULE_START */ struct mips_hi16 { @@ -38,45 +40,19 @@ struct mips_hi16 { Elf_Addr value; }; -static struct mips_hi16 *mips_hi16_list; - static LIST_HEAD(dbe_list); static DEFINE_SPINLOCK(dbe_lock); -void *module_alloc(unsigned long size) -{ #ifdef MODULE_START - struct vm_struct *area; - - size = PAGE_ALIGN(size); - if (!size) - return NULL; - - area = __get_vm_area(size, VM_ALLOC, MODULE_START, MODULE_END); - if (!area) - return NULL; - - return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL); -#else - if (size == 0) - return NULL; - return vmalloc(size); -#endif -} - -/* Free memory returned from module_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - -int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, - char *secstrings, struct module *mod) +void *module_alloc(unsigned long size) { - return 0; + return __vmalloc_node_range(size, 1, MODULE_START, MODULE_END, + GFP_KERNEL, PAGE_KERNEL, NUMA_NO_NODE, + __builtin_return_address(0)); } +#endif -static int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v) +int apply_r_mips_none(struct module *me, u32 *location, Elf_Addr v) { return 0; } @@ -88,13 +64,6 @@ static int apply_r_mips_32_rel(struct module *me, u32 *location, Elf_Addr v) return 0; } -static int apply_r_mips_32_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *location = v; - - return 0; -} - static int apply_r_mips_26_rel(struct module *me, u32 *location, Elf_Addr v) { if (v % 4) { @@ -111,27 +80,7 @@ static int apply_r_mips_26_rel(struct module *me, u32 *location, Elf_Addr v) } *location = (*location & ~0x03ffffff) | - ((*location + (v >> 2)) & 0x03ffffff); - - return 0; -} - -static int apply_r_mips_26_rela(struct module *me, u32 *location, Elf_Addr v) -{ - if (v % 4) { - pr_err("module %s: dangerous R_MIPS_26 RELArelocation\n", - me->name); - return -ENOEXEC; - } - - if ((v & 0xf0000000) != (((unsigned long)location + 4) & 0xf0000000)) { - printk(KERN_ERR - "module %s: relocation overflow\n", - me->name); - return -ENOEXEC; - } - - *location = (*location & ~0x03ffffff) | ((v >> 2) & 0x03ffffff); + ((*location + (v >> 2)) & 0x03ffffff); return 0; } @@ -151,32 +100,34 @@ static int apply_r_mips_hi16_rel(struct module *me, u32 *location, Elf_Addr v) n->addr = (Elf_Addr *)location; n->value = v; - n->next = mips_hi16_list; - mips_hi16_list = n; + n->next = me->arch.r_mips_hi16_list; + me->arch.r_mips_hi16_list = n; return 0; } -static int apply_r_mips_hi16_rela(struct module *me, u32 *location, Elf_Addr v) +static void free_relocation_chain(struct mips_hi16 *l) { - *location = (*location & 0xffff0000) | - ((((long long) v + 0x8000LL) >> 16) & 0xffff); + struct mips_hi16 *next; - return 0; + while (l) { + next = l->next; + kfree(l); + l = next; + } } static int apply_r_mips_lo16_rel(struct module *me, u32 *location, Elf_Addr v) { unsigned long insnlo = *location; + struct mips_hi16 *l; Elf_Addr val, vallo; - /* Sign extend the addend we extract from the lo insn. */ + /* Sign extend the addend we extract from the lo insn. */ vallo = ((insnlo & 0xffff) ^ 0x8000) - 0x8000; - if (mips_hi16_list != NULL) { - struct mips_hi16 *l; - - l = mips_hi16_list; + if (me->arch.r_mips_hi16_list != NULL) { + l = me->arch.r_mips_hi16_list; while (l != NULL) { struct mips_hi16 *next; unsigned long insn; @@ -211,11 +162,11 @@ static int apply_r_mips_lo16_rel(struct module *me, u32 *location, Elf_Addr v) l = next; } - mips_hi16_list = NULL; + me->arch.r_mips_hi16_list = NULL; } /* - * Ok, we're done with the HI16 relocs. Now deal with the LO16. + * Ok, we're done with the HI16 relocs. Now deal with the LO16. */ val = v + vallo; insnlo = (insnlo & ~0xffff) | (val & 0xffff); @@ -224,43 +175,14 @@ static int apply_r_mips_lo16_rel(struct module *me, u32 *location, Elf_Addr v) return 0; out_danger: + free_relocation_chain(l); + me->arch.r_mips_hi16_list = NULL; + pr_err("module %s: dangerous R_MIPS_LO16 REL relocation\n", me->name); return -ENOEXEC; } -static int apply_r_mips_lo16_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *location = (*location & 0xffff0000) | (v & 0xffff); - - return 0; -} - -static int apply_r_mips_64_rela(struct module *me, u32 *location, Elf_Addr v) -{ - *(Elf_Addr *)location = v; - - return 0; -} - -static int apply_r_mips_higher_rela(struct module *me, u32 *location, - Elf_Addr v) -{ - *location = (*location & 0xffff0000) | - ((((long long) v + 0x80008000LL) >> 32) & 0xffff); - - return 0; -} - -static int apply_r_mips_highest_rela(struct module *me, u32 *location, - Elf_Addr v) -{ - *location = (*location & 0xffff0000) | - ((((long long) v + 0x800080008000LL) >> 48) & 0xffff); - - return 0; -} - static int (*reloc_handlers_rel[]) (struct module *me, u32 *location, Elf_Addr v) = { [R_MIPS_NONE] = apply_r_mips_none, @@ -270,18 +192,6 @@ static int (*reloc_handlers_rel[]) (struct module *me, u32 *location, [R_MIPS_LO16] = apply_r_mips_lo16_rel }; -static int (*reloc_handlers_rela[]) (struct module *me, u32 *location, - Elf_Addr v) = { - [R_MIPS_NONE] = apply_r_mips_none, - [R_MIPS_32] = apply_r_mips_32_rela, - [R_MIPS_26] = apply_r_mips_26_rela, - [R_MIPS_HI16] = apply_r_mips_hi16_rela, - [R_MIPS_LO16] = apply_r_mips_lo16_rela, - [R_MIPS_64] = apply_r_mips_64_rela, - [R_MIPS_HIGHER] = apply_r_mips_higher_rela, - [R_MIPS_HIGHEST] = apply_r_mips_highest_rela -}; - int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, unsigned int relsec, struct module *me) @@ -296,6 +206,7 @@ int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, pr_debug("Applying relocate section %u to %u\n", relsec, sechdrs[relsec].sh_info); + me->arch.r_mips_hi16_list = NULL; for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { /* This is where to make the change */ location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr @@ -319,44 +230,17 @@ int apply_relocate(Elf_Shdr *sechdrs, const char *strtab, return res; } - return 0; -} - -int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, - unsigned int symindex, unsigned int relsec, - struct module *me) -{ - Elf_Mips_Rela *rel = (void *) sechdrs[relsec].sh_addr; - Elf_Sym *sym; - u32 *location; - unsigned int i; - Elf_Addr v; - int res; - - pr_debug("Applying relocate section %u to %u\n", relsec, - sechdrs[relsec].sh_info); - - for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { - /* This is where to make the change */ - location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr - + rel[i].r_offset; - /* This is the symbol it is referring to */ - sym = (Elf_Sym *)sechdrs[symindex].sh_addr - + ELF_MIPS_R_SYM(rel[i]); - if (IS_ERR_VALUE(sym->st_value)) { - /* Ignore unresolved weak symbol */ - if (ELF_ST_BIND(sym->st_info) == STB_WEAK) - continue; - printk(KERN_WARNING "%s: Unknown symbol %s\n", - me->name, strtab + sym->st_name); - return -ENOENT; - } - - v = sym->st_value + rel[i].r_addend; + /* + * Normally the hi16 list should be deallocated at this point. A + * malformed binary however could contain a series of R_MIPS_HI16 + * relocations not followed by a R_MIPS_LO16 relocation. In that + * case, free up the list and return an error. + */ + if (me->arch.r_mips_hi16_list) { + free_relocation_chain(me->arch.r_mips_hi16_list); + me->arch.r_mips_hi16_list = NULL; - res = reloc_handlers_rela[ELF_MIPS_R_TYPE(rel[i])](me, location, v); - if (res) - return res; + return -ENOEXEC; } return 0; @@ -378,7 +262,7 @@ const struct exception_table_entry *search_module_dbetables(unsigned long addr) spin_unlock_irqrestore(&dbe_lock, flags); /* Now, if we found one, we are running inside it now, hence - we cannot unload the module, hence no refcnt needed. */ + we cannot unload the module, hence no refcnt needed. */ return e; } @@ -390,6 +274,9 @@ int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *s; char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + /* Make jump label nops. */ + jump_label_apply_nops(me); + INIT_LIST_HEAD(&me->arch.dbe_list); for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { if (strcmp("__dbe_table", secstrings + s->sh_name) != 0) diff --git a/arch/mips/kernel/octeon_switch.S b/arch/mips/kernel/octeon_switch.S index dd18b26a358..f6547680c81 100644 --- a/arch/mips/kernel/octeon_switch.S +++ b/arch/mips/kernel/octeon_switch.S @@ -4,34 +4,21 @@ * for more details. * * Copyright (C) 1994, 1995, 1996, 1998, 1999, 2002, 2003 Ralf Baechle - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * Copyright (C) 1994, 1995, 1996, by Andreas Busse * Copyright (C) 1999 Silicon Graphics, Inc. * Copyright (C) 2000 MIPS Technologies, Inc. * written by Carsten Langgaard, carstenl@mips.com */ -#include <asm/asm.h> -#include <asm/cachectl.h> -#include <asm/fpregdef.h> -#include <asm/mipsregs.h> -#include <asm/asm-offsets.h> -#include <asm/page.h> -#include <asm/pgtable-bits.h> -#include <asm/regdef.h> -#include <asm/stackframe.h> -#include <asm/thread_info.h> - -#include <asm/asmmacro.h> - -/* - * Offset to the current process status flags, the first 32 bytes of the - * stack are not used. - */ -#define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS) +#define USE_ALTERNATE_RESUME_IMPL 1 + .set push + .set arch=mips64r2 +#include "r4k_switch.S" + .set pop /* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti) + * struct thread_info *next_ti, int usedfpu) */ .align 7 LEAF(resume) @@ -41,6 +28,34 @@ cpu_save_nonscratch a0 LONG_S ra, THREAD_REG31(a0) + /* + * check if we need to save FPU registers + */ + PTR_L t3, TASK_THREAD_INFO(a0) + LONG_L t0, TI_FLAGS(t3) + li t1, _TIF_USEDFPU + and t2, t0, t1 + beqz t2, 1f + nor t1, zero, t1 + + and t0, t0, t1 + LONG_S t0, TI_FLAGS(t3) + + /* + * clear saved user stack CU1 bit + */ + LONG_L t0, ST_OFF(t3) + li t1, ~ST0_CU1 + and t0, t0, t1 + LONG_S t0, ST_OFF(t3) + + .set push + .set arch=mips64r2 + fpu_save_double a0 t0 t1 # c0_status passed in t0 + # clobbers t1 + .set pop +1: + /* check if we need to save COP2 registers */ PTR_L t2, TASK_THREAD_INFO(a0) LONG_L t0, ST_OFF(t2) @@ -70,7 +85,7 @@ 1: #if CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 /* Check if we need to store CVMSEG state */ - mfc0 t0, $11,7 /* CvmMemCtl */ + mfc0 t0, $11,7 /* CvmMemCtl */ bbit0 t0, 6, 3f /* Is user access enabled? */ /* Store the CVMSEG state */ @@ -78,8 +93,8 @@ andi t0, 0x3f /* Multiply * (cache line size/sizeof(long)/2) */ sll t0, 7-LONGLOG-1 - li t1, -32768 /* Base address of CVMSEG */ - LONG_ADDI t2, a0, THREAD_CVMSEG /* Where to store CVMSEG to */ + li t1, -32768 /* Base address of CVMSEG */ + LONG_ADDI t2, a0, THREAD_CVMSEG /* Where to store CVMSEG to */ synciobdma 2: .set noreorder @@ -90,15 +105,22 @@ LONG_S t8, 0(t2) /* Store CVMSEG to thread storage */ LONG_ADDU t2, LONGSIZE*2 /* Increment loc in thread storage */ bnez t0, 2b /* Loop until we've copied it all */ - LONG_S t9, -LONGSIZE(t2)/* Store CVMSEG to thread storage */ + LONG_S t9, -LONGSIZE(t2)/* Store CVMSEG to thread storage */ .set reorder /* Disable access to CVMSEG */ - mfc0 t0, $11,7 /* CvmMemCtl */ + mfc0 t0, $11,7 /* CvmMemCtl */ xori t0, t0, 0x40 /* Bit 6 is CVMSEG user enable */ - mtc0 t0, $11,7 /* CvmMemCtl */ + mtc0 t0, $11,7 /* CvmMemCtl */ #endif 3: + +#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard + LONG_L t9, TASK_STACK_CANARY(a1) + LONG_S t9, 0(t8) +#endif + /* * The order of restoring the registers takes care of the race * updating $28, $29 and kernelsp without disabling ints. @@ -106,12 +128,7 @@ move $28, a2 cpu_restore_nonscratch a1 -#if (_THREAD_SIZE - 32) < 0x8000 - PTR_ADDIU t0, $28, _THREAD_SIZE - 32 -#else - PTR_LI t0, _THREAD_SIZE - 32 - PTR_ADDU t0, $28 -#endif + PTR_ADDU t0, $28, _THREAD_SIZE - 32 set_saved_sp t0, t1, t2 mfc0 t1, CP0_STATUS /* Do we really need this? */ @@ -134,7 +151,7 @@ dmfc0 t9, $9,7 /* CvmCtl register. */ - /* Save the COP2 CRC state */ + /* Save the COP2 CRC state */ dmfc2 t0, 0x0201 dmfc2 t1, 0x0202 dmfc2 t2, 0x0200 @@ -150,30 +167,30 @@ sd t0, OCTEON_CP2_LLM_DAT(a0) sd t1, OCTEON_CP2_LLM_DAT+8(a0) -1: bbit1 t9, 26, 3f /* done if CvmCtl[NOCRYPTO] set */ +1: bbit1 t9, 26, 3f /* done if CvmCtl[NOCRYPTO] set */ /* Save the COP2 crypto state */ - /* this part is mostly common to both pass 1 and later revisions */ - dmfc2 t0, 0x0084 - dmfc2 t1, 0x0080 - dmfc2 t2, 0x0081 - dmfc2 t3, 0x0082 + /* this part is mostly common to both pass 1 and later revisions */ + dmfc2 t0, 0x0084 + dmfc2 t1, 0x0080 + dmfc2 t2, 0x0081 + dmfc2 t3, 0x0082 sd t0, OCTEON_CP2_3DES_IV(a0) - dmfc2 t0, 0x0088 + dmfc2 t0, 0x0088 sd t1, OCTEON_CP2_3DES_KEY(a0) - dmfc2 t1, 0x0111 /* only necessary for pass 1 */ + dmfc2 t1, 0x0111 /* only necessary for pass 1 */ sd t2, OCTEON_CP2_3DES_KEY+8(a0) - dmfc2 t2, 0x0102 + dmfc2 t2, 0x0102 sd t3, OCTEON_CP2_3DES_KEY+16(a0) - dmfc2 t3, 0x0103 + dmfc2 t3, 0x0103 sd t0, OCTEON_CP2_3DES_RESULT(a0) - dmfc2 t0, 0x0104 - sd t1, OCTEON_CP2_AES_INP0(a0) /* only necessary for pass 1 */ - dmfc2 t1, 0x0105 + dmfc2 t0, 0x0104 + sd t1, OCTEON_CP2_AES_INP0(a0) /* only necessary for pass 1 */ + dmfc2 t1, 0x0105 sd t2, OCTEON_CP2_AES_IV(a0) dmfc2 t2, 0x0106 sd t3, OCTEON_CP2_AES_IV+8(a0) - dmfc2 t3, 0x0107 + dmfc2 t3, 0x0107 sd t0, OCTEON_CP2_AES_KEY(a0) dmfc2 t0, 0x0110 sd t1, OCTEON_CP2_AES_KEY+8(a0) @@ -181,7 +198,7 @@ sd t2, OCTEON_CP2_AES_KEY+16(a0) dmfc2 t2, 0x0101 sd t3, OCTEON_CP2_AES_KEY+24(a0) - mfc0 t3, $15,0 /* Get the processor ID register */ + mfc0 t3, $15,0 /* Get the processor ID register */ sd t0, OCTEON_CP2_AES_KEYLEN(a0) li t0, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ sd t1, OCTEON_CP2_AES_RESULT(a0) @@ -189,7 +206,7 @@ /* Skip to the Pass1 version of the remainder of the COP2 state */ beq t3, t0, 2f - /* the non-pass1 state when !CvmCtl[NOCRYPTO] */ + /* the non-pass1 state when !CvmCtl[NOCRYPTO] */ dmfc2 t1, 0x0240 dmfc2 t2, 0x0241 dmfc2 t3, 0x0242 @@ -215,7 +232,7 @@ sd t2, OCTEON_CP2_HSH_DATW+72(a0) dmfc2 t2, 0x024D sd t3, OCTEON_CP2_HSH_DATW+80(a0) - dmfc2 t3, 0x024E + dmfc2 t3, 0x024E sd t0, OCTEON_CP2_HSH_DATW+88(a0) dmfc2 t0, 0x0250 sd t1, OCTEON_CP2_HSH_DATW+96(a0) @@ -233,9 +250,9 @@ sd t3, OCTEON_CP2_HSH_IVW+24(a0) dmfc2 t3, 0x0257 sd t0, OCTEON_CP2_HSH_IVW+32(a0) - dmfc2 t0, 0x0258 + dmfc2 t0, 0x0258 sd t1, OCTEON_CP2_HSH_IVW+40(a0) - dmfc2 t1, 0x0259 + dmfc2 t1, 0x0259 sd t2, OCTEON_CP2_HSH_IVW+48(a0) dmfc2 t2, 0x025E sd t3, OCTEON_CP2_HSH_IVW+56(a0) @@ -248,7 +265,7 @@ sd t0, OCTEON_CP2_GFM_RESULT+8(a0) jr ra -2: /* pass 1 special stuff when !CvmCtl[NOCRYPTO] */ +2: /* pass 1 special stuff when !CvmCtl[NOCRYPTO] */ dmfc2 t3, 0x0040 dmfc2 t0, 0x0041 dmfc2 t1, 0x0042 @@ -270,7 +287,7 @@ sd t3, OCTEON_CP2_HSH_IVW+8(a0) sd t0, OCTEON_CP2_HSH_IVW+16(a0) -3: /* pass 1 or CvmCtl[NOCRYPTO] set */ +3: /* pass 1 or CvmCtl[NOCRYPTO] set */ jr ra END(octeon_cop2_save) @@ -281,19 +298,19 @@ .set push .set noreorder LEAF(octeon_cop2_restore) - /* First cache line was prefetched before the call */ - pref 4, 128(a0) + /* First cache line was prefetched before the call */ + pref 4, 128(a0) dmfc0 t9, $9,7 /* CvmCtl register. */ - pref 4, 256(a0) + pref 4, 256(a0) ld t0, OCTEON_CP2_CRC_IV(a0) - pref 4, 384(a0) + pref 4, 384(a0) ld t1, OCTEON_CP2_CRC_LENGTH(a0) ld t2, OCTEON_CP2_CRC_POLY(a0) /* Restore the COP2 CRC state */ dmtc2 t0, 0x0201 - dmtc2 t1, 0x1202 + dmtc2 t1, 0x1202 bbit1 t9, 28, 2f /* Skip LLM if CvmCtl[NODFA_CP2] is set */ dmtc2 t2, 0x4200 @@ -311,19 +328,19 @@ ld t0, OCTEON_CP2_3DES_IV(a0) ld t1, OCTEON_CP2_3DES_KEY(a0) ld t2, OCTEON_CP2_3DES_KEY+8(a0) - dmtc2 t0, 0x0084 + dmtc2 t0, 0x0084 ld t0, OCTEON_CP2_3DES_KEY+16(a0) - dmtc2 t1, 0x0080 + dmtc2 t1, 0x0080 ld t1, OCTEON_CP2_3DES_RESULT(a0) - dmtc2 t2, 0x0081 + dmtc2 t2, 0x0081 ld t2, OCTEON_CP2_AES_INP0(a0) /* only really needed for pass 1 */ dmtc2 t0, 0x0082 ld t0, OCTEON_CP2_AES_IV(a0) - dmtc2 t1, 0x0098 + dmtc2 t1, 0x0098 ld t1, OCTEON_CP2_AES_IV+8(a0) - dmtc2 t2, 0x010A /* only really needed for pass 1 */ + dmtc2 t2, 0x010A /* only really needed for pass 1 */ ld t2, OCTEON_CP2_AES_KEY(a0) - dmtc2 t0, 0x0102 + dmtc2 t0, 0x0102 ld t0, OCTEON_CP2_AES_KEY+8(a0) dmtc2 t1, 0x0103 ld t1, OCTEON_CP2_AES_KEY+16(a0) @@ -335,14 +352,14 @@ ld t1, OCTEON_CP2_AES_RESULT(a0) dmtc2 t2, 0x0107 ld t2, OCTEON_CP2_AES_RESULT+8(a0) - mfc0 t3, $15,0 /* Get the processor ID register */ + mfc0 t3, $15,0 /* Get the processor ID register */ dmtc2 t0, 0x0110 li t0, 0x000d0000 /* This is the processor ID of Octeon Pass1 */ dmtc2 t1, 0x0100 bne t0, t3, 3f /* Skip the next stuff for non-pass1 */ dmtc2 t2, 0x0101 - /* this code is specific for pass 1 */ + /* this code is specific for pass 1 */ ld t0, OCTEON_CP2_HSH_DATW(a0) ld t1, OCTEON_CP2_HSH_DATW+8(a0) ld t2, OCTEON_CP2_HSH_DATW+16(a0) @@ -362,10 +379,10 @@ ld t0, OCTEON_CP2_HSH_IVW+16(a0) dmtc2 t1, 0x0048 dmtc2 t2, 0x0049 - b done_restore /* unconditional branch */ + b done_restore /* unconditional branch */ dmtc2 t0, 0x004A -3: /* this is post-pass1 code */ +3: /* this is post-pass1 code */ ld t2, OCTEON_CP2_HSH_DATW(a0) ld t0, OCTEON_CP2_HSH_DATW+8(a0) ld t1, OCTEON_CP2_HSH_DATW+16(a0) @@ -434,7 +451,7 @@ done_restore: * sp is assumed to point to a struct pt_regs * * NOTE: This is called in SAVE_SOME in stackframe.h. It can only - * safely modify k0 and k1. + * safely modify k0 and k1. */ .align 7 .set push @@ -447,14 +464,14 @@ done_restore: /* Save the multiplier state */ v3mulu k0, $0, $0 v3mulu k1, $0, $0 - sd k0, PT_MTP(sp) /* PT_MTP has P0 */ + sd k0, PT_MTP(sp) /* PT_MTP has P0 */ v3mulu k0, $0, $0 sd k1, PT_MTP+8(sp) /* PT_MTP+8 has P1 */ ori k1, $0, 1 v3mulu k1, k1, $0 sd k0, PT_MTP+16(sp) /* PT_MTP+16 has P2 */ v3mulu k0, $0, $0 - sd k1, PT_MPL(sp) /* PT_MPL has MPL0 */ + sd k1, PT_MPL(sp) /* PT_MPL has MPL0 */ v3mulu k1, $0, $0 sd k0, PT_MPL+8(sp) /* PT_MPL+8 has MPL1 */ jr ra @@ -476,19 +493,19 @@ done_restore: .set noreorder LEAF(octeon_mult_restore) dmfc0 k1, $9,7 /* CvmCtl register. */ - ld v0, PT_MPL(sp) /* MPL0 */ - ld v1, PT_MPL+8(sp) /* MPL1 */ - ld k0, PT_MPL+16(sp) /* MPL2 */ + ld v0, PT_MPL(sp) /* MPL0 */ + ld v1, PT_MPL+8(sp) /* MPL1 */ + ld k0, PT_MPL+16(sp) /* MPL2 */ bbit1 k1, 27, 1f /* Skip CvmCtl[NOMUL] */ /* Normally falls through, so no time wasted here */ nop /* Restore the multiplier state */ - ld k1, PT_MTP+16(sp) /* P2 */ + ld k1, PT_MTP+16(sp) /* P2 */ MTM0 v0 /* MPL0 */ ld v0, PT_MTP+8(sp) /* P1 */ MTM1 v1 /* MPL1 */ - ld v1, PT_MTP(sp) /* P0 */ + ld v1, PT_MTP(sp) /* P0 */ MTM2 k0 /* MPL2 */ MTP2 k1 /* P2 */ MTP1 v0 /* P1 */ diff --git a/arch/mips/kernel/perf_event.c b/arch/mips/kernel/perf_event.c index 2b7f3f703b8..c1cf9c6c3f7 100644 --- a/arch/mips/kernel/perf_event.c +++ b/arch/mips/kernel/perf_event.c @@ -14,505 +14,16 @@ * published by the Free Software Foundation. */ -#include <linux/cpumask.h> -#include <linux/interrupt.h> -#include <linux/smp.h> -#include <linux/kernel.h> #include <linux/perf_event.h> -#include <linux/uaccess.h> -#include <asm/irq.h> -#include <asm/irq_regs.h> #include <asm/stacktrace.h> -#include <asm/time.h> /* For perf_irq */ - -/* These are for 32bit counters. For 64bit ones, define them accordingly. */ -#define MAX_PERIOD ((1ULL << 32) - 1) -#define VALID_COUNT 0x7fffffff -#define TOTAL_BITS 32 -#define HIGHEST_BIT 31 - -#define MIPS_MAX_HWEVENTS 4 - -struct cpu_hw_events { - /* Array of events on this cpu. */ - struct perf_event *events[MIPS_MAX_HWEVENTS]; - - /* - * Set the bit (indexed by the counter number) when the counter - * is used for an event. - */ - unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - - /* - * The borrowed MSB for the performance counter. A MIPS performance - * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit - * counters) as a factor of determining whether a counter overflow - * should be signaled. So here we use a separate MSB for each - * counter to make things easy. - */ - unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; - - /* - * Software copy of the control register for each performance counter. - * MIPS CPUs vary in performance counters. They use this differently, - * and even may not use it. - */ - unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; -}; -DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { - .saved_ctrl = {0}, -}; - -/* The description of MIPS performance events. */ -struct mips_perf_event { - unsigned int event_id; - /* - * MIPS performance counters are indexed starting from 0. - * CNTR_EVEN indicates the indexes of the counters to be used are - * even numbers. - */ - unsigned int cntr_mask; - #define CNTR_EVEN 0x55555555 - #define CNTR_ODD 0xaaaaaaaa -#ifdef CONFIG_MIPS_MT_SMP - enum { - T = 0, - V = 1, - P = 2, - } range; -#else - #define T - #define V - #define P -#endif -}; - -static struct mips_perf_event raw_event; -static DEFINE_MUTEX(raw_event_mutex); - -#define UNSUPPORTED_PERF_EVENT_ID 0xffffffff -#define C(x) PERF_COUNT_HW_CACHE_##x - -struct mips_pmu { - const char *name; - int irq; - irqreturn_t (*handle_irq)(int irq, void *dev); - int (*handle_shared_irq)(void); - void (*start)(void); - void (*stop)(void); - int (*alloc_counter)(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc); - u64 (*read_counter)(unsigned int idx); - void (*write_counter)(unsigned int idx, u64 val); - void (*enable_event)(struct hw_perf_event *evt, int idx); - void (*disable_event)(int idx); - const struct mips_perf_event *(*map_raw_event)(u64 config); - const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; - const struct mips_perf_event (*cache_event_map) - [PERF_COUNT_HW_CACHE_MAX] - [PERF_COUNT_HW_CACHE_OP_MAX] - [PERF_COUNT_HW_CACHE_RESULT_MAX]; - unsigned int num_counters; -}; - -static const struct mips_pmu *mipspmu; - -static int -mipspmu_event_set_period(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - s64 left = local64_read(&hwc->period_left); - s64 period = hwc->sample_period; - int ret = 0; - u64 uleft; - unsigned long flags; - - if (unlikely(left <= -period)) { - left = period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (unlikely(left <= 0)) { - left += period; - local64_set(&hwc->period_left, left); - hwc->last_period = period; - ret = 1; - } - - if (left > (s64)MAX_PERIOD) - left = MAX_PERIOD; - - local64_set(&hwc->prev_count, (u64)-left); - - local_irq_save(flags); - uleft = (u64)(-left) & MAX_PERIOD; - uleft > VALID_COUNT ? - set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); - mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); - local_irq_restore(flags); - - perf_event_update_userpage(event); - - return ret; -} - -static int mipspmu_enable(struct perf_event *event) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - struct hw_perf_event *hwc = &event->hw; - int idx; - int err = 0; - - /* To look for a free counter for this event. */ - idx = mipspmu->alloc_counter(cpuc, hwc); - if (idx < 0) { - err = idx; - goto out; - } - - /* - * If there is an event in the counter we are going to use then - * make sure it is disabled. - */ - event->hw.idx = idx; - mipspmu->disable_event(idx); - cpuc->events[idx] = event; - - /* Set the period for the event. */ - mipspmu_event_set_period(event, hwc, idx); - - /* Enable the event. */ - mipspmu->enable_event(hwc, idx); - - /* Propagate our changes to the userspace mapping. */ - perf_event_update_userpage(event); - -out: - return err; -} - -static void mipspmu_event_update(struct perf_event *event, - struct hw_perf_event *hwc, - int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - int shift = 64 - TOTAL_BITS; - s64 prev_raw_count, new_raw_count; - s64 delta; - -again: - prev_raw_count = local64_read(&hwc->prev_count); - local_irq_save(flags); - /* Make the counter value be a "real" one. */ - new_raw_count = mipspmu->read_counter(idx); - if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { - new_raw_count &= VALID_COUNT; - clear_bit(idx, cpuc->msbs); - } else - new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); - local_irq_restore(flags); - - if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, - new_raw_count) != prev_raw_count) - goto again; - - delta = (new_raw_count << shift) - (prev_raw_count << shift); - delta >>= shift; - - local64_add(delta, &event->count); - local64_sub(delta, &hwc->period_left); - - return; -} - -static void mipspmu_disable(struct perf_event *event) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - struct hw_perf_event *hwc = &event->hw; - int idx = hwc->idx; - - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - /* We are working on a local event. */ - mipspmu->disable_event(idx); - - barrier(); - - mipspmu_event_update(event, hwc, idx); - cpuc->events[idx] = NULL; - clear_bit(idx, cpuc->used_mask); - - perf_event_update_userpage(event); -} - -static void mipspmu_unthrottle(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - - mipspmu->enable_event(hwc, hwc->idx); -} - -static void mipspmu_read(struct perf_event *event) -{ - struct hw_perf_event *hwc = &event->hw; - - /* Don't read disabled counters! */ - if (hwc->idx < 0) - return; - - mipspmu_event_update(event, hwc, hwc->idx); -} - -static struct pmu pmu = { - .enable = mipspmu_enable, - .disable = mipspmu_disable, - .unthrottle = mipspmu_unthrottle, - .read = mipspmu_read, -}; - -static atomic_t active_events = ATOMIC_INIT(0); -static DEFINE_MUTEX(pmu_reserve_mutex); -static int (*save_perf_irq)(void); - -static int mipspmu_get_irq(void) -{ - int err; - - if (mipspmu->irq >= 0) { - /* Request my own irq handler. */ - err = request_irq(mipspmu->irq, mipspmu->handle_irq, - IRQF_DISABLED | IRQF_NOBALANCING, - "mips_perf_pmu", NULL); - if (err) { - pr_warning("Unable to request IRQ%d for MIPS " - "performance counters!\n", mipspmu->irq); - } - } else if (cp0_perfcount_irq < 0) { - /* - * We are sharing the irq number with the timer interrupt. - */ - save_perf_irq = perf_irq; - perf_irq = mipspmu->handle_shared_irq; - err = 0; - } else { - pr_warning("The platform hasn't properly defined its " - "interrupt controller.\n"); - err = -ENOENT; - } - - return err; -} - -static void mipspmu_free_irq(void) -{ - if (mipspmu->irq >= 0) - free_irq(mipspmu->irq, NULL); - else if (cp0_perfcount_irq < 0) - perf_irq = save_perf_irq; -} - -static inline unsigned int -mipspmu_perf_event_encode(const struct mips_perf_event *pev) -{ -/* - * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for - * event_id. - */ -#ifdef CONFIG_MIPS_MT_SMP - return ((unsigned int)pev->range << 24) | - (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#else - return (pev->cntr_mask & 0xffff00) | - (pev->event_id & 0xff); -#endif -} - -static const struct mips_perf_event * -mipspmu_map_general_event(int idx) -{ - const struct mips_perf_event *pev; - - pev = ((*mipspmu->general_event_map)[idx].event_id == - UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : - &(*mipspmu->general_event_map)[idx]); - - return pev; -} - -static const struct mips_perf_event * -mipspmu_map_cache_event(u64 config) -{ - unsigned int cache_type, cache_op, cache_result; - const struct mips_perf_event *pev; - - cache_type = (config >> 0) & 0xff; - if (cache_type >= PERF_COUNT_HW_CACHE_MAX) - return ERR_PTR(-EINVAL); - - cache_op = (config >> 8) & 0xff; - if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) - return ERR_PTR(-EINVAL); - - cache_result = (config >> 16) & 0xff; - if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) - return ERR_PTR(-EINVAL); - - pev = &((*mipspmu->cache_event_map) - [cache_type] - [cache_op] - [cache_result]); - - if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID) - return ERR_PTR(-EOPNOTSUPP); - - return pev; - -} - -static int validate_event(struct cpu_hw_events *cpuc, - struct perf_event *event) -{ - struct hw_perf_event fake_hwc = event->hw; - - if (event->pmu && event->pmu != &pmu) - return 0; - - return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; -} - -static int validate_group(struct perf_event *event) -{ - struct perf_event *sibling, *leader = event->group_leader; - struct cpu_hw_events fake_cpuc; - - memset(&fake_cpuc, 0, sizeof(fake_cpuc)); - - if (!validate_event(&fake_cpuc, leader)) - return -ENOSPC; - - list_for_each_entry(sibling, &leader->sibling_list, group_entry) { - if (!validate_event(&fake_cpuc, sibling)) - return -ENOSPC; - } - - if (!validate_event(&fake_cpuc, event)) - return -ENOSPC; - - return 0; -} - -/* - * mipsxx/rm9000/loongson2 have different performance counters, they have - * specific low-level init routines. - */ -static void reset_counters(void *arg); -static int __hw_perf_event_init(struct perf_event *event); - -static void hw_perf_event_destroy(struct perf_event *event) -{ - if (atomic_dec_and_mutex_lock(&active_events, - &pmu_reserve_mutex)) { - /* - * We must not call the destroy function with interrupts - * disabled. - */ - on_each_cpu(reset_counters, - (void *)(long)mipspmu->num_counters, 1); - mipspmu_free_irq(); - mutex_unlock(&pmu_reserve_mutex); - } -} - -const struct pmu *hw_perf_event_init(struct perf_event *event) -{ - int err = 0; - - if (!mipspmu || event->cpu >= nr_cpumask_bits || - (event->cpu >= 0 && !cpu_online(event->cpu))) - return ERR_PTR(-ENODEV); - - if (!atomic_inc_not_zero(&active_events)) { - if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { - atomic_dec(&active_events); - return ERR_PTR(-ENOSPC); - } - - mutex_lock(&pmu_reserve_mutex); - if (atomic_read(&active_events) == 0) - err = mipspmu_get_irq(); - - if (!err) - atomic_inc(&active_events); - mutex_unlock(&pmu_reserve_mutex); - } - - if (err) - return ERR_PTR(err); - - err = __hw_perf_event_init(event); - if (err) - hw_perf_event_destroy(event); - - return err ? ERR_PTR(err) : &pmu; -} - -void hw_perf_enable(void) -{ - if (mipspmu) - mipspmu->start(); -} - -void hw_perf_disable(void) -{ - if (mipspmu) - mipspmu->stop(); -} - -/* This is needed by specific irq handlers in perf_event_*.c */ -static void -handle_associated_event(struct cpu_hw_events *cpuc, - int idx, struct perf_sample_data *data, struct pt_regs *regs) -{ - struct perf_event *event = cpuc->events[idx]; - struct hw_perf_event *hwc = &event->hw; - - mipspmu_event_update(event, hwc, idx); - data->period = event->hw.last_period; - if (!mipspmu_event_set_period(event, hwc, idx)) - return; - - if (perf_event_overflow(event, 0, data, regs)) - mipspmu->disable_event(idx); -} - -#include "perf_event_mipsxx.c" /* Callchain handling code. */ -static inline void -callchain_store(struct perf_callchain_entry *entry, - u64 ip) -{ - if (entry->nr < PERF_MAX_STACK_DEPTH) - entry->ip[entry->nr++] = ip; -} /* * Leave userspace callchain empty for now. When we find a way to trace - * the user stack callchains, we add here. + * the user stack callchains, we will add it here. */ -static void -perf_callchain_user(struct pt_regs *regs, - struct perf_callchain_entry *entry) -{ -} static void save_raw_perf_callchain(struct perf_callchain_entry *entry, unsigned long reg29) @@ -523,23 +34,21 @@ static void save_raw_perf_callchain(struct perf_callchain_entry *entry, while (!kstack_end(sp)) { addr = *sp++; if (__kernel_text_address(addr)) { - callchain_store(entry, addr); + perf_callchain_store(entry, addr); if (entry->nr >= PERF_MAX_STACK_DEPTH) break; } } } -static void -perf_callchain_kernel(struct pt_regs *regs, - struct perf_callchain_entry *entry) +void perf_callchain_kernel(struct perf_callchain_entry *entry, + struct pt_regs *regs) { unsigned long sp = regs->regs[29]; #ifdef CONFIG_KALLSYMS unsigned long ra = regs->regs[31]; unsigned long pc = regs->cp0_epc; - callchain_store(entry, PERF_CONTEXT_KERNEL); if (raw_show_trace || !__kernel_text_address(pc)) { unsigned long stack_page = (unsigned long)task_stack_page(current); @@ -549,53 +58,12 @@ perf_callchain_kernel(struct pt_regs *regs, return; } do { - callchain_store(entry, pc); + perf_callchain_store(entry, pc); if (entry->nr >= PERF_MAX_STACK_DEPTH) break; pc = unwind_stack(current, &sp, pc, &ra); } while (pc); #else - callchain_store(entry, PERF_CONTEXT_KERNEL); save_raw_perf_callchain(entry, sp); #endif } - -static void -perf_do_callchain(struct pt_regs *regs, - struct perf_callchain_entry *entry) -{ - int is_user; - - if (!regs) - return; - - is_user = user_mode(regs); - - if (!current || !current->pid) - return; - - if (is_user && current->state != TASK_RUNNING) - return; - - if (!is_user) { - perf_callchain_kernel(regs, entry); - if (current->mm) - regs = task_pt_regs(current); - else - regs = NULL; - } - if (regs) - perf_callchain_user(regs, entry); -} - -static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry); - -struct perf_callchain_entry * -perf_callchain(struct pt_regs *regs) -{ - struct perf_callchain_entry *entry = &__get_cpu_var(pmc_irq_entry); - - entry->nr = 0; - perf_do_callchain(regs, entry); - return entry; -} diff --git a/arch/mips/kernel/perf_event_mipsxx.c b/arch/mips/kernel/perf_event_mipsxx.c index 5c7c6fc0756..4f2d9dece7a 100644 --- a/arch/mips/kernel/perf_event_mipsxx.c +++ b/arch/mips/kernel/perf_event_mipsxx.c @@ -1,22 +1,129 @@ -#if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) || \ - defined(CONFIG_CPU_R10000) || defined(CONFIG_CPU_SB1) +/* + * Linux performance counter support for MIPS. + * + * Copyright (C) 2010 MIPS Technologies, Inc. + * Copyright (C) 2011 Cavium Networks, Inc. + * Author: Deng-Cheng Zhu + * + * This code is based on the implementation for ARM, which is in turn + * based on the sparc64 perf event code and the x86 code. Performance + * counter access is based on the MIPS Oprofile code. And the callchain + * support references the code of MIPS stacktrace.c. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/cpumask.h> +#include <linux/interrupt.h> +#include <linux/smp.h> +#include <linux/kernel.h> +#include <linux/perf_event.h> +#include <linux/uaccess.h> + +#include <asm/irq.h> +#include <asm/irq_regs.h> +#include <asm/stacktrace.h> +#include <asm/time.h> /* For perf_irq */ + +#define MIPS_MAX_HWEVENTS 4 +#define MIPS_TCS_PER_COUNTER 2 +#define MIPS_CPUID_TO_COUNTER_MASK (MIPS_TCS_PER_COUNTER - 1) + +struct cpu_hw_events { + /* Array of events on this cpu. */ + struct perf_event *events[MIPS_MAX_HWEVENTS]; + + /* + * Set the bit (indexed by the counter number) when the counter + * is used for an event. + */ + unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; + + /* + * Software copy of the control register for each performance counter. + * MIPS CPUs vary in performance counters. They use this differently, + * and even may not use it. + */ + unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; +}; +DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { + .saved_ctrl = {0}, +}; + +/* The description of MIPS performance events. */ +struct mips_perf_event { + unsigned int event_id; + /* + * MIPS performance counters are indexed starting from 0. + * CNTR_EVEN indicates the indexes of the counters to be used are + * even numbers. + */ + unsigned int cntr_mask; + #define CNTR_EVEN 0x55555555 + #define CNTR_ODD 0xaaaaaaaa + #define CNTR_ALL 0xffffffff +#ifdef CONFIG_MIPS_MT_SMP + enum { + T = 0, + V = 1, + P = 2, + } range; +#else + #define T + #define V + #define P +#endif +}; + +static struct mips_perf_event raw_event; +static DEFINE_MUTEX(raw_event_mutex); + +#define C(x) PERF_COUNT_HW_CACHE_##x + +struct mips_pmu { + u64 max_period; + u64 valid_count; + u64 overflow; + const char *name; + int irq; + u64 (*read_counter)(unsigned int idx); + void (*write_counter)(unsigned int idx, u64 val); + const struct mips_perf_event *(*map_raw_event)(u64 config); + const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; + const struct mips_perf_event (*cache_event_map) + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + unsigned int num_counters; +}; + +static struct mips_pmu mipspmu; #define M_CONFIG1_PC (1 << 4) -#define M_PERFCTL_EXL (1UL << 0) -#define M_PERFCTL_KERNEL (1UL << 1) -#define M_PERFCTL_SUPERVISOR (1UL << 2) -#define M_PERFCTL_USER (1UL << 3) -#define M_PERFCTL_INTERRUPT_ENABLE (1UL << 4) +#define M_PERFCTL_EXL (1 << 0) +#define M_PERFCTL_KERNEL (1 << 1) +#define M_PERFCTL_SUPERVISOR (1 << 2) +#define M_PERFCTL_USER (1 << 3) +#define M_PERFCTL_INTERRUPT_ENABLE (1 << 4) #define M_PERFCTL_EVENT(event) (((event) & 0x3ff) << 5) -#define M_PERFCTL_VPEID(vpe) ((vpe) << 16) +#define M_PERFCTL_VPEID(vpe) ((vpe) << 16) + +#ifdef CONFIG_CPU_BMIPS5000 +#define M_PERFCTL_MT_EN(filter) 0 +#else /* !CONFIG_CPU_BMIPS5000 */ #define M_PERFCTL_MT_EN(filter) ((filter) << 20) -#define M_TC_EN_ALL M_PERFCTL_MT_EN(0) -#define M_TC_EN_VPE M_PERFCTL_MT_EN(1) -#define M_TC_EN_TC M_PERFCTL_MT_EN(2) -#define M_PERFCTL_TCID(tcid) ((tcid) << 22) -#define M_PERFCTL_WIDE (1UL << 30) -#define M_PERFCTL_MORE (1UL << 31) +#endif /* CONFIG_CPU_BMIPS5000 */ + +#define M_TC_EN_ALL M_PERFCTL_MT_EN(0) +#define M_TC_EN_VPE M_PERFCTL_MT_EN(1) +#define M_TC_EN_TC M_PERFCTL_MT_EN(2) +#define M_PERFCTL_TCID(tcid) ((tcid) << 22) +#define M_PERFCTL_WIDE (1 << 30) +#define M_PERFCTL_MORE (1 << 31) +#define M_PERFCTL_TC (1 << 30) #define M_PERFCTL_COUNT_EVENT_WHENEVER (M_PERFCTL_EXL | \ M_PERFCTL_KERNEL | \ @@ -31,253 +138,718 @@ #endif #define M_PERFCTL_EVENT_MASK 0xfe0 -#define M_COUNTER_OVERFLOW (1UL << 31) -#ifdef CONFIG_MIPS_MT_SMP +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS static int cpu_has_mipsmt_pertccounters; +static DEFINE_RWLOCK(pmuint_rwlock); + +#if defined(CONFIG_CPU_BMIPS5000) +#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ + 0 : (smp_processor_id() & MIPS_CPUID_TO_COUNTER_MASK)) +#else /* * FIXME: For VSMP, vpe_id() is redefined for Perf-events, because * cpu_data[cpuid].vpe_id reports 0 for _both_ CPUs. */ -#if defined(CONFIG_HW_PERF_EVENTS) #define vpe_id() (cpu_has_mipsmt_pertccounters ? \ - 0 : smp_processor_id()) -#else -#define vpe_id() (cpu_has_mipsmt_pertccounters ? \ - 0 : cpu_data[smp_processor_id()].vpe_id) + 0 : smp_processor_id()) #endif /* Copied from op_model_mipsxx.c */ -static inline unsigned int vpe_shift(void) +static unsigned int vpe_shift(void) { if (num_possible_cpus() > 1) return 1; return 0; } -#else /* !CONFIG_MIPS_MT_SMP */ -#define vpe_id() 0 -static inline unsigned int vpe_shift(void) -{ - return 0; -} -#endif /* CONFIG_MIPS_MT_SMP */ - -static inline unsigned int -counters_total_to_per_cpu(unsigned int counters) +static unsigned int counters_total_to_per_cpu(unsigned int counters) { return counters >> vpe_shift(); } -static inline unsigned int -counters_per_cpu_to_total(unsigned int counters) -{ - return counters << vpe_shift(); -} +#else /* !CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */ +#define vpe_id() 0 -#define __define_perf_accessors(r, n, np) \ - \ -static inline unsigned int r_c0_ ## r ## n(void) \ -{ \ - unsigned int cpu = vpe_id(); \ - \ - switch (cpu) { \ - case 0: \ - return read_c0_ ## r ## n(); \ - case 1: \ - return read_c0_ ## r ## np(); \ - default: \ - BUG(); \ - } \ - return 0; \ -} \ - \ -static inline void w_c0_ ## r ## n(unsigned int value) \ -{ \ - unsigned int cpu = vpe_id(); \ - \ - switch (cpu) { \ - case 0: \ - write_c0_ ## r ## n(value); \ - return; \ - case 1: \ - write_c0_ ## r ## np(value); \ - return; \ - default: \ - BUG(); \ - } \ - return; \ -} \ - -__define_perf_accessors(perfcntr, 0, 2) -__define_perf_accessors(perfcntr, 1, 3) -__define_perf_accessors(perfcntr, 2, 0) -__define_perf_accessors(perfcntr, 3, 1) - -__define_perf_accessors(perfctrl, 0, 2) -__define_perf_accessors(perfctrl, 1, 3) -__define_perf_accessors(perfctrl, 2, 0) -__define_perf_accessors(perfctrl, 3, 1) - -static inline int __n_counters(void) -{ - if (!(read_c0_config1() & M_CONFIG1_PC)) - return 0; - if (!(read_c0_perfctrl0() & M_PERFCTL_MORE)) - return 1; - if (!(read_c0_perfctrl1() & M_PERFCTL_MORE)) - return 2; - if (!(read_c0_perfctrl2() & M_PERFCTL_MORE)) - return 3; +#endif /* CONFIG_MIPS_PERF_SHARED_TC_COUNTERS */ - return 4; -} +static void resume_local_counters(void); +static void pause_local_counters(void); +static irqreturn_t mipsxx_pmu_handle_irq(int, void *); +static int mipsxx_pmu_handle_shared_irq(void); -static inline int n_counters(void) +static unsigned int mipsxx_pmu_swizzle_perf_idx(unsigned int idx) { - int counters; - - switch (current_cpu_type()) { - case CPU_R10000: - counters = 2; - break; + if (vpe_id() == 1) + idx = (idx + 2) & 3; + return idx; +} - case CPU_R12000: - case CPU_R14000: - counters = 4; - break; +static u64 mipsxx_pmu_read_counter(unsigned int idx) +{ + idx = mipsxx_pmu_swizzle_perf_idx(idx); + switch (idx) { + case 0: + /* + * The counters are unsigned, we must cast to truncate + * off the high bits. + */ + return (u32)read_c0_perfcntr0(); + case 1: + return (u32)read_c0_perfcntr1(); + case 2: + return (u32)read_c0_perfcntr2(); + case 3: + return (u32)read_c0_perfcntr3(); default: - counters = __n_counters(); + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; } - - return counters; } -static void reset_counters(void *arg) +static u64 mipsxx_pmu_read_counter_64(unsigned int idx) { - int counters = (int)(long)arg; - switch (counters) { - case 4: - w_c0_perfctrl3(0); - w_c0_perfcntr3(0); - case 3: - w_c0_perfctrl2(0); - w_c0_perfcntr2(0); - case 2: - w_c0_perfctrl1(0); - w_c0_perfcntr1(0); + idx = mipsxx_pmu_swizzle_perf_idx(idx); + + switch (idx) { + case 0: + return read_c0_perfcntr0_64(); case 1: - w_c0_perfctrl0(0); - w_c0_perfcntr0(0); + return read_c0_perfcntr1_64(); + case 2: + return read_c0_perfcntr2_64(); + case 3: + return read_c0_perfcntr3_64(); + default: + WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); + return 0; } } -static inline u64 -mipsxx_pmu_read_counter(unsigned int idx) +static void mipsxx_pmu_write_counter(unsigned int idx, u64 val) { + idx = mipsxx_pmu_swizzle_perf_idx(idx); + switch (idx) { case 0: - return r_c0_perfcntr0(); + write_c0_perfcntr0(val); + return; case 1: - return r_c0_perfcntr1(); + write_c0_perfcntr1(val); + return; case 2: - return r_c0_perfcntr2(); + write_c0_perfcntr2(val); + return; case 3: - return r_c0_perfcntr3(); - default: - WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); - return 0; + write_c0_perfcntr3(val); + return; } } -static inline void -mipsxx_pmu_write_counter(unsigned int idx, u64 val) +static void mipsxx_pmu_write_counter_64(unsigned int idx, u64 val) { + idx = mipsxx_pmu_swizzle_perf_idx(idx); + switch (idx) { case 0: - w_c0_perfcntr0(val); + write_c0_perfcntr0_64(val); return; case 1: - w_c0_perfcntr1(val); + write_c0_perfcntr1_64(val); return; case 2: - w_c0_perfcntr2(val); + write_c0_perfcntr2_64(val); return; case 3: - w_c0_perfcntr3(val); + write_c0_perfcntr3_64(val); return; } } -static inline unsigned int -mipsxx_pmu_read_control(unsigned int idx) +static unsigned int mipsxx_pmu_read_control(unsigned int idx) { + idx = mipsxx_pmu_swizzle_perf_idx(idx); + switch (idx) { case 0: - return r_c0_perfctrl0(); + return read_c0_perfctrl0(); case 1: - return r_c0_perfctrl1(); + return read_c0_perfctrl1(); case 2: - return r_c0_perfctrl2(); + return read_c0_perfctrl2(); case 3: - return r_c0_perfctrl3(); + return read_c0_perfctrl3(); default: WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); return 0; } } -static inline void -mipsxx_pmu_write_control(unsigned int idx, unsigned int val) +static void mipsxx_pmu_write_control(unsigned int idx, unsigned int val) { + idx = mipsxx_pmu_swizzle_perf_idx(idx); + switch (idx) { case 0: - w_c0_perfctrl0(val); + write_c0_perfctrl0(val); return; case 1: - w_c0_perfctrl1(val); + write_c0_perfctrl1(val); return; case 2: - w_c0_perfctrl2(val); + write_c0_perfctrl2(val); return; case 3: - w_c0_perfctrl3(val); + write_c0_perfctrl3(val); return; } } +static int mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, + struct hw_perf_event *hwc) +{ + int i; + + /* + * We only need to care the counter mask. The range has been + * checked definitely. + */ + unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; + + for (i = mipspmu.num_counters - 1; i >= 0; i--) { + /* + * Note that some MIPS perf events can be counted by both + * even and odd counters, wheresas many other are only by + * even _or_ odd counters. This introduces an issue that + * when the former kind of event takes the counter the + * latter kind of event wants to use, then the "counter + * allocation" for the latter event will fail. In fact if + * they can be dynamically swapped, they both feel happy. + * But here we leave this issue alone for now. + */ + if (test_bit(i, &cntr_mask) && + !test_and_set_bit(i, cpuc->used_mask)) + return i; + } + + return -EAGAIN; +} + +static void mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); + + cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | + (evt->config_base & M_PERFCTL_CONFIG_MASK) | + /* Make sure interrupt enabled. */ + M_PERFCTL_INTERRUPT_ENABLE; + if (IS_ENABLED(CONFIG_CPU_BMIPS5000)) + /* enable the counter for the calling thread */ + cpuc->saved_ctrl[idx] |= + (1 << (12 + vpe_id())) | M_PERFCTL_TC; + + /* + * We do not actually let the counter run. Leave it until start(). + */ +} + +static void mipsxx_pmu_disable_event(int idx) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + unsigned long flags; + + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); + + local_irq_save(flags); + cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & + ~M_PERFCTL_COUNT_EVENT_WHENEVER; + mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); + local_irq_restore(flags); +} + +static int mipspmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + u64 left = local64_read(&hwc->period_left); + u64 period = hwc->sample_period; + int ret = 0; + + if (unlikely((left + period) & (1ULL << 63))) { + /* left underflowed by more than period. */ + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } else if (unlikely((left + period) <= period)) { + /* left underflowed by less than period. */ + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (left > mipspmu.max_period) { + left = mipspmu.max_period; + local64_set(&hwc->period_left, left); + } + + local64_set(&hwc->prev_count, mipspmu.overflow - left); + + mipspmu.write_counter(idx, mipspmu.overflow - left); + + perf_event_update_userpage(event); + + return ret; +} + +static void mipspmu_event_update(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + u64 prev_raw_count, new_raw_count; + u64 delta; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = mipspmu.read_counter(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = new_raw_count - prev_raw_count; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); +} + +static void mipspmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + + hwc->state = 0; + + /* Set the period for the event. */ + mipspmu_event_set_period(event, hwc, hwc->idx); + + /* Enable the event. */ + mipsxx_pmu_enable_event(hwc, hwc->idx); +} + +static void mipspmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!(hwc->state & PERF_HES_STOPPED)) { + /* We are working on a local event. */ + mipsxx_pmu_disable_event(hwc->idx); + barrier(); + mipspmu_event_update(event, hwc, hwc->idx); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; + } +} + +static int mipspmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx; + int err = 0; + + perf_pmu_disable(event->pmu); + + /* To look for a free counter for this event. */ + idx = mipsxx_pmu_alloc_counter(cpuc, hwc); + if (idx < 0) { + err = idx; + goto out; + } + + /* + * If there is an event in the counter we are going to use then + * make sure it is disabled. + */ + event->hw.idx = idx; + mipsxx_pmu_disable_event(idx); + cpuc->events[idx] = event; + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + mipspmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + perf_pmu_enable(event->pmu); + return err; +} + +static void mipspmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + WARN_ON(idx < 0 || idx >= mipspmu.num_counters); + + mipspmu_stop(event, PERF_EF_UPDATE); + cpuc->events[idx] = NULL; + clear_bit(idx, cpuc->used_mask); + + perf_event_update_userpage(event); +} + +static void mipspmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* Don't read disabled counters! */ + if (hwc->idx < 0) + return; + + mipspmu_event_update(event, hwc, hwc->idx); +} + +static void mipspmu_enable(struct pmu *pmu) +{ +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS + write_unlock(&pmuint_rwlock); +#endif + resume_local_counters(); +} + +/* + * MIPS performance counters can be per-TC. The control registers can + * not be directly accessed accross CPUs. Hence if we want to do global + * control, we need cross CPU calls. on_each_cpu() can help us, but we + * can not make sure this function is called with interrupts enabled. So + * here we pause local counters and then grab a rwlock and leave the + * counters on other CPUs alone. If any counter interrupt raises while + * we own the write lock, simply pause local counters on that CPU and + * spin in the handler. Also we know we won't be switched to another + * CPU after pausing local counters and before grabbing the lock. + */ +static void mipspmu_disable(struct pmu *pmu) +{ + pause_local_counters(); +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS + write_lock(&pmuint_rwlock); +#endif +} + +static atomic_t active_events = ATOMIC_INIT(0); +static DEFINE_MUTEX(pmu_reserve_mutex); +static int (*save_perf_irq)(void); + +static int mipspmu_get_irq(void) +{ + int err; + + if (mipspmu.irq >= 0) { + /* Request my own irq handler. */ + err = request_irq(mipspmu.irq, mipsxx_pmu_handle_irq, + IRQF_PERCPU | IRQF_NOBALANCING, + "mips_perf_pmu", NULL); + if (err) { + pr_warning("Unable to request IRQ%d for MIPS " + "performance counters!\n", mipspmu.irq); + } + } else if (cp0_perfcount_irq < 0) { + /* + * We are sharing the irq number with the timer interrupt. + */ + save_perf_irq = perf_irq; + perf_irq = mipsxx_pmu_handle_shared_irq; + err = 0; + } else { + pr_warning("The platform hasn't properly defined its " + "interrupt controller.\n"); + err = -ENOENT; + } + + return err; +} + +static void mipspmu_free_irq(void) +{ + if (mipspmu.irq >= 0) + free_irq(mipspmu.irq, NULL); + else if (cp0_perfcount_irq < 0) + perf_irq = save_perf_irq; +} + +/* + * mipsxx/rm9000/loongson2 have different performance counters, they have + * specific low-level init routines. + */ +static void reset_counters(void *arg); +static int __hw_perf_event_init(struct perf_event *event); + +static void hw_perf_event_destroy(struct perf_event *event) +{ + if (atomic_dec_and_mutex_lock(&active_events, + &pmu_reserve_mutex)) { + /* + * We must not call the destroy function with interrupts + * disabled. + */ + on_each_cpu(reset_counters, + (void *)(long)mipspmu.num_counters, 1); + mipspmu_free_irq(); + mutex_unlock(&pmu_reserve_mutex); + } +} + +static int mipspmu_event_init(struct perf_event *event) +{ + int err = 0; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + + default: + return -ENOENT; + } + + if (event->cpu >= nr_cpumask_bits || + (event->cpu >= 0 && !cpu_online(event->cpu))) + return -ENODEV; + + if (!atomic_inc_not_zero(&active_events)) { + mutex_lock(&pmu_reserve_mutex); + if (atomic_read(&active_events) == 0) + err = mipspmu_get_irq(); + + if (!err) + atomic_inc(&active_events); + mutex_unlock(&pmu_reserve_mutex); + } + + if (err) + return err; + + return __hw_perf_event_init(event); +} + +static struct pmu pmu = { + .pmu_enable = mipspmu_enable, + .pmu_disable = mipspmu_disable, + .event_init = mipspmu_event_init, + .add = mipspmu_add, + .del = mipspmu_del, + .start = mipspmu_start, + .stop = mipspmu_stop, + .read = mipspmu_read, +}; + +static unsigned int mipspmu_perf_event_encode(const struct mips_perf_event *pev) +{ +/* + * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for + * event_id. + */ #ifdef CONFIG_MIPS_MT_SMP -static DEFINE_RWLOCK(pmuint_rwlock); + return ((unsigned int)pev->range << 24) | + (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); +#else + return (pev->cntr_mask & 0xffff00) | + (pev->event_id & 0xff); #endif +} + +static const struct mips_perf_event *mipspmu_map_general_event(int idx) +{ + + if ((*mipspmu.general_event_map)[idx].cntr_mask == 0) + return ERR_PTR(-EOPNOTSUPP); + return &(*mipspmu.general_event_map)[idx]; +} + +static const struct mips_perf_event *mipspmu_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct mips_perf_event *pev; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + pev = &((*mipspmu.cache_event_map) + [cache_type] + [cache_op] + [cache_result]); -/* 24K/34K/1004K cores can share the same event map. */ + if (pev->cntr_mask == 0) + return ERR_PTR(-EOPNOTSUPP); + + return pev; + +} + +static int validate_group(struct perf_event *event) +{ + struct perf_event *sibling, *leader = event->group_leader; + struct cpu_hw_events fake_cpuc; + + memset(&fake_cpuc, 0, sizeof(fake_cpuc)); + + if (mipsxx_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) + return -EINVAL; + + list_for_each_entry(sibling, &leader->sibling_list, group_entry) { + if (mipsxx_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) + return -EINVAL; + } + + if (mipsxx_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) + return -EINVAL; + + return 0; +} + +/* This is needed by specific irq handlers in perf_event_*.c */ +static void handle_associated_event(struct cpu_hw_events *cpuc, + int idx, struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc = &event->hw; + + mipspmu_event_update(event, hwc, idx); + data->period = event->hw.last_period; + if (!mipspmu_event_set_period(event, hwc, idx)) + return; + + if (perf_event_overflow(event, data, regs)) + mipsxx_pmu_disable_event(idx); +} + + +static int __n_counters(void) +{ + if (!(read_c0_config1() & M_CONFIG1_PC)) + return 0; + if (!(read_c0_perfctrl0() & M_PERFCTL_MORE)) + return 1; + if (!(read_c0_perfctrl1() & M_PERFCTL_MORE)) + return 2; + if (!(read_c0_perfctrl2() & M_PERFCTL_MORE)) + return 3; + + return 4; +} + +static int n_counters(void) +{ + int counters; + + switch (current_cpu_type()) { + case CPU_R10000: + counters = 2; + break; + + case CPU_R12000: + case CPU_R14000: + counters = 4; + break; + + default: + counters = __n_counters(); + } + + return counters; +} + +static void reset_counters(void *arg) +{ + int counters = (int)(long)arg; + switch (counters) { + case 4: + mipsxx_pmu_write_control(3, 0); + mipspmu.write_counter(3, 0); + case 3: + mipsxx_pmu_write_control(2, 0); + mipspmu.write_counter(2, 0); + case 2: + mipsxx_pmu_write_control(1, 0); + mipspmu.write_counter(1, 0); + case 1: + mipsxx_pmu_write_control(0, 0); + mipspmu.write_counter(0, 0); + } +} + +/* 24K/34K/1004K/interAptiv/loongson1 cores share the same event map. */ static const struct mips_perf_event mipsxxcore_event_map [PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, - [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, - [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02, CNTR_EVEN, T }, [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, - [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, }; -/* 74K core has different branch event code. */ -static const struct mips_perf_event mipsxx74Kcore_event_map +/* 74K/proAptiv core has different branch event code. */ +static const struct mips_perf_event mipsxxcore_event_map2 [PERF_COUNT_HW_MAX] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, P }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, - [PERF_COUNT_HW_CACHE_REFERENCES] = { UNSUPPORTED_PERF_EVENT_ID }, - [PERF_COUNT_HW_CACHE_MISSES] = { UNSUPPORTED_PERF_EVENT_ID }, [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x27, CNTR_EVEN, T }, [PERF_COUNT_HW_BRANCH_MISSES] = { 0x27, CNTR_ODD, T }, - [PERF_COUNT_HW_BUS_CYCLES] = { UNSUPPORTED_PERF_EVENT_ID }, }; -/* 24K/34K/1004K cores can share the same cache event map. */ +static const struct mips_perf_event octeon_event_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x03, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x2b, CNTR_ALL }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x2e, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x08, CNTR_ALL }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x09, CNTR_ALL }, + [PERF_COUNT_HW_BUS_CYCLES] = { 0x25, CNTR_ALL }, +}; + +static const struct mips_perf_event bmips5000_event_map + [PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, CNTR_EVEN | CNTR_ODD, T }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01, CNTR_EVEN | CNTR_ODD, T }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x02, CNTR_ODD, T }, +}; + +static const struct mips_perf_event xlp_event_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x01, CNTR_ALL }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x18, CNTR_ALL }, /* PAPI_TOT_INS */ + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ + [PERF_COUNT_HW_CACHE_MISSES] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x1b, CNTR_ALL }, /* PAPI_BR_CN */ + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x1c, CNTR_ALL }, /* PAPI_BR_MSP */ +}; + +/* 24K/34K/1004K/interAptiv/loongson1 cores share the same cache event map. */ static const struct mips_perf_event mipsxxcore_cache_map [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] @@ -297,10 +869,6 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_ACCESS)] = { 0x0a, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x0b, CNTR_EVEN | CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(L1I)] = { [C(OP_READ)] = { @@ -317,7 +885,6 @@ static const struct mips_perf_event mipsxxcore_cache_map * Note that MIPS has only "hit" events countable for * the prefetch operation. */ - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, }, }, [C(LL)] = { @@ -329,10 +896,6 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_ACCESS)] = { 0x15, CNTR_ODD, P }, [C(RESULT_MISS)] = { 0x16, CNTR_EVEN, P }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(DTLB)] = { [C(OP_READ)] = { @@ -343,10 +906,6 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_ACCESS)] = { 0x06, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x06, CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(ITLB)] = { [C(OP_READ)] = { @@ -357,10 +916,6 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_ACCESS)] = { 0x05, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x05, CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(BPU)] = { /* Using the same code for *HW_BRANCH* */ @@ -372,15 +927,11 @@ static const struct mips_perf_event mipsxxcore_cache_map [C(RESULT_ACCESS)] = { 0x02, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, }; -/* 74K core has completely different cache event map. */ -static const struct mips_perf_event mipsxx74Kcore_cache_map +/* 74K/proAptiv core has completely different cache event map. */ +static const struct mips_perf_event mipsxxcore_cache_map2 [PERF_COUNT_HW_CACHE_MAX] [PERF_COUNT_HW_CACHE_OP_MAX] [PERF_COUNT_HW_CACHE_RESULT_MAX] = { @@ -399,10 +950,6 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map [C(RESULT_ACCESS)] = { 0x17, CNTR_ODD, T }, [C(RESULT_MISS)] = { 0x18, CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(L1I)] = { [C(OP_READ)] = { @@ -419,38 +966,23 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map * Note that MIPS has only "hit" events countable for * the prefetch operation. */ - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, }, }, [C(LL)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, - [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, + [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN, P }, }, [C(OP_WRITE)] = { [C(RESULT_ACCESS)] = { 0x1c, CNTR_ODD, P }, - [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN | CNTR_ODD, P }, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, -}, -[C(DTLB)] = { - /* 74K core does not have specific DTLB events. */ - [C(OP_READ)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, - [C(OP_WRITE)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_MISS)] = { 0x1d, CNTR_EVEN, P }, }, }, +/* + * 74K core does not have specific DTLB events. proAptiv core has + * "speculative" DTLB events which are numbered 0x63 (even/odd) and + * not included here. One can use raw events if really needed. + */ [C(ITLB)] = { [C(OP_READ)] = { [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, @@ -460,10 +992,6 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map [C(RESULT_ACCESS)] = { 0x04, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x04, CNTR_ODD, T }, }, - [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, - }, }, [C(BPU)] = { /* Using the same code for *HW_BRANCH* */ @@ -475,17 +1003,169 @@ static const struct mips_perf_event mipsxx74Kcore_cache_map [C(RESULT_ACCESS)] = { 0x27, CNTR_EVEN, T }, [C(RESULT_MISS)] = { 0x27, CNTR_ODD, T }, }, +}, +}; + +/* BMIPS5000 */ +static const struct mips_perf_event bmips5000_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + /* + * Like some other architectures (e.g. ARM), the performance + * counters don't differentiate between read and write + * accesses/misses, so this isn't strictly correct, but it's the + * best we can do. Writes and reads get combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 12, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 12, CNTR_ODD, T }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 10, CNTR_EVEN, T }, + [C(RESULT_MISS)] = { 10, CNTR_ODD, T }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { 23, CNTR_EVEN, T }, + /* + * Note that MIPS has only "hit" events countable for + * the prefetch operation. + */ + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, + [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 28, CNTR_EVEN, P }, + [C(RESULT_MISS)] = { 28, CNTR_ODD, P }, + }, +}, +[C(BPU)] = { + /* Using the same code for *HW_BRANCH* */ + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x02, CNTR_ODD, T }, + }, +}, +}; + + +static const struct mips_perf_event octeon_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x2b, CNTR_ALL }, + [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x30, CNTR_ALL }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x18, CNTR_ALL }, + }, [C(OP_PREFETCH)] = { - [C(RESULT_ACCESS)] = { UNSUPPORTED_PERF_EVENT_ID }, - [C(RESULT_MISS)] = { UNSUPPORTED_PERF_EVENT_ID }, + [C(RESULT_ACCESS)] = { 0x19, CNTR_ALL }, + }, +}, +[C(DTLB)] = { + /* + * Only general DTLB misses are counted use the same event for + * read and write. + */ + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x35, CNTR_ALL }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, + }, +}, +}; + +static const struct mips_perf_event xlp_cache_map + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x31, CNTR_ALL }, /* PAPI_L1_DCR */ + [C(RESULT_MISS)] = { 0x30, CNTR_ALL }, /* PAPI_L1_LDM */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x2f, CNTR_ALL }, /* PAPI_L1_DCW */ + [C(RESULT_MISS)] = { 0x2e, CNTR_ALL }, /* PAPI_L1_STM */ + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x04, CNTR_ALL }, /* PAPI_L1_ICA */ + [C(RESULT_MISS)] = { 0x07, CNTR_ALL }, /* PAPI_L1_ICM */ + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x35, CNTR_ALL }, /* PAPI_L2_DCR */ + [C(RESULT_MISS)] = { 0x37, CNTR_ALL }, /* PAPI_L2_LDM */ + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x34, CNTR_ALL }, /* PAPI_L2_DCA */ + [C(RESULT_MISS)] = { 0x36, CNTR_ALL }, /* PAPI_L2_DCM */ + }, +}, +[C(DTLB)] = { + /* + * Only general DTLB misses are counted use the same event for + * read and write. + */ + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x2d, CNTR_ALL }, /* PAPI_TLB_DM */ + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ + }, + [C(OP_WRITE)] = { + [C(RESULT_MISS)] = { 0x08, CNTR_ALL }, /* PAPI_TLB_IM */ + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_MISS)] = { 0x25, CNTR_ALL }, }, }, }; #ifdef CONFIG_MIPS_MT_SMP -static void -check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) +static void check_and_calc_range(struct perf_event *event, + const struct mips_perf_event *pev) { struct hw_perf_event *hwc = &event->hw; @@ -508,9 +1188,8 @@ check_and_calc_range(struct perf_event *event, hwc->config_base |= M_TC_EN_ALL; } #else -static void -check_and_calc_range(struct perf_event *event, - const struct mips_perf_event *pev) +static void check_and_calc_range(struct perf_event *event, + const struct mips_perf_event *pev) { } #endif @@ -532,7 +1211,7 @@ static int __hw_perf_event_init(struct perf_event *event) } else if (PERF_TYPE_RAW == event->attr.type) { /* We are working on the global raw event. */ mutex_lock(&raw_event_mutex); - pev = mipspmu->map_raw_event(event->attr.config); + pev = mipspmu.map_raw_event(event->attr.config); } else { /* The event type is not (yet) supported. */ return -EOPNOTSUPP; @@ -577,83 +1256,61 @@ static int __hw_perf_event_init(struct perf_event *event) hwc->config = 0; if (!hwc->sample_period) { - hwc->sample_period = MAX_PERIOD; + hwc->sample_period = mipspmu.max_period; hwc->last_period = hwc->sample_period; local64_set(&hwc->period_left, hwc->sample_period); } err = 0; - if (event->group_leader != event) { + if (event->group_leader != event) err = validate_group(event); - if (err) - return -EINVAL; - } event->destroy = hw_perf_event_destroy; + if (err) + event->destroy(event); + return err; } static void pause_local_counters(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - int counters = mipspmu->num_counters; + int ctr = mipspmu.num_counters; unsigned long flags; local_irq_save(flags); - switch (counters) { - case 4: - cpuc->saved_ctrl[3] = r_c0_perfctrl3(); - w_c0_perfctrl3(cpuc->saved_ctrl[3] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 3: - cpuc->saved_ctrl[2] = r_c0_perfctrl2(); - w_c0_perfctrl2(cpuc->saved_ctrl[2] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 2: - cpuc->saved_ctrl[1] = r_c0_perfctrl1(); - w_c0_perfctrl1(cpuc->saved_ctrl[1] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - case 1: - cpuc->saved_ctrl[0] = r_c0_perfctrl0(); - w_c0_perfctrl0(cpuc->saved_ctrl[0] & - ~M_PERFCTL_COUNT_EVENT_WHENEVER); - } + do { + ctr--; + cpuc->saved_ctrl[ctr] = mipsxx_pmu_read_control(ctr); + mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] & + ~M_PERFCTL_COUNT_EVENT_WHENEVER); + } while (ctr > 0); local_irq_restore(flags); } static void resume_local_counters(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - int counters = mipspmu->num_counters; - unsigned long flags; + int ctr = mipspmu.num_counters; - local_irq_save(flags); - switch (counters) { - case 4: - w_c0_perfctrl3(cpuc->saved_ctrl[3]); - case 3: - w_c0_perfctrl2(cpuc->saved_ctrl[2]); - case 2: - w_c0_perfctrl1(cpuc->saved_ctrl[1]); - case 1: - w_c0_perfctrl0(cpuc->saved_ctrl[0]); - } - local_irq_restore(flags); + do { + ctr--; + mipsxx_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]); + } while (ctr > 0); } static int mipsxx_pmu_handle_shared_irq(void) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); struct perf_sample_data data; - unsigned int counters = mipspmu->num_counters; - unsigned int counter; + unsigned int counters = mipspmu.num_counters; + u64 counter; int handled = IRQ_NONE; struct pt_regs *regs; - if (cpu_has_mips_r2 && !(read_c0_cause() & (1 << 26))) + if (cpu_has_perf_cntr_intr_bit && !(read_c0_cause() & CAUSEF_PCI)) return handled; - /* * First we pause the local counters, so that when we are locked * here, the counters are all paused. When it gets locked due to @@ -662,25 +1319,21 @@ static int mipsxx_pmu_handle_shared_irq(void) * See also mipsxx_pmu_start(). */ pause_local_counters(); -#ifdef CONFIG_MIPS_MT_SMP +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS read_lock(&pmuint_rwlock); #endif regs = get_irq_regs(); - perf_sample_data_init(&data, 0); + perf_sample_data_init(&data, 0, 0); switch (counters) { #define HANDLE_COUNTER(n) \ case n + 1: \ if (test_bit(n, cpuc->used_mask)) { \ - counter = r_c0_perfcntr ## n(); \ - if (counter & M_COUNTER_OVERFLOW) { \ - w_c0_perfcntr ## n(counter & \ - VALID_COUNT); \ - if (test_and_change_bit(n, cpuc->msbs)) \ - handle_associated_event(cpuc, \ - n, &data, regs); \ + counter = mipspmu.read_counter(n); \ + if (counter & mipspmu.overflow) { \ + handle_associated_event(cpuc, n, &data, regs); \ handled = IRQ_HANDLED; \ } \ } @@ -696,128 +1349,25 @@ static int mipsxx_pmu_handle_shared_irq(void) * interrupt, not NMI. */ if (handled == IRQ_HANDLED) - perf_event_do_pending(); + irq_work_run(); -#ifdef CONFIG_MIPS_MT_SMP +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS read_unlock(&pmuint_rwlock); #endif resume_local_counters(); return handled; } -static irqreturn_t -mipsxx_pmu_handle_irq(int irq, void *dev) +static irqreturn_t mipsxx_pmu_handle_irq(int irq, void *dev) { return mipsxx_pmu_handle_shared_irq(); } -static void mipsxx_pmu_start(void) -{ -#ifdef CONFIG_MIPS_MT_SMP - write_unlock(&pmuint_rwlock); -#endif - resume_local_counters(); -} - -/* - * MIPS performance counters can be per-TC. The control registers can - * not be directly accessed accross CPUs. Hence if we want to do global - * control, we need cross CPU calls. on_each_cpu() can help us, but we - * can not make sure this function is called with interrupts enabled. So - * here we pause local counters and then grab a rwlock and leave the - * counters on other CPUs alone. If any counter interrupt raises while - * we own the write lock, simply pause local counters on that CPU and - * spin in the handler. Also we know we won't be switched to another - * CPU after pausing local counters and before grabbing the lock. - */ -static void mipsxx_pmu_stop(void) -{ - pause_local_counters(); -#ifdef CONFIG_MIPS_MT_SMP - write_lock(&pmuint_rwlock); -#endif -} - -static int -mipsxx_pmu_alloc_counter(struct cpu_hw_events *cpuc, - struct hw_perf_event *hwc) -{ - int i; - - /* - * We only need to care the counter mask. The range has been - * checked definitely. - */ - unsigned long cntr_mask = (hwc->event_base >> 8) & 0xffff; - - for (i = mipspmu->num_counters - 1; i >= 0; i--) { - /* - * Note that some MIPS perf events can be counted by both - * even and odd counters, wheresas many other are only by - * even _or_ odd counters. This introduces an issue that - * when the former kind of event takes the counter the - * latter kind of event wants to use, then the "counter - * allocation" for the latter event will fail. In fact if - * they can be dynamically swapped, they both feel happy. - * But here we leave this issue alone for now. - */ - if (test_bit(i, &cntr_mask) && - !test_and_set_bit(i, cpuc->used_mask)) - return i; - } - - return -EAGAIN; -} - -static void -mipsxx_pmu_enable_event(struct hw_perf_event *evt, int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - local_irq_save(flags); - cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base & 0xff) | - (evt->config_base & M_PERFCTL_CONFIG_MASK) | - /* Make sure interrupt enabled. */ - M_PERFCTL_INTERRUPT_ENABLE; - /* - * We do not actually let the counter run. Leave it until start(). - */ - local_irq_restore(flags); -} - -static void -mipsxx_pmu_disable_event(int idx) -{ - struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - unsigned long flags; - - WARN_ON(idx < 0 || idx >= mipspmu->num_counters); - - local_irq_save(flags); - cpuc->saved_ctrl[idx] = mipsxx_pmu_read_control(idx) & - ~M_PERFCTL_COUNT_EVENT_WHENEVER; - mipsxx_pmu_write_control(idx, cpuc->saved_ctrl[idx]); - local_irq_restore(flags); -} - /* 24K */ -#define IS_UNSUPPORTED_24K_EVENT(r, b) \ - ((b) == 12 || (r) == 151 || (r) == 152 || (b) == 26 || \ - (b) == 27 || (r) == 28 || (r) == 158 || (b) == 31 || \ - (b) == 32 || (b) == 34 || (b) == 36 || (r) == 168 || \ - (r) == 172 || (b) == 47 || ((b) >= 56 && (b) <= 63) || \ - ((b) >= 68 && (b) <= 127)) #define IS_BOTH_COUNTERS_24K_EVENT(b) \ ((b) == 0 || (b) == 1 || (b) == 11) /* 34K */ -#define IS_UNSUPPORTED_34K_EVENT(r, b) \ - ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 36 || \ - (b) == 38 || (r) == 175 || ((b) >= 56 && (b) <= 63) || \ - ((b) >= 68 && (b) <= 127)) #define IS_BOTH_COUNTERS_34K_EVENT(b) \ ((b) == 0 || (b) == 1 || (b) == 11) #ifdef CONFIG_MIPS_MT_SMP @@ -826,24 +1376,18 @@ mipsxx_pmu_disable_event(int idx) (b) == 25 || (b) == 39 || (r) == 44 || (r) == 174 || \ (r) == 176 || ((b) >= 50 && (b) <= 55) || \ ((b) >= 64 && (b) <= 67)) -#define IS_RANGE_V_34K_EVENT(r) ((r) == 47) +#define IS_RANGE_V_34K_EVENT(r) ((r) == 47) #endif /* 74K */ -#define IS_UNSUPPORTED_74K_EVENT(r, b) \ - ((r) == 5 || ((r) >= 135 && (r) <= 137) || \ - ((b) >= 10 && (b) <= 12) || (b) == 22 || (b) == 27 || \ - (b) == 33 || (b) == 34 || ((b) >= 47 && (b) <= 49) || \ - (r) == 178 || (b) == 55 || (b) == 57 || (b) == 60 || \ - (b) == 61 || (r) == 62 || (r) == 191 || \ - ((b) >= 64 && (b) <= 127)) #define IS_BOTH_COUNTERS_74K_EVENT(b) \ ((b) == 0 || (b) == 1) +/* proAptiv */ +#define IS_BOTH_COUNTERS_PROAPTIV_EVENT(b) \ + ((b) == 0 || (b) == 1) + /* 1004K */ -#define IS_UNSUPPORTED_1004K_EVENT(r, b) \ - ((b) == 12 || (r) == 27 || (r) == 158 || (b) == 38 || \ - (r) == 175 || (b) == 63 || ((b) >= 68 && (b) <= 127)) #define IS_BOTH_COUNTERS_1004K_EVENT(b) \ ((b) == 0 || (b) == 1 || (b) == 11) #ifdef CONFIG_MIPS_MT_SMP @@ -856,6 +1400,25 @@ mipsxx_pmu_disable_event(int idx) #define IS_RANGE_V_1004K_EVENT(r) ((r) == 47) #endif +/* interAptiv */ +#define IS_BOTH_COUNTERS_INTERAPTIV_EVENT(b) \ + ((b) == 0 || (b) == 1 || (b) == 11) +#ifdef CONFIG_MIPS_MT_SMP +/* The P/V/T info is not provided for "(b) == 38" in SUM, assume P. */ +#define IS_RANGE_P_INTERAPTIV_EVENT(r, b) \ + ((b) == 0 || (r) == 18 || (b) == 21 || (b) == 22 || \ + (b) == 25 || (b) == 36 || (b) == 38 || (b) == 39 || \ + (r) == 44 || (r) == 174 || (r) == 176 || ((b) >= 50 && \ + (b) <= 59) || (r) == 188 || (b) == 61 || (b) == 62 || \ + ((b) >= 64 && (b) <= 67)) +#define IS_RANGE_V_INTERAPTIV_EVENT(r) ((r) == 47 || (r) == 175) +#endif + +/* BMIPS5000 */ +#define IS_BOTH_COUNTERS_BMIPS5000_EVENT(b) \ + ((b) == 0 || (b) == 1) + + /* * User can use 0-255 raw events, where 0-127 for the events of even * counters, and 128-255 for odd counters. Note that bit 7 is used to @@ -864,17 +1427,15 @@ mipsxx_pmu_disable_event(int idx) * then 128 needs to be added to 15 as the input for the event config, * i.e., 143 (0x8F) to be used. */ -static const struct mips_perf_event * -mipsxx_pmu_map_raw_event(u64 config) +static const struct mips_perf_event *mipsxx_pmu_map_raw_event(u64 config) { unsigned int raw_id = config & 0xff; unsigned int base_id = raw_id & 0x7f; + raw_event.event_id = base_id; + switch (current_cpu_type()) { case CPU_24K: - if (IS_UNSUPPORTED_24K_EVENT(raw_id, base_id)) - return ERR_PTR(-EOPNOTSUPP); - raw_event.event_id = base_id; if (IS_BOTH_COUNTERS_24K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else @@ -889,9 +1450,6 @@ mipsxx_pmu_map_raw_event(u64 config) #endif break; case CPU_34K: - if (IS_UNSUPPORTED_34K_EVENT(raw_id, base_id)) - return ERR_PTR(-EOPNOTSUPP); - raw_event.event_id = base_id; if (IS_BOTH_COUNTERS_34K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else @@ -907,9 +1465,7 @@ mipsxx_pmu_map_raw_event(u64 config) #endif break; case CPU_74K: - if (IS_UNSUPPORTED_74K_EVENT(raw_id, base_id)) - return ERR_PTR(-EOPNOTSUPP); - raw_event.event_id = base_id; + case CPU_1074K: if (IS_BOTH_COUNTERS_74K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else @@ -919,10 +1475,17 @@ mipsxx_pmu_map_raw_event(u64 config) raw_event.range = P; #endif break; + case CPU_PROAPTIV: + if (IS_BOTH_COUNTERS_PROAPTIV_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; +#ifdef CONFIG_MIPS_MT_SMP + raw_event.range = P; +#endif + break; case CPU_1004K: - if (IS_UNSUPPORTED_1004K_EVENT(raw_id, base_id)) - return ERR_PTR(-EOPNOTSUPP); - raw_event.event_id = base_id; if (IS_BOTH_COUNTERS_1004K_EVENT(base_id)) raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; else @@ -937,45 +1500,84 @@ mipsxx_pmu_map_raw_event(u64 config) raw_event.range = T; #endif break; + case CPU_INTERAPTIV: + if (IS_BOTH_COUNTERS_INTERAPTIV_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; +#ifdef CONFIG_MIPS_MT_SMP + if (IS_RANGE_P_INTERAPTIV_EVENT(raw_id, base_id)) + raw_event.range = P; + else if (unlikely(IS_RANGE_V_INTERAPTIV_EVENT(raw_id))) + raw_event.range = V; + else + raw_event.range = T; +#endif + break; + case CPU_BMIPS5000: + if (IS_BOTH_COUNTERS_BMIPS5000_EVENT(base_id)) + raw_event.cntr_mask = CNTR_EVEN | CNTR_ODD; + else + raw_event.cntr_mask = + raw_id > 127 ? CNTR_ODD : CNTR_EVEN; } return &raw_event; } -static struct mips_pmu mipsxxcore_pmu = { - .handle_irq = mipsxx_pmu_handle_irq, - .handle_shared_irq = mipsxx_pmu_handle_shared_irq, - .start = mipsxx_pmu_start, - .stop = mipsxx_pmu_stop, - .alloc_counter = mipsxx_pmu_alloc_counter, - .read_counter = mipsxx_pmu_read_counter, - .write_counter = mipsxx_pmu_write_counter, - .enable_event = mipsxx_pmu_enable_event, - .disable_event = mipsxx_pmu_disable_event, - .map_raw_event = mipsxx_pmu_map_raw_event, - .general_event_map = &mipsxxcore_event_map, - .cache_event_map = &mipsxxcore_cache_map, -}; +static const struct mips_perf_event *octeon_pmu_map_raw_event(u64 config) +{ + unsigned int raw_id = config & 0xff; + unsigned int base_id = raw_id & 0x7f; -static struct mips_pmu mipsxx74Kcore_pmu = { - .handle_irq = mipsxx_pmu_handle_irq, - .handle_shared_irq = mipsxx_pmu_handle_shared_irq, - .start = mipsxx_pmu_start, - .stop = mipsxx_pmu_stop, - .alloc_counter = mipsxx_pmu_alloc_counter, - .read_counter = mipsxx_pmu_read_counter, - .write_counter = mipsxx_pmu_write_counter, - .enable_event = mipsxx_pmu_enable_event, - .disable_event = mipsxx_pmu_disable_event, - .map_raw_event = mipsxx_pmu_map_raw_event, - .general_event_map = &mipsxx74Kcore_event_map, - .cache_event_map = &mipsxx74Kcore_cache_map, -}; + + raw_event.cntr_mask = CNTR_ALL; + raw_event.event_id = base_id; + + if (current_cpu_type() == CPU_CAVIUM_OCTEON2) { + if (base_id > 0x42) + return ERR_PTR(-EOPNOTSUPP); + } else { + if (base_id > 0x3a) + return ERR_PTR(-EOPNOTSUPP); + } + + switch (base_id) { + case 0x00: + case 0x0f: + case 0x1e: + case 0x1f: + case 0x2f: + case 0x34: + case 0x3b ... 0x3f: + return ERR_PTR(-EOPNOTSUPP); + default: + break; + } + + return &raw_event; +} + +static const struct mips_perf_event *xlp_pmu_map_raw_event(u64 config) +{ + unsigned int raw_id = config & 0xff; + + /* Only 1-63 are defined */ + if ((raw_id < 0x01) || (raw_id > 0x3f)) + return ERR_PTR(-EOPNOTSUPP); + + raw_event.cntr_mask = CNTR_ALL; + raw_event.event_id = raw_id; + + return &raw_event; +} static int __init init_hw_perf_events(void) { int counters, irq; + int counter_bits; pr_info("Performance counters: "); @@ -985,7 +1587,7 @@ init_hw_perf_events(void) return -ENODEV; } -#ifdef CONFIG_MIPS_MT_SMP +#ifdef CONFIG_MIPS_PERF_SHARED_TC_COUNTERS cpu_has_mipsmt_pertccounters = read_c0_config7() & (1<<19); if (!cpu_has_mipsmt_pertccounters) counters = counters_total_to_per_cpu(counters); @@ -999,7 +1601,8 @@ init_hw_perf_events(void) irq = MSC01E_INT_BASE + MSC01E_INT_PERFCTR; } else { #endif - if (cp0_perfcount_irq >= 0) + if ((cp0_perfcount_irq >= 0) && + (cp0_compare_irq != cp0_perfcount_irq)) irq = MIPS_CPU_IRQ_BASE + cp0_perfcount_irq; else irq = -1; @@ -1007,32 +1610,67 @@ init_hw_perf_events(void) } #endif - on_each_cpu(reset_counters, (void *)(long)counters, 1); + mipspmu.map_raw_event = mipsxx_pmu_map_raw_event; switch (current_cpu_type()) { case CPU_24K: - mipsxxcore_pmu.name = "mips/24K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/24K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; break; case CPU_34K: - mipsxxcore_pmu.name = "mips/34K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/34K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; break; case CPU_74K: - mipsxx74Kcore_pmu.name = "mips/74K"; - mipsxx74Kcore_pmu.num_counters = counters; - mipsxx74Kcore_pmu.irq = irq; - mipspmu = &mipsxx74Kcore_pmu; + mipspmu.name = "mips/74K"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; + break; + case CPU_PROAPTIV: + mipspmu.name = "mips/proAptiv"; + mipspmu.general_event_map = &mipsxxcore_event_map2; + mipspmu.cache_event_map = &mipsxxcore_cache_map2; break; case CPU_1004K: - mipsxxcore_pmu.name = "mips/1004K"; - mipsxxcore_pmu.num_counters = counters; - mipsxxcore_pmu.irq = irq; - mipspmu = &mipsxxcore_pmu; + mipspmu.name = "mips/1004K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_1074K: + mipspmu.name = "mips/1074K"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_INTERAPTIV: + mipspmu.name = "mips/interAptiv"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_LOONGSON1: + mipspmu.name = "mips/loongson1"; + mipspmu.general_event_map = &mipsxxcore_event_map; + mipspmu.cache_event_map = &mipsxxcore_cache_map; + break; + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + mipspmu.name = "octeon"; + mipspmu.general_event_map = &octeon_event_map; + mipspmu.cache_event_map = &octeon_cache_map; + mipspmu.map_raw_event = octeon_pmu_map_raw_event; + break; + case CPU_BMIPS5000: + mipspmu.name = "BMIPS5000"; + mipspmu.general_event_map = &bmips5000_event_map; + mipspmu.cache_event_map = &bmips5000_cache_map; + break; + case CPU_XLP: + mipspmu.name = "xlp"; + mipspmu.general_event_map = &xlp_event_map; + mipspmu.cache_event_map = &xlp_cache_map; + mipspmu.map_raw_event = xlp_pmu_map_raw_event; break; default: pr_cont("Either hardware does not support performance " @@ -1040,13 +1678,33 @@ init_hw_perf_events(void) return -ENODEV; } - if (mipspmu) - pr_cont("%s PMU enabled, %d counters available to each " - "CPU, irq %d%s\n", mipspmu->name, counters, irq, - irq < 0 ? " (share with timer interrupt)" : ""); + mipspmu.num_counters = counters; + mipspmu.irq = irq; + + if (read_c0_perfctrl0() & M_PERFCTL_WIDE) { + mipspmu.max_period = (1ULL << 63) - 1; + mipspmu.valid_count = (1ULL << 63) - 1; + mipspmu.overflow = 1ULL << 63; + mipspmu.read_counter = mipsxx_pmu_read_counter_64; + mipspmu.write_counter = mipsxx_pmu_write_counter_64; + counter_bits = 64; + } else { + mipspmu.max_period = (1ULL << 31) - 1; + mipspmu.valid_count = (1ULL << 31) - 1; + mipspmu.overflow = 1ULL << 31; + mipspmu.read_counter = mipsxx_pmu_read_counter; + mipspmu.write_counter = mipsxx_pmu_write_counter; + counter_bits = 32; + } + + on_each_cpu(reset_counters, (void *)(long)counters, 1); + + pr_cont("%s PMU enabled, %d %d-bit counters available to each " + "CPU, irq %d%s\n", mipspmu.name, counters, counter_bits, irq, + irq < 0 ? " (share with timer interrupt)" : ""); + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); return 0; } -arch_initcall(init_hw_perf_events); - -#endif /* defined(CONFIG_CPU_MIPS32)... */ +early_initcall(init_hw_perf_events); diff --git a/arch/mips/kernel/pm-cps.c b/arch/mips/kernel/pm-cps.c new file mode 100644 index 00000000000..c4c2069d3a2 --- /dev/null +++ b/arch/mips/kernel/pm-cps.c @@ -0,0 +1,716 @@ +/* + * Copyright (C) 2014 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/init.h> +#include <linux/percpu.h> +#include <linux/slab.h> + +#include <asm/asm-offsets.h> +#include <asm/cacheflush.h> +#include <asm/cacheops.h> +#include <asm/idle.h> +#include <asm/mips-cm.h> +#include <asm/mips-cpc.h> +#include <asm/mipsmtregs.h> +#include <asm/pm.h> +#include <asm/pm-cps.h> +#include <asm/smp-cps.h> +#include <asm/uasm.h> + +/* + * cps_nc_entry_fn - type of a generated non-coherent state entry function + * @online: the count of online coupled VPEs + * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count + * + * The code entering & exiting non-coherent states is generated at runtime + * using uasm, in order to ensure that the compiler cannot insert a stray + * memory access at an unfortunate time and to allow the generation of optimal + * core-specific code particularly for cache routines. If coupled_coherence + * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state, + * returns the number of VPEs that were in the wait state at the point this + * VPE left it. Returns garbage if coupled_coherence is zero or this is not + * the entry function for CPS_PM_NC_WAIT. + */ +typedef unsigned (*cps_nc_entry_fn)(unsigned online, u32 *nc_ready_count); + +/* + * The entry point of the generated non-coherent idle state entry/exit + * functions. Actually per-core rather than per-CPU. + */ +static DEFINE_PER_CPU_READ_MOSTLY(cps_nc_entry_fn[CPS_PM_STATE_COUNT], + nc_asm_enter); + +/* Bitmap indicating which states are supported by the system */ +DECLARE_BITMAP(state_support, CPS_PM_STATE_COUNT); + +/* + * Indicates the number of coupled VPEs ready to operate in a non-coherent + * state. Actually per-core rather than per-CPU. + */ +static DEFINE_PER_CPU_ALIGNED(u32*, ready_count); +static DEFINE_PER_CPU_ALIGNED(void*, ready_count_alloc); + +/* Indicates online CPUs coupled with the current CPU */ +static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled); + +/* + * Used to synchronize entry to deep idle states. Actually per-core rather + * than per-CPU. + */ +static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier); + +/* Saved CPU state across the CPS_PM_POWER_GATED state */ +DEFINE_PER_CPU_ALIGNED(struct mips_static_suspend_state, cps_cpu_state); + +/* A somewhat arbitrary number of labels & relocs for uasm */ +static struct uasm_label labels[32] __initdata; +static struct uasm_reloc relocs[32] __initdata; + +/* CPU dependant sync types */ +static unsigned stype_intervention; +static unsigned stype_memory; +static unsigned stype_ordering; + +enum mips_reg { + zero, at, v0, v1, a0, a1, a2, a3, + t0, t1, t2, t3, t4, t5, t6, t7, + s0, s1, s2, s3, s4, s5, s6, s7, + t8, t9, k0, k1, gp, sp, fp, ra, +}; + +bool cps_pm_support_state(enum cps_pm_state state) +{ + return test_bit(state, state_support); +} + +static void coupled_barrier(atomic_t *a, unsigned online) +{ + /* + * This function is effectively the same as + * cpuidle_coupled_parallel_barrier, which can't be used here since + * there's no cpuidle device. + */ + + if (!coupled_coherence) + return; + + smp_mb__before_atomic(); + atomic_inc(a); + + while (atomic_read(a) < online) + cpu_relax(); + + if (atomic_inc_return(a) == online * 2) { + atomic_set(a, 0); + return; + } + + while (atomic_read(a) > online) + cpu_relax(); +} + +int cps_pm_enter_state(enum cps_pm_state state) +{ + unsigned cpu = smp_processor_id(); + unsigned core = current_cpu_data.core; + unsigned online, left; + cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled); + u32 *core_ready_count, *nc_core_ready_count; + void *nc_addr; + cps_nc_entry_fn entry; + struct core_boot_config *core_cfg; + struct vpe_boot_config *vpe_cfg; + + /* Check that there is an entry function for this state */ + entry = per_cpu(nc_asm_enter, core)[state]; + if (!entry) + return -EINVAL; + + /* Calculate which coupled CPUs (VPEs) are online */ +#ifdef CONFIG_MIPS_MT + if (cpu_online(cpu)) { + cpumask_and(coupled_mask, cpu_online_mask, + &cpu_sibling_map[cpu]); + online = cpumask_weight(coupled_mask); + cpumask_clear_cpu(cpu, coupled_mask); + } else +#endif + { + cpumask_clear(coupled_mask); + online = 1; + } + + /* Setup the VPE to run mips_cps_pm_restore when started again */ + if (config_enabled(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { + core_cfg = &mips_cps_core_bootcfg[core]; + vpe_cfg = &core_cfg->vpe_config[current_cpu_data.vpe_id]; + vpe_cfg->pc = (unsigned long)mips_cps_pm_restore; + vpe_cfg->gp = (unsigned long)current_thread_info(); + vpe_cfg->sp = 0; + } + + /* Indicate that this CPU might not be coherent */ + cpumask_clear_cpu(cpu, &cpu_coherent_mask); + smp_mb__after_atomic(); + + /* Create a non-coherent mapping of the core ready_count */ + core_ready_count = per_cpu(ready_count, core); + nc_addr = kmap_noncoherent(virt_to_page(core_ready_count), + (unsigned long)core_ready_count); + nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK); + nc_core_ready_count = nc_addr; + + /* Ensure ready_count is zero-initialised before the assembly runs */ + ACCESS_ONCE(*nc_core_ready_count) = 0; + coupled_barrier(&per_cpu(pm_barrier, core), online); + + /* Run the generated entry code */ + left = entry(online, nc_core_ready_count); + + /* Remove the non-coherent mapping of ready_count */ + kunmap_noncoherent(); + + /* Indicate that this CPU is definitely coherent */ + cpumask_set_cpu(cpu, &cpu_coherent_mask); + + /* + * If this VPE is the first to leave the non-coherent wait state then + * it needs to wake up any coupled VPEs still running their wait + * instruction so that they return to cpuidle, which can then complete + * coordination between the coupled VPEs & provide the governor with + * a chance to reflect on the length of time the VPEs were in the + * idle state. + */ + if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online)) + arch_send_call_function_ipi_mask(coupled_mask); + + return 0; +} + +static void __init cps_gen_cache_routine(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + const struct cache_desc *cache, + unsigned op, int lbl) +{ + unsigned cache_size = cache->ways << cache->waybit; + unsigned i; + const unsigned unroll_lines = 32; + + /* If the cache isn't present this function has it easy */ + if (cache->flags & MIPS_CACHE_NOT_PRESENT) + return; + + /* Load base address */ + UASM_i_LA(pp, t0, (long)CKSEG0); + + /* Calculate end address */ + if (cache_size < 0x8000) + uasm_i_addiu(pp, t1, t0, cache_size); + else + UASM_i_LA(pp, t1, (long)(CKSEG0 + cache_size)); + + /* Start of cache op loop */ + uasm_build_label(pl, *pp, lbl); + + /* Generate the cache ops */ + for (i = 0; i < unroll_lines; i++) + uasm_i_cache(pp, op, i * cache->linesz, t0); + + /* Update the base address */ + uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz); + + /* Loop if we haven't reached the end address yet */ + uasm_il_bne(pp, pr, t0, t1, lbl); + uasm_i_nop(pp); +} + +static int __init cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + const struct cpuinfo_mips *cpu_info, + int lbl) +{ + unsigned i, fsb_size = 8; + unsigned num_loads = (fsb_size * 3) / 2; + unsigned line_stride = 2; + unsigned line_size = cpu_info->dcache.linesz; + unsigned perf_counter, perf_event; + unsigned revision = cpu_info->processor_id & PRID_REV_MASK; + + /* + * Determine whether this CPU requires an FSB flush, and if so which + * performance counter/event reflect stalls due to a full FSB. + */ + switch (__get_cpu_type(cpu_info->cputype)) { + case CPU_INTERAPTIV: + perf_counter = 1; + perf_event = 51; + break; + + case CPU_PROAPTIV: + /* Newer proAptiv cores don't require this workaround */ + if (revision >= PRID_REV_ENCODE_332(1, 1, 0)) + return 0; + + /* On older ones it's unavailable */ + return -1; + + /* CPUs which do not require the workaround */ + case CPU_P5600: + return 0; + + default: + WARN_ONCE(1, "pm-cps: FSB flush unsupported for this CPU\n"); + return -1; + } + + /* + * Ensure that the fill/store buffer (FSB) is not holding the results + * of a prefetch, since if it is then the CPC sequencer may become + * stuck in the D3 (ClrBus) state whilst entering a low power state. + */ + + /* Preserve perf counter setup */ + uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ + + /* Setup perf counter to count FSB full pipeline stalls */ + uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf); + uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_ehb(pp); + uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */ + uasm_i_ehb(pp); + + /* Base address for loads */ + UASM_i_LA(pp, t0, (long)CKSEG0); + + /* Start of clear loop */ + uasm_build_label(pl, *pp, lbl); + + /* Perform some loads to fill the FSB */ + for (i = 0; i < num_loads; i++) + uasm_i_lw(pp, zero, i * line_size * line_stride, t0); + + /* + * Invalidate the new D-cache entries so that the cache will need + * refilling (via the FSB) if the loop is executed again. + */ + for (i = 0; i < num_loads; i++) { + uasm_i_cache(pp, Hit_Invalidate_D, + i * line_size * line_stride, t0); + uasm_i_cache(pp, Hit_Writeback_Inv_SD, + i * line_size * line_stride, t0); + } + + /* Completion barrier */ + uasm_i_sync(pp, stype_memory); + uasm_i_ehb(pp); + + /* Check whether the pipeline stalled due to the FSB being full */ + uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */ + + /* Loop if it didn't */ + uasm_il_beqz(pp, pr, t1, lbl); + uasm_i_nop(pp); + + /* Restore perf counter 1. The count may well now be wrong... */ + uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */ + uasm_i_ehb(pp); + uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */ + uasm_i_ehb(pp); + + return 0; +} + +static void __init cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl, + struct uasm_reloc **pr, + unsigned r_addr, int lbl) +{ + uasm_i_lui(pp, t0, uasm_rel_hi(0x80000000)); + uasm_build_label(pl, *pp, lbl); + uasm_i_ll(pp, t1, 0, r_addr); + uasm_i_or(pp, t1, t1, t0); + uasm_i_sc(pp, t1, 0, r_addr); + uasm_il_beqz(pp, pr, t1, lbl); + uasm_i_nop(pp); +} + +static void * __init cps_gen_entry_code(unsigned cpu, enum cps_pm_state state) +{ + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + u32 *buf, *p; + const unsigned r_online = a0; + const unsigned r_nc_count = a1; + const unsigned r_pcohctl = t7; + const unsigned max_instrs = 256; + unsigned cpc_cmd; + int err; + enum { + lbl_incready = 1, + lbl_poll_cont, + lbl_secondary_hang, + lbl_disable_coherence, + lbl_flush_fsb, + lbl_invicache, + lbl_flushdcache, + lbl_hang, + lbl_set_cont, + lbl_secondary_cont, + lbl_decready, + }; + + /* Allocate a buffer to hold the generated code */ + p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL); + if (!buf) + return NULL; + + /* Clear labels & relocs ready for (re)use */ + memset(labels, 0, sizeof(labels)); + memset(relocs, 0, sizeof(relocs)); + + if (config_enabled(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) { + /* + * Save CPU state. Note the non-standard calling convention + * with the return address placed in v0 to avoid clobbering + * the ra register before it is saved. + */ + UASM_i_LA(&p, t0, (long)mips_cps_pm_save); + uasm_i_jalr(&p, v0, t0); + uasm_i_nop(&p); + } + + /* + * Load addresses of required CM & CPC registers. This is done early + * because they're needed in both the enable & disable coherence steps + * but in the coupled case the enable step will only run on one VPE. + */ + UASM_i_LA(&p, r_pcohctl, (long)addr_gcr_cl_coherence()); + + if (coupled_coherence) { + /* Increment ready_count */ + uasm_i_sync(&p, stype_ordering); + uasm_build_label(&l, p, lbl_incready); + uasm_i_ll(&p, t1, 0, r_nc_count); + uasm_i_addiu(&p, t2, t1, 1); + uasm_i_sc(&p, t2, 0, r_nc_count); + uasm_il_beqz(&p, &r, t2, lbl_incready); + uasm_i_addiu(&p, t1, t1, 1); + + /* Ordering barrier */ + uasm_i_sync(&p, stype_ordering); + + /* + * If this is the last VPE to become ready for non-coherence + * then it should branch below. + */ + uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence); + uasm_i_nop(&p); + + if (state < CPS_PM_POWER_GATED) { + /* + * Otherwise this is not the last VPE to become ready + * for non-coherence. It needs to wait until coherence + * has been disabled before proceeding, which it will do + * by polling for the top bit of ready_count being set. + */ + uasm_i_addiu(&p, t1, zero, -1); + uasm_build_label(&l, p, lbl_poll_cont); + uasm_i_lw(&p, t0, 0, r_nc_count); + uasm_il_bltz(&p, &r, t0, lbl_secondary_cont); + uasm_i_ehb(&p); + uasm_i_yield(&p, zero, t1); + uasm_il_b(&p, &r, lbl_poll_cont); + uasm_i_nop(&p); + } else { + /* + * The core will lose power & this VPE will not continue + * so it can simply halt here. + */ + uasm_i_addiu(&p, t0, zero, TCHALT_H); + uasm_i_mtc0(&p, t0, 2, 4); + uasm_build_label(&l, p, lbl_secondary_hang); + uasm_il_b(&p, &r, lbl_secondary_hang); + uasm_i_nop(&p); + } + } + + /* + * This is the point of no return - this VPE will now proceed to + * disable coherence. At this point we *must* be sure that no other + * VPE within the core will interfere with the L1 dcache. + */ + uasm_build_label(&l, p, lbl_disable_coherence); + + /* Invalidate the L1 icache */ + cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache, + Index_Invalidate_I, lbl_invicache); + + /* Writeback & invalidate the L1 dcache */ + cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache, + Index_Writeback_Inv_D, lbl_flushdcache); + + /* Completion barrier */ + uasm_i_sync(&p, stype_memory); + uasm_i_ehb(&p); + + /* + * Disable all but self interventions. The load from COHCTL is defined + * by the interAptiv & proAptiv SUMs as ensuring that the operation + * resulting from the preceeding store is complete. + */ + uasm_i_addiu(&p, t0, zero, 1 << cpu_data[cpu].core); + uasm_i_sw(&p, t0, 0, r_pcohctl); + uasm_i_lw(&p, t0, 0, r_pcohctl); + + /* Sync to ensure previous interventions are complete */ + uasm_i_sync(&p, stype_intervention); + uasm_i_ehb(&p); + + /* Disable coherence */ + uasm_i_sw(&p, zero, 0, r_pcohctl); + uasm_i_lw(&p, t0, 0, r_pcohctl); + + if (state >= CPS_PM_CLOCK_GATED) { + err = cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu], + lbl_flush_fsb); + if (err) + goto out_err; + + /* Determine the CPC command to issue */ + switch (state) { + case CPS_PM_CLOCK_GATED: + cpc_cmd = CPC_Cx_CMD_CLOCKOFF; + break; + case CPS_PM_POWER_GATED: + cpc_cmd = CPC_Cx_CMD_PWRDOWN; + break; + default: + BUG(); + goto out_err; + } + + /* Issue the CPC command */ + UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd()); + uasm_i_addiu(&p, t1, zero, cpc_cmd); + uasm_i_sw(&p, t1, 0, t0); + + if (state == CPS_PM_POWER_GATED) { + /* If anything goes wrong just hang */ + uasm_build_label(&l, p, lbl_hang); + uasm_il_b(&p, &r, lbl_hang); + uasm_i_nop(&p); + + /* + * There's no point generating more code, the core is + * powered down & if powered back up will run from the + * reset vector not from here. + */ + goto gen_done; + } + + /* Completion barrier */ + uasm_i_sync(&p, stype_memory); + uasm_i_ehb(&p); + } + + if (state == CPS_PM_NC_WAIT) { + /* + * At this point it is safe for all VPEs to proceed with + * execution. This VPE will set the top bit of ready_count + * to indicate to the other VPEs that they may continue. + */ + if (coupled_coherence) + cps_gen_set_top_bit(&p, &l, &r, r_nc_count, + lbl_set_cont); + + /* + * VPEs which did not disable coherence will continue + * executing, after coherence has been disabled, from this + * point. + */ + uasm_build_label(&l, p, lbl_secondary_cont); + + /* Now perform our wait */ + uasm_i_wait(&p, 0); + } + + /* + * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs + * will run this. The first will actually re-enable coherence & the + * rest will just be performing a rather unusual nop. + */ + uasm_i_addiu(&p, t0, zero, CM_GCR_Cx_COHERENCE_COHDOMAINEN_MSK); + uasm_i_sw(&p, t0, 0, r_pcohctl); + uasm_i_lw(&p, t0, 0, r_pcohctl); + + /* Completion barrier */ + uasm_i_sync(&p, stype_memory); + uasm_i_ehb(&p); + + if (coupled_coherence && (state == CPS_PM_NC_WAIT)) { + /* Decrement ready_count */ + uasm_build_label(&l, p, lbl_decready); + uasm_i_sync(&p, stype_ordering); + uasm_i_ll(&p, t1, 0, r_nc_count); + uasm_i_addiu(&p, t2, t1, -1); + uasm_i_sc(&p, t2, 0, r_nc_count); + uasm_il_beqz(&p, &r, t2, lbl_decready); + uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1); + + /* Ordering barrier */ + uasm_i_sync(&p, stype_ordering); + } + + if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) { + /* + * At this point it is safe for all VPEs to proceed with + * execution. This VPE will set the top bit of ready_count + * to indicate to the other VPEs that they may continue. + */ + cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont); + + /* + * This core will be reliant upon another core sending a + * power-up command to the CPC in order to resume operation. + * Thus an arbitrary VPE can't trigger the core leaving the + * idle state and the one that disables coherence might as well + * be the one to re-enable it. The rest will continue from here + * after that has been done. + */ + uasm_build_label(&l, p, lbl_secondary_cont); + + /* Ordering barrier */ + uasm_i_sync(&p, stype_ordering); + } + + /* The core is coherent, time to return to C code */ + uasm_i_jr(&p, ra); + uasm_i_nop(&p); + +gen_done: + /* Ensure the code didn't exceed the resources allocated for it */ + BUG_ON((p - buf) > max_instrs); + BUG_ON((l - labels) > ARRAY_SIZE(labels)); + BUG_ON((r - relocs) > ARRAY_SIZE(relocs)); + + /* Patch branch offsets */ + uasm_resolve_relocs(relocs, labels); + + /* Flush the icache */ + local_flush_icache_range((unsigned long)buf, (unsigned long)p); + + return buf; +out_err: + kfree(buf); + return NULL; +} + +static int __init cps_gen_core_entries(unsigned cpu) +{ + enum cps_pm_state state; + unsigned core = cpu_data[cpu].core; + unsigned dlinesz = cpu_data[cpu].dcache.linesz; + void *entry_fn, *core_rc; + + for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) { + if (per_cpu(nc_asm_enter, core)[state]) + continue; + if (!test_bit(state, state_support)) + continue; + + entry_fn = cps_gen_entry_code(cpu, state); + if (!entry_fn) { + pr_err("Failed to generate core %u state %u entry\n", + core, state); + clear_bit(state, state_support); + } + + per_cpu(nc_asm_enter, core)[state] = entry_fn; + } + + if (!per_cpu(ready_count, core)) { + core_rc = kmalloc(dlinesz * 2, GFP_KERNEL); + if (!core_rc) { + pr_err("Failed allocate core %u ready_count\n", core); + return -ENOMEM; + } + per_cpu(ready_count_alloc, core) = core_rc; + + /* Ensure ready_count is aligned to a cacheline boundary */ + core_rc += dlinesz - 1; + core_rc = (void *)((unsigned long)core_rc & ~(dlinesz - 1)); + per_cpu(ready_count, core) = core_rc; + } + + return 0; +} + +static int __init cps_pm_init(void) +{ + unsigned cpu; + int err; + + /* Detect appropriate sync types for the system */ + switch (current_cpu_data.cputype) { + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_M5150: + case CPU_P5600: + stype_intervention = 0x2; + stype_memory = 0x3; + stype_ordering = 0x10; + break; + + default: + pr_warn("Power management is using heavyweight sync 0\n"); + } + + /* A CM is required for all non-coherent states */ + if (!mips_cm_present()) { + pr_warn("pm-cps: no CM, non-coherent states unavailable\n"); + goto out; + } + + /* + * If interrupts were enabled whilst running a wait instruction on a + * non-coherent core then the VPE may end up processing interrupts + * whilst non-coherent. That would be bad. + */ + if (cpu_wait == r4k_wait_irqoff) + set_bit(CPS_PM_NC_WAIT, state_support); + else + pr_warn("pm-cps: non-coherent wait unavailable\n"); + + /* Detect whether a CPC is present */ + if (mips_cpc_present()) { + /* Detect whether clock gating is implemented */ + if (read_cpc_cl_stat_conf() & CPC_Cx_STAT_CONF_CLKGAT_IMPL_MSK) + set_bit(CPS_PM_CLOCK_GATED, state_support); + else + pr_warn("pm-cps: CPC does not support clock gating\n"); + + /* Power gating is available with CPS SMP & any CPC */ + if (mips_cps_smp_in_use()) + set_bit(CPS_PM_POWER_GATED, state_support); + else + pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n"); + } else { + pr_warn("pm-cps: no CPC, clock & power gating unavailable\n"); + } + + for_each_present_cpu(cpu) { + err = cps_gen_core_entries(cpu); + if (err) + return err; + } +out: + return 0; +} +arch_initcall(cps_pm_init); diff --git a/arch/mips/kernel/pm.c b/arch/mips/kernel/pm.c new file mode 100644 index 00000000000..fefdf39d3df --- /dev/null +++ b/arch/mips/kernel/pm.c @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 Imagination Technologies Ltd. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * CPU PM notifiers for saving/restoring general CPU state. + */ + +#include <linux/cpu_pm.h> +#include <linux/init.h> + +#include <asm/dsp.h> +#include <asm/fpu.h> +#include <asm/mmu_context.h> +#include <asm/pm.h> +#include <asm/watch.h> + +/* Used by PM helper macros in asm/pm.h */ +struct mips_static_suspend_state mips_static_suspend_state; + +/** + * mips_cpu_save() - Save general CPU state. + * Ensures that general CPU context is saved, notably FPU and DSP. + */ +static int mips_cpu_save(void) +{ + /* Save FPU state */ + lose_fpu(1); + + /* Save DSP state */ + save_dsp(current); + + return 0; +} + +/** + * mips_cpu_restore() - Restore general CPU state. + * Restores important CPU context. + */ +static void mips_cpu_restore(void) +{ + unsigned int cpu = smp_processor_id(); + + /* Restore ASID */ + if (current->mm) + write_c0_entryhi(cpu_asid(cpu, current->mm)); + + /* Restore DSP state */ + restore_dsp(current); + + /* Restore UserLocal */ + if (cpu_has_userlocal) + write_c0_userlocal(current_thread_info()->tp_value); + + /* Restore watch registers */ + __restore_watch(); +} + +/** + * mips_pm_notifier() - Notifier for preserving general CPU context. + * @self: Notifier block. + * @cmd: CPU PM event. + * @v: Private data (unused). + * + * This is called when a CPU power management event occurs, and is used to + * ensure that important CPU context is preserved across a CPU power down. + */ +static int mips_pm_notifier(struct notifier_block *self, unsigned long cmd, + void *v) +{ + int ret; + + switch (cmd) { + case CPU_PM_ENTER: + ret = mips_cpu_save(); + if (ret) + return NOTIFY_STOP; + break; + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + mips_cpu_restore(); + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block mips_pm_notifier_block = { + .notifier_call = mips_pm_notifier, +}; + +static int __init mips_pm_init(void) +{ + return cpu_pm_register_notifier(&mips_pm_notifier_block); +} +arch_initcall(mips_pm_init); diff --git a/arch/mips/kernel/proc.c b/arch/mips/kernel/proc.c index 26109c4d517..037a44d962f 100644 --- a/arch/mips/kernel/proc.c +++ b/arch/mips/kernel/proc.c @@ -1,7 +1,7 @@ /* * Copyright (C) 1995, 1996, 2001 Ralf Baechle * Copyright (C) 2001, 2004 MIPS Technologies, Inc. - * Copyright (C) 2004 Maciej W. Rozycki + * Copyright (C) 2004 Maciej W. Rozycki */ #include <linux/delay.h> #include <linux/kernel.h> @@ -10,13 +10,31 @@ #include <asm/bootinfo.h> #include <asm/cpu.h> #include <asm/cpu-features.h> +#include <asm/idle.h> #include <asm/mipsregs.h> #include <asm/processor.h> +#include <asm/prom.h> unsigned int vced_count, vcei_count; +/* + * * No lock; only written during early bootup by CPU 0. + * */ +static RAW_NOTIFIER_HEAD(proc_cpuinfo_chain); + +int __ref register_proc_cpuinfo_notifier(struct notifier_block *nb) +{ + return raw_notifier_chain_register(&proc_cpuinfo_chain, nb); +} + +int proc_cpuinfo_notifier_call_chain(unsigned long val, void *v) +{ + return raw_notifier_call_chain(&proc_cpuinfo_chain, val, v); +} + static int show_cpuinfo(struct seq_file *m, void *v) { + struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args; unsigned long n = (unsigned long) v - 1; unsigned int version = cpu_data[n].processor_id; unsigned int fp_vers = cpu_data[n].fpu_id; @@ -24,57 +42,100 @@ static int show_cpuinfo(struct seq_file *m, void *v) int i; #ifdef CONFIG_SMP - if (!cpu_isset(n, cpu_online_map)) + if (!cpu_online(n)) return 0; #endif /* * For the first processor also print the system type */ - if (n == 0) + if (n == 0) { seq_printf(m, "system type\t\t: %s\n", get_system_type()); + if (mips_get_machine_name()) + seq_printf(m, "machine\t\t\t: %s\n", + mips_get_machine_name()); + } seq_printf(m, "processor\t\t: %ld\n", n); sprintf(fmt, "cpu model\t\t: %%s V%%d.%%d%s\n", - cpu_data[n].options & MIPS_CPU_FPU ? " FPU V%d.%d" : ""); + cpu_data[n].options & MIPS_CPU_FPU ? " FPU V%d.%d" : ""); seq_printf(m, fmt, __cpu_name[n], - (version >> 4) & 0x0f, version & 0x0f, - (fp_vers >> 4) & 0x0f, fp_vers & 0x0f); + (version >> 4) & 0x0f, version & 0x0f, + (fp_vers >> 4) & 0x0f, fp_vers & 0x0f); seq_printf(m, "BogoMIPS\t\t: %u.%02u\n", - cpu_data[n].udelay_val / (500000/HZ), - (cpu_data[n].udelay_val / (5000/HZ)) % 100); + cpu_data[n].udelay_val / (500000/HZ), + (cpu_data[n].udelay_val / (5000/HZ)) % 100); seq_printf(m, "wait instruction\t: %s\n", cpu_wait ? "yes" : "no"); seq_printf(m, "microsecond timers\t: %s\n", - cpu_has_counter ? "yes" : "no"); + cpu_has_counter ? "yes" : "no"); seq_printf(m, "tlb_entries\t\t: %d\n", cpu_data[n].tlbsize); seq_printf(m, "extra interrupt vector\t: %s\n", - cpu_has_divec ? "yes" : "no"); + cpu_has_divec ? "yes" : "no"); seq_printf(m, "hardware watchpoint\t: %s", - cpu_has_watch ? "yes, " : "no\n"); + cpu_has_watch ? "yes, " : "no\n"); if (cpu_has_watch) { seq_printf(m, "count: %d, address/irw mask: [", - cpu_data[n].watch_reg_count); + cpu_data[n].watch_reg_count); for (i = 0; i < cpu_data[n].watch_reg_count; i++) seq_printf(m, "%s0x%04x", i ? ", " : "" , - cpu_data[n].watch_reg_masks[i]); + cpu_data[n].watch_reg_masks[i]); seq_printf(m, "]\n"); } - seq_printf(m, "ASEs implemented\t:%s%s%s%s%s%s\n", - cpu_has_mips16 ? " mips16" : "", - cpu_has_mdmx ? " mdmx" : "", - cpu_has_mips3d ? " mips3d" : "", - cpu_has_smartmips ? " smartmips" : "", - cpu_has_dsp ? " dsp" : "", - cpu_has_mipsmt ? " mt" : "" - ); + + seq_printf(m, "isa\t\t\t: mips1"); + if (cpu_has_mips_2) + seq_printf(m, "%s", " mips2"); + if (cpu_has_mips_3) + seq_printf(m, "%s", " mips3"); + if (cpu_has_mips_4) + seq_printf(m, "%s", " mips4"); + if (cpu_has_mips_5) + seq_printf(m, "%s", " mips5"); + if (cpu_has_mips32r1) + seq_printf(m, "%s", " mips32r1"); + if (cpu_has_mips32r2) + seq_printf(m, "%s", " mips32r2"); + if (cpu_has_mips64r1) + seq_printf(m, "%s", " mips64r1"); + if (cpu_has_mips64r2) + seq_printf(m, "%s", " mips64r2"); + seq_printf(m, "\n"); + + seq_printf(m, "ASEs implemented\t:"); + if (cpu_has_mips16) seq_printf(m, "%s", " mips16"); + if (cpu_has_mdmx) seq_printf(m, "%s", " mdmx"); + if (cpu_has_mips3d) seq_printf(m, "%s", " mips3d"); + if (cpu_has_smartmips) seq_printf(m, "%s", " smartmips"); + if (cpu_has_dsp) seq_printf(m, "%s", " dsp"); + if (cpu_has_dsp2) seq_printf(m, "%s", " dsp2"); + if (cpu_has_mipsmt) seq_printf(m, "%s", " mt"); + if (cpu_has_mmips) seq_printf(m, "%s", " micromips"); + if (cpu_has_vz) seq_printf(m, "%s", " vz"); + if (cpu_has_msa) seq_printf(m, "%s", " msa"); + if (cpu_has_eva) seq_printf(m, "%s", " eva"); + seq_printf(m, "\n"); + + if (cpu_has_mmips) { + seq_printf(m, "micromips kernel\t: %s\n", + (read_c0_config3() & MIPS_CONF3_ISA_OE) ? "yes" : "no"); + } seq_printf(m, "shadow register sets\t: %d\n", - cpu_data[n].srsets); + cpu_data[n].srsets); + seq_printf(m, "kscratch registers\t: %d\n", + hweight8(cpu_data[n].kscratch_mask)); seq_printf(m, "core\t\t\t: %d\n", cpu_data[n].core); sprintf(fmt, "VCE%%c exceptions\t\t: %s\n", - cpu_has_vce ? "%u" : "not available"); + cpu_has_vce ? "%u" : "not available"); seq_printf(m, fmt, 'D', vced_count); seq_printf(m, fmt, 'I', vcei_count); + + proc_cpuinfo_notifier_args.m = m; + proc_cpuinfo_notifier_args.n = n; + + raw_notifier_call_chain(&proc_cpuinfo_chain, 0, + &proc_cpuinfo_notifier_args); + seq_printf(m, "\n"); return 0; diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index 99960940d4a..0a1ec0f3bef 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -7,15 +7,16 @@ * Copyright (C) 2005, 2006 by Ralf Baechle (ralf@linux-mips.org) * Copyright (C) 1999, 2000 Silicon Graphics, Inc. * Copyright (C) 2004 Thiemo Seufer + * Copyright (C) 2013 Imagination Technologies Ltd. */ #include <linux/errno.h> -#include <linux/module.h> #include <linux/sched.h> #include <linux/tick.h> #include <linux/kernel.h> #include <linux/mm.h> #include <linux/stddef.h> #include <linux/unistd.h> +#include <linux/export.h> #include <linux/ptrace.h> #include <linux/mman.h> #include <linux/personality.h> @@ -31,8 +32,8 @@ #include <asm/cpu.h> #include <asm/dsp.h> #include <asm/fpu.h> +#include <asm/msa.h> #include <asm/pgtable.h> -#include <asm/system.h> #include <asm/mipsregs.h> #include <asm/processor.h> #include <asm/uaccess.h> @@ -42,49 +43,17 @@ #include <asm/inst.h> #include <asm/stacktrace.h> -/* - * The idle thread. There's no useful work to be done, so just try to conserve - * power and have a low exit latency (ie sit in a loop waiting for somebody to - * say that they'd like to reschedule) - */ -void __noreturn cpu_idle(void) -{ - int cpu; - - /* CPU is going idle. */ - cpu = smp_processor_id(); - - /* endless idle loop with no priority at all */ - while (1) { - tick_nohz_stop_sched_tick(1); - while (!need_resched() && cpu_online(cpu)) { -#ifdef CONFIG_MIPS_MT_SMTC - extern void smtc_idle_loop_hook(void); - - smtc_idle_loop_hook(); -#endif - - if (cpu_wait) { - /* Don't trace irqs off for idle */ - stop_critical_timings(); - (*cpu_wait)(); - start_critical_timings(); - } - } #ifdef CONFIG_HOTPLUG_CPU - if (!cpu_online(cpu) && !cpu_isset(cpu, cpu_callin_map) && - (system_state == SYSTEM_RUNNING || - system_state == SYSTEM_BOOTING)) - play_dead(); -#endif - tick_nohz_restart_sched_tick(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); - } +void arch_cpu_idle_dead(void) +{ + /* What the heck is this check doing ? */ + if (!cpu_isset(smp_processor_id(), cpu_callin_map)) + play_dead(); } +#endif asmlinkage void ret_from_fork(void); +asmlinkage void ret_from_kernel_thread(void); void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) { @@ -92,18 +61,15 @@ void start_thread(struct pt_regs * regs, unsigned long pc, unsigned long sp) /* New thread loses kernel privileges. */ status = regs->cp0_status & ~(ST0_CU0|ST0_CU1|ST0_FR|KU_MASK); -#ifdef CONFIG_64BIT - status |= test_thread_flag(TIF_32BIT_REGS) ? 0 : ST0_FR; -#endif status |= KU_USER; regs->cp0_status = status; clear_used_math(); clear_fpu_owner(); - if (cpu_has_dsp) - __init_dsp(); + init_dsp(); + clear_thread_flag(TIF_MSA_CTX_LIVE); + disable_msa(); regs->cp0_epc = pc; regs->regs[29] = sp; - current_thread_info()->addr_limit = USER_DS; } void exit_thread(void) @@ -115,10 +81,10 @@ void flush_thread(void) } int copy_thread(unsigned long clone_flags, unsigned long usp, - unsigned long unused, struct task_struct *p, struct pt_regs *regs) + unsigned long arg, struct task_struct *p) { struct thread_info *ti = task_thread_info(p); - struct pt_regs *childregs; + struct pt_regs *childregs, *regs = current_pt_regs(); unsigned long childksp; p->set_child_tid = p->clear_child_tid = NULL; @@ -126,7 +92,9 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, preempt_disable(); - if (is_fpu_owner()) + if (is_msa_enabled()) + save_msa(p); + else if (is_fpu_owner()) save_fp(p); if (cpu_has_dsp) @@ -138,20 +106,31 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, childregs = (struct pt_regs *) childksp - 1; /* Put the stack after the struct pt_regs. */ childksp = (unsigned long) childregs; - *childregs = *regs; - childregs->regs[7] = 0; /* Clear error flag */ - - childregs->regs[2] = 0; /* Child gets zero as return value */ - regs->regs[2] = p->pid; - - if (childregs->cp0_status & ST0_CU0) { - childregs->regs[28] = (unsigned long) ti; - childregs->regs[29] = childksp; + p->thread.cp0_status = read_c0_status() & ~(ST0_CU2|ST0_CU1); + if (unlikely(p->flags & PF_KTHREAD)) { + unsigned long status = p->thread.cp0_status; + memset(childregs, 0, sizeof(struct pt_regs)); ti->addr_limit = KERNEL_DS; - } else { - childregs->regs[29] = usp; - ti->addr_limit = USER_DS; + p->thread.reg16 = usp; /* fn */ + p->thread.reg17 = arg; + p->thread.reg29 = childksp; + p->thread.reg31 = (unsigned long) ret_from_kernel_thread; +#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) + status = (status & ~(ST0_KUP | ST0_IEP | ST0_IEC)) | + ((status & (ST0_KUC | ST0_IEC)) << 2); +#else + status |= ST0_EXL; +#endif + childregs->cp0_status = status; + return 0; } + *childregs = *regs; + childregs->regs[7] = 0; /* Clear error flag */ + childregs->regs[2] = 0; /* Child gets zero as return value */ + if (usp) + childregs->regs[29] = usp; + ti->addr_limit = USER_DS; + p->thread.reg29 = (unsigned long) childregs; p->thread.reg31 = (unsigned long) ret_from_fork; @@ -159,16 +138,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, * New tasks lose permission to use the fpu. This accelerates context * switching for most programs since they don't use the fpu. */ - p->thread.cp0_status = read_c0_status() & ~(ST0_CU2|ST0_CU1); childregs->cp0_status &= ~(ST0_CU2|ST0_CU1); -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC restores TCStatus after Status, and the CU bits - * are aliased there. - */ - childregs->cp0_tcstatus &= ~(ST0_CU2|ST0_CU1); -#endif clear_tsk_thread_flag(p, TIF_USEDFPU); #ifdef CONFIG_MIPS_MT_FPAFF @@ -184,7 +155,13 @@ int copy_thread(unsigned long clone_flags, unsigned long usp, /* Fill in the fpu structure for a core dump.. */ int dump_fpu(struct pt_regs *regs, elf_fpregset_t *r) { - memcpy(r, ¤t->thread.fpu, sizeof(current->thread.fpu)); + int i; + + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&r[i], ¤t->thread.fpu.fpr[i], sizeof(*r)); + + memcpy(&r[NUM_FPU_REGS], ¤t->thread.fpu.fcr31, + sizeof(current->thread.fpu.fcr31)); return 1; } @@ -219,43 +196,23 @@ int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs) int dump_task_fpu(struct task_struct *t, elf_fpregset_t *fpr) { - memcpy(fpr, &t->thread.fpu, sizeof(current->thread.fpu)); - - return 1; -} + int i; -/* - * Create a kernel thread - */ -static void __noreturn kernel_thread_helper(void *arg, int (*fn)(void *)) -{ - do_exit(fn(arg)); -} + for (i = 0; i < NUM_FPU_REGS; i++) + memcpy(&fpr[i], &t->thread.fpu.fpr[i], sizeof(*fpr)); -long kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) -{ - struct pt_regs regs; + memcpy(&fpr[NUM_FPU_REGS], &t->thread.fpu.fcr31, + sizeof(t->thread.fpu.fcr31)); - memset(®s, 0, sizeof(regs)); + return 1; +} - regs.regs[4] = (unsigned long) arg; - regs.regs[5] = (unsigned long) fn; - regs.cp0_epc = (unsigned long) kernel_thread_helper; - regs.cp0_status = read_c0_status(); -#if defined(CONFIG_CPU_R3000) || defined(CONFIG_CPU_TX39XX) - regs.cp0_status = (regs.cp0_status & ~(ST0_KUP | ST0_IEP | ST0_IEC)) | - ((regs.cp0_status & (ST0_KUC | ST0_IEC)) << 2); -#else - regs.cp0_status |= ST0_EXL; +#ifdef CONFIG_CC_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); #endif - /* Ok, create the new process.. */ - return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, ®s, 0, NULL, NULL); -} - -/* - * - */ struct mips_frame_info { void *func; unsigned long func_size; @@ -263,36 +220,122 @@ struct mips_frame_info { int pc_offset; }; +#define J_TARGET(pc,target) \ + (((unsigned long)(pc) & 0xf0000000) | ((target) << 2)) + static inline int is_ra_save_ins(union mips_instruction *ip) { +#ifdef CONFIG_CPU_MICROMIPS + union mips_instruction mmi; + + /* + * swsp ra,offset + * swm16 reglist,offset(sp) + * swm32 reglist,offset(sp) + * sw32 ra,offset(sp) + * jradiussp - NOT SUPPORTED + * + * microMIPS is way more fun... + */ + if (mm_insn_16bit(ip->halfword[0])) { + mmi.word = (ip->halfword[0] << 16); + return ((mmi.mm16_r5_format.opcode == mm_swsp16_op && + mmi.mm16_r5_format.rt == 31) || + (mmi.mm16_m_format.opcode == mm_pool16c_op && + mmi.mm16_m_format.func == mm_swm16_op)); + } + else { + mmi.halfword[0] = ip->halfword[1]; + mmi.halfword[1] = ip->halfword[0]; + return ((mmi.mm_m_format.opcode == mm_pool32b_op && + mmi.mm_m_format.rd > 9 && + mmi.mm_m_format.base == 29 && + mmi.mm_m_format.func == mm_swm32_func) || + (mmi.i_format.opcode == mm_sw32_op && + mmi.i_format.rs == 29 && + mmi.i_format.rt == 31)); + } +#else /* sw / sd $ra, offset($sp) */ return (ip->i_format.opcode == sw_op || ip->i_format.opcode == sd_op) && ip->i_format.rs == 29 && ip->i_format.rt == 31; +#endif } -static inline int is_jal_jalr_jr_ins(union mips_instruction *ip) +static inline int is_jump_ins(union mips_instruction *ip) { +#ifdef CONFIG_CPU_MICROMIPS + /* + * jr16,jrc,jalr16,jalr16 + * jal + * jalr/jr,jalr.hb/jr.hb,jalrs,jalrs.hb + * jraddiusp - NOT SUPPORTED + * + * microMIPS is kind of more fun... + */ + union mips_instruction mmi; + + mmi.word = (ip->halfword[0] << 16); + + if ((mmi.mm16_r5_format.opcode == mm_pool16c_op && + (mmi.mm16_r5_format.rt & mm_jr16_op) == mm_jr16_op) || + ip->j_format.opcode == mm_jal32_op) + return 1; + if (ip->r_format.opcode != mm_pool32a_op || + ip->r_format.func != mm_pool32axf_op) + return 0; + return (((ip->u_format.uimmediate >> 6) & mm_jalr_op) == mm_jalr_op); +#else + if (ip->j_format.opcode == j_op) + return 1; if (ip->j_format.opcode == jal_op) return 1; if (ip->r_format.opcode != spec_op) return 0; return ip->r_format.func == jalr_op || ip->r_format.func == jr_op; +#endif } static inline int is_sp_move_ins(union mips_instruction *ip) { +#ifdef CONFIG_CPU_MICROMIPS + /* + * addiusp -imm + * addius5 sp,-imm + * addiu32 sp,sp,-imm + * jradiussp - NOT SUPPORTED + * + * microMIPS is not more fun... + */ + if (mm_insn_16bit(ip->halfword[0])) { + union mips_instruction mmi; + + mmi.word = (ip->halfword[0] << 16); + return ((mmi.mm16_r3_format.opcode == mm_pool16d_op && + mmi.mm16_r3_format.simmediate && mm_addiusp_func) || + (mmi.mm16_r5_format.opcode == mm_pool16d_op && + mmi.mm16_r5_format.rt == 29)); + } + return (ip->mm_i_format.opcode == mm_addiu32_op && + ip->mm_i_format.rt == 29 && ip->mm_i_format.rs == 29); +#else /* addiu/daddiu sp,sp,-imm */ if (ip->i_format.rs != 29 || ip->i_format.rt != 29) return 0; if (ip->i_format.opcode == addiu_op || ip->i_format.opcode == daddiu_op) return 1; +#endif return 0; } static int get_frame_info(struct mips_frame_info *info) { +#ifdef CONFIG_CPU_MICROMIPS + union mips_instruction *ip = (void *) (((char *) info->func) - 1); +#else union mips_instruction *ip = info->func; +#endif unsigned max_insns = info->func_size / sizeof(union mips_instruction); unsigned i; @@ -308,11 +351,30 @@ static int get_frame_info(struct mips_frame_info *info) for (i = 0; i < max_insns; i++, ip++) { - if (is_jal_jalr_jr_ins(ip)) + if (is_jump_ins(ip)) break; if (!info->frame_size) { if (is_sp_move_ins(ip)) + { +#ifdef CONFIG_CPU_MICROMIPS + if (mm_insn_16bit(ip->halfword[0])) + { + unsigned short tmp; + + if (ip->halfword[0] & mm_addiusp_func) + { + tmp = (((ip->halfword[0] >> 1) & 0x1ff) << 2); + info->frame_size = -(signed short)(tmp | ((tmp & 0x100) ? 0xfe00 : 0)); + } else { + tmp = (ip->halfword[0] >> 1); + info->frame_size = -(signed short)(tmp & 0xf); + } + ip = (void *) &ip->halfword[1]; + ip--; + } else +#endif info->frame_size = - ip->i_format.simmediate; + } continue; } if (info->pc_offset == -1 && is_ra_save_ins(ip)) { @@ -332,15 +394,42 @@ err: static struct mips_frame_info schedule_mfi __read_mostly; +#ifdef CONFIG_KALLSYMS +static unsigned long get___schedule_addr(void) +{ + return kallsyms_lookup_name("__schedule"); +} +#else +static unsigned long get___schedule_addr(void) +{ + union mips_instruction *ip = (void *)schedule; + int max_insns = 8; + int i; + + for (i = 0; i < max_insns; i++, ip++) { + if (ip->j_format.opcode == j_op) + return J_TARGET(ip, ip->j_format.target); + } + return 0; +} +#endif + static int __init frame_info_init(void) { unsigned long size = 0; #ifdef CONFIG_KALLSYMS unsigned long ofs; +#endif + unsigned long addr; - kallsyms_lookup_size_offset((unsigned long)schedule, &size, &ofs); + addr = get___schedule_addr(); + if (!addr) + addr = (unsigned long)schedule; + +#ifdef CONFIG_KALLSYMS + kallsyms_lookup_size_offset(addr, &size, &ofs); #endif - schedule_mfi.func = schedule; + schedule_mfi.func = (void *)addr; schedule_mfi.func_size = size; get_frame_info(&schedule_mfi); @@ -374,18 +463,18 @@ unsigned long thread_saved_pc(struct task_struct *tsk) #ifdef CONFIG_KALLSYMS -/* used by show_backtrace() */ -unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, - unsigned long pc, unsigned long *ra) +/* generic stack unwinding function */ +unsigned long notrace unwind_stack_by_address(unsigned long stack_page, + unsigned long *sp, + unsigned long pc, + unsigned long *ra) { - unsigned long stack_page; struct mips_frame_info info; unsigned long size, ofs; int leaf; extern void ret_from_irq(void); extern void ret_from_exception(void); - stack_page = (unsigned long)task_stack_page(task); if (!stack_page) return 0; @@ -411,7 +500,7 @@ unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, if (!kallsyms_lookup_size_offset(pc, &size, &ofs)) return 0; /* - * Return ra if an exception occured at the first instruction + * Return ra if an exception occurred at the first instruction */ if (unlikely(ofs == 0)) { pc = *ra; @@ -444,6 +533,15 @@ unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, *ra = 0; return __kernel_text_address(pc) ? pc : 0; } +EXPORT_SYMBOL(unwind_stack_by_address); + +/* used by show_backtrace() */ +unsigned long unwind_stack(struct task_struct *task, unsigned long *sp, + unsigned long pc, unsigned long *ra) +{ + unsigned long stack_page = (unsigned long)task_stack_page(task); + return unwind_stack_by_address(stack_page, sp, pc, ra); +} #endif /* diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c index e000b278f02..5d39bb85bf3 100644 --- a/arch/mips/kernel/prom.c +++ b/arch/mips/kernel/prom.c @@ -9,104 +9,49 @@ */ #include <linux/init.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/errno.h> #include <linux/types.h> #include <linux/bootmem.h> -#include <linux/initrd.h> #include <linux/debugfs.h> #include <linux/of.h> #include <linux/of_fdt.h> -#include <linux/of_irq.h> -#include <linux/of_platform.h> #include <asm/page.h> #include <asm/prom.h> -int __init early_init_dt_scan_memory_arch(unsigned long node, - const char *uname, int depth, - void *data) -{ - return early_init_dt_scan_memory(node, uname, depth, data); -} - -void __init early_init_dt_add_memory_arch(u64 base, u64 size) -{ - return add_memory_region(base, size, BOOT_MEM_RAM); -} +static char mips_machine_name[64] = "Unknown"; -int __init reserve_mem_mach(unsigned long addr, unsigned long size) +__init void mips_set_machine_name(const char *name) { - return reserve_bootmem(addr, size, BOOTMEM_DEFAULT); -} - -void __init free_mem_mach(unsigned long addr, unsigned long size) -{ - return free_bootmem(addr, size); -} + if (name == NULL) + return; -u64 __init early_init_dt_alloc_memory_arch(u64 size, u64 align) -{ - return virt_to_phys( - __alloc_bootmem(size, align, __pa(MAX_DMA_ADDRESS)) - ); + strlcpy(mips_machine_name, name, sizeof(mips_machine_name)); + pr_info("MIPS: machine is %s\n", mips_get_machine_name()); } -#ifdef CONFIG_BLK_DEV_INITRD -void __init early_init_dt_setup_initrd_arch(unsigned long start, - unsigned long end) +char *mips_get_machine_name(void) { - initrd_start = (unsigned long)__va(start); - initrd_end = (unsigned long)__va(end); - initrd_below_start_ok = 1; + return mips_machine_name; } -#endif -/* - * irq_create_of_mapping - Hook to resolve OF irq specifier into a Linux irq# - * - * Currently the mapping mechanism is trivial; simple flat hwirq numbers are - * mapped 1:1 onto Linux irq numbers. Cascaded irq controllers are not - * supported. - */ -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) +#ifdef CONFIG_OF +void __init early_init_dt_add_memory_arch(u64 base, u64 size) { - return intspec[0]; + return add_memory_region(base, size, BOOT_MEM_RAM); } -EXPORT_SYMBOL_GPL(irq_create_of_mapping); -void __init early_init_devtree(void *params) +void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) { - /* Setup flat device-tree pointer */ - initial_boot_params = params; - - /* Retrieve various informations from the /chosen node of the - * device-tree, including the platform type, initrd location and - * size, and more ... - */ - of_scan_flat_dt(early_init_dt_scan_chosen, NULL); - - /* Scan memory nodes */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); - of_scan_flat_dt(early_init_dt_scan_memory_arch, NULL); + return __alloc_bootmem(size, align, __pa(MAX_DMA_ADDRESS)); } -void __init device_tree_init(void) +void __init __dt_setup_arch(void *bph) { - unsigned long base, size; - - if (!initial_boot_params) + if (!early_init_dt_scan(bph)) return; - base = virt_to_phys((void *)initial_boot_params); - size = initial_boot_params->totalsize; - - /* Before we do anything, lets reserve the dt blob */ - reserve_mem_mach(base, size); - - unflatten_device_tree(); - - /* free the space reserved for the dt blob */ - free_mem_mach(base, size); + mips_set_machine_name(of_flat_dt_get_machine_name()); } +#endif diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index d21c388c011..f639ccd5060 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -15,16 +15,21 @@ * binaries. */ #include <linux/compiler.h> +#include <linux/context_tracking.h> +#include <linux/elf.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/errno.h> #include <linux/ptrace.h> +#include <linux/regset.h> #include <linux/smp.h> #include <linux/user.h> #include <linux/security.h> +#include <linux/tracehook.h> #include <linux/audit.h> #include <linux/seccomp.h> +#include <linux/ftrace.h> #include <asm/byteorder.h> #include <asm/cpu.h> @@ -34,11 +39,14 @@ #include <asm/mipsmtregs.h> #include <asm/pgtable.h> #include <asm/page.h> -#include <asm/system.h> +#include <asm/syscall.h> #include <asm/uaccess.h> #include <asm/bootinfo.h> #include <asm/reg.h> +#define CREATE_TRACE_POINTS +#include <trace/events/syscalls.h> + /* * Called by kernel/ptrace.c when detaching.. * @@ -51,7 +59,7 @@ void ptrace_disable(struct task_struct *child) } /* - * Read a general register set. We always use the 64-bit format, even + * Read a general register set. We always use the 64-bit format, even * for 32-bit kernels and for 32-bit processes on a 64-bit kernel. * Registers are sign extended to fill the available space. */ @@ -106,51 +114,30 @@ int ptrace_setregs(struct task_struct *child, __s64 __user *data) int ptrace_getfpregs(struct task_struct *child, __u32 __user *data) { int i; - unsigned int tmp; if (!access_ok(VERIFY_WRITE, data, 33 * 8)) return -EIO; if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); + union fpureg *fregs = get_fpu_regs(child); for (i = 0; i < 32; i++) - __put_user(fregs[i], i + (__u64 __user *) data); + __put_user(get_fpr64(&fregs[i], 0), + i + (__u64 __user *)data); } else { for (i = 0; i < 32; i++) __put_user((__u64) -1, i + (__u64 __user *) data); } __put_user(child->thread.fpu.fcr31, data + 64); - - preempt_disable(); - if (cpu_has_fpu) { - unsigned int flags; - - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0" : "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0" : "=r" (tmp)); - write_c0_status(flags); - } - } else { - tmp = 0; - } - preempt_enable(); - __put_user(tmp, data + 65); + __put_user(current_cpu_data.fpu_id, data + 65); return 0; } int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) { - fpureg_t *fregs; + union fpureg *fregs; + u64 fpr_val; int i; if (!access_ok(VERIFY_READ, data, 33 * 8)) @@ -158,8 +145,10 @@ int ptrace_setfpregs(struct task_struct *child, __u32 __user *data) fregs = get_fpu_regs(child); - for (i = 0; i < 32; i++) - __get_user(fregs[i], i + (__u64 __user *) data); + for (i = 0; i < 32; i++) { + __get_user(fpr_val, i + (__u64 __user *)data); + set_fpr64(&fregs[i], 0, fpr_val); + } __get_user(child->thread.fpu.fcr31, data + 64); @@ -174,7 +163,7 @@ int ptrace_get_watch_regs(struct task_struct *child, enum pt_watch_style style; int i; - if (!cpu_has_watch || current_cpu_data.watch_reg_use_cnt == 0) + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; if (!access_ok(VERIFY_WRITE, addr, sizeof(struct pt_watch_regs))) return -EIO; @@ -188,14 +177,14 @@ int ptrace_get_watch_regs(struct task_struct *child, #endif __put_user(style, &addr->style); - __put_user(current_cpu_data.watch_reg_use_cnt, + __put_user(boot_cpu_data.watch_reg_use_cnt, &addr->WATCH_STYLE.num_valid); - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { __put_user(child->thread.watch.mips3264.watchlo[i], &addr->WATCH_STYLE.watchlo[i]); __put_user(child->thread.watch.mips3264.watchhi[i] & 0xfff, &addr->WATCH_STYLE.watchhi[i]); - __put_user(current_cpu_data.watch_reg_masks[i], + __put_user(boot_cpu_data.watch_reg_masks[i], &addr->WATCH_STYLE.watch_masks[i]); } for (; i < 8; i++) { @@ -215,12 +204,12 @@ int ptrace_set_watch_regs(struct task_struct *child, unsigned long lt[NUM_WATCH_REGS]; u16 ht[NUM_WATCH_REGS]; - if (!cpu_has_watch || current_cpu_data.watch_reg_use_cnt == 0) + if (!cpu_has_watch || boot_cpu_data.watch_reg_use_cnt == 0) return -EIO; if (!access_ok(VERIFY_READ, addr, sizeof(struct pt_watch_regs))) return -EIO; /* Check the values. */ - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { __get_user(lt[i], &addr->WATCH_STYLE.watchlo[i]); #ifdef CONFIG_32BIT if (lt[i] & __UA_LIMIT) @@ -239,7 +228,7 @@ int ptrace_set_watch_regs(struct task_struct *child, return -EINVAL; } /* Install them. */ - for (i = 0; i < current_cpu_data.watch_reg_use_cnt; i++) { + for (i = 0; i < boot_cpu_data.watch_reg_use_cnt; i++) { if (lt[i] & 7) watch_active = 1; child->thread.watch.mips3264.watchlo[i] = lt[i]; @@ -255,6 +244,167 @@ int ptrace_set_watch_regs(struct task_struct *child, return 0; } +/* regset get/set implementations */ + +static int gpr_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + regs, 0, sizeof(*regs)); +} + +static int gpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs newregs; + int ret; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &newregs, + 0, sizeof(newregs)); + if (ret) + return ret; + + *task_pt_regs(target) = newregs; + + return 0; +} + +static int fpr_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + unsigned i; + int err; + u64 fpr_val; + + /* XXX fcr31 */ + + if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t)) + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu, + 0, sizeof(elf_fpregset_t)); + + for (i = 0; i < NUM_FPU_REGS; i++) { + fpr_val = get_fpr64(&target->thread.fpu.fpr[i], 0); + err = user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &fpr_val, i * sizeof(elf_fpreg_t), + (i + 1) * sizeof(elf_fpreg_t)); + if (err) + return err; + } + + return 0; +} + +static int fpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned i; + int err; + u64 fpr_val; + + /* XXX fcr31 */ + + if (sizeof(target->thread.fpu.fpr[i]) == sizeof(elf_fpreg_t)) + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fpu, + 0, sizeof(elf_fpregset_t)); + + for (i = 0; i < NUM_FPU_REGS; i++) { + err = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fpr_val, i * sizeof(elf_fpreg_t), + (i + 1) * sizeof(elf_fpreg_t)); + if (err) + return err; + set_fpr64(&target->thread.fpu.fpr[i], 0, fpr_val); + } + + return 0; +} + +enum mips_regset { + REGSET_GPR, + REGSET_FPR, +}; + +static const struct user_regset mips_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(unsigned int), + .align = sizeof(unsigned int), + .get = gpr_get, + .set = gpr_set, + }, + [REGSET_FPR] = { + .core_note_type = NT_PRFPREG, + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .get = fpr_get, + .set = fpr_set, + }, +}; + +static const struct user_regset_view user_mips_view = { + .name = "mips", + .e_machine = ELF_ARCH, + .ei_osabi = ELF_OSABI, + .regsets = mips_regsets, + .n = ARRAY_SIZE(mips_regsets), +}; + +static const struct user_regset mips64_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(unsigned long), + .align = sizeof(unsigned long), + .get = gpr_get, + .set = gpr_set, + }, + [REGSET_FPR] = { + .core_note_type = NT_PRFPREG, + .n = ELF_NFPREG, + .size = sizeof(elf_fpreg_t), + .align = sizeof(elf_fpreg_t), + .get = fpr_get, + .set = fpr_set, + }, +}; + +static const struct user_regset_view user_mips64_view = { + .name = "mips", + .e_machine = ELF_ARCH, + .ei_osabi = ELF_OSABI, + .regsets = mips64_regsets, + .n = ARRAY_SIZE(mips_regsets), +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ +#ifdef CONFIG_32BIT + return &user_mips_view; +#endif + +#ifdef CONFIG_MIPS32_O32 + if (test_thread_flag(TIF_32BIT_REGS)) + return &user_mips_view; +#endif + + return &user_mips64_view; +} + long arch_ptrace(struct task_struct *child, long request, unsigned long addr, unsigned long data) { @@ -273,6 +423,7 @@ long arch_ptrace(struct task_struct *child, long request, /* Read the word at location addr in the USER area. */ case PTRACE_PEEKUSR: { struct pt_regs *regs; + union fpureg *fregs; unsigned long tmp = 0; regs = task_pt_regs(child); @@ -283,26 +434,26 @@ long arch_ptrace(struct task_struct *child, long request, tmp = regs->regs[addr]; break; case FPR_BASE ... FPR_BASE + 31: - if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); + if (!tsk_used_math(child)) { + /* FP not yet used */ + tmp = -1; + break; + } + fregs = get_fpu_regs(child); #ifdef CONFIG_32BIT + if (test_thread_flag(TIF_32BIT_FPREGS)) { /* * The odd registers are actually the high * order bits of the values stored in the even * registers - unless we're using r2k_switch.S. */ - if (addr & 1) - tmp = (unsigned long) (fregs[((addr & ~1) - 32)] >> 32); - else - tmp = (unsigned long) (fregs[(addr - 32)] & 0xffffffff); -#endif -#ifdef CONFIG_64BIT - tmp = fregs[addr - FPR_BASE]; -#endif - } else { - tmp = -1; /* FP not yet used */ + tmp = get_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1); + break; } +#endif + tmp = get_fpr32(&fregs[addr - FPR_BASE], 0); break; case PC: tmp = regs->cp0_epc; @@ -327,44 +478,10 @@ long arch_ptrace(struct task_struct *child, long request, case FPC_CSR: tmp = child->thread.fpu.fcr31; break; - case FPC_EIR: { /* implementation / version register */ - unsigned int flags; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned long irqflags; - unsigned int mtflags; -#endif /* CONFIG_MIPS_MT_SMTC */ - - preempt_disable(); - if (!cpu_has_fpu) { - preempt_enable(); - break; - } - -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-write of Status must be atomic */ - local_irq_save(irqflags); - mtflags = dmt(); -#endif /* CONFIG_MIPS_MT_SMTC */ - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - } -#ifdef CONFIG_MIPS_MT_SMTC - emt(mtflags); - local_irq_restore(irqflags); -#endif /* CONFIG_MIPS_MT_SMTC */ - preempt_enable(); + case FPC_EIR: + /* implementation / version register */ + tmp = current_cpu_data.fpu_id; break; - } case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -410,7 +527,7 @@ long arch_ptrace(struct task_struct *child, long request, regs->regs[addr] = data; break; case FPR_BASE ... FPR_BASE + 31: { - fpureg_t *fregs = get_fpu_regs(child); + union fpureg *fregs = get_fpu_regs(child); if (!tsk_used_math(child)) { /* FP not yet used */ @@ -419,22 +536,18 @@ long arch_ptrace(struct task_struct *child, long request, child->thread.fpu.fcr31 = 0; } #ifdef CONFIG_32BIT - /* - * The odd registers are actually the high order bits - * of the values stored in the even registers - unless - * we're using r2k_switch.S. - */ - if (addr & 1) { - fregs[(addr & ~1) - FPR_BASE] &= 0xffffffff; - fregs[(addr & ~1) - FPR_BASE] |= ((unsigned long long) data) << 32; - } else { - fregs[addr - FPR_BASE] &= ~0xffffffffLL; - fregs[addr - FPR_BASE] |= data; + if (test_thread_flag(TIF_32BIT_FPREGS)) { + /* + * The odd registers are actually the high + * order bits of the values stored in the even + * registers - unless we're using r2k_switch.S. + */ + set_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1, data); + break; } #endif -#ifdef CONFIG_64BIT - fregs[addr - FPR_BASE] = data; -#endif + set_fpr64(&fregs[addr - FPR_BASE], 0, data); break; } case PC: @@ -517,56 +630,52 @@ long arch_ptrace(struct task_struct *child, long request, return ret; } -static inline int audit_arch(void) -{ - int arch = EM_MIPS; -#ifdef CONFIG_64BIT - arch |= __AUDIT_ARCH_64BIT; -#endif -#if defined(__LITTLE_ENDIAN) - arch |= __AUDIT_ARCH_LE; -#endif - return arch; -} - /* * Notification of system call entry/exit * - triggered by current->work.syscall_trace */ -asmlinkage void do_syscall_trace(struct pt_regs *regs, int entryexit) +asmlinkage long syscall_trace_enter(struct pt_regs *regs, long syscall) { - /* do the secure computing check first */ - if (!entryexit) - secure_computing(regs->regs[2]); + long ret = 0; + user_exit(); - if (unlikely(current->audit_context) && entryexit) - audit_syscall_exit(AUDITSC_RESULT(regs->regs[2]), - regs->regs[2]); + if (secure_computing(syscall) == -1) + return -1; - if (!(current->ptrace & PT_PTRACED)) - goto out; + if (test_thread_flag(TIF_SYSCALL_TRACE) && + tracehook_report_syscall_entry(regs)) + ret = -1; - if (!test_thread_flag(TIF_SYSCALL_TRACE)) - goto out; + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->regs[2]); - /* The 0x80 provides a way for the tracing parent to distinguish - between a syscall stop and SIGTRAP delivery */ - ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD) ? - 0x80 : 0)); + audit_syscall_entry(syscall_get_arch(), + syscall, + regs->regs[4], regs->regs[5], + regs->regs[6], regs->regs[7]); + return syscall; +} - /* - * this isn't the same as continuing with a signal, but it will do - * for normal use. strace only continues with a signal if the - * stopping signal is not SIGTRAP. -brl +/* + * Notification of system call entry/exit + * - triggered by current->work.syscall_trace + */ +asmlinkage void syscall_trace_leave(struct pt_regs *regs) +{ + /* + * We may come here right after calling schedule_user() + * or do_notify_resume(), in which case we can be in RCU + * user mode. */ - if (current->exit_code) { - send_sig(current->exit_code, current, 1); - current->exit_code = 0; - } + user_exit(); + + audit_syscall_exit(regs); + + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs->regs[2]); + + if (test_thread_flag(TIF_SYSCALL_TRACE)) + tracehook_report_syscall_exit(regs, 0); -out: - if (unlikely(current->audit_context) && !entryexit) - audit_syscall_entry(audit_arch(), regs->regs[2], - regs->regs[4], regs->regs[5], - regs->regs[6], regs->regs[7]); + user_enter(); } diff --git a/arch/mips/kernel/ptrace32.c b/arch/mips/kernel/ptrace32.c index 32644b4a071..b40c3ca60ee 100644 --- a/arch/mips/kernel/ptrace32.c +++ b/arch/mips/kernel/ptrace32.c @@ -32,7 +32,6 @@ #include <asm/mipsmtregs.h> #include <asm/pgtable.h> #include <asm/page.h> -#include <asm/system.h> #include <asm/uaccess.h> #include <asm/bootinfo.h> @@ -81,6 +80,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, /* Read the word at location addr in the USER area. */ case PTRACE_PEEKUSR: { struct pt_regs *regs; + union fpureg *fregs; unsigned int tmp; regs = task_pt_regs(child); @@ -91,21 +91,23 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, tmp = regs->regs[addr]; break; case FPR_BASE ... FPR_BASE + 31: - if (tsk_used_math(child)) { - fpureg_t *fregs = get_fpu_regs(child); - + if (!tsk_used_math(child)) { + /* FP not yet used */ + tmp = -1; + break; + } + fregs = get_fpu_regs(child); + if (test_thread_flag(TIF_32BIT_FPREGS)) { /* * The odd registers are actually the high * order bits of the values stored in the even * registers - unless we're using r2k_switch.S. */ - if (addr & 1) - tmp = (unsigned long) (fregs[((addr & ~1) - 32)] >> 32); - else - tmp = (unsigned long) (fregs[(addr - 32)] & 0xffffffff); - } else { - tmp = -1; /* FP not yet used */ + tmp = get_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1); + break; } + tmp = get_fpr32(&fregs[addr - FPR_BASE], 0); break; case PC: tmp = regs->cp0_epc; @@ -125,46 +127,10 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, case FPC_CSR: tmp = child->thread.fpu.fcr31; break; - case FPC_EIR: { /* implementation / version register */ - unsigned int flags; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned int irqflags; - unsigned int mtflags; -#endif /* CONFIG_MIPS_MT_SMTC */ - - preempt_disable(); - if (!cpu_has_fpu) { - preempt_enable(); - tmp = 0; - break; - } - -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-write of Status must be atomic */ - local_irq_save(irqflags); - mtflags = dmt(); -#endif /* CONFIG_MIPS_MT_SMTC */ - - if (cpu_has_mipsmt) { - unsigned int vpflags = dvpe(); - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - evpe(vpflags); - } else { - flags = read_c0_status(); - __enable_fpu(); - __asm__ __volatile__("cfc1\t%0,$0": "=r" (tmp)); - write_c0_status(flags); - } -#ifdef CONFIG_MIPS_MT_SMTC - emt(mtflags); - local_irq_restore(irqflags); -#endif /* CONFIG_MIPS_MT_SMTC */ - preempt_enable(); + case FPC_EIR: + /* implementation / version register */ + tmp = current_cpu_data.fpu_id; break; - } case DSP_BASE ... DSP_BASE + 5: { dspreg_t *dregs; @@ -229,7 +195,7 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, regs->regs[addr] = data; break; case FPR_BASE ... FPR_BASE + 31: { - fpureg_t *fregs = get_fpu_regs(child); + union fpureg *fregs = get_fpu_regs(child); if (!tsk_used_math(child)) { /* FP not yet used */ @@ -237,20 +203,17 @@ long compat_arch_ptrace(struct task_struct *child, compat_long_t request, sizeof(child->thread.fpu)); child->thread.fpu.fcr31 = 0; } - /* - * The odd registers are actually the high order bits - * of the values stored in the even registers - unless - * we're using r2k_switch.S. - */ - if (addr & 1) { - fregs[(addr & ~1) - FPR_BASE] &= 0xffffffff; - fregs[(addr & ~1) - FPR_BASE] |= ((unsigned long long) data) << 32; - } else { - fregs[addr - FPR_BASE] &= ~0xffffffffLL; - /* Must cast, lest sign extension fill upper - bits! */ - fregs[addr - FPR_BASE] |= (unsigned int)data; + if (test_thread_flag(TIF_32BIT_FPREGS)) { + /* + * The odd registers are actually the high + * order bits of the values stored in the even + * registers - unless we're using r2k_switch.S. + */ + set_fpr32(&fregs[(addr & ~1) - FPR_BASE], + addr & 1, data); + break; } + set_fpr64(&fregs[addr - FPR_BASE], 0, data); break; } case PC: diff --git a/arch/mips/kernel/r2300_fpu.S b/arch/mips/kernel/r2300_fpu.S index ac68e68339d..f31063dbdae 100644 --- a/arch/mips/kernel/r2300_fpu.S +++ b/arch/mips/kernel/r2300_fpu.S @@ -6,7 +6,7 @@ * Copyright (C) 1996, 1998 by Ralf Baechle * * Multi-arch abstraction and asm macros for easier reading: - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * * Further modifications to make this work: * Copyright (c) 1998 Harald Koerfgen @@ -30,38 +30,38 @@ LEAF(_save_fp_context) li v0, 0 # assume success cfc1 t1,fcr31 - EX(swc1 $f0,(SC_FPREGS+0)(a0)) - EX(swc1 $f1,(SC_FPREGS+8)(a0)) - EX(swc1 $f2,(SC_FPREGS+16)(a0)) - EX(swc1 $f3,(SC_FPREGS+24)(a0)) - EX(swc1 $f4,(SC_FPREGS+32)(a0)) - EX(swc1 $f5,(SC_FPREGS+40)(a0)) - EX(swc1 $f6,(SC_FPREGS+48)(a0)) - EX(swc1 $f7,(SC_FPREGS+56)(a0)) - EX(swc1 $f8,(SC_FPREGS+64)(a0)) - EX(swc1 $f9,(SC_FPREGS+72)(a0)) - EX(swc1 $f10,(SC_FPREGS+80)(a0)) - EX(swc1 $f11,(SC_FPREGS+88)(a0)) - EX(swc1 $f12,(SC_FPREGS+96)(a0)) - EX(swc1 $f13,(SC_FPREGS+104)(a0)) - EX(swc1 $f14,(SC_FPREGS+112)(a0)) - EX(swc1 $f15,(SC_FPREGS+120)(a0)) - EX(swc1 $f16,(SC_FPREGS+128)(a0)) - EX(swc1 $f17,(SC_FPREGS+136)(a0)) - EX(swc1 $f18,(SC_FPREGS+144)(a0)) - EX(swc1 $f19,(SC_FPREGS+152)(a0)) - EX(swc1 $f20,(SC_FPREGS+160)(a0)) - EX(swc1 $f21,(SC_FPREGS+168)(a0)) - EX(swc1 $f22,(SC_FPREGS+176)(a0)) - EX(swc1 $f23,(SC_FPREGS+184)(a0)) - EX(swc1 $f24,(SC_FPREGS+192)(a0)) - EX(swc1 $f25,(SC_FPREGS+200)(a0)) - EX(swc1 $f26,(SC_FPREGS+208)(a0)) - EX(swc1 $f27,(SC_FPREGS+216)(a0)) - EX(swc1 $f28,(SC_FPREGS+224)(a0)) - EX(swc1 $f29,(SC_FPREGS+232)(a0)) - EX(swc1 $f30,(SC_FPREGS+240)(a0)) - EX(swc1 $f31,(SC_FPREGS+248)(a0)) + EX(swc1 $f0,(SC_FPREGS+0)(a0)) + EX(swc1 $f1,(SC_FPREGS+8)(a0)) + EX(swc1 $f2,(SC_FPREGS+16)(a0)) + EX(swc1 $f3,(SC_FPREGS+24)(a0)) + EX(swc1 $f4,(SC_FPREGS+32)(a0)) + EX(swc1 $f5,(SC_FPREGS+40)(a0)) + EX(swc1 $f6,(SC_FPREGS+48)(a0)) + EX(swc1 $f7,(SC_FPREGS+56)(a0)) + EX(swc1 $f8,(SC_FPREGS+64)(a0)) + EX(swc1 $f9,(SC_FPREGS+72)(a0)) + EX(swc1 $f10,(SC_FPREGS+80)(a0)) + EX(swc1 $f11,(SC_FPREGS+88)(a0)) + EX(swc1 $f12,(SC_FPREGS+96)(a0)) + EX(swc1 $f13,(SC_FPREGS+104)(a0)) + EX(swc1 $f14,(SC_FPREGS+112)(a0)) + EX(swc1 $f15,(SC_FPREGS+120)(a0)) + EX(swc1 $f16,(SC_FPREGS+128)(a0)) + EX(swc1 $f17,(SC_FPREGS+136)(a0)) + EX(swc1 $f18,(SC_FPREGS+144)(a0)) + EX(swc1 $f19,(SC_FPREGS+152)(a0)) + EX(swc1 $f20,(SC_FPREGS+160)(a0)) + EX(swc1 $f21,(SC_FPREGS+168)(a0)) + EX(swc1 $f22,(SC_FPREGS+176)(a0)) + EX(swc1 $f23,(SC_FPREGS+184)(a0)) + EX(swc1 $f24,(SC_FPREGS+192)(a0)) + EX(swc1 $f25,(SC_FPREGS+200)(a0)) + EX(swc1 $f26,(SC_FPREGS+208)(a0)) + EX(swc1 $f27,(SC_FPREGS+216)(a0)) + EX(swc1 $f28,(SC_FPREGS+224)(a0)) + EX(swc1 $f29,(SC_FPREGS+232)(a0)) + EX(swc1 $f30,(SC_FPREGS+240)(a0)) + EX(swc1 $f31,(SC_FPREGS+248)(a0)) EX(sw t1,(SC_FPC_CSR)(a0)) cfc1 t0,$0 # implementation/version jr ra @@ -82,38 +82,38 @@ LEAF(_save_fp_context) LEAF(_restore_fp_context) li v0, 0 # assume success EX(lw t0,(SC_FPC_CSR)(a0)) - EX(lwc1 $f0,(SC_FPREGS+0)(a0)) - EX(lwc1 $f1,(SC_FPREGS+8)(a0)) - EX(lwc1 $f2,(SC_FPREGS+16)(a0)) - EX(lwc1 $f3,(SC_FPREGS+24)(a0)) - EX(lwc1 $f4,(SC_FPREGS+32)(a0)) - EX(lwc1 $f5,(SC_FPREGS+40)(a0)) - EX(lwc1 $f6,(SC_FPREGS+48)(a0)) - EX(lwc1 $f7,(SC_FPREGS+56)(a0)) - EX(lwc1 $f8,(SC_FPREGS+64)(a0)) - EX(lwc1 $f9,(SC_FPREGS+72)(a0)) - EX(lwc1 $f10,(SC_FPREGS+80)(a0)) - EX(lwc1 $f11,(SC_FPREGS+88)(a0)) - EX(lwc1 $f12,(SC_FPREGS+96)(a0)) - EX(lwc1 $f13,(SC_FPREGS+104)(a0)) - EX(lwc1 $f14,(SC_FPREGS+112)(a0)) - EX(lwc1 $f15,(SC_FPREGS+120)(a0)) - EX(lwc1 $f16,(SC_FPREGS+128)(a0)) - EX(lwc1 $f17,(SC_FPREGS+136)(a0)) - EX(lwc1 $f18,(SC_FPREGS+144)(a0)) - EX(lwc1 $f19,(SC_FPREGS+152)(a0)) - EX(lwc1 $f20,(SC_FPREGS+160)(a0)) - EX(lwc1 $f21,(SC_FPREGS+168)(a0)) - EX(lwc1 $f22,(SC_FPREGS+176)(a0)) - EX(lwc1 $f23,(SC_FPREGS+184)(a0)) - EX(lwc1 $f24,(SC_FPREGS+192)(a0)) - EX(lwc1 $f25,(SC_FPREGS+200)(a0)) - EX(lwc1 $f26,(SC_FPREGS+208)(a0)) - EX(lwc1 $f27,(SC_FPREGS+216)(a0)) - EX(lwc1 $f28,(SC_FPREGS+224)(a0)) - EX(lwc1 $f29,(SC_FPREGS+232)(a0)) - EX(lwc1 $f30,(SC_FPREGS+240)(a0)) - EX(lwc1 $f31,(SC_FPREGS+248)(a0)) + EX(lwc1 $f0,(SC_FPREGS+0)(a0)) + EX(lwc1 $f1,(SC_FPREGS+8)(a0)) + EX(lwc1 $f2,(SC_FPREGS+16)(a0)) + EX(lwc1 $f3,(SC_FPREGS+24)(a0)) + EX(lwc1 $f4,(SC_FPREGS+32)(a0)) + EX(lwc1 $f5,(SC_FPREGS+40)(a0)) + EX(lwc1 $f6,(SC_FPREGS+48)(a0)) + EX(lwc1 $f7,(SC_FPREGS+56)(a0)) + EX(lwc1 $f8,(SC_FPREGS+64)(a0)) + EX(lwc1 $f9,(SC_FPREGS+72)(a0)) + EX(lwc1 $f10,(SC_FPREGS+80)(a0)) + EX(lwc1 $f11,(SC_FPREGS+88)(a0)) + EX(lwc1 $f12,(SC_FPREGS+96)(a0)) + EX(lwc1 $f13,(SC_FPREGS+104)(a0)) + EX(lwc1 $f14,(SC_FPREGS+112)(a0)) + EX(lwc1 $f15,(SC_FPREGS+120)(a0)) + EX(lwc1 $f16,(SC_FPREGS+128)(a0)) + EX(lwc1 $f17,(SC_FPREGS+136)(a0)) + EX(lwc1 $f18,(SC_FPREGS+144)(a0)) + EX(lwc1 $f19,(SC_FPREGS+152)(a0)) + EX(lwc1 $f20,(SC_FPREGS+160)(a0)) + EX(lwc1 $f21,(SC_FPREGS+168)(a0)) + EX(lwc1 $f22,(SC_FPREGS+176)(a0)) + EX(lwc1 $f23,(SC_FPREGS+184)(a0)) + EX(lwc1 $f24,(SC_FPREGS+192)(a0)) + EX(lwc1 $f25,(SC_FPREGS+200)(a0)) + EX(lwc1 $f26,(SC_FPREGS+208)(a0)) + EX(lwc1 $f27,(SC_FPREGS+216)(a0)) + EX(lwc1 $f28,(SC_FPREGS+224)(a0)) + EX(lwc1 $f29,(SC_FPREGS+232)(a0)) + EX(lwc1 $f30,(SC_FPREGS+240)(a0)) + EX(lwc1 $f31,(SC_FPREGS+248)(a0)) jr ra ctc1 t0,fcr31 END(_restore_fp_context) diff --git a/arch/mips/kernel/r2300_switch.S b/arch/mips/kernel/r2300_switch.S index 698414b7a25..20b7b040e76 100644 --- a/arch/mips/kernel/r2300_switch.S +++ b/arch/mips/kernel/r2300_switch.S @@ -5,7 +5,7 @@ * Copyright (C) 1994, 1995, 1996 by Andreas Busse * * Multi-cpu abstraction and macros for easier reading: - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * * Further modifications to make this work: * Copyright (c) 1998-2000 Harald Koerfgen @@ -15,7 +15,6 @@ #include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/asm-offsets.h> -#include <asm/page.h> #include <asm/regdef.h> #include <asm/stackframe.h> #include <asm/thread_info.h> @@ -43,7 +42,7 @@ /* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti) ) + * struct thread_info *next_ti, int usedfpu) */ LEAF(resume) mfc0 t1, CP0_STATUS @@ -51,18 +50,9 @@ LEAF(resume) cpu_save_nonscratch a0 sw ra, THREAD_REG31(a0) - /* - * check if we need to save FPU registers - */ - lw t3, TASK_THREAD_INFO(a0) - lw t0, TI_FLAGS(t3) - li t1, _TIF_USEDFPU - and t2, t0, t1 - beqz t2, 1f - nor t1, zero, t1 + beqz a3, 1f - and t0, t0, t1 - sw t0, TI_FLAGS(t3) + PTR_L t3, TASK_THREAD_INFO(a0) /* * clear saved user stack CU1 bit @@ -75,6 +65,13 @@ LEAF(resume) fpu_save_single a0, t0 # clobbers t0 1: + +#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard + LONG_L t9, TASK_STACK_CANARY(a1) + LONG_S t9, 0(t8) +#endif + /* * The order of restoring the registers takes care of the race * updating $28, $29 and kernelsp without disabling ints. diff --git a/arch/mips/kernel/r4k_fpu.S b/arch/mips/kernel/r4k_fpu.S index dbd42adc52e..8352523568e 100644 --- a/arch/mips/kernel/r4k_fpu.S +++ b/arch/mips/kernel/r4k_fpu.S @@ -6,7 +6,7 @@ * Copyright (C) 1996, 98, 99, 2000, 01 Ralf Baechle * * Multi-arch abstraction and asm macros for easier reading: - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * * Carsten Langgaard, carstenl@mips.com * Copyright (C) 2000 MIPS Technologies, Inc. @@ -30,12 +30,20 @@ .endm .set noreorder - .set mips3 + .set arch=r4000 LEAF(_save_fp_context) cfc1 t1, fcr31 -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2) + .set push +#ifdef CONFIG_CPU_MIPS32_R2 + .set mips64r2 + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip storing odd if FR=0 + nop +#endif /* Store the 16 odd double precision registers */ EX sdc1 $f1, SC_FPREGS+8(a0) EX sdc1 $f3, SC_FPREGS+24(a0) @@ -53,6 +61,7 @@ LEAF(_save_fp_context) EX sdc1 $f27, SC_FPREGS+216(a0) EX sdc1 $f29, SC_FPREGS+232(a0) EX sdc1 $f31, SC_FPREGS+248(a0) +1: .set pop #endif /* Store the 16 even double precision registers */ @@ -82,7 +91,31 @@ LEAF(_save_fp_context) LEAF(_save_fp_context32) cfc1 t1, fcr31 - EX sdc1 $f0, SC32_FPREGS+0(a0) + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip storing odd if FR=0 + nop + + /* Store the 16 odd double precision registers */ + EX sdc1 $f1, SC32_FPREGS+8(a0) + EX sdc1 $f3, SC32_FPREGS+24(a0) + EX sdc1 $f5, SC32_FPREGS+40(a0) + EX sdc1 $f7, SC32_FPREGS+56(a0) + EX sdc1 $f9, SC32_FPREGS+72(a0) + EX sdc1 $f11, SC32_FPREGS+88(a0) + EX sdc1 $f13, SC32_FPREGS+104(a0) + EX sdc1 $f15, SC32_FPREGS+120(a0) + EX sdc1 $f17, SC32_FPREGS+136(a0) + EX sdc1 $f19, SC32_FPREGS+152(a0) + EX sdc1 $f21, SC32_FPREGS+168(a0) + EX sdc1 $f23, SC32_FPREGS+184(a0) + EX sdc1 $f25, SC32_FPREGS+200(a0) + EX sdc1 $f27, SC32_FPREGS+216(a0) + EX sdc1 $f29, SC32_FPREGS+232(a0) + EX sdc1 $f31, SC32_FPREGS+248(a0) + + /* Store the 16 even double precision registers */ +1: EX sdc1 $f0, SC32_FPREGS+0(a0) EX sdc1 $f2, SC32_FPREGS+16(a0) EX sdc1 $f4, SC32_FPREGS+32(a0) EX sdc1 $f6, SC32_FPREGS+48(a0) @@ -113,8 +146,17 @@ LEAF(_save_fp_context32) * - cp1 status/control register */ LEAF(_restore_fp_context) - EX lw t0, SC_FPC_CSR(a0) -#ifdef CONFIG_64BIT + EX lw t1, SC_FPC_CSR(a0) + +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2) + .set push +#ifdef CONFIG_CPU_MIPS32_R2 + .set mips64r2 + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip loading odd if FR=0 + nop +#endif EX ldc1 $f1, SC_FPREGS+8(a0) EX ldc1 $f3, SC_FPREGS+24(a0) EX ldc1 $f5, SC_FPREGS+40(a0) @@ -131,6 +173,7 @@ LEAF(_restore_fp_context) EX ldc1 $f27, SC_FPREGS+216(a0) EX ldc1 $f29, SC_FPREGS+232(a0) EX ldc1 $f31, SC_FPREGS+248(a0) +1: .set pop #endif EX ldc1 $f0, SC_FPREGS+0(a0) EX ldc1 $f2, SC_FPREGS+16(a0) @@ -148,7 +191,7 @@ LEAF(_restore_fp_context) EX ldc1 $f26, SC_FPREGS+208(a0) EX ldc1 $f28, SC_FPREGS+224(a0) EX ldc1 $f30, SC_FPREGS+240(a0) - ctc1 t0, fcr31 + ctc1 t1, fcr31 jr ra li v0, 0 # success END(_restore_fp_context) @@ -156,8 +199,31 @@ LEAF(_restore_fp_context) #ifdef CONFIG_MIPS32_COMPAT LEAF(_restore_fp_context32) /* Restore an o32 sigcontext. */ - EX lw t0, SC32_FPC_CSR(a0) - EX ldc1 $f0, SC32_FPREGS+0(a0) + EX lw t1, SC32_FPC_CSR(a0) + + mfc0 t0, CP0_STATUS + sll t0, t0, 5 + bgez t0, 1f # skip loading odd if FR=0 + nop + + EX ldc1 $f1, SC32_FPREGS+8(a0) + EX ldc1 $f3, SC32_FPREGS+24(a0) + EX ldc1 $f5, SC32_FPREGS+40(a0) + EX ldc1 $f7, SC32_FPREGS+56(a0) + EX ldc1 $f9, SC32_FPREGS+72(a0) + EX ldc1 $f11, SC32_FPREGS+88(a0) + EX ldc1 $f13, SC32_FPREGS+104(a0) + EX ldc1 $f15, SC32_FPREGS+120(a0) + EX ldc1 $f17, SC32_FPREGS+136(a0) + EX ldc1 $f19, SC32_FPREGS+152(a0) + EX ldc1 $f21, SC32_FPREGS+168(a0) + EX ldc1 $f23, SC32_FPREGS+184(a0) + EX ldc1 $f25, SC32_FPREGS+200(a0) + EX ldc1 $f27, SC32_FPREGS+216(a0) + EX ldc1 $f29, SC32_FPREGS+232(a0) + EX ldc1 $f31, SC32_FPREGS+248(a0) + +1: EX ldc1 $f0, SC32_FPREGS+0(a0) EX ldc1 $f2, SC32_FPREGS+16(a0) EX ldc1 $f4, SC32_FPREGS+32(a0) EX ldc1 $f6, SC32_FPREGS+48(a0) @@ -173,7 +239,7 @@ LEAF(_restore_fp_context32) EX ldc1 $f26, SC32_FPREGS+208(a0) EX ldc1 $f28, SC32_FPREGS+224(a0) EX ldc1 $f30, SC32_FPREGS+240(a0) - ctc1 t0, fcr31 + ctc1 t1, fcr31 jr ra li v0, 0 # success END(_restore_fp_context32) diff --git a/arch/mips/kernel/r4k_switch.S b/arch/mips/kernel/r4k_switch.S index 8893ee1a236..81ca3f70fe2 100644 --- a/arch/mips/kernel/r4k_switch.S +++ b/arch/mips/kernel/r4k_switch.S @@ -4,7 +4,7 @@ * for more details. * * Copyright (C) 1994, 1995, 1996, 1998, 1999, 2002, 2003 Ralf Baechle - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) * Copyright (C) 1994, 1995, 1996, by Andreas Busse * Copyright (C) 1999 Silicon Graphics, Inc. * Copyright (C) 2000 MIPS Technologies, Inc. @@ -15,7 +15,6 @@ #include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/asm-offsets.h> -#include <asm/page.h> #include <asm/pgtable-bits.h> #include <asm/regdef.h> #include <asm/stackframe.h> @@ -29,19 +28,10 @@ */ #define ST_OFF (_THREAD_SIZE - 32 - PT_SIZE + PT_STATUS) -/* - * FPU context is saved iff the process has used it's FPU in the current - * time slice as indicated by _TIF_USEDFPU. In any case, the CU1 bit for user - * space STATUS register should be 0, so that a process *always* starts its - * userland with FPU disabled after each context switch. - * - * FPU will be enabled as soon as the process accesses FPU again, through - * do_cpu() trap. - */ - +#ifndef USE_ALTERNATE_RESUME_IMPL /* * task_struct *resume(task_struct *prev, task_struct *next, - * struct thread_info *next_ti) + * struct thread_info *next_ti, s32 fp_save) */ .align 5 LEAF(resume) @@ -51,29 +41,43 @@ LONG_S ra, THREAD_REG31(a0) /* - * check if we need to save FPU registers + * Check whether we need to save any FP context. FP context is saved + * iff the process has used the context with the scalar FPU or the MSA + * ASE in the current time slice, as indicated by _TIF_USEDFPU and + * _TIF_USEDMSA respectively. switch_to will have set fp_save + * accordingly to an FP_SAVE_ enum value. */ - PTR_L t3, TASK_THREAD_INFO(a0) - LONG_L t0, TI_FLAGS(t3) - li t1, _TIF_USEDFPU - and t2, t0, t1 - beqz t2, 1f - nor t1, zero, t1 - - and t0, t0, t1 - LONG_S t0, TI_FLAGS(t3) + beqz a3, 2f /* - * clear saved user stack CU1 bit + * We do. Clear the saved CU1 bit for prev, such that next time it is + * scheduled it will start in userland with the FPU disabled. If the + * task uses the FPU then it will be enabled again via the do_cpu trap. + * This allows us to lazily restore the FP context. */ + PTR_L t3, TASK_THREAD_INFO(a0) LONG_L t0, ST_OFF(t3) li t1, ~ST0_CU1 and t0, t0, t1 LONG_S t0, ST_OFF(t3) + /* Check whether we're saving scalar or vector context. */ + bgtz a3, 1f + + /* Save 128b MSA vector context. */ + msa_save_all a0 + b 2f + +1: /* Save 32b/64b scalar FP context. */ fpu_save_double a0 t0 t1 # c0_status passed in t0 # clobbers t1 -1: +2: + +#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) + PTR_LA t8, __stack_chk_guard + LONG_L t9, TASK_STACK_CANARY(a1) + LONG_S t9, 0(t8) +#endif /* * The order of restoring the registers takes care of the race @@ -84,18 +88,6 @@ PTR_ADDU t0, $28, _THREAD_SIZE - 32 set_saved_sp t0, t1, t2 -#ifdef CONFIG_MIPS_MT_SMTC - /* Read-modify-writes of Status must be atomic on a VPE */ - mfc0 t2, CP0_TCSTATUS - ori t1, t2, TCSTATUS_IXMT - mtc0 t1, CP0_TCSTATUS - andi t2, t2, TCSTATUS_IXMT - _ehb - DMT 8 # dmt t0 - move t1,ra - jal mips_ihb - move ra,t1 -#endif /* CONFIG_MIPS_MT_SMTC */ mfc0 t1, CP0_STATUS /* Do we really need this? */ li a3, 0xff01 and t1, a3 @@ -104,27 +96,17 @@ and a2, a3 or a2, t1 mtc0 a2, CP0_STATUS -#ifdef CONFIG_MIPS_MT_SMTC - _ehb - andi t0, t0, VPECONTROL_TE - beqz t0, 1f - emt -1: - mfc0 t1, CP0_TCSTATUS - xori t1, t1, TCSTATUS_IXMT - or t1, t1, t2 - mtc0 t1, CP0_TCSTATUS - _ehb -#endif /* CONFIG_MIPS_MT_SMTC */ move v0, a0 jr ra END(resume) +#endif /* USE_ALTERNATE_RESUME_IMPL */ + /* * Save a thread's fp context. */ LEAF(_save_fp) -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2) mfc0 t0, CP0_STATUS #endif fpu_save_double a0 t0 t1 # clobbers t1 @@ -135,13 +117,33 @@ LEAF(_save_fp) * Restore a thread's fp context. */ LEAF(_restore_fp) -#ifdef CONFIG_64BIT +#if defined(CONFIG_64BIT) || defined(CONFIG_CPU_MIPS32_R2) mfc0 t0, CP0_STATUS #endif fpu_restore_double a0 t0 t1 # clobbers t1 jr ra END(_restore_fp) +#ifdef CONFIG_CPU_HAS_MSA + +/* + * Save a thread's MSA vector context. + */ +LEAF(_save_msa) + msa_save_all a0 + jr ra + END(_save_msa) + +/* + * Restore a thread's MSA vector context. + */ +LEAF(_restore_msa) + msa_restore_all a0 + jr ra + END(_restore_msa) + +#endif + /* * Load the FPU with signalling NANS. This bit pattern we're using has * the property that no matter whether considered as single or as double @@ -153,19 +155,10 @@ LEAF(_restore_fp) #define FPU_DEFAULT 0x00000000 LEAF(_init_fpu) -#ifdef CONFIG_MIPS_MT_SMTC - /* Rather than manipulate per-VPE Status, set per-TC bit in TCStatus */ - mfc0 t0, CP0_TCSTATUS - /* Bit position is the same for Status, TCStatus */ - li t1, ST0_CU1 - or t0, t1 - mtc0 t0, CP0_TCSTATUS -#else /* Normal MIPS CU1 enable */ mfc0 t0, CP0_STATUS li t1, ST0_CU1 or t0, t1 mtc0 t0, CP0_STATUS -#endif /* CONFIG_MIPS_MT_SMTC */ enable_fpu_hazard li t1, FPU_DEFAULT @@ -229,8 +222,49 @@ LEAF(_init_fpu) mtc1 t1, $f29 mtc1 t1, $f30 mtc1 t1, $f31 + +#ifdef CONFIG_CPU_MIPS32_R2 + .set push + .set mips64r2 + sll t0, t0, 5 # is Status.FR set? + bgez t0, 1f # no: skip setting upper 32b + + mthc1 t1, $f0 + mthc1 t1, $f1 + mthc1 t1, $f2 + mthc1 t1, $f3 + mthc1 t1, $f4 + mthc1 t1, $f5 + mthc1 t1, $f6 + mthc1 t1, $f7 + mthc1 t1, $f8 + mthc1 t1, $f9 + mthc1 t1, $f10 + mthc1 t1, $f11 + mthc1 t1, $f12 + mthc1 t1, $f13 + mthc1 t1, $f14 + mthc1 t1, $f15 + mthc1 t1, $f16 + mthc1 t1, $f17 + mthc1 t1, $f18 + mthc1 t1, $f19 + mthc1 t1, $f20 + mthc1 t1, $f21 + mthc1 t1, $f22 + mthc1 t1, $f23 + mthc1 t1, $f24 + mthc1 t1, $f25 + mthc1 t1, $f26 + mthc1 t1, $f27 + mthc1 t1, $f28 + mthc1 t1, $f29 + mthc1 t1, $f30 + mthc1 t1, $f31 +1: .set pop +#endif /* CONFIG_CPU_MIPS32_R2 */ #else - .set mips3 + .set arch=r4000 dmtc1 t1, $f0 dmtc1 t1, $f2 dmtc1 t1, $f4 diff --git a/arch/mips/kernel/r6000_fpu.S b/arch/mips/kernel/r6000_fpu.S index 43cda53f5af..da0fbe46d83 100644 --- a/arch/mips/kernel/r6000_fpu.S +++ b/arch/mips/kernel/r6000_fpu.S @@ -8,7 +8,7 @@ * Copyright (C) 1996 by Ralf Baechle * * Multi-arch abstraction and asm macros for easier reading: - * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) */ #include <asm/asm.h> #include <asm/fpregdef.h> diff --git a/arch/mips/kernel/relocate_kernel.S b/arch/mips/kernel/relocate_kernel.S index 87481f916a6..74bab9ddd0e 100644 --- a/arch/mips/kernel/relocate_kernel.S +++ b/arch/mips/kernel/relocate_kernel.S @@ -9,12 +9,16 @@ #include <asm/asm.h> #include <asm/asmmacro.h> #include <asm/regdef.h> -#include <asm/page.h> #include <asm/mipsregs.h> #include <asm/stackframe.h> #include <asm/addrspace.h> LEAF(relocate_new_kernel) + PTR_L a0, arg0 + PTR_L a1, arg1 + PTR_L a2, arg2 + PTR_L a3, arg3 + PTR_L s0, kexec_indirection_page PTR_L s1, kexec_start_address @@ -22,15 +26,20 @@ process_entry: PTR_L s2, (s0) PTR_ADD s0, s0, SZREG + /* + * In case of a kdump/crash kernel, the indirection page is not + * populated as the kernel is directly copied to a reserved location + */ + beqz s2, done + /* destination page */ and s3, s2, 0x1 beq s3, zero, 1f and s4, s2, ~0x1 /* store destination addr in s4 */ - move a0, s4 b process_entry 1: - /* indirection page, update s0 */ + /* indirection page, update s0 */ and s3, s2, 0x2 beq s3, zero, 1f and s0, s2, ~0x2 @@ -46,7 +55,7 @@ process_entry: and s3, s2, 0x8 beq s3, zero, process_entry and s2, s2, ~0x8 - li s6, (1 << PAGE_SHIFT) / SZREG + li s6, (1 << _PAGE_SHIFT) / SZREG copy_word: /* copy page word by word */ @@ -60,10 +69,111 @@ copy_word: b process_entry done: +#ifdef CONFIG_SMP + /* kexec_flag reset is signal to other CPUs what kernel + was moved to it's location. Note - we need relocated address + of kexec_flag. */ + + bal 1f + 1: move t1,ra; + PTR_LA t2,1b + PTR_LA t0,kexec_flag + PTR_SUB t0,t0,t2; + PTR_ADD t0,t1,t0; + LONG_S zero,(t0) +#endif + +#ifdef CONFIG_CPU_CAVIUM_OCTEON + /* We need to flush I-cache before jumping to new kernel. + * Unfortunatelly, this code is cpu-specific. + */ + .set push + .set noreorder + syncw + syncw + synci 0($0) + .set pop +#else + sync +#endif /* jump to kexec_start_address */ j s1 END(relocate_new_kernel) +#ifdef CONFIG_SMP +/* + * Other CPUs should wait until code is relocated and + * then start at entry (?) point. + */ +LEAF(kexec_smp_wait) + PTR_L a0, s_arg0 + PTR_L a1, s_arg1 + PTR_L a2, s_arg2 + PTR_L a3, s_arg3 + PTR_L s1, kexec_start_address + + /* Non-relocated address works for args and kexec_start_address ( old + * kernel is not overwritten). But we need relocated address of + * kexec_flag. + */ + + bal 1f +1: move t1,ra; + PTR_LA t2,1b + PTR_LA t0,kexec_flag + PTR_SUB t0,t0,t2; + PTR_ADD t0,t1,t0; + +1: LONG_L s0, (t0) + bne s0, zero,1b + +#ifdef CONFIG_CPU_CAVIUM_OCTEON + .set push + .set noreorder + synci 0($0) + .set pop +#else + sync +#endif + j s1 + END(kexec_smp_wait) +#endif + +#ifdef __mips64 + /* all PTR's must be aligned to 8 byte in 64-bit mode */ + .align 3 +#endif + +/* All parameters to new kernel are passed in registers a0-a3. + * kexec_args[0..3] are uses to prepare register values. + */ + +kexec_args: + EXPORT(kexec_args) +arg0: PTR 0x0 +arg1: PTR 0x0 +arg2: PTR 0x0 +arg3: PTR 0x0 + .size kexec_args,PTRSIZE*4 + +#ifdef CONFIG_SMP +/* + * Secondary CPUs may have different kernel parameters in + * their registers a0-a3. secondary_kexec_args[0..3] are used + * to prepare register values. + */ +secondary_kexec_args: + EXPORT(secondary_kexec_args) +s_arg0: PTR 0x0 +s_arg1: PTR 0x0 +s_arg2: PTR 0x0 +s_arg3: PTR 0x0 + .size secondary_kexec_args,PTRSIZE*4 +kexec_flag: + LONG 0x1 + +#endif + kexec_start_address: EXPORT(kexec_start_address) PTR 0x0 diff --git a/arch/mips/kernel/reset.c b/arch/mips/kernel/reset.c index 060563a712b..07fc5244aed 100644 --- a/arch/mips/kernel/reset.c +++ b/arch/mips/kernel/reset.c @@ -7,7 +7,7 @@ * Copyright (C) 2001 MIPS Technologies, Inc. */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/pm.h> #include <linux/types.h> #include <linux/reboot.h> diff --git a/arch/mips/kernel/rtlx-cmp.c b/arch/mips/kernel/rtlx-cmp.c new file mode 100644 index 00000000000..758fb3cd232 --- /dev/null +++ b/arch/mips/kernel/rtlx-cmp.c @@ -0,0 +1,119 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/err.h> +#include <linux/wait.h> +#include <linux/sched.h> +#include <linux/smp.h> + +#include <asm/mips_mt.h> +#include <asm/vpe.h> +#include <asm/rtlx.h> + +static int major; + +static void rtlx_interrupt(void) +{ + int i; + struct rtlx_info *info; + struct rtlx_info **p = vpe_get_shared(aprp_cpu_index()); + + if (p == NULL || *p == NULL) + return; + + info = *p; + + if (info->ap_int_pending == 1 && smp_processor_id() == 0) { + for (i = 0; i < RTLX_CHANNELS; i++) { + wake_up(&channel_wqs[i].lx_queue); + wake_up(&channel_wqs[i].rt_queue); + } + info->ap_int_pending = 0; + } +} + +void _interrupt_sp(void) +{ + smp_send_reschedule(aprp_cpu_index()); +} + +int __init rtlx_module_init(void) +{ + struct device *dev; + int i, err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (num_possible_cpus() - aprp_cpu_index() < 1) { + pr_warn("No TCs reserved for AP/SP, not initializing RTLX.\n" + "Pass maxcpus=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + major = register_chrdev(0, RTLX_MODULE_NAME, &rtlx_fops); + if (major < 0) { + pr_err("rtlx_module_init: unable to register device\n"); + return major; + } + + /* initialise the wait queues */ + for (i = 0; i < RTLX_CHANNELS; i++) { + init_waitqueue_head(&channel_wqs[i].rt_queue); + init_waitqueue_head(&channel_wqs[i].lx_queue); + atomic_set(&channel_wqs[i].in_open, 0); + mutex_init(&channel_wqs[i].mutex); + + dev = device_create(mt_class, NULL, MKDEV(major, i), NULL, + "%s%d", RTLX_MODULE_NAME, i); + if (IS_ERR(dev)) { + err = PTR_ERR(dev); + goto out_chrdev; + } + } + + /* set up notifiers */ + rtlx_notify.start = rtlx_starting; + rtlx_notify.stop = rtlx_stopping; + vpe_notify(aprp_cpu_index(), &rtlx_notify); + + if (cpu_has_vint) { + aprp_hook = rtlx_interrupt; + } else { + pr_err("APRP RTLX init on non-vectored-interrupt processor\n"); + err = -ENODEV; + goto out_class; + } + + return 0; + +out_class: + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(mt_class, MKDEV(major, i)); +out_chrdev: + unregister_chrdev(major, RTLX_MODULE_NAME); + + return err; +} + +void __exit rtlx_module_exit(void) +{ + int i; + + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(mt_class, MKDEV(major, i)); + + unregister_chrdev(major, RTLX_MODULE_NAME); + + aprp_hook = NULL; +} diff --git a/arch/mips/kernel/rtlx-mt.c b/arch/mips/kernel/rtlx-mt.c new file mode 100644 index 00000000000..5a66b975989 --- /dev/null +++ b/arch/mips/kernel/rtlx-mt.c @@ -0,0 +1,150 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/err.h> +#include <linux/wait.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/irq.h> + +#include <asm/mips_mt.h> +#include <asm/vpe.h> +#include <asm/rtlx.h> + +static int major; + +static void rtlx_dispatch(void) +{ + if (read_c0_cause() & read_c0_status() & C_SW0) + do_IRQ(MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ); +} + +/* + * Interrupt handler may be called before rtlx_init has otherwise had + * a chance to run. + */ +static irqreturn_t rtlx_interrupt(int irq, void *dev_id) +{ + unsigned int vpeflags; + unsigned long flags; + int i; + + local_irq_save(flags); + vpeflags = dvpe(); + set_c0_status(0x100 << MIPS_CPU_RTLX_IRQ); + irq_enable_hazard(); + evpe(vpeflags); + local_irq_restore(flags); + + for (i = 0; i < RTLX_CHANNELS; i++) { + wake_up(&channel_wqs[i].lx_queue); + wake_up(&channel_wqs[i].rt_queue); + } + + return IRQ_HANDLED; +} + +static struct irqaction rtlx_irq = { + .handler = rtlx_interrupt, + .name = "RTLX", +}; + +static int rtlx_irq_num = MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ; + +void _interrupt_sp(void) +{ + unsigned long flags; + + local_irq_save(flags); + dvpe(); + settc(1); + write_vpe_c0_cause(read_vpe_c0_cause() | C_SW0); + evpe(EVPE_ENABLE); + local_irq_restore(flags); +} + +int __init rtlx_module_init(void) +{ + struct device *dev; + int i, err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (aprp_cpu_index() == 0) { + pr_warn("No TCs reserved for AP/SP, not initializing RTLX.\n" + "Pass maxtcs=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + major = register_chrdev(0, RTLX_MODULE_NAME, &rtlx_fops); + if (major < 0) { + pr_err("rtlx_module_init: unable to register device\n"); + return major; + } + + /* initialise the wait queues */ + for (i = 0; i < RTLX_CHANNELS; i++) { + init_waitqueue_head(&channel_wqs[i].rt_queue); + init_waitqueue_head(&channel_wqs[i].lx_queue); + atomic_set(&channel_wqs[i].in_open, 0); + mutex_init(&channel_wqs[i].mutex); + + dev = device_create(mt_class, NULL, MKDEV(major, i), NULL, + "%s%d", RTLX_MODULE_NAME, i); + if (IS_ERR(dev)) { + err = PTR_ERR(dev); + goto out_chrdev; + } + } + + /* set up notifiers */ + rtlx_notify.start = rtlx_starting; + rtlx_notify.stop = rtlx_stopping; + vpe_notify(aprp_cpu_index(), &rtlx_notify); + + if (cpu_has_vint) { + aprp_hook = rtlx_dispatch; + } else { + pr_err("APRP RTLX init on non-vectored-interrupt processor\n"); + err = -ENODEV; + goto out_class; + } + + rtlx_irq.dev_id = rtlx; + err = setup_irq(rtlx_irq_num, &rtlx_irq); + if (err) + goto out_class; + + return 0; + +out_class: + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(mt_class, MKDEV(major, i)); +out_chrdev: + unregister_chrdev(major, RTLX_MODULE_NAME); + + return err; +} + +void __exit rtlx_module_exit(void) +{ + int i; + + for (i = 0; i < RTLX_CHANNELS; i++) + device_destroy(mt_class, MKDEV(major, i)); + + unregister_chrdev(major, RTLX_MODULE_NAME); + + aprp_hook = NULL; +} diff --git a/arch/mips/kernel/rtlx.c b/arch/mips/kernel/rtlx.c index 557ef72472e..31b1b763cb2 100644 --- a/arch/mips/kernel/rtlx.c +++ b/arch/mips/kernel/rtlx.c @@ -1,115 +1,51 @@ /* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * * Copyright (C) 2005 MIPS Technologies, Inc. All rights reserved. * Copyright (C) 2005, 06 Ralf Baechle (ralf@linux-mips.org) - * - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - * + * Copyright (C) 2013 Imagination Technologies Ltd. */ - -#include <linux/device.h> #include <linux/kernel.h> -#include <linux/module.h> #include <linux/fs.h> -#include <linux/init.h> -#include <asm/uaccess.h> -#include <linux/list.h> -#include <linux/vmalloc.h> -#include <linux/elf.h> -#include <linux/seq_file.h> #include <linux/syscalls.h> #include <linux/moduleloader.h> -#include <linux/interrupt.h> -#include <linux/poll.h> -#include <linux/sched.h> -#include <linux/wait.h> +#include <linux/atomic.h> #include <asm/mipsmtregs.h> #include <asm/mips_mt.h> -#include <asm/cacheflush.h> -#include <asm/atomic.h> -#include <asm/cpu.h> #include <asm/processor.h> -#include <asm/system.h> -#include <asm/vpe.h> #include <asm/rtlx.h> +#include <asm/setup.h> +#include <asm/vpe.h> -static struct rtlx_info *rtlx; -static int major; -static char module_name[] = "rtlx"; - -static struct chan_waitqueues { - wait_queue_head_t rt_queue; - wait_queue_head_t lx_queue; - atomic_t in_open; - struct mutex mutex; -} channel_wqs[RTLX_CHANNELS]; - -static struct vpe_notifications notify; static int sp_stopping; - -extern void *vpe_get_shared(int index); - -static void rtlx_dispatch(void) -{ - do_IRQ(MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ); -} - - -/* Interrupt handler may be called before rtlx_init has otherwise had - a chance to run. -*/ -static irqreturn_t rtlx_interrupt(int irq, void *dev_id) -{ - unsigned int vpeflags; - unsigned long flags; - int i; - - /* Ought not to be strictly necessary for SMTC builds */ - local_irq_save(flags); - vpeflags = dvpe(); - set_c0_status(0x100 << MIPS_CPU_RTLX_IRQ); - irq_enable_hazard(); - evpe(vpeflags); - local_irq_restore(flags); - - for (i = 0; i < RTLX_CHANNELS; i++) { - wake_up(&channel_wqs[i].lx_queue); - wake_up(&channel_wqs[i].rt_queue); - } - - return IRQ_HANDLED; -} +struct rtlx_info *rtlx; +struct chan_waitqueues channel_wqs[RTLX_CHANNELS]; +struct vpe_notifications rtlx_notify; +void (*aprp_hook)(void) = NULL; +EXPORT_SYMBOL(aprp_hook); static void __used dump_rtlx(void) { int i; - printk("id 0x%lx state %d\n", rtlx->id, rtlx->state); + pr_info("id 0x%lx state %d\n", rtlx->id, rtlx->state); for (i = 0; i < RTLX_CHANNELS; i++) { struct rtlx_channel *chan = &rtlx->channel[i]; - printk(" rt_state %d lx_state %d buffer_size %d\n", - chan->rt_state, chan->lx_state, chan->buffer_size); + pr_info(" rt_state %d lx_state %d buffer_size %d\n", + chan->rt_state, chan->lx_state, chan->buffer_size); - printk(" rt_read %d rt_write %d\n", - chan->rt_read, chan->rt_write); + pr_info(" rt_read %d rt_write %d\n", + chan->rt_read, chan->rt_write); - printk(" lx_read %d lx_write %d\n", - chan->lx_read, chan->lx_write); + pr_info(" lx_read %d lx_write %d\n", + chan->lx_read, chan->lx_write); - printk(" rt_buffer <%s>\n", chan->rt_buffer); - printk(" lx_buffer <%s>\n", chan->lx_buffer); + pr_info(" rt_buffer <%s>\n", chan->rt_buffer); + pr_info(" lx_buffer <%s>\n", chan->lx_buffer); } } @@ -117,8 +53,7 @@ static void __used dump_rtlx(void) static int rtlx_init(struct rtlx_info *rtlxi) { if (rtlxi->id != RTLX_ID) { - printk(KERN_ERR "no valid RTLX id at 0x%p 0x%lx\n", - rtlxi, rtlxi->id); + pr_err("no valid RTLX id at 0x%p 0x%lx\n", rtlxi, rtlxi->id); return -ENOEXEC; } @@ -128,20 +63,20 @@ static int rtlx_init(struct rtlx_info *rtlxi) } /* notifications */ -static void starting(int vpe) +void rtlx_starting(int vpe) { int i; sp_stopping = 0; /* force a reload of rtlx */ - rtlx=NULL; + rtlx = NULL; /* wake up any sleeping rtlx_open's */ for (i = 0; i < RTLX_CHANNELS; i++) wake_up_interruptible(&channel_wqs[i].lx_queue); } -static void stopping(int vpe) +void rtlx_stopping(int vpe) { int i; @@ -159,30 +94,30 @@ int rtlx_open(int index, int can_sleep) int ret = 0; if (index >= RTLX_CHANNELS) { - printk(KERN_DEBUG "rtlx_open index out of range\n"); + pr_debug(KERN_DEBUG "rtlx_open index out of range\n"); return -ENOSYS; } if (atomic_inc_return(&channel_wqs[index].in_open) > 1) { - printk(KERN_DEBUG "rtlx_open channel %d already opened\n", - index); + pr_debug(KERN_DEBUG "rtlx_open channel %d already opened\n", index); ret = -EBUSY; goto out_fail; } if (rtlx == NULL) { - if( (p = vpe_get_shared(tclimit)) == NULL) { - if (can_sleep) { - __wait_event_interruptible(channel_wqs[index].lx_queue, - (p = vpe_get_shared(tclimit)), ret); - if (ret) + p = vpe_get_shared(aprp_cpu_index()); + if (p == NULL) { + if (can_sleep) { + ret = __wait_event_interruptible( + channel_wqs[index].lx_queue, + (p = vpe_get_shared(aprp_cpu_index()))); + if (ret) + goto out_fail; + } else { + pr_debug("No SP program loaded, and device opened with O_NONBLOCK\n"); + ret = -ENOSYS; goto out_fail; - } else { - printk(KERN_DEBUG "No SP program loaded, and device " - "opened with O_NONBLOCK\n"); - ret = -ENOSYS; - goto out_fail; - } + } } smp_rmb(); @@ -204,24 +139,24 @@ int rtlx_open(int index, int can_sleep) ret = -ERESTARTSYS; goto out_fail; } - finish_wait(&channel_wqs[index].lx_queue, &wait); + finish_wait(&channel_wqs[index].lx_queue, + &wait); } else { - pr_err(" *vpe_get_shared is NULL. " - "Has an SP program been loaded?\n"); + pr_err(" *vpe_get_shared is NULL. Has an SP program been loaded?\n"); ret = -ENOSYS; goto out_fail; } } if ((unsigned int)*p < KSEG0) { - printk(KERN_WARNING "vpe_get_shared returned an " - "invalid pointer maybe an error code %d\n", - (int)*p); + pr_warn("vpe_get_shared returned an invalid pointer maybe an error code %d\n", + (int)*p); ret = -ENOSYS; goto out_fail; } - if ((ret = rtlx_init(*p)) < 0) + ret = rtlx_init(*p); + if (ret < 0) goto out_ret; } @@ -254,21 +189,20 @@ int rtlx_release(int index) unsigned int rtlx_read_poll(int index, int can_sleep) { - struct rtlx_channel *chan; + struct rtlx_channel *chan; - if (rtlx == NULL) - return 0; + if (rtlx == NULL) + return 0; - chan = &rtlx->channel[index]; + chan = &rtlx->channel[index]; /* data available to read? */ if (chan->lx_read == chan->lx_write) { if (can_sleep) { - int ret = 0; - - __wait_event_interruptible(channel_wqs[index].lx_queue, + int ret = __wait_event_interruptible( + channel_wqs[index].lx_queue, (chan->lx_read != chan->lx_write) || - sp_stopping, ret); + sp_stopping); if (ret) return ret; @@ -353,7 +287,7 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) size_t fl; if (rtlx == NULL) - return(-ENOSYS); + return -ENOSYS; rt = &rtlx->channel[index]; @@ -362,8 +296,8 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) rt_read = rt->rt_read; /* total number of bytes to copy */ - count = min(count, (size_t)write_spacefree(rt_read, rt->rt_write, - rt->buffer_size)); + count = min_t(size_t, count, write_spacefree(rt_read, rt->rt_write, + rt->buffer_size)); /* first bit from write pointer to the end of the buffer, or count */ fl = min(count, (size_t) rt->buffer_size - rt->rt_write); @@ -373,9 +307,8 @@ ssize_t rtlx_write(int index, const void __user *buffer, size_t count) goto out; /* if there's any left copy to the beginning of the buffer */ - if (count - fl) { + if (count - fl) failed = copy_from_user(rt->rt_buffer, buffer + fl, count - fl); - } out: count -= failed; @@ -385,6 +318,8 @@ out: smp_wmb(); mutex_unlock(&channel_wqs[index].mutex); + _interrupt_sp(); + return count; } @@ -399,13 +334,11 @@ static int file_release(struct inode *inode, struct file *filp) return rtlx_release(iminor(inode)); } -static unsigned int file_poll(struct file *file, poll_table * wait) +static unsigned int file_poll(struct file *file, poll_table *wait) { - int minor; + int minor = iminor(file_inode(file)); unsigned int mask = 0; - minor = iminor(file->f_path.dentry->d_inode); - poll_wait(file, &channel_wqs[minor].rt_queue, wait); poll_wait(file, &channel_wqs[minor].lx_queue, wait); @@ -423,38 +356,32 @@ static unsigned int file_poll(struct file *file, poll_table * wait) return mask; } -static ssize_t file_read(struct file *file, char __user * buffer, size_t count, - loff_t * ppos) +static ssize_t file_read(struct file *file, char __user *buffer, size_t count, + loff_t *ppos) { - int minor = iminor(file->f_path.dentry->d_inode); + int minor = iminor(file_inode(file)); /* data available? */ - if (!rtlx_read_poll(minor, (file->f_flags & O_NONBLOCK) ? 0 : 1)) { - return 0; // -EAGAIN makes cat whinge - } + if (!rtlx_read_poll(minor, (file->f_flags & O_NONBLOCK) ? 0 : 1)) + return 0; /* -EAGAIN makes 'cat' whine */ return rtlx_read(minor, buffer, count); } -static ssize_t file_write(struct file *file, const char __user * buffer, - size_t count, loff_t * ppos) +static ssize_t file_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) { - int minor; - struct rtlx_channel *rt; - - minor = iminor(file->f_path.dentry->d_inode); - rt = &rtlx->channel[minor]; + int minor = iminor(file_inode(file)); /* any space left... */ if (!rtlx_write_poll(minor)) { - int ret = 0; + int ret; if (file->f_flags & O_NONBLOCK) return -EAGAIN; - __wait_event_interruptible(channel_wqs[minor].rt_queue, - rtlx_write_poll(minor), - ret); + ret = __wait_event_interruptible(channel_wqs[minor].rt_queue, + rtlx_write_poll(minor)); if (ret) return ret; } @@ -462,7 +389,7 @@ static ssize_t file_write(struct file *file, const char __user * buffer, return rtlx_write(minor, buffer, count); } -static const struct file_operations rtlx_fops = { +const struct file_operations rtlx_fops = { .owner = THIS_MODULE, .open = file_open, .release = file_release, @@ -472,91 +399,6 @@ static const struct file_operations rtlx_fops = { .llseek = noop_llseek, }; -static struct irqaction rtlx_irq = { - .handler = rtlx_interrupt, - .flags = IRQF_DISABLED, - .name = "RTLX", -}; - -static int rtlx_irq_num = MIPS_CPU_IRQ_BASE + MIPS_CPU_RTLX_IRQ; - -static char register_chrdev_failed[] __initdata = - KERN_ERR "rtlx_module_init: unable to register device\n"; - -static int __init rtlx_module_init(void) -{ - struct device *dev; - int i, err; - - if (!cpu_has_mipsmt) { - printk("VPE loader: not a MIPS MT capable processor\n"); - return -ENODEV; - } - - if (tclimit == 0) { - printk(KERN_WARNING "No TCs reserved for AP/SP, not " - "initializing RTLX.\nPass maxtcs=<n> argument as kernel " - "argument\n"); - - return -ENODEV; - } - - major = register_chrdev(0, module_name, &rtlx_fops); - if (major < 0) { - printk(register_chrdev_failed); - return major; - } - - /* initialise the wait queues */ - for (i = 0; i < RTLX_CHANNELS; i++) { - init_waitqueue_head(&channel_wqs[i].rt_queue); - init_waitqueue_head(&channel_wqs[i].lx_queue); - atomic_set(&channel_wqs[i].in_open, 0); - mutex_init(&channel_wqs[i].mutex); - - dev = device_create(mt_class, NULL, MKDEV(major, i), NULL, - "%s%d", module_name, i); - if (IS_ERR(dev)) { - err = PTR_ERR(dev); - goto out_chrdev; - } - } - - /* set up notifiers */ - notify.start = starting; - notify.stop = stopping; - vpe_notify(tclimit, ¬ify); - - if (cpu_has_vint) - set_vi_handler(MIPS_CPU_RTLX_IRQ, rtlx_dispatch); - else { - pr_err("APRP RTLX init on non-vectored-interrupt processor\n"); - err = -ENODEV; - goto out_chrdev; - } - - rtlx_irq.dev_id = rtlx; - setup_irq(rtlx_irq_num, &rtlx_irq); - - return 0; - -out_chrdev: - for (i = 0; i < RTLX_CHANNELS; i++) - device_destroy(mt_class, MKDEV(major, i)); - - return err; -} - -static void __exit rtlx_module_exit(void) -{ - int i; - - for (i = 0; i < RTLX_CHANNELS; i++) - device_destroy(mt_class, MKDEV(major, i)); - - unregister_chrdev(major, module_name); -} - module_init(rtlx_module_init); module_exit(rtlx_module_exit); diff --git a/arch/mips/kernel/scall32-o32.S b/arch/mips/kernel/scall32-o32.S index fbaabad0e6e..3245474f19d 100644 --- a/arch/mips/kernel/scall32-o32.S +++ b/arch/mips/kernel/scall32-o32.S @@ -6,6 +6,7 @@ * Copyright (C) 1995-99, 2000- 02, 06 Ralf Baechle <ralf@linux-mips.org> * Copyright (C) 2001 MIPS Technologies, Inc. * Copyright (C) 2004 Thiemo Seufer + * Copyright (C) 2014 Imagination Technologies Ltd. */ #include <linux/errno.h> #include <asm/asm.h> @@ -24,7 +25,7 @@ /* Highest syscall used of any syscall flavour */ #define MAX_SYSCALL_NO __NR_O32_Linux + __NR_O32_Linux_syscalls - .align 5 + .align 5 NESTED(handle_sys, PT_SIZE, sp) .set noat SAVE_SOME @@ -40,21 +41,62 @@ NESTED(handle_sys, PT_SIZE, sp) sw t1, PT_EPC(sp) beqz t0, illegal_syscall - sll t0, v0, 3 + sll t0, v0, 2 la t1, sys_call_table addu t1, t0 lw t2, (t1) # syscall routine - lw t3, 4(t1) # >= 0 if we need stack arguments beqz t2, illegal_syscall sw a3, PT_R26(sp) # save a3 for syscall restarting - bgez t3, stackargs -stack_done: + /* + * More than four arguments. Try to deal with it by copying the + * stack arguments from the user stack to the kernel stack. + * This Sucks (TM). + */ + lw t0, PT_R29(sp) # get old user stack pointer + + /* + * We intentionally keep the kernel stack a little below the top of + * userspace so we don't have to do a slower byte accurate check here. + */ + lw t5, TI_ADDR_LIMIT($28) + addu t4, t0, 32 + and t5, t4 + bltz t5, bad_stack # -> sp is bad + + /* + * Ok, copy the args from the luser stack to the kernel stack. + * t3 is the precomputed number of instruction bytes needed to + * load or store arguments 6-8. + */ + + .set push + .set noreorder + .set nomacro + +1: user_lw(t5, 16(t0)) # argument #5 from usp +4: user_lw(t6, 20(t0)) # argument #6 from usp +3: user_lw(t7, 24(t0)) # argument #7 from usp +2: user_lw(t8, 28(t0)) # argument #8 from usp + + sw t5, 16(sp) # argument #5 to ksp + sw t6, 20(sp) # argument #6 to ksp + sw t7, 24(sp) # argument #7 to ksp + sw t8, 28(sp) # argument #8 to ksp + .set pop + + .section __ex_table,"a" + PTR 1b,bad_stack + PTR 2b,bad_stack + PTR 3b,bad_stack + PTR 4b,bad_stack + .previous + lw t0, TI_FLAGS($28) # syscall tracing enabled? - li t1, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + li t1, _TIF_WORK_SYSCALL_ENTRY and t0, t1 - bnez t0, syscall_trace_entry # -> yes + bnez t0, syscall_trace_entry # -> yes jalr t2 # Do The Real Thing (TM) @@ -69,18 +111,7 @@ stack_done: 1: sw v0, PT_R2(sp) # result o32_syscall_exit: - local_irq_disable # make sure need_resched and - # signals dont change between - # sampling and return - lw a2, TI_FLAGS($28) # current->work - li t0, _TIF_ALLWORK_MASK - and t0, a2 - bnez t0, o32_syscall_exit_work - - j restore_partial - -o32_syscall_exit_work: - j syscall_exit_work_partial + j syscall_exit_partial /* ------------------------------------------------------------------------ */ @@ -88,8 +119,18 @@ syscall_trace_entry: SAVE_STATIC move s0, t2 move a0, sp - li a1, 0 - jal do_syscall_trace + + /* + * syscall number is in v0 unless we called syscall(__NR_###) + * where the real syscall number is in a0 + */ + addiu a1, v0, __NR_O32_Linux + bnez v0, 1f /* __NR_syscall at offset 0 */ + lw a1, PT_R4(sp) + +1: jal syscall_trace_enter + + bltz v0, 2f # seccomp failed? Skip syscall move t0, s0 RESTORE_STATIC @@ -109,62 +150,11 @@ syscall_trace_entry: sw t1, PT_R0(sp) # save it for syscall restarting 1: sw v0, PT_R2(sp) # result - j syscall_exit +2: j syscall_exit /* ------------------------------------------------------------------------ */ /* - * More than four arguments. Try to deal with it by copying the - * stack arguments from the user stack to the kernel stack. - * This Sucks (TM). - */ -stackargs: - lw t0, PT_R29(sp) # get old user stack pointer - - /* - * We intentionally keep the kernel stack a little below the top of - * userspace so we don't have to do a slower byte accurate check here. - */ - lw t5, TI_ADDR_LIMIT($28) - addu t4, t0, 32 - and t5, t4 - bltz t5, bad_stack # -> sp is bad - - /* Ok, copy the args from the luser stack to the kernel stack. - * t3 is the precomputed number of instruction bytes needed to - * load or store arguments 6-8. - */ - - la t1, 5f # load up to 3 arguments - subu t1, t3 -1: lw t5, 16(t0) # argument #5 from usp - .set push - .set noreorder - .set nomacro - jr t1 - addiu t1, 6f - 5f - -2: lw t8, 28(t0) # argument #8 from usp -3: lw t7, 24(t0) # argument #7 from usp -4: lw t6, 20(t0) # argument #6 from usp -5: jr t1 - sw t5, 16(sp) # argument #5 to ksp - - sw t8, 28(sp) # argument #8 to ksp - sw t7, 24(sp) # argument #7 to ksp - sw t6, 20(sp) # argument #6 to ksp -6: j stack_done # go back - nop - .set pop - - .section __ex_table,"a" - PTR 1b,bad_stack - PTR 2b,bad_stack - PTR 3b,bad_stack - PTR 4b,bad_stack - .previous - - /* * The stackpointer for a call with more than 4 arguments is bad. * We probably should handle this case a bit more drastic. */ @@ -190,7 +180,7 @@ illegal_syscall: subu t0, a0, __NR_O32_Linux # check syscall number sltiu v0, t0, __NR_O32_Linux_syscalls + 1 beqz t0, einval # do not recurse - sll t1, t0, 3 + sll t1, t0, 2 beqz v0, einval lw t2, sys_call_table(t1) # syscall routine @@ -217,387 +207,374 @@ illegal_syscall: jr t2 /* Unreached */ -einval: li v0, -ENOSYS +einval: li v0, -ENOSYS jr ra END(sys_syscall) - .macro fifty ptr, nargs, from=1, to=50 - sys \ptr \nargs - .if \to-\from - fifty \ptr,\nargs,"(\from+1)",\to - .endif - .endm - - .macro mille ptr, nargs, from=1, to=20 - fifty \ptr,\nargs - .if \to-\from - mille \ptr,\nargs,"(\from+1)",\to - .endif - .endm - - .macro syscalltable - sys sys_syscall 8 /* 4000 */ - sys sys_exit 1 - sys sys_fork 0 - sys sys_read 3 - sys sys_write 3 - sys sys_open 3 /* 4005 */ - sys sys_close 1 - sys sys_waitpid 3 - sys sys_creat 2 - sys sys_link 2 - sys sys_unlink 1 /* 4010 */ - sys sys_execve 0 - sys sys_chdir 1 - sys sys_time 1 - sys sys_mknod 3 - sys sys_chmod 2 /* 4015 */ - sys sys_lchown 3 - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 /* was sys_stat */ - sys sys_lseek 3 - sys sys_getpid 0 /* 4020 */ - sys sys_mount 5 - sys sys_oldumount 1 - sys sys_setuid 1 - sys sys_getuid 0 - sys sys_stime 1 /* 4025 */ - sys sys_ptrace 4 - sys sys_alarm 1 - sys sys_ni_syscall 0 /* was sys_fstat */ - sys sys_pause 0 - sys sys_utime 2 /* 4030 */ - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 - sys sys_access 2 - sys sys_nice 1 - sys sys_ni_syscall 0 /* 4035 */ - sys sys_sync 0 - sys sys_kill 2 - sys sys_rename 2 - sys sys_mkdir 2 - sys sys_rmdir 1 /* 4040 */ - sys sys_dup 1 - sys sysm_pipe 0 - sys sys_times 1 - sys sys_ni_syscall 0 - sys sys_brk 1 /* 4045 */ - sys sys_setgid 1 - sys sys_getgid 0 - sys sys_ni_syscall 0 /* was signal(2) */ - sys sys_geteuid 0 - sys sys_getegid 0 /* 4050 */ - sys sys_acct 1 - sys sys_umount 2 - sys sys_ni_syscall 0 - sys sys_ioctl 3 - sys sys_fcntl 3 /* 4055 */ - sys sys_ni_syscall 2 - sys sys_setpgid 2 - sys sys_ni_syscall 0 - sys sys_olduname 1 - sys sys_umask 1 /* 4060 */ - sys sys_chroot 1 - sys sys_ustat 2 - sys sys_dup2 2 - sys sys_getppid 0 - sys sys_getpgrp 0 /* 4065 */ - sys sys_setsid 0 - sys sys_sigaction 3 - sys sys_sgetmask 0 - sys sys_ssetmask 1 - sys sys_setreuid 2 /* 4070 */ - sys sys_setregid 2 - sys sys_sigsuspend 0 - sys sys_sigpending 1 - sys sys_sethostname 2 - sys sys_setrlimit 2 /* 4075 */ - sys sys_getrlimit 2 - sys sys_getrusage 2 - sys sys_gettimeofday 2 - sys sys_settimeofday 2 - sys sys_getgroups 2 /* 4080 */ - sys sys_setgroups 2 - sys sys_ni_syscall 0 /* old_select */ - sys sys_symlink 2 - sys sys_ni_syscall 0 /* was sys_lstat */ - sys sys_readlink 3 /* 4085 */ - sys sys_uselib 1 - sys sys_swapon 2 - sys sys_reboot 3 - sys sys_old_readdir 3 - sys sys_mips_mmap 6 /* 4090 */ - sys sys_munmap 2 - sys sys_truncate 2 - sys sys_ftruncate 2 - sys sys_fchmod 2 - sys sys_fchown 3 /* 4095 */ - sys sys_getpriority 2 - sys sys_setpriority 3 - sys sys_ni_syscall 0 - sys sys_statfs 2 - sys sys_fstatfs 2 /* 4100 */ - sys sys_ni_syscall 0 /* was ioperm(2) */ - sys sys_socketcall 2 - sys sys_syslog 3 - sys sys_setitimer 3 - sys sys_getitimer 2 /* 4105 */ - sys sys_newstat 2 - sys sys_newlstat 2 - sys sys_newfstat 2 - sys sys_uname 1 - sys sys_ni_syscall 0 /* 4110 was iopl(2) */ - sys sys_vhangup 0 - sys sys_ni_syscall 0 /* was sys_idle() */ - sys sys_ni_syscall 0 /* was sys_vm86 */ - sys sys_wait4 4 - sys sys_swapoff 1 /* 4115 */ - sys sys_sysinfo 1 - sys sys_ipc 6 - sys sys_fsync 1 - sys sys_sigreturn 0 - sys sys_clone 0 /* 4120 */ - sys sys_setdomainname 2 - sys sys_newuname 1 - sys sys_ni_syscall 0 /* sys_modify_ldt */ - sys sys_adjtimex 1 - sys sys_mprotect 3 /* 4125 */ - sys sys_sigprocmask 3 - sys sys_ni_syscall 0 /* was create_module */ - sys sys_init_module 5 - sys sys_delete_module 1 - sys sys_ni_syscall 0 /* 4130 was get_kernel_syms */ - sys sys_quotactl 4 - sys sys_getpgid 1 - sys sys_fchdir 1 - sys sys_bdflush 2 - sys sys_sysfs 3 /* 4135 */ - sys sys_personality 1 - sys sys_ni_syscall 0 /* for afs_syscall */ - sys sys_setfsuid 1 - sys sys_setfsgid 1 - sys sys_llseek 5 /* 4140 */ - sys sys_getdents 3 - sys sys_select 5 - sys sys_flock 2 - sys sys_msync 3 - sys sys_readv 3 /* 4145 */ - sys sys_writev 3 - sys sys_cacheflush 3 - sys sys_cachectl 3 - sys sys_sysmips 4 - sys sys_ni_syscall 0 /* 4150 */ - sys sys_getsid 1 - sys sys_fdatasync 1 - sys sys_sysctl 1 - sys sys_mlock 2 - sys sys_munlock 2 /* 4155 */ - sys sys_mlockall 1 - sys sys_munlockall 0 - sys sys_sched_setparam 2 - sys sys_sched_getparam 2 - sys sys_sched_setscheduler 3 /* 4160 */ - sys sys_sched_getscheduler 1 - sys sys_sched_yield 0 - sys sys_sched_get_priority_max 1 - sys sys_sched_get_priority_min 1 - sys sys_sched_rr_get_interval 2 /* 4165 */ - sys sys_nanosleep, 2 - sys sys_mremap, 5 - sys sys_accept 3 - sys sys_bind 3 - sys sys_connect 3 /* 4170 */ - sys sys_getpeername 3 - sys sys_getsockname 3 - sys sys_getsockopt 5 - sys sys_listen 2 - sys sys_recv 4 /* 4175 */ - sys sys_recvfrom 6 - sys sys_recvmsg 3 - sys sys_send 4 - sys sys_sendmsg 3 - sys sys_sendto 6 /* 4180 */ - sys sys_setsockopt 5 - sys sys_shutdown 2 - sys sys_socket 3 - sys sys_socketpair 4 - sys sys_setresuid 3 /* 4185 */ - sys sys_getresuid 3 - sys sys_ni_syscall 0 /* was sys_query_module */ - sys sys_poll 3 - sys sys_nfsservctl 3 - sys sys_setresgid 3 /* 4190 */ - sys sys_getresgid 3 - sys sys_prctl 5 - sys sys_rt_sigreturn 0 - sys sys_rt_sigaction 4 - sys sys_rt_sigprocmask 4 /* 4195 */ - sys sys_rt_sigpending 2 - sys sys_rt_sigtimedwait 4 - sys sys_rt_sigqueueinfo 3 - sys sys_rt_sigsuspend 0 - sys sys_pread64 6 /* 4200 */ - sys sys_pwrite64 6 - sys sys_chown 3 - sys sys_getcwd 2 - sys sys_capget 2 - sys sys_capset 2 /* 4205 */ - sys sys_sigaltstack 0 - sys sys_sendfile 4 - sys sys_ni_syscall 0 - sys sys_ni_syscall 0 - sys sys_mips_mmap2 6 /* 4210 */ - sys sys_truncate64 4 - sys sys_ftruncate64 4 - sys sys_stat64 2 - sys sys_lstat64 2 - sys sys_fstat64 2 /* 4215 */ - sys sys_pivot_root 2 - sys sys_mincore 3 - sys sys_madvise 3 - sys sys_getdents64 3 - sys sys_fcntl64 3 /* 4220 */ - sys sys_ni_syscall 0 - sys sys_gettid 0 - sys sys_readahead 5 - sys sys_setxattr 5 - sys sys_lsetxattr 5 /* 4225 */ - sys sys_fsetxattr 5 - sys sys_getxattr 4 - sys sys_lgetxattr 4 - sys sys_fgetxattr 4 - sys sys_listxattr 3 /* 4230 */ - sys sys_llistxattr 3 - sys sys_flistxattr 3 - sys sys_removexattr 2 - sys sys_lremovexattr 2 - sys sys_fremovexattr 2 /* 4235 */ - sys sys_tkill 2 - sys sys_sendfile64 5 - sys sys_futex 6 + .align 2 + .type sys_call_table, @object +EXPORT(sys_call_table) + PTR sys_syscall /* 4000 */ + PTR sys_exit + PTR __sys_fork + PTR sys_read + PTR sys_write + PTR sys_open /* 4005 */ + PTR sys_close + PTR sys_waitpid + PTR sys_creat + PTR sys_link + PTR sys_unlink /* 4010 */ + PTR sys_execve + PTR sys_chdir + PTR sys_time + PTR sys_mknod + PTR sys_chmod /* 4015 */ + PTR sys_lchown + PTR sys_ni_syscall + PTR sys_ni_syscall /* was sys_stat */ + PTR sys_lseek + PTR sys_getpid /* 4020 */ + PTR sys_mount + PTR sys_oldumount + PTR sys_setuid + PTR sys_getuid + PTR sys_stime /* 4025 */ + PTR sys_ptrace + PTR sys_alarm + PTR sys_ni_syscall /* was sys_fstat */ + PTR sys_pause + PTR sys_utime /* 4030 */ + PTR sys_ni_syscall + PTR sys_ni_syscall + PTR sys_access + PTR sys_nice + PTR sys_ni_syscall /* 4035 */ + PTR sys_sync + PTR sys_kill + PTR sys_rename + PTR sys_mkdir + PTR sys_rmdir /* 4040 */ + PTR sys_dup + PTR sysm_pipe + PTR sys_times + PTR sys_ni_syscall + PTR sys_brk /* 4045 */ + PTR sys_setgid + PTR sys_getgid + PTR sys_ni_syscall /* was signal(2) */ + PTR sys_geteuid + PTR sys_getegid /* 4050 */ + PTR sys_acct + PTR sys_umount + PTR sys_ni_syscall + PTR sys_ioctl + PTR sys_fcntl /* 4055 */ + PTR sys_ni_syscall + PTR sys_setpgid + PTR sys_ni_syscall + PTR sys_olduname + PTR sys_umask /* 4060 */ + PTR sys_chroot + PTR sys_ustat + PTR sys_dup2 + PTR sys_getppid + PTR sys_getpgrp /* 4065 */ + PTR sys_setsid + PTR sys_sigaction + PTR sys_sgetmask + PTR sys_ssetmask + PTR sys_setreuid /* 4070 */ + PTR sys_setregid + PTR sys_sigsuspend + PTR sys_sigpending + PTR sys_sethostname + PTR sys_setrlimit /* 4075 */ + PTR sys_getrlimit + PTR sys_getrusage + PTR sys_gettimeofday + PTR sys_settimeofday + PTR sys_getgroups /* 4080 */ + PTR sys_setgroups + PTR sys_ni_syscall /* old_select */ + PTR sys_symlink + PTR sys_ni_syscall /* was sys_lstat */ + PTR sys_readlink /* 4085 */ + PTR sys_uselib + PTR sys_swapon + PTR sys_reboot + PTR sys_old_readdir + PTR sys_mips_mmap /* 4090 */ + PTR sys_munmap + PTR sys_truncate + PTR sys_ftruncate + PTR sys_fchmod + PTR sys_fchown /* 4095 */ + PTR sys_getpriority + PTR sys_setpriority + PTR sys_ni_syscall + PTR sys_statfs + PTR sys_fstatfs /* 4100 */ + PTR sys_ni_syscall /* was ioperm(2) */ + PTR sys_socketcall + PTR sys_syslog + PTR sys_setitimer + PTR sys_getitimer /* 4105 */ + PTR sys_newstat + PTR sys_newlstat + PTR sys_newfstat + PTR sys_uname + PTR sys_ni_syscall /* 4110 was iopl(2) */ + PTR sys_vhangup + PTR sys_ni_syscall /* was sys_idle() */ + PTR sys_ni_syscall /* was sys_vm86 */ + PTR sys_wait4 + PTR sys_swapoff /* 4115 */ + PTR sys_sysinfo + PTR sys_ipc + PTR sys_fsync + PTR sys_sigreturn + PTR __sys_clone /* 4120 */ + PTR sys_setdomainname + PTR sys_newuname + PTR sys_ni_syscall /* sys_modify_ldt */ + PTR sys_adjtimex + PTR sys_mprotect /* 4125 */ + PTR sys_sigprocmask + PTR sys_ni_syscall /* was create_module */ + PTR sys_init_module + PTR sys_delete_module + PTR sys_ni_syscall /* 4130 was get_kernel_syms */ + PTR sys_quotactl + PTR sys_getpgid + PTR sys_fchdir + PTR sys_bdflush + PTR sys_sysfs /* 4135 */ + PTR sys_personality + PTR sys_ni_syscall /* for afs_syscall */ + PTR sys_setfsuid + PTR sys_setfsgid + PTR sys_llseek /* 4140 */ + PTR sys_getdents + PTR sys_select + PTR sys_flock + PTR sys_msync + PTR sys_readv /* 4145 */ + PTR sys_writev + PTR sys_cacheflush + PTR sys_cachectl + PTR sys_sysmips + PTR sys_ni_syscall /* 4150 */ + PTR sys_getsid + PTR sys_fdatasync + PTR sys_sysctl + PTR sys_mlock + PTR sys_munlock /* 4155 */ + PTR sys_mlockall + PTR sys_munlockall + PTR sys_sched_setparam + PTR sys_sched_getparam + PTR sys_sched_setscheduler /* 4160 */ + PTR sys_sched_getscheduler + PTR sys_sched_yield + PTR sys_sched_get_priority_max + PTR sys_sched_get_priority_min + PTR sys_sched_rr_get_interval /* 4165 */ + PTR sys_nanosleep + PTR sys_mremap + PTR sys_accept + PTR sys_bind + PTR sys_connect /* 4170 */ + PTR sys_getpeername + PTR sys_getsockname + PTR sys_getsockopt + PTR sys_listen + PTR sys_recv /* 4175 */ + PTR sys_recvfrom + PTR sys_recvmsg + PTR sys_send + PTR sys_sendmsg + PTR sys_sendto /* 4180 */ + PTR sys_setsockopt + PTR sys_shutdown + PTR sys_socket + PTR sys_socketpair + PTR sys_setresuid /* 4185 */ + PTR sys_getresuid + PTR sys_ni_syscall /* was sys_query_module */ + PTR sys_poll + PTR sys_ni_syscall /* was nfsservctl */ + PTR sys_setresgid /* 4190 */ + PTR sys_getresgid + PTR sys_prctl + PTR sys_rt_sigreturn + PTR sys_rt_sigaction + PTR sys_rt_sigprocmask /* 4195 */ + PTR sys_rt_sigpending + PTR sys_rt_sigtimedwait + PTR sys_rt_sigqueueinfo + PTR sys_rt_sigsuspend + PTR sys_pread64 /* 4200 */ + PTR sys_pwrite64 + PTR sys_chown + PTR sys_getcwd + PTR sys_capget + PTR sys_capset /* 4205 */ + PTR sys_sigaltstack + PTR sys_sendfile + PTR sys_ni_syscall + PTR sys_ni_syscall + PTR sys_mips_mmap2 /* 4210 */ + PTR sys_truncate64 + PTR sys_ftruncate64 + PTR sys_stat64 + PTR sys_lstat64 + PTR sys_fstat64 /* 4215 */ + PTR sys_pivot_root + PTR sys_mincore + PTR sys_madvise + PTR sys_getdents64 + PTR sys_fcntl64 /* 4220 */ + PTR sys_ni_syscall + PTR sys_gettid + PTR sys_readahead + PTR sys_setxattr + PTR sys_lsetxattr /* 4225 */ + PTR sys_fsetxattr + PTR sys_getxattr + PTR sys_lgetxattr + PTR sys_fgetxattr + PTR sys_listxattr /* 4230 */ + PTR sys_llistxattr + PTR sys_flistxattr + PTR sys_removexattr + PTR sys_lremovexattr + PTR sys_fremovexattr /* 4235 */ + PTR sys_tkill + PTR sys_sendfile64 + PTR sys_futex #ifdef CONFIG_MIPS_MT_FPAFF /* * For FPU affinity scheduling on MIPS MT processors, we need to * intercept sys_sched_xxxaffinity() calls until we get a proper hook - * in kernel/sched.c. Considered only temporary we only support these - * hooks for the 32-bit kernel - there is no MIPS64 MT processor atm. + * in kernel/sched/core.c. Considered only temporary we only support + * these hooks for the 32-bit kernel - there is no MIPS64 MT processor + * atm. */ - sys mipsmt_sys_sched_setaffinity 3 - sys mipsmt_sys_sched_getaffinity 3 + PTR mipsmt_sys_sched_setaffinity + PTR mipsmt_sys_sched_getaffinity #else - sys sys_sched_setaffinity 3 - sys sys_sched_getaffinity 3 /* 4240 */ + PTR sys_sched_setaffinity + PTR sys_sched_getaffinity /* 4240 */ #endif /* CONFIG_MIPS_MT_FPAFF */ - sys sys_io_setup 2 - sys sys_io_destroy 1 - sys sys_io_getevents 5 - sys sys_io_submit 3 - sys sys_io_cancel 3 /* 4245 */ - sys sys_exit_group 1 - sys sys_lookup_dcookie 4 - sys sys_epoll_create 1 - sys sys_epoll_ctl 4 - sys sys_epoll_wait 3 /* 4250 */ - sys sys_remap_file_pages 5 - sys sys_set_tid_address 1 - sys sys_restart_syscall 0 - sys sys_fadvise64_64 7 - sys sys_statfs64 3 /* 4255 */ - sys sys_fstatfs64 2 - sys sys_timer_create 3 - sys sys_timer_settime 4 - sys sys_timer_gettime 2 - sys sys_timer_getoverrun 1 /* 4260 */ - sys sys_timer_delete 1 - sys sys_clock_settime 2 - sys sys_clock_gettime 2 - sys sys_clock_getres 2 - sys sys_clock_nanosleep 4 /* 4265 */ - sys sys_tgkill 3 - sys sys_utimes 2 - sys sys_mbind 4 - sys sys_ni_syscall 0 /* sys_get_mempolicy */ - sys sys_ni_syscall 0 /* 4270 sys_set_mempolicy */ - sys sys_mq_open 4 - sys sys_mq_unlink 1 - sys sys_mq_timedsend 5 - sys sys_mq_timedreceive 5 - sys sys_mq_notify 2 /* 4275 */ - sys sys_mq_getsetattr 3 - sys sys_ni_syscall 0 /* sys_vserver */ - sys sys_waitid 5 - sys sys_ni_syscall 0 /* available, was setaltroot */ - sys sys_add_key 5 /* 4280 */ - sys sys_request_key 4 - sys sys_keyctl 5 - sys sys_set_thread_area 1 - sys sys_inotify_init 0 - sys sys_inotify_add_watch 3 /* 4285 */ - sys sys_inotify_rm_watch 2 - sys sys_migrate_pages 4 - sys sys_openat 4 - sys sys_mkdirat 3 - sys sys_mknodat 4 /* 4290 */ - sys sys_fchownat 5 - sys sys_futimesat 3 - sys sys_fstatat64 4 - sys sys_unlinkat 3 - sys sys_renameat 4 /* 4295 */ - sys sys_linkat 5 - sys sys_symlinkat 3 - sys sys_readlinkat 4 - sys sys_fchmodat 3 - sys sys_faccessat 3 /* 4300 */ - sys sys_pselect6 6 - sys sys_ppoll 5 - sys sys_unshare 1 - sys sys_splice 6 - sys sys_sync_file_range 7 /* 4305 */ - sys sys_tee 4 - sys sys_vmsplice 4 - sys sys_move_pages 6 - sys sys_set_robust_list 2 - sys sys_get_robust_list 3 /* 4310 */ - sys sys_kexec_load 4 - sys sys_getcpu 3 - sys sys_epoll_pwait 6 - sys sys_ioprio_set 3 - sys sys_ioprio_get 2 /* 4315 */ - sys sys_utimensat 4 - sys sys_signalfd 3 - sys sys_ni_syscall 0 - sys sys_eventfd 1 - sys sys_fallocate 6 /* 4320 */ - sys sys_timerfd_create 2 - sys sys_timerfd_gettime 2 - sys sys_timerfd_settime 4 - sys sys_signalfd4 4 - sys sys_eventfd2 2 /* 4325 */ - sys sys_epoll_create1 1 - sys sys_dup3 3 - sys sys_pipe2 2 - sys sys_inotify_init1 1 - sys sys_preadv 6 /* 4330 */ - sys sys_pwritev 6 - sys sys_rt_tgsigqueueinfo 4 - sys sys_perf_event_open 5 - sys sys_accept4 4 - sys sys_recvmmsg 5 /* 4335 */ - sys sys_fanotify_init 2 - sys sys_fanotify_mark 6 - sys sys_prlimit64 4 - .endm - - /* We pre-compute the number of _instruction_ bytes needed to - load or store the arguments 6-8. Negative values are ignored. */ - - .macro sys function, nargs - PTR \function - LONG (\nargs << 2) - (5 << 2) - .endm - - .align 3 - .type sys_call_table,@object -EXPORT(sys_call_table) - syscalltable - .size sys_call_table, . - sys_call_table + PTR sys_io_setup + PTR sys_io_destroy + PTR sys_io_getevents + PTR sys_io_submit + PTR sys_io_cancel /* 4245 */ + PTR sys_exit_group + PTR sys_lookup_dcookie + PTR sys_epoll_create + PTR sys_epoll_ctl + PTR sys_epoll_wait /* 4250 */ + PTR sys_remap_file_pages + PTR sys_set_tid_address + PTR sys_restart_syscall + PTR sys_fadvise64_64 + PTR sys_statfs64 /* 4255 */ + PTR sys_fstatfs64 + PTR sys_timer_create + PTR sys_timer_settime + PTR sys_timer_gettime + PTR sys_timer_getoverrun /* 4260 */ + PTR sys_timer_delete + PTR sys_clock_settime + PTR sys_clock_gettime + PTR sys_clock_getres + PTR sys_clock_nanosleep /* 4265 */ + PTR sys_tgkill + PTR sys_utimes + PTR sys_mbind + PTR sys_ni_syscall /* sys_get_mempolicy */ + PTR sys_ni_syscall /* 4270 sys_set_mempolicy */ + PTR sys_mq_open + PTR sys_mq_unlink + PTR sys_mq_timedsend + PTR sys_mq_timedreceive + PTR sys_mq_notify /* 4275 */ + PTR sys_mq_getsetattr + PTR sys_ni_syscall /* sys_vserver */ + PTR sys_waitid + PTR sys_ni_syscall /* available, was setaltroot */ + PTR sys_add_key /* 4280 */ + PTR sys_request_key + PTR sys_keyctl + PTR sys_set_thread_area + PTR sys_inotify_init + PTR sys_inotify_add_watch /* 4285 */ + PTR sys_inotify_rm_watch + PTR sys_migrate_pages + PTR sys_openat + PTR sys_mkdirat + PTR sys_mknodat /* 4290 */ + PTR sys_fchownat + PTR sys_futimesat + PTR sys_fstatat64 + PTR sys_unlinkat + PTR sys_renameat /* 4295 */ + PTR sys_linkat + PTR sys_symlinkat + PTR sys_readlinkat + PTR sys_fchmodat + PTR sys_faccessat /* 4300 */ + PTR sys_pselect6 + PTR sys_ppoll + PTR sys_unshare + PTR sys_splice + PTR sys_sync_file_range /* 4305 */ + PTR sys_tee + PTR sys_vmsplice + PTR sys_move_pages + PTR sys_set_robust_list + PTR sys_get_robust_list /* 4310 */ + PTR sys_kexec_load + PTR sys_getcpu + PTR sys_epoll_pwait + PTR sys_ioprio_set + PTR sys_ioprio_get /* 4315 */ + PTR sys_utimensat + PTR sys_signalfd + PTR sys_ni_syscall /* was timerfd */ + PTR sys_eventfd + PTR sys_fallocate /* 4320 */ + PTR sys_timerfd_create + PTR sys_timerfd_gettime + PTR sys_timerfd_settime + PTR sys_signalfd4 + PTR sys_eventfd2 /* 4325 */ + PTR sys_epoll_create1 + PTR sys_dup3 + PTR sys_pipe2 + PTR sys_inotify_init1 + PTR sys_preadv /* 4330 */ + PTR sys_pwritev + PTR sys_rt_tgsigqueueinfo + PTR sys_perf_event_open + PTR sys_accept4 + PTR sys_recvmmsg /* 4335 */ + PTR sys_fanotify_init + PTR sys_fanotify_mark + PTR sys_prlimit64 + PTR sys_name_to_handle_at + PTR sys_open_by_handle_at /* 4340 */ + PTR sys_clock_adjtime + PTR sys_syncfs + PTR sys_sendmmsg + PTR sys_setns + PTR sys_process_vm_readv /* 4345 */ + PTR sys_process_vm_writev + PTR sys_kcmp + PTR sys_finit_module + PTR sys_sched_setattr + PTR sys_sched_getattr /* 4350 */ + PTR sys_renameat2 diff --git a/arch/mips/kernel/scall64-64.S b/arch/mips/kernel/scall64-64.S index 3f417928320..be2fedd4ae3 100644 --- a/arch/mips/kernel/scall64-64.S +++ b/arch/mips/kernel/scall64-64.S @@ -25,7 +25,7 @@ #define handle_sys64 handle_sys #endif - .align 5 + .align 5 NESTED(handle_sys64, PT_SIZE, sp) #if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) /* @@ -40,7 +40,7 @@ NESTED(handle_sys64, PT_SIZE, sp) #endif dsubu t0, v0, __NR_64_Linux # check syscall number - sltiu t0, t0, __NR_64_Linux_syscalls + 1 + sltiu t0, t0, __NR_64_Linux_syscalls + 1 #if !defined(CONFIG_MIPS32_O32) && !defined(CONFIG_MIPS32_N32) ld t1, PT_EPC(sp) # skip syscall on return daddiu t1, 4 # skip to next instruction @@ -54,7 +54,7 @@ NESTED(handle_sys64, PT_SIZE, sp) sd a3, PT_R26(sp) # save a3 for syscall restarting - li t1, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + li t1, _TIF_WORK_SYSCALL_ENTRY LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? and t0, t1, t0 bnez t0, syscall_trace_entry @@ -72,18 +72,7 @@ NESTED(handle_sys64, PT_SIZE, sp) 1: sd v0, PT_R2(sp) # result n64_syscall_exit: - local_irq_disable # make sure need_resched and - # signals dont change between - # sampling and return - LONG_L a2, TI_FLAGS($28) # current->work - li t0, _TIF_ALLWORK_MASK - and t0, a2, t0 - bnez t0, n64_syscall_exit_work - - j restore_partial - -n64_syscall_exit_work: - j syscall_exit_work_partial + j syscall_exit_partial /* ------------------------------------------------------------------------ */ @@ -91,8 +80,10 @@ syscall_trace_entry: SAVE_STATIC move s0, t2 move a0, sp - li a1, 0 - jal do_syscall_trace + daddiu a1, v0, __NR_64_Linux + jal syscall_trace_enter + + bltz v0, 2f # seccomp failed? Skip syscall move t0, s0 RESTORE_STATIC @@ -114,7 +105,7 @@ syscall_trace_entry: sd t1, PT_R0(sp) # save it for syscall restarting 1: sd v0, PT_R2(sp) # result - j syscall_exit +2: j syscall_exit illegal_syscall: /* This also isn't a 64-bit syscall, throw an error. */ @@ -126,7 +117,8 @@ illegal_syscall: END(handle_sys64) .align 3 -sys_call_table: + .type sys_call_table, @object +EXPORT(sys_call_table) PTR sys_read /* 5000 */ PTR sys_write PTR sys_open @@ -182,8 +174,8 @@ sys_call_table: PTR sys_socketpair PTR sys_setsockopt PTR sys_getsockopt - PTR sys_clone /* 5055 */ - PTR sys_fork + PTR __sys_clone /* 5055 */ + PTR __sys_fork PTR sys_execve PTR sys_exit PTR sys_wait4 @@ -300,9 +292,9 @@ sys_call_table: PTR sys_ni_syscall /* 5170, was get_kernel_syms */ PTR sys_ni_syscall /* was query_module */ PTR sys_quotactl - PTR sys_nfsservctl + PTR sys_ni_syscall /* was nfsservctl */ PTR sys_ni_syscall /* res. for getpmsg */ - PTR sys_ni_syscall /* 5175 for putpmsg */ + PTR sys_ni_syscall /* 5175 for putpmsg */ PTR sys_ni_syscall /* res. for afs_syscall */ PTR sys_ni_syscall /* res. for security */ PTR sys_gettid @@ -404,7 +396,7 @@ sys_call_table: PTR sys_ioprio_get PTR sys_utimensat /* 5275 */ PTR sys_signalfd - PTR sys_ni_syscall + PTR sys_ni_syscall /* was timerfd */ PTR sys_eventfd PTR sys_fallocate PTR sys_timerfd_create /* 5280 */ @@ -425,4 +417,18 @@ sys_call_table: PTR sys_fanotify_init /* 5295 */ PTR sys_fanotify_mark PTR sys_prlimit64 + PTR sys_name_to_handle_at + PTR sys_open_by_handle_at + PTR sys_clock_adjtime /* 5300 */ + PTR sys_syncfs + PTR sys_sendmmsg + PTR sys_setns + PTR sys_process_vm_readv + PTR sys_process_vm_writev /* 5305 */ + PTR sys_kcmp + PTR sys_finit_module + PTR sys_getdents64 + PTR sys_sched_setattr + PTR sys_sched_getattr /* 5310 */ + PTR sys_renameat2 .size sys_call_table,.-sys_call_table diff --git a/arch/mips/kernel/scall64-n32.S b/arch/mips/kernel/scall64-n32.S index f08ece6d8ac..c1dbcda4b81 100644 --- a/arch/mips/kernel/scall64-n32.S +++ b/arch/mips/kernel/scall64-n32.S @@ -17,18 +17,12 @@ #include <asm/thread_info.h> #include <asm/unistd.h> -/* This duplicates the definition from <linux/sched.h> */ -#define PT_TRACESYS 0x00000002 /* tracing system calls */ - -/* This duplicates the definition from <asm/signal.h> */ -#define SIGILL 4 /* Illegal instruction (ANSI). */ - #ifndef CONFIG_MIPS32_O32 /* No O32, so define handle_sys here */ #define handle_sysn32 handle_sys #endif - .align 5 + .align 5 NESTED(handle_sysn32, PT_SIZE, sp) #ifndef CONFIG_MIPS32_O32 .set noat @@ -39,7 +33,7 @@ NESTED(handle_sysn32, PT_SIZE, sp) #endif dsubu t0, v0, __NR_N32_Linux # check syscall number - sltiu t0, t0, __NR_N32_Linux_syscalls + 1 + sltiu t0, t0, __NR_N32_Linux_syscalls + 1 #ifndef CONFIG_MIPS32_O32 ld t1, PT_EPC(sp) # skip syscall on return @@ -53,7 +47,7 @@ NESTED(handle_sysn32, PT_SIZE, sp) sd a3, PT_R26(sp) # save a3 for syscall restarting - li t1, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + li t1, _TIF_WORK_SYSCALL_ENTRY LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? and t0, t1, t0 bnez t0, n32_syscall_trace_entry @@ -70,18 +64,7 @@ NESTED(handle_sysn32, PT_SIZE, sp) sd t1, PT_R0(sp) # save it for syscall restarting 1: sd v0, PT_R2(sp) # result - local_irq_disable # make sure need_resched and - # signals dont change between - # sampling and return - LONG_L a2, TI_FLAGS($28) # current->work - li t0, _TIF_ALLWORK_MASK - and t0, a2, t0 - bnez t0, n32_syscall_exit_work - - j restore_partial - -n32_syscall_exit_work: - j syscall_exit_work_partial + j syscall_exit_partial /* ------------------------------------------------------------------------ */ @@ -89,8 +72,10 @@ n32_syscall_trace_entry: SAVE_STATIC move s0, t2 move a0, sp - li a1, 0 - jal do_syscall_trace + daddiu a1, v0, __NR_N32_Linux + jal syscall_trace_enter + + bltz v0, 2f # seccomp failed? Skip syscall move t0, s0 RESTORE_STATIC @@ -112,7 +97,7 @@ n32_syscall_trace_entry: sd t1, PT_R0(sp) # save it for syscall restarting 1: sd v0, PT_R2(sp) # result - j syscall_exit +2: j syscall_exit not_n32_scall: /* This is not an n32 compatibility syscall, pass it on to @@ -121,6 +106,7 @@ not_n32_scall: END(handle_sysn32) + .type sysn32_call_table, @object EXPORT(sysn32_call_table) PTR sys_read /* 6000 */ PTR sys_write @@ -135,8 +121,8 @@ EXPORT(sysn32_call_table) PTR sys_mprotect /* 6010 */ PTR sys_munmap PTR sys_brk - PTR sys_32_rt_sigaction - PTR sys_32_rt_sigprocmask + PTR compat_sys_rt_sigaction + PTR compat_sys_rt_sigprocmask PTR compat_sys_ioctl /* 6015 */ PTR sys_pread64 PTR sys_pwrite64 @@ -161,7 +147,7 @@ EXPORT(sysn32_call_table) PTR compat_sys_setitimer PTR sys_alarm PTR sys_getpid - PTR sys_32_sendfile + PTR compat_sys_sendfile PTR sys_socket /* 6040 */ PTR sys_connect PTR sys_accept @@ -177,20 +163,20 @@ EXPORT(sysn32_call_table) PTR sys_socketpair PTR compat_sys_setsockopt PTR sys_getsockopt - PTR sys_clone /* 6055 */ - PTR sys_fork - PTR sys32_execve + PTR __sys_clone /* 6055 */ + PTR __sys_fork + PTR compat_sys_execve PTR sys_exit PTR compat_sys_wait4 PTR sys_kill /* 6060 */ PTR sys_newuname PTR sys_semget PTR sys_semop - PTR sys_n32_semctl + PTR compat_sys_semctl PTR sys_shmdt /* 6065 */ PTR sys_msgget - PTR sys_n32_msgsnd - PTR sys_n32_msgrcv + PTR compat_sys_msgsnd + PTR compat_sys_msgrcv PTR compat_sys_msgctl PTR compat_sys_fcntl /* 6070 */ PTR sys_flock @@ -247,11 +233,11 @@ EXPORT(sysn32_call_table) PTR sys_getsid PTR sys_capget PTR sys_capset - PTR sys_32_rt_sigpending /* 6125 */ + PTR compat_sys_rt_sigpending /* 6125 */ PTR compat_sys_rt_sigtimedwait - PTR sys_32_rt_sigqueueinfo - PTR sysn32_rt_sigsuspend - PTR sys32_sigaltstack + PTR compat_sys_rt_sigqueueinfo + PTR compat_sys_rt_sigsuspend + PTR compat_sys_sigaltstack PTR compat_sys_utime /* 6130 */ PTR sys_mknod PTR sys_32_personality @@ -267,7 +253,7 @@ EXPORT(sysn32_call_table) PTR sys_sched_getscheduler PTR sys_sched_get_priority_max PTR sys_sched_get_priority_min - PTR sys_32_sched_rr_get_interval /* 6145 */ + PTR compat_sys_sched_rr_get_interval /* 6145 */ PTR sys_mlock PTR sys_munlock PTR sys_mlockall @@ -295,9 +281,9 @@ EXPORT(sysn32_call_table) PTR sys_ni_syscall /* 6170, was get_kernel_syms */ PTR sys_ni_syscall /* was query_module */ PTR sys_quotactl - PTR compat_sys_nfsservctl + PTR sys_ni_syscall /* was nfsservctl */ PTR sys_ni_syscall /* res. for getpmsg */ - PTR sys_ni_syscall /* 6175 for putpmsg */ + PTR sys_ni_syscall /* 6175 for putpmsg */ PTR sys_ni_syscall /* res. for afs_syscall */ PTR sys_ni_syscall /* res. for security */ PTR sys_gettid @@ -403,26 +389,39 @@ EXPORT(sysn32_call_table) PTR sys_ioprio_get PTR compat_sys_utimensat PTR compat_sys_signalfd /* 6280 */ - PTR sys_ni_syscall + PTR sys_ni_syscall /* was timerfd */ PTR sys_eventfd PTR sys_fallocate PTR sys_timerfd_create PTR compat_sys_timerfd_gettime /* 6285 */ PTR compat_sys_timerfd_settime - PTR sys_signalfd4 + PTR compat_sys_signalfd4 PTR sys_eventfd2 PTR sys_epoll_create1 PTR sys_dup3 /* 6290 */ PTR sys_pipe2 PTR sys_inotify_init1 - PTR sys_preadv - PTR sys_pwritev + PTR compat_sys_preadv + PTR compat_sys_pwritev PTR compat_sys_rt_tgsigqueueinfo /* 6295 */ PTR sys_perf_event_open PTR sys_accept4 - PTR compat_sys_recvmmsg - PTR sys_getdents64 + PTR compat_sys_recvmmsg + PTR sys_getdents64 PTR sys_fanotify_init /* 6300 */ PTR sys_fanotify_mark PTR sys_prlimit64 + PTR sys_name_to_handle_at + PTR sys_open_by_handle_at + PTR compat_sys_clock_adjtime /* 6305 */ + PTR sys_syncfs + PTR compat_sys_sendmmsg + PTR sys_setns + PTR compat_sys_process_vm_readv + PTR compat_sys_process_vm_writev /* 6310 */ + PTR sys_kcmp + PTR sys_finit_module + PTR sys_sched_setattr + PTR sys_sched_getattr + PTR sys_renameat2 /* 6315 */ .size sysn32_call_table,.-sysn32_call_table diff --git a/arch/mips/kernel/scall64-o32.S b/arch/mips/kernel/scall64-o32.S index 78d768a3e19..f1343ccd7ed 100644 --- a/arch/mips/kernel/scall64-o32.S +++ b/arch/mips/kernel/scall64-o32.S @@ -10,7 +10,7 @@ * * Hairy, the userspace application uses a different argument passing * convention than the kernel, so we have to translate things from o32 - * to ABI64 calling convention. 64-bit syscalls are also processed + * to ABI64 calling convention. 64-bit syscalls are also processed * here for now. */ #include <linux/errno.h> @@ -24,7 +24,7 @@ #include <asm/unistd.h> #include <asm/sysmips.h> - .align 5 + .align 5 NESTED(handle_sys, PT_SIZE, sp) .set noat SAVE_SOME @@ -53,7 +53,7 @@ NESTED(handle_sys, PT_SIZE, sp) sll a3, a3, 0 dsll t0, v0, 3 # offset into table - ld t2, (sys_call_table - (__NR_O32_Linux * 8))(t0) + ld t2, (sys32_call_table - (__NR_O32_Linux * 8))(t0) sd a3, PT_R26(sp) # save a3 for syscall restarting @@ -81,7 +81,7 @@ NESTED(handle_sys, PT_SIZE, sp) PTR 4b, bad_stack .previous - li t1, _TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT + li t1, _TIF_WORK_SYSCALL_ENTRY LONG_L t0, TI_FLAGS($28) # syscall tracing enabled? and t0, t1, t0 bnez t0, trace_a_syscall @@ -99,18 +99,7 @@ NESTED(handle_sys, PT_SIZE, sp) 1: sd v0, PT_R2(sp) # result o32_syscall_exit: - local_irq_disable # make need_resched and - # signals dont change between - # sampling and return - LONG_L a2, TI_FLAGS($28) - li t0, _TIF_ALLWORK_MASK - and t0, a2, t0 - bnez t0, o32_syscall_exit_work - - j restore_partial - -o32_syscall_exit_work: - j syscall_exit_work_partial + j syscall_exit_partial /* ------------------------------------------------------------------------ */ @@ -123,8 +112,20 @@ trace_a_syscall: move s0, t2 # Save syscall pointer move a0, sp - li a1, 0 - jal do_syscall_trace + /* + * syscall number is in v0 unless we called syscall(__NR_###) + * where the real syscall number is in a0 + * note: NR_syscall is the first O32 syscall but the macro is + * only defined when compiling with -mabi=32 (CONFIG_32BIT) + * therefore __NR_O32_Linux is used (4000) + */ + addiu a1, v0, __NR_O32_Linux + bnez v0, 1f /* __NR_syscall at offset 0 */ + lw a1, PT_R4(sp) + +1: jal syscall_trace_enter + + bltz v0, 2f # seccomp failed? Skip syscall move t0, s0 RESTORE_STATIC @@ -148,7 +149,7 @@ trace_a_syscall: sd t1, PT_R0(sp) # save it for syscall restarting 1: sd v0, PT_R2(sp) # result - j syscall_exit +2: j syscall_exit /* ------------------------------------------------------------------------ */ @@ -180,7 +181,7 @@ LEAF(sys32_syscall) beqz t0, einval # do not recurse dsll t1, t0, 3 beqz v0, einval - ld t2, sys_call_table(t1) # syscall routine + ld t2, sys32_call_table(t1) # syscall routine move a0, a1 # shift argument registers move a1, a2 @@ -197,16 +198,16 @@ LEAF(sys32_syscall) jr t2 /* Unreached */ -einval: li v0, -ENOSYS +einval: li v0, -ENOSYS jr ra END(sys32_syscall) .align 3 - .type sys_call_table,@object -sys_call_table: + .type sys32_call_table,@object +EXPORT(sys32_call_table) PTR sys32_syscall /* 4000 */ PTR sys_exit - PTR sys_fork + PTR __sys_fork PTR sys_read PTR sys_write PTR compat_sys_open /* 4005 */ @@ -215,7 +216,7 @@ sys_call_table: PTR sys_creat PTR sys_link PTR sys_unlink /* 4010 */ - PTR sys32_execve + PTR compat_sys_execve PTR sys_chdir PTR compat_sys_time PTR sys_mknod @@ -296,8 +297,8 @@ sys_call_table: PTR compat_sys_old_readdir PTR sys_mips_mmap /* 4090 */ PTR sys_munmap - PTR sys_truncate - PTR sys_ftruncate + PTR compat_sys_truncate + PTR compat_sys_ftruncate PTR sys_fchmod PTR sys_fchown /* 4095 */ PTR sys_getpriority @@ -321,10 +322,10 @@ sys_call_table: PTR compat_sys_wait4 PTR sys_swapoff /* 4115 */ PTR compat_sys_sysinfo - PTR sys_32_ipc + PTR compat_sys_ipc PTR sys_fsync PTR sys32_sigreturn - PTR sys32_clone /* 4120 */ + PTR __sys_clone /* 4120 */ PTR sys_setdomainname PTR sys_newuname PTR sys_ni_syscall /* sys_modify_ldt */ @@ -341,7 +342,7 @@ sys_call_table: PTR sys_bdflush PTR sys_sysfs /* 4135 */ PTR sys_32_personality - PTR sys_ni_syscall /* for afs_syscall */ + PTR sys_ni_syscall /* for afs_syscall */ PTR sys_setfsuid PTR sys_setfsgid PTR sys_32_llseek /* 4140 */ @@ -364,12 +365,12 @@ sys_call_table: PTR sys_munlockall PTR sys_sched_setparam PTR sys_sched_getparam - PTR sys_sched_setscheduler /* 4160 */ + PTR sys_sched_setscheduler /* 4160 */ PTR sys_sched_getscheduler PTR sys_sched_yield PTR sys_sched_get_priority_max PTR sys_sched_get_priority_min - PTR sys_32_sched_rr_get_interval /* 4165 */ + PTR compat_sys_sched_rr_get_interval /* 4165 */ PTR compat_sys_nanosleep PTR sys_mremap PTR sys_accept @@ -393,25 +394,25 @@ sys_call_table: PTR sys_getresuid PTR sys_ni_syscall /* was query_module */ PTR sys_poll - PTR compat_sys_nfsservctl + PTR sys_ni_syscall /* was nfsservctl */ PTR sys_setresgid /* 4190 */ PTR sys_getresgid PTR sys_prctl PTR sys32_rt_sigreturn - PTR sys_32_rt_sigaction - PTR sys_32_rt_sigprocmask /* 4195 */ - PTR sys_32_rt_sigpending + PTR compat_sys_rt_sigaction + PTR compat_sys_rt_sigprocmask /* 4195 */ + PTR compat_sys_rt_sigpending PTR compat_sys_rt_sigtimedwait - PTR sys_32_rt_sigqueueinfo - PTR sys32_rt_sigsuspend + PTR compat_sys_rt_sigqueueinfo + PTR compat_sys_rt_sigsuspend PTR sys_32_pread /* 4200 */ PTR sys_32_pwrite PTR sys_chown PTR sys_getcwd PTR sys_capget PTR sys_capset /* 4205 */ - PTR sys32_sigaltstack - PTR sys_32_sendfile + PTR compat_sys_sigaltstack + PTR compat_sys_sendfile PTR sys_ni_syscall PTR sys_ni_syscall PTR sys_mips_mmap2 /* 4210 */ @@ -451,7 +452,7 @@ sys_call_table: PTR compat_sys_io_submit PTR sys_io_cancel /* 4245 */ PTR sys_exit_group - PTR sys32_lookup_dcookie + PTR compat_sys_lookup_dcookie PTR sys_epoll_create PTR sys_epoll_ctl PTR sys_epoll_wait /* 4250 */ @@ -482,7 +483,7 @@ sys_call_table: PTR compat_sys_mq_notify /* 4275 */ PTR compat_sys_mq_getsetattr PTR sys_ni_syscall /* sys_vserver */ - PTR sys_32_waitid + PTR compat_sys_waitid PTR sys_ni_syscall /* available, was setaltroot */ PTR sys_add_key /* 4280 */ PTR sys_request_key @@ -522,7 +523,7 @@ sys_call_table: PTR sys_ioprio_get /* 4315 */ PTR compat_sys_utimensat PTR compat_sys_signalfd - PTR sys_ni_syscall + PTR sys_ni_syscall /* was timerfd */ PTR sys_eventfd PTR sys32_fallocate /* 4320 */ PTR sys_timerfd_create @@ -541,6 +542,19 @@ sys_call_table: PTR sys_accept4 PTR compat_sys_recvmmsg /* 4335 */ PTR sys_fanotify_init - PTR sys_32_fanotify_mark + PTR compat_sys_fanotify_mark PTR sys_prlimit64 - .size sys_call_table,.-sys_call_table + PTR sys_name_to_handle_at + PTR compat_sys_open_by_handle_at /* 4340 */ + PTR compat_sys_clock_adjtime + PTR sys_syncfs + PTR compat_sys_sendmmsg + PTR sys_setns + PTR compat_sys_process_vm_readv /* 4345 */ + PTR compat_sys_process_vm_writev + PTR sys_kcmp + PTR sys_finit_module + PTR sys_sched_setattr + PTR sys_sched_getattr /* 4350 */ + PTR sys_renameat2 + .size sys32_call_table,.-sys32_call_table diff --git a/arch/mips/kernel/segment.c b/arch/mips/kernel/segment.c new file mode 100644 index 00000000000..076ead2a985 --- /dev/null +++ b/arch/mips/kernel/segment.c @@ -0,0 +1,110 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2013 Imagination Technologies Ltd. + */ + +#include <linux/kernel.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> +#include <asm/cpu.h> +#include <asm/mipsregs.h> + +static void build_segment_config(char *str, unsigned int cfg) +{ + unsigned int am; + static const char * const am_str[] = { + "UK", "MK", "MSK", "MUSK", "MUSUK", "USK", + "RSRVD", "UUSK"}; + + /* Segment access mode. */ + am = (cfg & MIPS_SEGCFG_AM) >> MIPS_SEGCFG_AM_SHIFT; + str += sprintf(str, "%-5s", am_str[am]); + + /* + * Access modes MK, MSK and MUSK are mapped segments. Therefore + * there is no direct physical address mapping. + */ + if ((am == 0) || (am > 3)) { + str += sprintf(str, " %03lx", + ((cfg & MIPS_SEGCFG_PA) >> MIPS_SEGCFG_PA_SHIFT)); + str += sprintf(str, " %01ld", + ((cfg & MIPS_SEGCFG_C) >> MIPS_SEGCFG_C_SHIFT)); + } else { + str += sprintf(str, " UND"); + str += sprintf(str, " U"); + } + + /* Exception configuration. */ + str += sprintf(str, " %01ld\n", + ((cfg & MIPS_SEGCFG_EU) >> MIPS_SEGCFG_EU_SHIFT)); +} + +static int show_segments(struct seq_file *m, void *v) +{ + unsigned int segcfg; + char str[42]; + + seq_puts(m, "Segment Virtual Size Access Mode Physical Caching EU\n"); + seq_puts(m, "------- ------- ---- ----------- -------- ------- --\n"); + + segcfg = read_c0_segctl0(); + build_segment_config(str, segcfg); + seq_printf(m, " 0 e0000000 512M %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 1 c0000000 512M %s", str); + + segcfg = read_c0_segctl1(); + build_segment_config(str, segcfg); + seq_printf(m, " 2 a0000000 512M %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 3 80000000 512M %s", str); + + segcfg = read_c0_segctl2(); + build_segment_config(str, segcfg); + seq_printf(m, " 4 40000000 1G %s", str); + + segcfg >>= 16; + build_segment_config(str, segcfg); + seq_printf(m, " 5 00000000 1G %s\n", str); + + return 0; +} + +static int segments_open(struct inode *inode, struct file *file) +{ + return single_open(file, show_segments, NULL); +} + +static const struct file_operations segments_fops = { + .open = segments_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init segments_info(void) +{ + extern struct dentry *mips_debugfs_dir; + struct dentry *segments; + + if (cpu_has_segments) { + if (!mips_debugfs_dir) + return -ENODEV; + + segments = debugfs_create_file("segments", S_IRUGO, + mips_debugfs_dir, NULL, + &segments_fops); + if (!segments) + return -ENOMEM; + } + return 0; +} + +device_initcall(segments_info); diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c index acd3f2c49c0..a842154d57d 100644 --- a/arch/mips/kernel/setup.c +++ b/arch/mips/kernel/setup.c @@ -8,12 +8,13 @@ * Copyright (C) 1994, 95, 96, 97, 98, 99, 2000, 01, 02, 03 Ralf Baechle * Copyright (C) 1996 Stoned Elipot * Copyright (C) 1999 Silicon Graphics, Inc. - * Copyright (C) 2000, 2001, 2002, 2007 Maciej W. Rozycki + * Copyright (C) 2000, 2001, 2002, 2007 Maciej W. Rozycki */ #include <linux/init.h> #include <linux/ioport.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/screen_info.h> +#include <linux/memblock.h> #include <linux/bootmem.h> #include <linux/initrd.h> #include <linux/root_dev.h> @@ -21,6 +22,8 @@ #include <linux/console.h> #include <linux/pfn.h> #include <linux/debugfs.h> +#include <linux/kexec.h> +#include <linux/sizes.h> #include <asm/addrspace.h> #include <asm/bootinfo.h> @@ -30,7 +33,6 @@ #include <asm/sections.h> #include <asm/setup.h> #include <asm/smp-ops.h> -#include <asm/system.h> #include <asm/prom.h> struct cpuinfo_mips cpu_data[NR_CPUS] __read_mostly; @@ -70,16 +72,18 @@ static char __initdata builtin_cmdline[COMMAND_LINE_SIZE] = CONFIG_CMDLINE; * mips_io_port_base is the begin of the address space to which x86 style * I/O ports are mapped. */ -const unsigned long mips_io_port_base __read_mostly = -1; +const unsigned long mips_io_port_base = -1; EXPORT_SYMBOL(mips_io_port_base); static struct resource code_resource = { .name = "Kernel code", }; static struct resource data_resource = { .name = "Kernel data", }; +static void *detect_magic __initdata = detect_memory_region; + void __init add_memory_region(phys_t start, phys_t size, long type) { int x = boot_mem_map.nr_map; - struct boot_mem_map_entry *prev = boot_mem_map.map + x - 1; + int i; /* Sanity check */ if (start + size < start) { @@ -88,15 +92,29 @@ void __init add_memory_region(phys_t start, phys_t size, long type) } /* - * Try to merge with previous entry if any. This is far less than - * perfect but is sufficient for most real world cases. + * Try to merge with existing entry, if any. */ - if (x && prev->addr + prev->size == start && prev->type == type) { - prev->size += size; + for (i = 0; i < boot_mem_map.nr_map; i++) { + struct boot_mem_map_entry *entry = boot_mem_map.map + i; + unsigned long top; + + if (entry->type != type) + continue; + + if (start + size < entry->addr) + continue; /* no overlap */ + + if (entry->addr + entry->size < start) + continue; /* no overlap */ + + top = max(entry->addr + entry->size, start + size); + entry->addr = min(entry->addr, start); + entry->size = top - entry->addr; + return; } - if (x == BOOT_MEM_MAP_MAX) { + if (boot_mem_map.nr_map == BOOT_MEM_MAP_MAX) { pr_err("Ooops! Too many entries in the memory map!\n"); return; } @@ -107,6 +125,25 @@ void __init add_memory_region(phys_t start, phys_t size, long type) boot_mem_map.nr_map++; } +void __init detect_memory_region(phys_t start, phys_t sz_min, phys_t sz_max) +{ + void *dm = &detect_magic; + phys_t size; + + for (size = sz_min; size < sz_max; size <<= 1) { + if (!memcmp(dm, dm + size, sizeof(detect_magic))) + break; + } + + pr_debug("Memory: %lluMB of RAM detected at 0x%llx (min: %lluMB, max: %lluMB)\n", + ((unsigned long long) size) / SZ_1M, + (unsigned long long) start, + ((unsigned long long) sz_min) / SZ_1M, + ((unsigned long long) sz_max) / SZ_1M); + + add_memory_region(start, size, BOOT_MEM_RAM); +} + static void __init print_memory_map(void) { int i; @@ -121,6 +158,9 @@ static void __init print_memory_map(void) case BOOT_MEM_RAM: printk(KERN_CONT "(usable)\n"); break; + case BOOT_MEM_INIT_RAM: + printk(KERN_CONT "(usable after init)\n"); + break; case BOOT_MEM_ROM_DATA: printk(KERN_CONT "(ROM data)\n"); break; @@ -260,12 +300,13 @@ static void __init bootmem_init(void) int i; /* - * Init any data related to initrd. It's a nop if INITRD is - * not selected. Once that done we can determine the low bound - * of usable memory. + * Sanity check any INITRD first. We don't take it into account + * for bootmem setup initially, rely on the end-of-kernel-code + * as our memory range starting point. Once bootmem is inited we + * will reserve the area used for the initrd. */ - reserved_end = max(init_initrd(), - (unsigned long) PFN_UP(__pa_symbol(&_end))); + init_initrd(); + reserved_end = (unsigned long) PFN_UP(__pa_symbol(&_end)); /* * max_low_pfn is not a number of pages. The number of pages @@ -322,6 +363,14 @@ static void __init bootmem_init(void) max_low_pfn = PFN_DOWN(HIGHMEM_START); } +#ifdef CONFIG_BLK_DEV_INITRD + /* + * mapstart should be after initrd_end + */ + if (initrd_end) + mapstart = max(mapstart, (unsigned long)PFN_UP(__pa(initrd_end))); +#endif + /* * Initialize the boot-time allocator with low memory only. */ @@ -352,7 +401,7 @@ static void __init bootmem_init(void) continue; #endif - add_active_range(0, start, end); + memblock_add_node(PFN_PHYS(start), PFN_PHYS(end - start), 0); } /* @@ -361,15 +410,24 @@ static void __init bootmem_init(void) for (i = 0; i < boot_mem_map.nr_map; i++) { unsigned long start, end, size; + start = PFN_UP(boot_mem_map.map[i].addr); + end = PFN_DOWN(boot_mem_map.map[i].addr + + boot_mem_map.map[i].size); + /* * Reserve usable memory. */ - if (boot_mem_map.map[i].type != BOOT_MEM_RAM) + switch (boot_mem_map.map[i].type) { + case BOOT_MEM_RAM: + break; + case BOOT_MEM_INIT_RAM: + memory_present(0, start, end); + continue; + default: + /* Not usable memory */ continue; + } - start = PFN_UP(boot_mem_map.map[i].addr); - end = PFN_DOWN(boot_mem_map.map[i].addr - + boot_mem_map.map[i].size); /* * We are rounding up the start address of usable memory * and at the end of the usable range downwards. @@ -422,7 +480,7 @@ static void __init bootmem_init(void) * At this stage the bootmem allocator is ready to use. * * NOTE: historically plat_mem_setup did the entire platform initialization. - * This was rather impractical because it meant plat_mem_setup had to + * This was rather impractical because it meant plat_mem_setup had to * get away without any kind of memory allocator. To keep old code from * breaking plat_setup was just renamed to plat_setup and a second platform * initialization hook for anything else was introduced. @@ -442,7 +500,7 @@ static int __init early_parse_mem(char *p) if (usermem == 0) { boot_mem_map.nr_map = 0; usermem = 1; - } + } start = 0; size = memparse(p, &p); if (*p == '@') @@ -453,6 +511,102 @@ static int __init early_parse_mem(char *p) } early_param("mem", early_parse_mem); +#ifdef CONFIG_PROC_VMCORE +unsigned long setup_elfcorehdr, setup_elfcorehdr_size; +static int __init early_parse_elfcorehdr(char *p) +{ + int i; + + setup_elfcorehdr = memparse(p, &p); + + for (i = 0; i < boot_mem_map.nr_map; i++) { + unsigned long start = boot_mem_map.map[i].addr; + unsigned long end = (boot_mem_map.map[i].addr + + boot_mem_map.map[i].size); + if (setup_elfcorehdr >= start && setup_elfcorehdr < end) { + /* + * Reserve from the elf core header to the end of + * the memory segment, that should all be kdump + * reserved memory. + */ + setup_elfcorehdr_size = end - setup_elfcorehdr; + break; + } + } + /* + * If we don't find it in the memory map, then we shouldn't + * have to worry about it, as the new kernel won't use it. + */ + return 0; +} +early_param("elfcorehdr", early_parse_elfcorehdr); +#endif + +static void __init arch_mem_addpart(phys_t mem, phys_t end, int type) +{ + phys_t size; + int i; + + size = end - mem; + if (!size) + return; + + /* Make sure it is in the boot_mem_map */ + for (i = 0; i < boot_mem_map.nr_map; i++) { + if (mem >= boot_mem_map.map[i].addr && + mem < (boot_mem_map.map[i].addr + + boot_mem_map.map[i].size)) + return; + } + add_memory_region(mem, size, type); +} + +#ifdef CONFIG_KEXEC +static inline unsigned long long get_total_mem(void) +{ + unsigned long long total; + + total = max_pfn - min_low_pfn; + return total << PAGE_SHIFT; +} + +static void __init mips_parse_crashkernel(void) +{ + unsigned long long total_mem; + unsigned long long crash_size, crash_base; + int ret; + + total_mem = get_total_mem(); + ret = parse_crashkernel(boot_command_line, total_mem, + &crash_size, &crash_base); + if (ret != 0 || crash_size <= 0) + return; + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; +} + +static void __init request_crashkernel(struct resource *res) +{ + int ret; + + ret = request_resource(res, &crashk_res); + if (!ret) + pr_info("Reserving %ldMB of memory at %ldMB for crashkernel\n", + (unsigned long)((crashk_res.end - + crashk_res.start + 1) >> 20), + (unsigned long)(crashk_res.start >> 20)); +} +#else /* !defined(CONFIG_KEXEC) */ +static void __init mips_parse_crashkernel(void) +{ +} + +static void __init request_crashkernel(struct resource *res) +{ +} +#endif /* !defined(CONFIG_KEXEC) */ + static void __init arch_mem_init(char **cmdline_p) { extern void plat_mem_setup(void); @@ -460,6 +614,19 @@ static void __init arch_mem_init(char **cmdline_p) /* call board setup routine */ plat_mem_setup(); + /* + * Make sure all kernel memory is in the maps. The "UP" and + * "DOWN" are opposite for initdata since if it crosses over + * into another memory section you don't want that to be + * freed when the initdata is freed. + */ + arch_mem_addpart(PFN_DOWN(__pa_symbol(&_text)) << PAGE_SHIFT, + PFN_UP(__pa_symbol(&_edata)) << PAGE_SHIFT, + BOOT_MEM_RAM); + arch_mem_addpart(PFN_UP(__pa_symbol(&__init_begin)) << PAGE_SHIFT, + PFN_DOWN(__pa_symbol(&__init_end)) << PAGE_SHIFT, + BOOT_MEM_INIT_RAM); + pr_info("Determined physical RAM map:\n"); print_memory_map(); @@ -488,6 +655,22 @@ static void __init arch_mem_init(char **cmdline_p) } bootmem_init(); +#ifdef CONFIG_PROC_VMCORE + if (setup_elfcorehdr && setup_elfcorehdr_size) { + printk(KERN_INFO "kdump reserved memory at %lx-%lx\n", + setup_elfcorehdr, setup_elfcorehdr_size); + reserve_bootmem(setup_elfcorehdr, setup_elfcorehdr_size, + BOOTMEM_DEFAULT); + } +#endif + + mips_parse_crashkernel(); +#ifdef CONFIG_KEXEC + if (crashk_res.start != crashk_res.end) + reserve_bootmem(crashk_res.start, + crashk_res.end - crashk_res.start + 1, + BOOTMEM_DEFAULT); +#endif device_tree_init(); sparse_init(); plat_swiotlb_setup(); @@ -506,9 +689,6 @@ static void __init resource_init(void) data_resource.start = __pa_symbol(&_etext); data_resource.end = __pa_symbol(&_edata) - 1; - /* - * Request address space for all standard RAM. - */ for (i = 0; i < boot_mem_map.nr_map; i++) { struct resource *res; unsigned long start, end; @@ -523,6 +703,7 @@ static void __init resource_init(void) res = alloc_bootmem(sizeof(struct resource)); switch (boot_mem_map.map[i].type) { case BOOT_MEM_RAM: + case BOOT_MEM_INIT_RAM: case BOOT_MEM_ROM_DATA: res->name = "System RAM"; break; @@ -544,6 +725,7 @@ static void __init resource_init(void) */ request_resource(res, &code_resource); request_resource(res, &data_resource); + request_crashkernel(res); } } @@ -570,6 +752,8 @@ void __init setup_arch(char **cmdline_p) resource_init(); plat_smp_setup(); + + cpu_cache_init(); } unsigned long kernelsp[NR_CPUS]; diff --git a/arch/mips/kernel/signal-common.h b/arch/mips/kernel/signal-common.h index 10263b40598..9c60d09e62a 100644 --- a/arch/mips/kernel/signal-common.h +++ b/arch/mips/kernel/signal-common.h @@ -19,8 +19,6 @@ # define DEBUGP(fmt, args...) #endif -#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) - /* * Determine which stack to use.. */ diff --git a/arch/mips/kernel/signal.c b/arch/mips/kernel/signal.c index 5922342bca3..9e60d117e41 100644 --- a/arch/mips/kernel/signal.c +++ b/arch/mips/kernel/signal.c @@ -6,8 +6,11 @@ * Copyright (C) 1991, 1992 Linus Torvalds * Copyright (C) 1994 - 2000 Ralf Baechle * Copyright (C) 1999, 2000 Silicon Graphics, Inc. + * Copyright (C) 2014, Imagination Technologies Ltd. */ #include <linux/cache.h> +#include <linux/context_tracking.h> +#include <linux/irqflags.h> #include <linux/sched.h> #include <linux/mm.h> #include <linux/personality.h> @@ -33,6 +36,8 @@ #include <asm/cpu-features.h> #include <asm/war.h> #include <asm/vdso.h> +#include <asm/dsp.h> +#include <asm/inst.h> #include "signal-common.h" @@ -42,9 +47,6 @@ static int (*restore_fp_context)(struct sigcontext __user *sc); extern asmlinkage int _save_fp_context(struct sigcontext __user *sc); extern asmlinkage int _restore_fp_context(struct sigcontext __user *sc); -extern asmlinkage int fpu_emulator_save_context(struct sigcontext __user *sc); -extern asmlinkage int fpu_emulator_restore_context(struct sigcontext __user *sc); - struct sigframe { u32 sf_ass[4]; /* argument save space for o32 */ u32 sf_pad[2]; /* Was: signal trampoline */ @@ -60,16 +62,55 @@ struct rt_sigframe { }; /* + * Thread saved context copy to/from a signal context presumed to be on the + * user stack, and therefore accessed with appropriate macros from uaccess.h. + */ +static int copy_fp_to_sigcontext(struct sigcontext __user *sc) +{ + int i; + int err = 0; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= + __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + &sc->sc_fpregs[i]); + } + err |= __put_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + + return err; +} + +static int copy_fp_from_sigcontext(struct sigcontext __user *sc) +{ + int i; + int err = 0; + u64 fpr_val; + + for (i = 0; i < NUM_FPU_REGS; i++) { + err |= __get_user(fpr_val, &sc->sc_fpregs[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + } + err |= __get_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + + return err; +} + +/* * Helper routines */ static int protected_save_fp_context(struct sigcontext __user *sc) { int err; +#ifndef CONFIG_EVA while (1) { lock_fpu_owner(); - own_fpu_inatomic(1); - err = save_fp_context(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = save_fp_context(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_to_sigcontext(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ @@ -79,17 +120,30 @@ static int protected_save_fp_context(struct sigcontext __user *sc) if (err) break; /* really bad sigcontext */ } +#else + /* + * EVA does not have FPU EVA instructions so saving fpu context directly + * does not work. + */ + lose_fpu(1); + err = save_fp_context(sc); /* this might fail */ +#endif return err; } static int protected_restore_fp_context(struct sigcontext __user *sc) { - int err, tmp; + int err, tmp __maybe_unused; +#ifndef CONFIG_EVA while (1) { lock_fpu_owner(); - own_fpu_inatomic(0); - err = restore_fp_context(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = restore_fp_context(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_from_sigcontext(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ @@ -99,6 +153,14 @@ static int protected_restore_fp_context(struct sigcontext __user *sc) if (err) break; /* really bad sigcontext */ } +#else + /* + * EVA does not have FPU EVA instructions so restoring fpu context + * directly does not work. + */ + lose_fpu(0); + err = restore_fp_context(sc); /* this might fail */ +#endif return err; } @@ -245,57 +307,12 @@ void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, */ #ifdef CONFIG_TRAD_SIGNALS -asmlinkage int sys_sigsuspend(nabi_no_regargs struct pt_regs regs) +SYSCALL_DEFINE1(sigsuspend, sigset_t __user *, uset) { - sigset_t newset; - sigset_t __user *uset; - - uset = (sigset_t __user *) regs.regs[4]; - if (copy_from_user(&newset, uset, sizeof(sigset_t))) - return -EFAULT; - sigdelsetmask(&newset, ~_BLOCKABLE); - - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - current->blocked = newset; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - return -ERESTARTNOHAND; + return sys_rt_sigsuspend(uset, sizeof(sigset_t)); } #endif -asmlinkage int sys_rt_sigsuspend(nabi_no_regargs struct pt_regs regs) -{ - sigset_t newset; - sigset_t __user *unewset; - size_t sigsetsize; - - /* XXX Don't preclude handling different sized sigset_t's. */ - sigsetsize = regs.regs[5]; - if (sigsetsize != sizeof(sigset_t)) - return -EINVAL; - - unewset = (sigset_t __user *) regs.regs[4]; - if (copy_from_user(&newset, unewset, sizeof(newset))) - return -EFAULT; - sigdelsetmask(&newset, ~_BLOCKABLE); - - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - current->blocked = newset; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - return -ERESTARTNOHAND; -} - #ifdef CONFIG_TRAD_SIGNALS SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, struct sigaction __user *, oact) @@ -337,15 +354,6 @@ SYSCALL_DEFINE3(sigaction, int, sig, const struct sigaction __user *, act, } #endif -asmlinkage int sys_sigaltstack(nabi_no_regargs struct pt_regs regs) -{ - const stack_t __user *uss = (const stack_t __user *) regs.regs[4]; - stack_t __user *uoss = (stack_t __user *) regs.regs[5]; - unsigned long usp = regs.regs[29]; - - return do_sigaltstack(uss, uoss, usp); -} - #ifdef CONFIG_TRAD_SIGNALS asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) { @@ -359,11 +367,7 @@ asmlinkage void sys_sigreturn(nabi_no_regargs struct pt_regs regs) if (__copy_from_user(&blocked, &frame->sf_mask, sizeof(blocked))) goto badframe; - sigdelsetmask(&blocked, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = blocked; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&blocked); sig = restore_sigcontext(®s, &frame->sf_sc); if (sig < 0) @@ -398,11 +402,7 @@ asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) if (__copy_from_user(&set, &frame->rs_uc.uc_sigmask, sizeof(set))) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); if (sig < 0) @@ -410,9 +410,8 @@ asmlinkage void sys_rt_sigreturn(nabi_no_regargs struct pt_regs regs) else if (sig) force_sig(sig, current); - /* It is more difficult to avoid calling this function than to - call it and ignore errors. */ - do_sigaltstack(&frame->rs_uc.uc_stack, NULL, regs.regs[29]); + if (restore_altstack(&frame->rs_uc.uc_stack)) + goto badframe; /* * Don't let your children do this ... @@ -473,7 +472,7 @@ give_sigsegv: #endif static int setup_rt_frame(void *sig_return, struct k_sigaction *ka, - struct pt_regs *regs, int signr, sigset_t *set, + struct pt_regs *regs, int signr, sigset_t *set, siginfo_t *info) { struct rt_sigframe __user *frame; @@ -486,15 +485,10 @@ static int setup_rt_frame(void *sig_return, struct k_sigaction *ka, /* Create siginfo. */ err |= copy_siginfo_to_user(&frame->rs_info, info); - /* Create the ucontext. */ + /* Create the ucontext. */ err |= __put_user(0, &frame->rs_uc.uc_flags); err |= __put_user(NULL, &frame->rs_uc.uc_link); - err |= __put_user((void __user *)current->sas_ss_sp, - &frame->rs_uc.uc_stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->regs[29]), - &frame->rs_uc.uc_stack.ss_flags); - err |= __put_user(current->sas_ss_size, - &frame->rs_uc.uc_stack.ss_size); + err |= __save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); err |= setup_sigcontext(regs, &frame->rs_uc.uc_mcontext); err |= __copy_to_user(&frame->rs_uc.uc_sigmask, set, sizeof(*set)); @@ -534,18 +528,27 @@ struct mips_abi mips_abi = { .setup_frame = setup_frame, .signal_return_offset = offsetof(struct mips_vdso, signal_trampoline), #endif - .setup_rt_frame = setup_rt_frame, + .setup_rt_frame = setup_rt_frame, .rt_signal_return_offset = offsetof(struct mips_vdso, rt_signal_trampoline), .restart = __NR_restart_syscall }; -static int handle_signal(unsigned long sig, siginfo_t *info, - struct k_sigaction *ka, sigset_t *oldset, struct pt_regs *regs) +static void handle_signal(unsigned long sig, siginfo_t *info, + struct k_sigaction *ka, struct pt_regs *regs) { + sigset_t *oldset = sigmask_to_save(); int ret; struct mips_abi *abi = current->thread.abi; +#ifdef CONFIG_CPU_MICROMIPS + void *vdso; + unsigned int tmp = (unsigned int)current->mm->context.vdso; + + set_isa16_mode(tmp); + vdso = (void *)tmp; +#else void *vdso = current->mm->context.vdso; +#endif if (regs->regs[0]) { switch(regs->regs[2]) { @@ -565,7 +568,7 @@ static int handle_signal(unsigned long sig, siginfo_t *info, regs->cp0_epc -= 4; } - regs->regs[0] = 0; /* Don't deal with this again. */ + regs->regs[0] = 0; /* Don't deal with this again. */ } if (sig_uses_siginfo(ka)) @@ -576,79 +579,48 @@ static int handle_signal(unsigned long sig, siginfo_t *info, ka, regs, sig, oldset); if (ret) - return ret; - - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked, ¤t->blocked, &ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NODEFER)) - sigaddset(¤t->blocked, sig); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + return; - return ret; + signal_delivered(sig, info, ka, regs, 0); } static void do_signal(struct pt_regs *regs) { struct k_sigaction ka; - sigset_t *oldset; siginfo_t info; int signr; - /* - * We want the common case to go fast, which is why we may in certain - * cases get here from kernel mode. Just return without doing anything - * if so. - */ - if (!user_mode(regs)) - return; - - if (test_thread_flag(TIF_RESTORE_SIGMASK)) - oldset = ¤t->saved_sigmask; - else - oldset = ¤t->blocked; - signr = get_signal_to_deliver(&info, &ka, regs, NULL); if (signr > 0) { - /* Whee! Actually deliver the signal. */ - if (handle_signal(signr, &info, &ka, oldset, regs) == 0) { - /* - * A signal was successfully delivered; the saved - * sigmask will have been stored in the signal frame, - * and will be restored by sigreturn, so we can simply - * clear the TIF_RESTORE_SIGMASK flag. - */ - if (test_thread_flag(TIF_RESTORE_SIGMASK)) - clear_thread_flag(TIF_RESTORE_SIGMASK); - } - + /* Whee! Actually deliver the signal. */ + handle_signal(signr, &info, &ka, regs); return; } if (regs->regs[0]) { - if (regs->regs[2] == ERESTARTNOHAND || - regs->regs[2] == ERESTARTSYS || - regs->regs[2] == ERESTARTNOINTR) { + switch (regs->regs[2]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: regs->regs[2] = regs->regs[0]; regs->regs[7] = regs->regs[26]; regs->cp0_epc -= 4; - } - if (regs->regs[2] == ERESTART_RESTARTBLOCK) { + break; + + case ERESTART_RESTARTBLOCK: regs->regs[2] = current->thread.abi->restart; regs->regs[7] = regs->regs[26]; regs->cp0_epc -= 4; + break; } - regs->regs[0] = 0; /* Don't deal with this again. */ + regs->regs[0] = 0; /* Don't deal with this again. */ } /* * If there's no signal to deliver, we just put the saved sigmask * back */ - if (test_thread_flag(TIF_RESTORE_SIGMASK)) { - clear_thread_flag(TIF_RESTORE_SIGMASK); - sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); - } + restore_saved_sigmask(); } /* @@ -658,36 +630,43 @@ static void do_signal(struct pt_regs *regs) asmlinkage void do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) { + local_irq_enable(); + + user_exit(); + /* deal with pending signal delivery */ - if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) + if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); - if (current->replacement_session_keyring) - key_replace_session_keyring(); } + + user_enter(); } #ifdef CONFIG_SMP +#ifndef CONFIG_EVA static int smp_save_fp_context(struct sigcontext __user *sc) { return raw_cpu_has_fpu ? _save_fp_context(sc) - : fpu_emulator_save_context(sc); + : copy_fp_to_sigcontext(sc); } static int smp_restore_fp_context(struct sigcontext __user *sc) { return raw_cpu_has_fpu ? _restore_fp_context(sc) - : fpu_emulator_restore_context(sc); + : copy_fp_from_sigcontext(sc); } +#endif /* CONFIG_EVA */ #endif static int signal_setup(void) { +#ifndef CONFIG_EVA #ifdef CONFIG_SMP /* For now just do the cpu_has_fpu check when the functions are invoked */ save_fp_context = smp_save_fp_context; @@ -697,9 +676,13 @@ static int signal_setup(void) save_fp_context = _save_fp_context; restore_fp_context = _restore_fp_context; } else { - save_fp_context = fpu_emulator_save_context; - restore_fp_context = fpu_emulator_restore_context; + save_fp_context = copy_fp_from_sigcontext; + restore_fp_context = copy_fp_to_sigcontext; } +#endif /* CONFIG_SMP */ +#else + save_fp_context = copy_fp_from_sigcontext;; + restore_fp_context = copy_fp_to_sigcontext; #endif return 0; diff --git a/arch/mips/kernel/signal32.c b/arch/mips/kernel/signal32.c index a0ed0e052b2..bae2e6ee210 100644 --- a/arch/mips/kernel/signal32.c +++ b/arch/mips/kernel/signal32.c @@ -29,10 +29,10 @@ #include <asm/cacheflush.h> #include <asm/sim.h> #include <asm/ucontext.h> -#include <asm/system.h> #include <asm/fpu.h> #include <asm/war.h> #include <asm/vdso.h> +#include <asm/dsp.h> #include "signal-common.h" @@ -42,38 +42,22 @@ static int (*restore_fp_context32)(struct sigcontext32 __user *sc); extern asmlinkage int _save_fp_context32(struct sigcontext32 __user *sc); extern asmlinkage int _restore_fp_context32(struct sigcontext32 __user *sc); -extern asmlinkage int fpu_emulator_save_context32(struct sigcontext32 __user *sc); -extern asmlinkage int fpu_emulator_restore_context32(struct sigcontext32 __user *sc); - /* * Including <asm/unistd.h> would give use the 64-bit syscall numbers ... */ -#define __NR_O32_restart_syscall 4253 +#define __NR_O32_restart_syscall 4253 /* 32-bit compatibility types */ typedef unsigned int __sighandler32_t; typedef void (*vfptr_t)(void); -struct sigaction32 { - unsigned int sa_flags; - __sighandler32_t sa_handler; - compat_sigset_t sa_mask; -}; - -/* IRIX compatible stack_t */ -typedef struct sigaltstack32 { - s32 ss_sp; - compat_size_t ss_size; - int ss_flags; -} stack32_t; - struct ucontext32 { - u32 uc_flags; - s32 uc_link; - stack32_t uc_stack; + u32 uc_flags; + s32 uc_link; + compat_stack_t uc_stack; struct sigcontext32 uc_mcontext; - compat_sigset_t uc_sigmask; /* mask last for extensibility */ + compat_sigset_t uc_sigmask; /* mask last for extensibility */ }; struct sigframe32 { @@ -91,6 +75,42 @@ struct rt_sigframe32 { }; /* + * Thread saved context copy to/from a signal context presumed to be on the + * user stack, and therefore accessed with appropriate macros from uaccess.h. + */ +static int copy_fp_to_sigcontext32(struct sigcontext32 __user *sc) +{ + int i; + int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; + + for (i = 0; i < NUM_FPU_REGS; i += inc) { + err |= + __put_user(get_fpr64(¤t->thread.fpu.fpr[i], 0), + &sc->sc_fpregs[i]); + } + err |= __put_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + + return err; +} + +static int copy_fp_from_sigcontext32(struct sigcontext32 __user *sc) +{ + int i; + int err = 0; + int inc = test_thread_flag(TIF_32BIT_FPREGS) ? 2 : 1; + u64 fpr_val; + + for (i = 0; i < NUM_FPU_REGS; i += inc) { + err |= __get_user(fpr_val, &sc->sc_fpregs[i]); + set_fpr64(¤t->thread.fpu.fpr[i], 0, fpr_val); + } + err |= __get_user(current->thread.fpu.fcr31, &sc->sc_fpc_csr); + + return err; +} + +/* * sigcontext handlers */ static int protected_save_fp_context32(struct sigcontext32 __user *sc) @@ -98,9 +118,13 @@ static int protected_save_fp_context32(struct sigcontext32 __user *sc) int err; while (1) { lock_fpu_owner(); - own_fpu_inatomic(1); - err = save_fp_context32(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = save_fp_context32(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_to_sigcontext32(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ @@ -115,12 +139,16 @@ static int protected_save_fp_context32(struct sigcontext32 __user *sc) static int protected_restore_fp_context32(struct sigcontext32 __user *sc) { - int err, tmp; + int err, tmp __maybe_unused; while (1) { lock_fpu_owner(); - own_fpu_inatomic(0); - err = restore_fp_context32(sc); /* this might fail */ - unlock_fpu_owner(); + if (is_fpu_owner()) { + err = restore_fp_context32(sc); + unlock_fpu_owner(); + } else { + unlock_fpu_owner(); + err = copy_fp_from_sigcontext32(sc); + } if (likely(!err)) break; /* touch the sigcontext and try again */ @@ -280,58 +308,13 @@ static inline int get_sigset(sigset_t *kbuf, const compat_sigset_t __user *ubuf) * Atomically swap in the new signal mask, and wait for a signal. */ -asmlinkage int sys32_sigsuspend(nabi_no_regargs struct pt_regs regs) -{ - compat_sigset_t __user *uset; - sigset_t newset; - - uset = (compat_sigset_t __user *) regs.regs[4]; - if (get_sigset(&newset, uset)) - return -EFAULT; - sigdelsetmask(&newset, ~_BLOCKABLE); - - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - current->blocked = newset; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - return -ERESTARTNOHAND; -} - -asmlinkage int sys32_rt_sigsuspend(nabi_no_regargs struct pt_regs regs) +asmlinkage int sys32_sigsuspend(compat_sigset_t __user *uset) { - compat_sigset_t __user *uset; - sigset_t newset; - size_t sigsetsize; - - /* XXX Don't preclude handling different sized sigset_t's. */ - sigsetsize = regs.regs[5]; - if (sigsetsize != sizeof(compat_sigset_t)) - return -EINVAL; - - uset = (compat_sigset_t __user *) regs.regs[4]; - if (get_sigset(&newset, uset)) - return -EFAULT; - sigdelsetmask(&newset, ~_BLOCKABLE); - - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - current->blocked = newset; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - return -ERESTARTNOHAND; + return compat_sys_rt_sigsuspend(uset, sizeof(compat_sigset_t)); } -SYSCALL_DEFINE3(32_sigaction, long, sig, const struct sigaction32 __user *, act, - struct sigaction32 __user *, oact) +SYSCALL_DEFINE3(32_sigaction, long, sig, const struct compat_sigaction __user *, act, + struct compat_sigaction __user *, oact) { struct k_sigaction new_ka, old_ka; int ret; @@ -360,7 +343,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct sigaction32 __user *, act, return -EFAULT; err |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); err |= __put_user((u32)(u64)old_ka.sa.sa_handler, - &oact->sa_handler); + &oact->sa_handler); err |= __put_user(old_ka.sa.sa_mask.sig[0], oact->sa_mask.sig); err |= __put_user(0, &oact->sa_mask.sig[1]); err |= __put_user(0, &oact->sa_mask.sig[2]); @@ -372,46 +355,7 @@ SYSCALL_DEFINE3(32_sigaction, long, sig, const struct sigaction32 __user *, act, return ret; } -asmlinkage int sys32_sigaltstack(nabi_no_regargs struct pt_regs regs) -{ - const stack32_t __user *uss = (const stack32_t __user *) regs.regs[4]; - stack32_t __user *uoss = (stack32_t __user *) regs.regs[5]; - unsigned long usp = regs.regs[29]; - stack_t kss, koss; - int ret, err = 0; - mm_segment_t old_fs = get_fs(); - s32 sp; - - if (uss) { - if (!access_ok(VERIFY_READ, uss, sizeof(*uss))) - return -EFAULT; - err |= __get_user(sp, &uss->ss_sp); - kss.ss_sp = (void __user *) (long) sp; - err |= __get_user(kss.ss_size, &uss->ss_size); - err |= __get_user(kss.ss_flags, &uss->ss_flags); - if (err) - return -EFAULT; - } - - set_fs(KERNEL_DS); - ret = do_sigaltstack(uss ? (stack_t __user *)&kss : NULL, - uoss ? (stack_t __user *)&koss : NULL, usp); - set_fs(old_fs); - - if (!ret && uoss) { - if (!access_ok(VERIFY_WRITE, uoss, sizeof(*uoss))) - return -EFAULT; - sp = (int) (unsigned long) koss.ss_sp; - err |= __put_user(sp, &uoss->ss_sp); - err |= __put_user(koss.ss_size, &uoss->ss_size); - err |= __put_user(koss.ss_flags, &uoss->ss_flags); - if (err) - return -EFAULT; - } - return ret; -} - -int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) +int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) { int err; @@ -487,11 +431,7 @@ asmlinkage void sys32_sigreturn(nabi_no_regargs struct pt_regs regs) if (__copy_conv_sigset_from_user(&blocked, &frame->sf_mask)) goto badframe; - sigdelsetmask(&blocked, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = blocked; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&blocked); sig = restore_sigcontext32(®s, &frame->sf_sc); if (sig < 0) @@ -516,10 +456,7 @@ badframe: asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) { struct rt_sigframe32 __user *frame; - mm_segment_t old_fs; sigset_t set; - stack_t st; - s32 sp; int sig; frame = (struct rt_sigframe32 __user *) regs.regs[29]; @@ -528,11 +465,7 @@ asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); sig = restore_sigcontext32(®s, &frame->rs_uc.uc_mcontext); if (sig < 0) @@ -540,21 +473,8 @@ asmlinkage void sys32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) else if (sig) force_sig(sig, current); - /* The ucontext contains a stack32_t, so we must convert! */ - if (__get_user(sp, &frame->rs_uc.uc_stack.ss_sp)) + if (compat_restore_altstack(&frame->rs_uc.uc_stack)) goto badframe; - st.ss_sp = (void __user *)(long) sp; - if (__get_user(st.ss_size, &frame->rs_uc.uc_stack.ss_size)) - goto badframe; - if (__get_user(st.ss_flags, &frame->rs_uc.uc_stack.ss_flags)) - goto badframe; - - /* It is more difficult to avoid calling this function than to - call it and ignore errors. */ - old_fs = get_fs(); - set_fs(KERNEL_DS); - do_sigaltstack((stack_t __user *)&st, NULL, regs.regs[29]); - set_fs(old_fs); /* * Don't let your children do this ... @@ -620,7 +540,6 @@ static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka, { struct rt_sigframe32 __user *frame; int err = 0; - s32 sp; frame = get_sigframe(ka, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) @@ -629,16 +548,10 @@ static int setup_rt_frame_32(void *sig_return, struct k_sigaction *ka, /* Convert (siginfo_t -> compat_siginfo_t) and copy to user. */ err |= copy_siginfo_to_user32(&frame->rs_info, info); - /* Create the ucontext. */ + /* Create the ucontext. */ err |= __put_user(0, &frame->rs_uc.uc_flags); err |= __put_user(0, &frame->rs_uc.uc_link); - sp = (int) (long) current->sas_ss_sp; - err |= __put_user(sp, - &frame->rs_uc.uc_stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->regs[29]), - &frame->rs_uc.uc_stack.ss_flags); - err |= __put_user(current->sas_ss_size, - &frame->rs_uc.uc_stack.ss_size); + err |= __compat_save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); err |= setup_sigcontext32(regs, &frame->rs_uc.uc_mcontext); err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set); @@ -680,145 +593,20 @@ struct mips_abi mips_abi_32 = { .setup_frame = setup_frame_32, .signal_return_offset = offsetof(struct mips_vdso, o32_signal_trampoline), - .setup_rt_frame = setup_rt_frame_32, + .setup_rt_frame = setup_rt_frame_32, .rt_signal_return_offset = offsetof(struct mips_vdso, o32_rt_signal_trampoline), .restart = __NR_O32_restart_syscall }; -SYSCALL_DEFINE4(32_rt_sigaction, int, sig, - const struct sigaction32 __user *, act, - struct sigaction32 __user *, oact, unsigned int, sigsetsize) -{ - struct k_sigaction new_sa, old_sa; - int ret = -EINVAL; - - /* XXX: Don't preclude handling different sized sigset_t's. */ - if (sigsetsize != sizeof(sigset_t)) - goto out; - - if (act) { - s32 handler; - int err = 0; - - if (!access_ok(VERIFY_READ, act, sizeof(*act))) - return -EFAULT; - err |= __get_user(handler, &act->sa_handler); - new_sa.sa.sa_handler = (void __user *)(s64)handler; - err |= __get_user(new_sa.sa.sa_flags, &act->sa_flags); - err |= get_sigset(&new_sa.sa.sa_mask, &act->sa_mask); - if (err) - return -EFAULT; - } - - ret = do_sigaction(sig, act ? &new_sa : NULL, oact ? &old_sa : NULL); - - if (!ret && oact) { - int err = 0; - - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact))) - return -EFAULT; - - err |= __put_user((u32)(u64)old_sa.sa.sa_handler, - &oact->sa_handler); - err |= __put_user(old_sa.sa.sa_flags, &oact->sa_flags); - err |= put_sigset(&old_sa.sa.sa_mask, &oact->sa_mask); - if (err) - return -EFAULT; - } -out: - return ret; -} - -SYSCALL_DEFINE4(32_rt_sigprocmask, int, how, compat_sigset_t __user *, set, - compat_sigset_t __user *, oset, unsigned int, sigsetsize) -{ - sigset_t old_set, new_set; - int ret; - mm_segment_t old_fs = get_fs(); - - if (set && get_sigset(&new_set, set)) - return -EFAULT; - - set_fs(KERNEL_DS); - ret = sys_rt_sigprocmask(how, set ? (sigset_t __user *)&new_set : NULL, - oset ? (sigset_t __user *)&old_set : NULL, - sigsetsize); - set_fs(old_fs); - - if (!ret && oset && put_sigset(&old_set, oset)) - return -EFAULT; - - return ret; -} - -SYSCALL_DEFINE2(32_rt_sigpending, compat_sigset_t __user *, uset, - unsigned int, sigsetsize) -{ - int ret; - sigset_t set; - mm_segment_t old_fs = get_fs(); - - set_fs(KERNEL_DS); - ret = sys_rt_sigpending((sigset_t __user *)&set, sigsetsize); - set_fs(old_fs); - - if (!ret && put_sigset(&set, uset)) - return -EFAULT; - - return ret; -} - -SYSCALL_DEFINE3(32_rt_sigqueueinfo, int, pid, int, sig, - compat_siginfo_t __user *, uinfo) -{ - siginfo_t info; - int ret; - mm_segment_t old_fs = get_fs(); - - if (copy_from_user(&info, uinfo, 3*sizeof(int)) || - copy_from_user(info._sifields._pad, uinfo->_sifields._pad, SI_PAD_SIZE)) - return -EFAULT; - set_fs(KERNEL_DS); - ret = sys_rt_sigqueueinfo(pid, sig, (siginfo_t __user *)&info); - set_fs(old_fs); - return ret; -} - -SYSCALL_DEFINE5(32_waitid, int, which, compat_pid_t, pid, - compat_siginfo_t __user *, uinfo, int, options, - struct compat_rusage __user *, uru) -{ - siginfo_t info; - struct rusage ru; - long ret; - mm_segment_t old_fs = get_fs(); - - info.si_signo = 0; - set_fs(KERNEL_DS); - ret = sys_waitid(which, pid, (siginfo_t __user *) &info, options, - uru ? (struct rusage __user *) &ru : NULL); - set_fs(old_fs); - - if (ret < 0 || info.si_signo == 0) - return ret; - - if (uru && (ret = put_compat_rusage(&ru, uru))) - return ret; - - BUG_ON(info.si_code & __SI_MASK); - info.si_code |= __SI_CHLD; - return copy_siginfo_to_user32(uinfo, &info); -} - static int signal32_init(void) { if (cpu_has_fpu) { save_fp_context32 = _save_fp_context32; restore_fp_context32 = _restore_fp_context32; } else { - save_fp_context32 = fpu_emulator_save_context32; - restore_fp_context32 = fpu_emulator_restore_context32; + save_fp_context32 = copy_fp_to_sigcontext32; + restore_fp_context32 = copy_fp_from_sigcontext32; } return 0; diff --git a/arch/mips/kernel/signal_n32.c b/arch/mips/kernel/signal_n32.c index ee24d814d5b..b2241bb9cac 100644 --- a/arch/mips/kernel/signal_n32.c +++ b/arch/mips/kernel/signal_n32.c @@ -35,7 +35,6 @@ #include <asm/sim.h> #include <asm/uaccess.h> #include <asm/ucontext.h> -#include <asm/system.h> #include <asm/fpu.h> #include <asm/cpu-features.h> #include <asm/war.h> @@ -51,20 +50,12 @@ extern int setup_sigcontext(struct pt_regs *, struct sigcontext __user *); extern int restore_sigcontext(struct pt_regs *, struct sigcontext __user *); - -/* IRIX compatible stack_t */ -typedef struct sigaltstack32 { - s32 ss_sp; - compat_size_t ss_size; - int ss_flags; -} stack32_t; - struct ucontextn32 { - u32 uc_flags; - s32 uc_link; - stack32_t uc_stack; + u32 uc_flags; + s32 uc_link; + compat_stack_t uc_stack; struct sigcontext uc_mcontext; - compat_sigset_t uc_sigmask; /* mask last for extensibility */ + compat_sigset_t uc_sigmask; /* mask last for extensibility */ }; struct rt_sigframe_n32 { @@ -74,45 +65,10 @@ struct rt_sigframe_n32 { struct ucontextn32 rs_uc; }; -extern void sigset_from_compat(sigset_t *set, compat_sigset_t *compat); - -asmlinkage int sysn32_rt_sigsuspend(nabi_no_regargs struct pt_regs regs) -{ - compat_sigset_t __user *unewset; - compat_sigset_t uset; - size_t sigsetsize; - sigset_t newset; - - /* XXX Don't preclude handling different sized sigset_t's. */ - sigsetsize = regs.regs[5]; - if (sigsetsize != sizeof(sigset_t)) - return -EINVAL; - - unewset = (compat_sigset_t __user *) regs.regs[4]; - if (copy_from_user(&uset, unewset, sizeof(uset))) - return -EFAULT; - sigset_from_compat(&newset, &uset); - sigdelsetmask(&newset, ~_BLOCKABLE); - - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - current->blocked = newset; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - return -ERESTARTNOHAND; -} - asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) { struct rt_sigframe_n32 __user *frame; - mm_segment_t old_fs; sigset_t set; - stack_t st; - s32 sp; int sig; frame = (struct rt_sigframe_n32 __user *) regs.regs[29]; @@ -121,11 +77,7 @@ asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) if (__copy_conv_sigset_from_user(&set, &frame->rs_uc.uc_sigmask)) goto badframe; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); sig = restore_sigcontext(®s, &frame->rs_uc.uc_mcontext); if (sig < 0) @@ -133,23 +85,9 @@ asmlinkage void sysn32_rt_sigreturn(nabi_no_regargs struct pt_regs regs) else if (sig) force_sig(sig, current); - /* The ucontext contains a stack32_t, so we must convert! */ - if (__get_user(sp, &frame->rs_uc.uc_stack.ss_sp)) - goto badframe; - st.ss_sp = (void __user *)(long) sp; - if (__get_user(st.ss_size, &frame->rs_uc.uc_stack.ss_size)) - goto badframe; - if (__get_user(st.ss_flags, &frame->rs_uc.uc_stack.ss_flags)) + if (compat_restore_altstack(&frame->rs_uc.uc_stack)) goto badframe; - /* It is more difficult to avoid calling this function than to - call it and ignore errors. */ - old_fs = get_fs(); - set_fs(KERNEL_DS); - do_sigaltstack((stack_t __user *)&st, NULL, regs.regs[29]); - set_fs(old_fs); - - /* * Don't let your children do this ... */ @@ -169,7 +107,6 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka, { struct rt_sigframe_n32 __user *frame; int err = 0; - s32 sp; frame = get_sigframe(ka, regs, sizeof(*frame)); if (!access_ok(VERIFY_WRITE, frame, sizeof (*frame))) @@ -178,16 +115,10 @@ static int setup_rt_frame_n32(void *sig_return, struct k_sigaction *ka, /* Create siginfo. */ err |= copy_siginfo_to_user32(&frame->rs_info, info); - /* Create the ucontext. */ + /* Create the ucontext. */ err |= __put_user(0, &frame->rs_uc.uc_flags); err |= __put_user(0, &frame->rs_uc.uc_link); - sp = (int) (long) current->sas_ss_sp; - err |= __put_user(sp, - &frame->rs_uc.uc_stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->regs[29]), - &frame->rs_uc.uc_stack.ss_flags); - err |= __put_user(current->sas_ss_size, - &frame->rs_uc.uc_stack.ss_size); + err |= __compat_save_altstack(&frame->rs_uc.uc_stack, regs->regs[29]); err |= setup_sigcontext(regs, &frame->rs_uc.uc_mcontext); err |= __copy_conv_sigset_to_user(&frame->rs_uc.uc_sigmask, set); @@ -223,7 +154,7 @@ give_sigsegv: } struct mips_abi mips_abi_n32 = { - .setup_rt_frame = setup_rt_frame_n32, + .setup_rt_frame = setup_rt_frame_n32, .rt_signal_return_offset = offsetof(struct mips_vdso, n32_rt_signal_trampoline), .restart = __NR_N32_restart_syscall diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c new file mode 100644 index 00000000000..df9e2bd9b2c --- /dev/null +++ b/arch/mips/kernel/smp-bmips.c @@ -0,0 +1,535 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2011 by Kevin Cernekee (cernekee@gmail.com) + * + * SMP support for BMIPS + */ + +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/delay.h> +#include <linux/smp.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/reboot.h> +#include <linux/io.h> +#include <linux/compiler.h> +#include <linux/linkage.h> +#include <linux/bug.h> +#include <linux/kernel.h> + +#include <asm/time.h> +#include <asm/pgtable.h> +#include <asm/processor.h> +#include <asm/bootinfo.h> +#include <asm/pmon.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/mipsregs.h> +#include <asm/bmips.h> +#include <asm/traps.h> +#include <asm/barrier.h> + +static int __maybe_unused max_cpus = 1; + +/* these may be configured by the platform code */ +int bmips_smp_enabled = 1; +int bmips_cpu_offset; +cpumask_t bmips_booted_mask; + +#ifdef CONFIG_SMP + +/* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ +unsigned long bmips_smp_boot_sp; +unsigned long bmips_smp_boot_gp; + +static void bmips43xx_send_ipi_single(int cpu, unsigned int action); +static void bmips5000_send_ipi_single(int cpu, unsigned int action); +static irqreturn_t bmips43xx_ipi_interrupt(int irq, void *dev_id); +static irqreturn_t bmips5000_ipi_interrupt(int irq, void *dev_id); + +/* SW interrupts 0,1 are used for interprocessor signaling */ +#define IPI0_IRQ (MIPS_CPU_IRQ_BASE + 0) +#define IPI1_IRQ (MIPS_CPU_IRQ_BASE + 1) + +#define CPUNUM(cpu, shift) (((cpu) + bmips_cpu_offset) << (shift)) +#define ACTION_CLR_IPI(cpu, ipi) (0x2000 | CPUNUM(cpu, 9) | ((ipi) << 8)) +#define ACTION_SET_IPI(cpu, ipi) (0x3000 | CPUNUM(cpu, 9) | ((ipi) << 8)) +#define ACTION_BOOT_THREAD(cpu) (0x08 | CPUNUM(cpu, 0)) + +static void __init bmips_smp_setup(void) +{ + int i, cpu = 1, boot_cpu = 0; + int cpu_hw_intr; + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + /* arbitration priority */ + clear_c0_brcm_cmt_ctrl(0x30); + + /* NBK and weak order flags */ + set_c0_brcm_config_0(0x30000); + + /* Find out if we are running on TP0 or TP1 */ + boot_cpu = !!(read_c0_brcm_cmt_local() & (1 << 31)); + + /* + * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other + * thread + * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output + * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output + */ + if (boot_cpu == 0) + cpu_hw_intr = 0x02; + else + cpu_hw_intr = 0x1d; + + change_c0_brcm_cmt_intr(0xf8018000, + (cpu_hw_intr << 27) | (0x03 << 15)); + + /* single core, 2 threads (2 pipelines) */ + max_cpus = 2; + + break; + case CPU_BMIPS5000: + /* enable raceless SW interrupts */ + set_c0_brcm_config(0x03 << 22); + + /* route HW interrupt 0 to CPU0, HW interrupt 1 to CPU1 */ + change_c0_brcm_mode(0x1f << 27, 0x02 << 27); + + /* N cores, 2 threads per core */ + max_cpus = (((read_c0_brcm_config() >> 6) & 0x03) + 1) << 1; + + /* clear any pending SW interrupts */ + for (i = 0; i < max_cpus; i++) { + write_c0_brcm_action(ACTION_CLR_IPI(i, 0)); + write_c0_brcm_action(ACTION_CLR_IPI(i, 1)); + } + + break; + default: + max_cpus = 1; + } + + if (!bmips_smp_enabled) + max_cpus = 1; + + /* this can be overridden by the BSP */ + if (!board_ebase_setup) + board_ebase_setup = &bmips_ebase_setup; + + __cpu_number_map[boot_cpu] = 0; + __cpu_logical_map[0] = boot_cpu; + + for (i = 0; i < max_cpus; i++) { + if (i != boot_cpu) { + __cpu_number_map[i] = cpu; + __cpu_logical_map[cpu] = i; + cpu++; + } + set_cpu_possible(i, 1); + set_cpu_present(i, 1); + } +} + +/* + * IPI IRQ setup - runs on CPU0 + */ +static void bmips_prepare_cpus(unsigned int max_cpus) +{ + irqreturn_t (*bmips_ipi_interrupt)(int irq, void *dev_id); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + bmips_ipi_interrupt = bmips43xx_ipi_interrupt; + break; + case CPU_BMIPS5000: + bmips_ipi_interrupt = bmips5000_ipi_interrupt; + break; + default: + return; + } + + if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, + "smp_ipi0", NULL)) + panic("Can't request IPI0 interrupt"); + if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, + "smp_ipi1", NULL)) + panic("Can't request IPI1 interrupt"); +} + +/* + * Tell the hardware to boot CPUx - runs on CPU0 + */ +static void bmips_boot_secondary(int cpu, struct task_struct *idle) +{ + bmips_smp_boot_sp = __KSTK_TOS(idle); + bmips_smp_boot_gp = (unsigned long)task_thread_info(idle); + mb(); + + /* + * Initial boot sequence for secondary CPU: + * bmips_reset_nmi_vec @ a000_0000 -> + * bmips_smp_entry -> + * plat_wired_tlb_setup (cached function call; optional) -> + * start_secondary (cached jump) + * + * Warm restart sequence: + * play_dead WAIT loop -> + * bmips_smp_int_vec @ BMIPS_WARM_RESTART_VEC -> + * eret to play_dead -> + * bmips_secondary_reentry -> + * start_secondary + */ + + pr_info("SMP: Booting CPU%d...\n", cpu); + + if (cpumask_test_cpu(cpu, &bmips_booted_mask)) { + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + bmips43xx_send_ipi_single(cpu, 0); + break; + case CPU_BMIPS5000: + bmips5000_send_ipi_single(cpu, 0); + break; + } + } + else { + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + /* Reset slave TP1 if booting from TP0 */ + if (cpu_logical_map(cpu) == 1) + set_c0_brcm_cmt_ctrl(0x01); + break; + case CPU_BMIPS5000: + if (cpu & 0x01) + write_c0_brcm_action(ACTION_BOOT_THREAD(cpu)); + else { + /* + * core N thread 0 was already booted; just + * pulse the NMI line + */ + bmips_write_zscm_reg(0x210, 0xc0000000); + udelay(10); + bmips_write_zscm_reg(0x210, 0x00); + } + break; + } + cpumask_set_cpu(cpu, &bmips_booted_mask); + } +} + +/* + * Early setup - runs on secondary CPU after cache probe + */ +static void bmips_init_secondary(void) +{ + /* move NMI vector to kseg0, in case XKS01 is enabled */ + + void __iomem *cbr; + unsigned long old_vec; + unsigned long relo_vector; + int boot_cpu; + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + case CPU_BMIPS4380: + cbr = BMIPS_GET_CBR(); + + boot_cpu = !!(read_c0_brcm_cmt_local() & (1 << 31)); + relo_vector = boot_cpu ? BMIPS_RELO_VECTOR_CONTROL_0 : + BMIPS_RELO_VECTOR_CONTROL_1; + + old_vec = __raw_readl(cbr + relo_vector); + __raw_writel(old_vec & ~0x20000000, cbr + relo_vector); + + clear_c0_cause(smp_processor_id() ? C_SW1 : C_SW0); + break; + case CPU_BMIPS5000: + write_c0_brcm_bootvec(read_c0_brcm_bootvec() & + (smp_processor_id() & 0x01 ? ~0x20000000 : ~0x2000)); + + write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), 0)); + break; + } +} + +/* + * Late setup - runs on secondary CPU before entering the idle loop + */ +static void bmips_smp_finish(void) +{ + pr_info("SMP: CPU%d is running\n", smp_processor_id()); + + /* make sure there won't be a timer interrupt for a little while */ + write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ); + + irq_enable_hazard(); + set_c0_status(IE_SW0 | IE_SW1 | IE_IRQ1 | IE_IRQ5 | ST0_IE); + irq_enable_hazard(); +} + +/* + * BMIPS5000 raceless IPIs + * + * Each CPU has two inbound SW IRQs which are independent of all other CPUs. + * IPI0 is used for SMP_RESCHEDULE_YOURSELF + * IPI1 is used for SMP_CALL_FUNCTION + */ + +static void bmips5000_send_ipi_single(int cpu, unsigned int action) +{ + write_c0_brcm_action(ACTION_SET_IPI(cpu, action == SMP_CALL_FUNCTION)); +} + +static irqreturn_t bmips5000_ipi_interrupt(int irq, void *dev_id) +{ + int action = irq - IPI0_IRQ; + + write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), action)); + + if (action == 0) + scheduler_ipi(); + else + smp_call_function_interrupt(); + + return IRQ_HANDLED; +} + +static void bmips5000_send_ipi_mask(const struct cpumask *mask, + unsigned int action) +{ + unsigned int i; + + for_each_cpu(i, mask) + bmips5000_send_ipi_single(i, action); +} + +/* + * BMIPS43xx racey IPIs + * + * We use one inbound SW IRQ for each CPU. + * + * A spinlock must be held in order to keep CPUx from accidentally clearing + * an incoming IPI when it writes CP0 CAUSE to raise an IPI on CPUy. The + * same spinlock is used to protect the action masks. + */ + +static DEFINE_SPINLOCK(ipi_lock); +static DEFINE_PER_CPU(int, ipi_action_mask); + +static void bmips43xx_send_ipi_single(int cpu, unsigned int action) +{ + unsigned long flags; + + spin_lock_irqsave(&ipi_lock, flags); + set_c0_cause(cpu ? C_SW1 : C_SW0); + per_cpu(ipi_action_mask, cpu) |= action; + irq_enable_hazard(); + spin_unlock_irqrestore(&ipi_lock, flags); +} + +static irqreturn_t bmips43xx_ipi_interrupt(int irq, void *dev_id) +{ + unsigned long flags; + int action, cpu = irq - IPI0_IRQ; + + spin_lock_irqsave(&ipi_lock, flags); + action = __get_cpu_var(ipi_action_mask); + per_cpu(ipi_action_mask, cpu) = 0; + clear_c0_cause(cpu ? C_SW1 : C_SW0); + spin_unlock_irqrestore(&ipi_lock, flags); + + if (action & SMP_RESCHEDULE_YOURSELF) + scheduler_ipi(); + if (action & SMP_CALL_FUNCTION) + smp_call_function_interrupt(); + + return IRQ_HANDLED; +} + +static void bmips43xx_send_ipi_mask(const struct cpumask *mask, + unsigned int action) +{ + unsigned int i; + + for_each_cpu(i, mask) + bmips43xx_send_ipi_single(i, action); +} + +#ifdef CONFIG_HOTPLUG_CPU + +static int bmips_cpu_disable(void) +{ + unsigned int cpu = smp_processor_id(); + + if (cpu == 0) + return -EBUSY; + + pr_info("SMP: CPU%d is offline\n", cpu); + + set_cpu_online(cpu, false); + cpu_clear(cpu, cpu_callin_map); + + local_flush_tlb_all(); + local_flush_icache_range(0, ~0); + + return 0; +} + +static void bmips_cpu_die(unsigned int cpu) +{ +} + +void __ref play_dead(void) +{ + idle_task_exit(); + + /* flush data cache */ + _dma_cache_wback_inv(0, ~0); + + /* + * Wakeup is on SW0 or SW1; disable everything else + * Use BEV !IV (BMIPS_WARM_RESTART_VEC) to avoid the regular Linux + * IRQ handlers; this clears ST0_IE and returns immediately. + */ + clear_c0_cause(CAUSEF_IV | C_SW0 | C_SW1); + change_c0_status(IE_IRQ5 | IE_IRQ1 | IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV, + IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV); + irq_disable_hazard(); + + /* + * wait for SW interrupt from bmips_boot_secondary(), then jump + * back to start_secondary() + */ + __asm__ __volatile__( + " wait\n" + " j bmips_secondary_reentry\n" + : : : "memory"); +} + +#endif /* CONFIG_HOTPLUG_CPU */ + +struct plat_smp_ops bmips43xx_smp_ops = { + .smp_setup = bmips_smp_setup, + .prepare_cpus = bmips_prepare_cpus, + .boot_secondary = bmips_boot_secondary, + .smp_finish = bmips_smp_finish, + .init_secondary = bmips_init_secondary, + .send_ipi_single = bmips43xx_send_ipi_single, + .send_ipi_mask = bmips43xx_send_ipi_mask, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = bmips_cpu_disable, + .cpu_die = bmips_cpu_die, +#endif +}; + +struct plat_smp_ops bmips5000_smp_ops = { + .smp_setup = bmips_smp_setup, + .prepare_cpus = bmips_prepare_cpus, + .boot_secondary = bmips_boot_secondary, + .smp_finish = bmips_smp_finish, + .init_secondary = bmips_init_secondary, + .send_ipi_single = bmips5000_send_ipi_single, + .send_ipi_mask = bmips5000_send_ipi_mask, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = bmips_cpu_disable, + .cpu_die = bmips_cpu_die, +#endif +}; + +#endif /* CONFIG_SMP */ + +/*********************************************************************** + * BMIPS vector relocation + * This is primarily used for SMP boot, but it is applicable to some + * UP BMIPS systems as well. + ***********************************************************************/ + +static void bmips_wr_vec(unsigned long dst, char *start, char *end) +{ + memcpy((void *)dst, start, end - start); + dma_cache_wback((unsigned long)start, end - start); + local_flush_icache_range(dst, dst + (end - start)); + instruction_hazard(); +} + +static inline void bmips_nmi_handler_setup(void) +{ + bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, + &bmips_reset_nmi_vec_end); + bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec, + &bmips_smp_int_vec_end); +} + +void bmips_ebase_setup(void) +{ + unsigned long new_ebase = ebase; + void __iomem __maybe_unused *cbr; + + BUG_ON(ebase != CKSEG0); + + switch (current_cpu_type()) { + case CPU_BMIPS4350: + /* + * BMIPS4350 cannot relocate the normal vectors, but it + * can relocate the BEV=1 vectors. So CPU1 starts up at + * the relocated BEV=1, IV=0 general exception vector @ + * 0xa000_0380. + * + * set_uncached_handler() is used here because: + * - CPU1 will run this from uncached space + * - None of the cacheflush functions are set up yet + */ + set_uncached_handler(BMIPS_WARM_RESTART_VEC - CKSEG0, + &bmips_smp_int_vec, 0x80); + __sync(); + return; + case CPU_BMIPS4380: + /* + * 0x8000_0000: reset/NMI (initially in kseg1) + * 0x8000_0400: normal vectors + */ + new_ebase = 0x80000400; + cbr = BMIPS_GET_CBR(); + __raw_writel(0x80080800, cbr + BMIPS_RELO_VECTOR_CONTROL_0); + __raw_writel(0xa0080800, cbr + BMIPS_RELO_VECTOR_CONTROL_1); + break; + case CPU_BMIPS5000: + /* + * 0x8000_0000: reset/NMI (initially in kseg1) + * 0x8000_1000: normal vectors + */ + new_ebase = 0x80001000; + write_c0_brcm_bootvec(0xa0088008); + write_c0_ebase(new_ebase); + if (max_cpus > 2) + bmips_write_zscm_reg(0xa0, 0xa008a008); + break; + default: + return; + } + + board_nmi_handler_setup = &bmips_nmi_handler_setup; + ebase = new_ebase; +} + +asmlinkage void __weak plat_wired_tlb_setup(void) +{ + /* + * Called when starting/restarting a secondary CPU. + * Kernel stacks and other important data might only be accessible + * once the wired entries are present. + */ +} diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c index cc81771b882..fc8a5155342 100644 --- a/arch/mips/kernel/smp-cmp.c +++ b/arch/mips/kernel/smp-cmp.c @@ -25,11 +25,10 @@ #include <linux/interrupt.h> #include <linux/compiler.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/cacheflush.h> #include <asm/cpu.h> #include <asm/processor.h> -#include <asm/system.h> #include <asm/hardirq.h> #include <asm/mmu_context.h> #include <asm/smp.h> @@ -40,57 +39,9 @@ #include <asm/amon.h> #include <asm/gic.h> -static void ipi_call_function(unsigned int cpu) -{ - pr_debug("CPU%d: %s cpu %d status %08x\n", - smp_processor_id(), __func__, cpu, read_c0_status()); - - gic_send_ipi(plat_ipi_call_int_xlate(cpu)); -} - - -static void ipi_resched(unsigned int cpu) -{ - pr_debug("CPU%d: %s cpu %d status %08x\n", - smp_processor_id(), __func__, cpu, read_c0_status()); - - gic_send_ipi(plat_ipi_resched_int_xlate(cpu)); -} - -/* - * FIXME: This isn't restricted to CMP - * The SMVP kernel could use GIC interrupts if available - */ -void cmp_send_ipi_single(int cpu, unsigned int action) -{ - unsigned long flags; - - local_irq_save(flags); - - switch (action) { - case SMP_CALL_FUNCTION: - ipi_call_function(cpu); - break; - - case SMP_RESCHEDULE_YOURSELF: - ipi_resched(cpu); - break; - } - - local_irq_restore(flags); -} - -static void cmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) -{ - unsigned int i; - - for_each_cpu(i, mask) - cmp_send_ipi_single(i, action); -} - static void cmp_init_secondary(void) { - struct cpuinfo_mips *c = ¤t_cpu_data; + struct cpuinfo_mips *c __maybe_unused = ¤t_cpu_data; /* Assume GIC is present */ change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 | STATUSF_IP6 | @@ -98,12 +49,10 @@ static void cmp_init_secondary(void) /* Enable per-cpu interrupts: platform specific */ - c->core = (read_c0_ebase() >> 1) & 0xff; -#if defined(CONFIG_MIPS_MT_SMP) || defined(CONFIG_MIPS_MT_SMTC) - c->vpe_id = (read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & TCBIND_CURVPE; -#endif -#ifdef CONFIG_MIPS_MT_SMTC - c->tc_id = (read_c0_tcbind() >> TCBIND_CURTC_SHIFT) & TCBIND_CURTC; +#ifdef CONFIG_MIPS_MT_SMP + if (cpu_has_mipsmt) + c->vpe_id = (read_c0_tcbind() >> TCBIND_CURVPE_SHIFT) & + TCBIND_CURVPE; #endif } @@ -123,11 +72,6 @@ static void cmp_smp_finish(void) local_irq_enable(); } -static void cmp_cpus_done(void) -{ - pr_debug("SMPCMP: CPU%d: %s\n", smp_processor_id(), __func__); -} - /* * Setup the PC, SP, and GP of a secondary processor and start it running * smp_bootstrap is the place to resume from @@ -173,14 +117,17 @@ void __init cmp_smp_setup(void) if (amon_cpu_avail(i)) { set_cpu_possible(i, true); __cpu_number_map[i] = ++ncpu; - __cpu_logical_map[ncpu] = i; + __cpu_logical_map[ncpu] = i; } } if (cpu_has_mipsmt) { - unsigned int nvpe, mvpconf0 = read_c0_mvpconf0(); + unsigned int nvpe = 1; +#ifdef CONFIG_MIPS_MT_SMP + unsigned int mvpconf0 = read_c0_mvpconf0(); - nvpe = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; + nvpe = ((mvpconf0 & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; +#endif smp_num_siblings = nvpe; } pr_info("Detected %i available secondary CPU(s)\n", ncpu); @@ -191,19 +138,21 @@ void __init cmp_prepare_cpus(unsigned int max_cpus) pr_debug("SMPCMP: CPU%d: %s max_cpus=%d\n", smp_processor_id(), __func__, max_cpus); +#ifdef CONFIG_MIPS_MT /* * FIXME: some of these options are per-system, some per-core and * some per-cpu */ mips_mt_set_cpuoptions(); +#endif + } struct plat_smp_ops cmp_smp_ops = { - .send_ipi_single = cmp_send_ipi_single, - .send_ipi_mask = cmp_send_ipi_mask, + .send_ipi_single = gic_send_ipi_single, + .send_ipi_mask = gic_send_ipi_mask, .init_secondary = cmp_init_secondary, .smp_finish = cmp_smp_finish, - .cpus_done = cmp_cpus_done, .boot_secondary = cmp_boot_secondary, .smp_setup = cmp_smp_setup, .prepare_cpus = cmp_prepare_cpus, diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c new file mode 100644 index 00000000000..949f2c6827a --- /dev/null +++ b/arch/mips/kernel/smp-cps.c @@ -0,0 +1,466 @@ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/io.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/smp.h> +#include <linux/types.h> + +#include <asm/cacheflush.h> +#include <asm/gic.h> +#include <asm/mips-cm.h> +#include <asm/mips-cpc.h> +#include <asm/mips_mt.h> +#include <asm/mipsregs.h> +#include <asm/pm-cps.h> +#include <asm/smp-cps.h> +#include <asm/time.h> +#include <asm/uasm.h> + +static DECLARE_BITMAP(core_power, NR_CPUS); + +struct core_boot_config *mips_cps_core_bootcfg; + +static unsigned core_vpe_count(unsigned core) +{ + unsigned cfg; + + if (!config_enabled(CONFIG_MIPS_MT_SMP) || !cpu_has_mipsmt) + return 1; + + write_gcr_cl_other(core << CM_GCR_Cx_OTHER_CORENUM_SHF); + cfg = read_gcr_co_config() & CM_GCR_Cx_CONFIG_PVPE_MSK; + return (cfg >> CM_GCR_Cx_CONFIG_PVPE_SHF) + 1; +} + +static void __init cps_smp_setup(void) +{ + unsigned int ncores, nvpes, core_vpes; + int c, v; + + /* Detect & record VPE topology */ + ncores = mips_cm_numcores(); + pr_info("VPE topology "); + for (c = nvpes = 0; c < ncores; c++) { + core_vpes = core_vpe_count(c); + pr_cont("%c%u", c ? ',' : '{', core_vpes); + + /* Use the number of VPEs in core 0 for smp_num_siblings */ + if (!c) + smp_num_siblings = core_vpes; + + for (v = 0; v < min_t(int, core_vpes, NR_CPUS - nvpes); v++) { + cpu_data[nvpes + v].core = c; +#ifdef CONFIG_MIPS_MT_SMP + cpu_data[nvpes + v].vpe_id = v; +#endif + } + + nvpes += core_vpes; + } + pr_cont("} total %u\n", nvpes); + + /* Indicate present CPUs (CPU being synonymous with VPE) */ + for (v = 0; v < min_t(unsigned, nvpes, NR_CPUS); v++) { + set_cpu_possible(v, true); + set_cpu_present(v, true); + __cpu_number_map[v] = v; + __cpu_logical_map[v] = v; + } + + /* Set a coherent default CCA (CWB) */ + change_c0_config(CONF_CM_CMASK, 0x5); + + /* Core 0 is powered up (we're running on it) */ + bitmap_set(core_power, 0, 1); + + /* Initialise core 0 */ + mips_cps_core_init(); + + /* Make core 0 coherent with everything */ + write_gcr_cl_coherence(0xff); +} + +static void __init cps_prepare_cpus(unsigned int max_cpus) +{ + unsigned ncores, core_vpes, c, cca; + bool cca_unsuitable; + u32 *entry_code; + + mips_mt_set_cpuoptions(); + + /* Detect whether the CCA is unsuited to multi-core SMP */ + cca = read_c0_config() & CONF_CM_CMASK; + switch (cca) { + case 0x4: /* CWBE */ + case 0x5: /* CWB */ + /* The CCA is coherent, multi-core is fine */ + cca_unsuitable = false; + break; + + default: + /* CCA is not coherent, multi-core is not usable */ + cca_unsuitable = true; + } + + /* Warn the user if the CCA prevents multi-core */ + ncores = mips_cm_numcores(); + if (cca_unsuitable && ncores > 1) { + pr_warn("Using only one core due to unsuitable CCA 0x%x\n", + cca); + + for_each_present_cpu(c) { + if (cpu_data[c].core) + set_cpu_present(c, false); + } + } + + /* + * Patch the start of mips_cps_core_entry to provide: + * + * v0 = CM base address + * s0 = kseg0 CCA + */ + entry_code = (u32 *)&mips_cps_core_entry; + UASM_i_LA(&entry_code, 3, (long)mips_cm_base); + uasm_i_addiu(&entry_code, 16, 0, cca); + dma_cache_wback_inv((unsigned long)&mips_cps_core_entry, + (void *)entry_code - (void *)&mips_cps_core_entry); + + /* Allocate core boot configuration structs */ + mips_cps_core_bootcfg = kcalloc(ncores, sizeof(*mips_cps_core_bootcfg), + GFP_KERNEL); + if (!mips_cps_core_bootcfg) { + pr_err("Failed to allocate boot config for %u cores\n", ncores); + goto err_out; + } + + /* Allocate VPE boot configuration structs */ + for (c = 0; c < ncores; c++) { + core_vpes = core_vpe_count(c); + mips_cps_core_bootcfg[c].vpe_config = kcalloc(core_vpes, + sizeof(*mips_cps_core_bootcfg[c].vpe_config), + GFP_KERNEL); + if (!mips_cps_core_bootcfg[c].vpe_config) { + pr_err("Failed to allocate %u VPE boot configs\n", + core_vpes); + goto err_out; + } + } + + /* Mark this CPU as booted */ + atomic_set(&mips_cps_core_bootcfg[current_cpu_data.core].vpe_mask, + 1 << cpu_vpe_id(¤t_cpu_data)); + + return; +err_out: + /* Clean up allocations */ + if (mips_cps_core_bootcfg) { + for (c = 0; c < ncores; c++) + kfree(mips_cps_core_bootcfg[c].vpe_config); + kfree(mips_cps_core_bootcfg); + mips_cps_core_bootcfg = NULL; + } + + /* Effectively disable SMP by declaring CPUs not present */ + for_each_possible_cpu(c) { + if (c == 0) + continue; + set_cpu_present(c, false); + } +} + +static void boot_core(unsigned core) +{ + u32 access; + + /* Select the appropriate core */ + write_gcr_cl_other(core << CM_GCR_Cx_OTHER_CORENUM_SHF); + + /* Set its reset vector */ + write_gcr_co_reset_base(CKSEG1ADDR((unsigned long)mips_cps_core_entry)); + + /* Ensure its coherency is disabled */ + write_gcr_co_coherence(0); + + /* Ensure the core can access the GCRs */ + access = read_gcr_access(); + access |= 1 << (CM_GCR_ACCESS_ACCESSEN_SHF + core); + write_gcr_access(access); + + if (mips_cpc_present()) { + /* Reset the core */ + mips_cpc_lock_other(core); + write_cpc_co_cmd(CPC_Cx_CMD_RESET); + mips_cpc_unlock_other(); + } else { + /* Take the core out of reset */ + write_gcr_co_reset_release(0); + } + + /* The core is now powered up */ + bitmap_set(core_power, core, 1); +} + +static void remote_vpe_boot(void *dummy) +{ + mips_cps_boot_vpes(); +} + +static void cps_boot_secondary(int cpu, struct task_struct *idle) +{ + unsigned core = cpu_data[cpu].core; + unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]); + struct core_boot_config *core_cfg = &mips_cps_core_bootcfg[core]; + struct vpe_boot_config *vpe_cfg = &core_cfg->vpe_config[vpe_id]; + unsigned int remote; + int err; + + vpe_cfg->pc = (unsigned long)&smp_bootstrap; + vpe_cfg->sp = __KSTK_TOS(idle); + vpe_cfg->gp = (unsigned long)task_thread_info(idle); + + atomic_or(1 << cpu_vpe_id(&cpu_data[cpu]), &core_cfg->vpe_mask); + + preempt_disable(); + + if (!test_bit(core, core_power)) { + /* Boot a VPE on a powered down core */ + boot_core(core); + goto out; + } + + if (core != current_cpu_data.core) { + /* Boot a VPE on another powered up core */ + for (remote = 0; remote < NR_CPUS; remote++) { + if (cpu_data[remote].core != core) + continue; + if (cpu_online(remote)) + break; + } + BUG_ON(remote >= NR_CPUS); + + err = smp_call_function_single(remote, remote_vpe_boot, + NULL, 1); + if (err) + panic("Failed to call remote CPU\n"); + goto out; + } + + BUG_ON(!cpu_has_mipsmt); + + /* Boot a VPE on this core */ + mips_cps_boot_vpes(); +out: + preempt_enable(); +} + +static void cps_init_secondary(void) +{ + /* Disable MT - we only want to run 1 TC per VPE */ + if (cpu_has_mipsmt) + dmt(); + + change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 | + STATUSF_IP6 | STATUSF_IP7); +} + +static void cps_smp_finish(void) +{ + write_c0_compare(read_c0_count() + (8 * mips_hpt_frequency / HZ)); + +#ifdef CONFIG_MIPS_MT_FPAFF + /* If we have an FPU, enroll ourselves in the FPU-full mask */ + if (cpu_has_fpu) + cpu_set(smp_processor_id(), mt_fpu_cpumask); +#endif /* CONFIG_MIPS_MT_FPAFF */ + + local_irq_enable(); +} + +#ifdef CONFIG_HOTPLUG_CPU + +static int cps_cpu_disable(void) +{ + unsigned cpu = smp_processor_id(); + struct core_boot_config *core_cfg; + + if (!cpu) + return -EBUSY; + + if (!cps_pm_support_state(CPS_PM_POWER_GATED)) + return -EINVAL; + + core_cfg = &mips_cps_core_bootcfg[current_cpu_data.core]; + atomic_sub(1 << cpu_vpe_id(¤t_cpu_data), &core_cfg->vpe_mask); + smp_mb__after_atomic(); + set_cpu_online(cpu, false); + cpu_clear(cpu, cpu_callin_map); + + return 0; +} + +static DECLARE_COMPLETION(cpu_death_chosen); +static unsigned cpu_death_sibling; +static enum { + CPU_DEATH_HALT, + CPU_DEATH_POWER, +} cpu_death; + +void play_dead(void) +{ + unsigned cpu, core; + + local_irq_disable(); + idle_task_exit(); + cpu = smp_processor_id(); + cpu_death = CPU_DEATH_POWER; + + if (cpu_has_mipsmt) { + core = cpu_data[cpu].core; + + /* Look for another online VPE within the core */ + for_each_online_cpu(cpu_death_sibling) { + if (cpu_data[cpu_death_sibling].core != core) + continue; + + /* + * There is an online VPE within the core. Just halt + * this TC and leave the core alone. + */ + cpu_death = CPU_DEATH_HALT; + break; + } + } + + /* This CPU has chosen its way out */ + complete(&cpu_death_chosen); + + if (cpu_death == CPU_DEATH_HALT) { + /* Halt this TC */ + write_c0_tchalt(TCHALT_H); + instruction_hazard(); + } else { + /* Power down the core */ + cps_pm_enter_state(CPS_PM_POWER_GATED); + } + + /* This should never be reached */ + panic("Failed to offline CPU %u", cpu); +} + +static void wait_for_sibling_halt(void *ptr_cpu) +{ + unsigned cpu = (unsigned)ptr_cpu; + unsigned vpe_id = cpu_data[cpu].vpe_id; + unsigned halted; + unsigned long flags; + + do { + local_irq_save(flags); + settc(vpe_id); + halted = read_tc_c0_tchalt(); + local_irq_restore(flags); + } while (!(halted & TCHALT_H)); +} + +static void cps_cpu_die(unsigned int cpu) +{ + unsigned core = cpu_data[cpu].core; + unsigned stat; + int err; + + /* Wait for the cpu to choose its way out */ + if (!wait_for_completion_timeout(&cpu_death_chosen, + msecs_to_jiffies(5000))) { + pr_err("CPU%u: didn't offline\n", cpu); + return; + } + + /* + * Now wait for the CPU to actually offline. Without doing this that + * offlining may race with one or more of: + * + * - Onlining the CPU again. + * - Powering down the core if another VPE within it is offlined. + * - A sibling VPE entering a non-coherent state. + * + * In the non-MT halt case (ie. infinite loop) the CPU is doing nothing + * with which we could race, so do nothing. + */ + if (cpu_death == CPU_DEATH_POWER) { + /* + * Wait for the core to enter a powered down or clock gated + * state, the latter happening when a JTAG probe is connected + * in which case the CPC will refuse to power down the core. + */ + do { + mips_cpc_lock_other(core); + stat = read_cpc_co_stat_conf(); + stat &= CPC_Cx_STAT_CONF_SEQSTATE_MSK; + mips_cpc_unlock_other(); + } while (stat != CPC_Cx_STAT_CONF_SEQSTATE_D0 && + stat != CPC_Cx_STAT_CONF_SEQSTATE_D2 && + stat != CPC_Cx_STAT_CONF_SEQSTATE_U2); + + /* Indicate the core is powered off */ + bitmap_clear(core_power, core, 1); + } else if (cpu_has_mipsmt) { + /* + * Have a CPU with access to the offlined CPUs registers wait + * for its TC to halt. + */ + err = smp_call_function_single(cpu_death_sibling, + wait_for_sibling_halt, + (void *)cpu, 1); + if (err) + panic("Failed to call remote sibling CPU\n"); + } +} + +#endif /* CONFIG_HOTPLUG_CPU */ + +static struct plat_smp_ops cps_smp_ops = { + .smp_setup = cps_smp_setup, + .prepare_cpus = cps_prepare_cpus, + .boot_secondary = cps_boot_secondary, + .init_secondary = cps_init_secondary, + .smp_finish = cps_smp_finish, + .send_ipi_single = gic_send_ipi_single, + .send_ipi_mask = gic_send_ipi_mask, +#ifdef CONFIG_HOTPLUG_CPU + .cpu_disable = cps_cpu_disable, + .cpu_die = cps_cpu_die, +#endif +}; + +bool mips_cps_smp_in_use(void) +{ + extern struct plat_smp_ops *mp_ops; + return mp_ops == &cps_smp_ops; +} + +int register_cps_smp_ops(void) +{ + if (!mips_cm_present()) { + pr_warn("MIPS CPS SMP unable to proceed without a CM\n"); + return -ENODEV; + } + + /* check we have a GIC - we need one for IPIs */ + if (!(read_gcr_gic_status() & CM_GCR_GIC_STATUS_EX_MSK)) { + pr_warn("MIPS CPS SMP unable to proceed without a GIC\n"); + return -ENODEV; + } + + register_smp_ops(&cps_smp_ops); + return 0; +} diff --git a/arch/mips/kernel/smp-gic.c b/arch/mips/kernel/smp-gic.c new file mode 100644 index 00000000000..3b21a96d1cc --- /dev/null +++ b/arch/mips/kernel/smp-gic.c @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2013 Imagination Technologies + * Author: Paul Burton <paul.burton@imgtec.com> + * + * Based on smp-cmp.c: + * Copyright (C) 2007 MIPS Technologies, Inc. + * Author: Chris Dearman (chris@mips.com) + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/printk.h> + +#include <asm/gic.h> +#include <asm/mips-cpc.h> +#include <asm/smp-ops.h> + +void gic_send_ipi_single(int cpu, unsigned int action) +{ + unsigned long flags; + unsigned int intr; + unsigned int core = cpu_data[cpu].core; + + pr_debug("CPU%d: %s cpu %d action %u status %08x\n", + smp_processor_id(), __func__, cpu, action, read_c0_status()); + + local_irq_save(flags); + + switch (action) { + case SMP_CALL_FUNCTION: + intr = plat_ipi_call_int_xlate(cpu); + break; + + case SMP_RESCHEDULE_YOURSELF: + intr = plat_ipi_resched_int_xlate(cpu); + break; + + default: + BUG(); + } + + gic_send_ipi(intr); + + if (mips_cpc_present() && (core != current_cpu_data.core)) { + while (!cpumask_test_cpu(cpu, &cpu_coherent_mask)) { + mips_cpc_lock_other(core); + write_cpc_co_cmd(CPC_Cx_CMD_PWRUP); + mips_cpc_unlock_other(); + } + } + + local_irq_restore(flags); +} + +void gic_send_ipi_mask(const struct cpumask *mask, unsigned int action) +{ + unsigned int i; + + for_each_cpu(i, mask) + gic_send_ipi_single(i, action); +} diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 43e7cdc5ded..3babf6e4f89 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -24,17 +24,17 @@ #include <linux/compiler.h> #include <linux/smp.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/cacheflush.h> #include <asm/cpu.h> #include <asm/processor.h> -#include <asm/system.h> #include <asm/hardirq.h> #include <asm/mmu_context.h> #include <asm/time.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/mips_mt.h> +#include <asm/gic.h> static void __init smvp_copy_vpe_config(void) { @@ -71,8 +71,9 @@ static unsigned int __init smvp_vpe_init(unsigned int tc, unsigned int mvpconf0, /* Record this as available CPU */ set_cpu_possible(tc, true); + set_cpu_present(tc, true); __cpu_number_map[tc] = ++ncpu; - __cpu_logical_map[ncpu] = tc; + __cpu_logical_map[ncpu] = tc; } /* Disable multi-threading with TC's */ @@ -118,9 +119,15 @@ static void vsmp_send_ipi_single(int cpu, unsigned int action) unsigned long flags; int vpflags; +#ifdef CONFIG_IRQ_GIC + if (gic_present) { + gic_send_ipi_single(cpu, action); + return; + } +#endif local_irq_save(flags); - vpflags = dvpe(); /* cant access the other CPU's registers whilst MVPE enabled */ + vpflags = dvpe(); /* can't access the other CPU's registers whilst MVPE enabled */ switch (action) { case SMP_CALL_FUNCTION: @@ -149,20 +156,20 @@ static void vsmp_send_ipi_mask(const struct cpumask *mask, unsigned int action) vsmp_send_ipi_single(i, action); } -static void __cpuinit vsmp_init_secondary(void) +static void vsmp_init_secondary(void) { - extern int gic_present; - - /* This is Malta specific: IPI,performance and timer inetrrupts */ +#ifdef CONFIG_IRQ_GIC + /* This is Malta specific: IPI,performance and timer interrupts */ if (gic_present) change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 | STATUSF_IP6 | STATUSF_IP7); else +#endif change_c0_status(ST0_IM, STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP6 | STATUSF_IP7); } -static void __cpuinit vsmp_smp_finish(void) +static void vsmp_smp_finish(void) { /* CDFIXME: remove this? */ write_c0_compare(read_c0_count() + (8* mips_hpt_frequency/HZ)); @@ -176,10 +183,6 @@ static void __cpuinit vsmp_smp_finish(void) local_irq_enable(); } -static void vsmp_cpus_done(void) -{ -} - /* * Setup the PC, SP, and GP of a secondary processor and start it * running! @@ -188,7 +191,7 @@ static void vsmp_cpus_done(void) * (unsigned long)idle->thread_info the gp * assumes a 1:1 mapping of TC => VPE */ -static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) +static void vsmp_boot_secondary(int cpu, struct task_struct *idle) { struct thread_info *gp = task_thread_info(idle); dvpe(); @@ -214,7 +217,7 @@ static void __cpuinit vsmp_boot_secondary(int cpu, struct task_struct *idle) write_tc_gpr_gp((unsigned long)gp); flush_icache_range((unsigned long)gp, - (unsigned long)(gp + sizeof(struct thread_info))); + (unsigned long)(gp + sizeof(struct thread_info))); /* finally out of configuration and into chaos */ clear_c0_mvpcontrol(MVPCONTROL_VPC); @@ -280,8 +283,29 @@ struct plat_smp_ops vsmp_smp_ops = { .send_ipi_mask = vsmp_send_ipi_mask, .init_secondary = vsmp_init_secondary, .smp_finish = vsmp_smp_finish, - .cpus_done = vsmp_cpus_done, .boot_secondary = vsmp_boot_secondary, .smp_setup = vsmp_smp_setup, .prepare_cpus = vsmp_prepare_cpus, }; + +static int proc_cpuinfo_chain_call(struct notifier_block *nfb, + unsigned long action_unused, void *data) +{ + struct proc_cpuinfo_notifier_args *pcn = data; + struct seq_file *m = pcn->m; + unsigned long n = pcn->n; + + if (!cpu_has_mipsmt) + return NOTIFY_OK; + + seq_printf(m, "VPE\t\t\t: %d\n", cpu_data[n].vpe_id); + + return NOTIFY_OK; +} + +static int __init proc_cpuinfo_notifier_init(void) +{ + return proc_cpuinfo_notifier(proc_cpuinfo_chain_call, 0); +} + +subsys_initcall(proc_cpuinfo_notifier_init); diff --git a/arch/mips/kernel/smp-up.c b/arch/mips/kernel/smp-up.c index 00500fea275..17878d71ef2 100644 --- a/arch/mips/kernel/smp-up.c +++ b/arch/mips/kernel/smp-up.c @@ -28,23 +28,18 @@ static inline void up_send_ipi_mask(const struct cpumask *mask, * After we've done initial boot, this function is called to allow the * board code to clean up state, if needed */ -static void __cpuinit up_init_secondary(void) +static void up_init_secondary(void) { } -static void __cpuinit up_smp_finish(void) -{ -} - -/* Hook for after all CPUs are online */ -static void up_cpus_done(void) +static void up_smp_finish(void) { } /* * Firmware CPU startup hook */ -static void __cpuinit up_boot_secondary(int cpu, struct task_struct *idle) +static void up_boot_secondary(int cpu, struct task_struct *idle) { } @@ -73,7 +68,6 @@ struct plat_smp_ops up_smp_ops = { .send_ipi_mask = up_send_ipi_mask, .init_secondary = up_init_secondary, .smp_finish = up_smp_finish, - .cpus_done = up_cpus_done, .boot_secondary = up_boot_secondary, .smp_setup = up_smp_setup, .prepare_cpus = up_prepare_cpus, diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 383aeb95cb4..9bad52ede90 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -34,17 +34,14 @@ #include <linux/err.h> #include <linux/ftrace.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/cpu.h> #include <asm/processor.h> +#include <asm/idle.h> #include <asm/r4k-timer.h> -#include <asm/system.h> #include <asm/mmu_context.h> #include <asm/time.h> - -#ifdef CONFIG_MIPS_MT_SMTC -#include <asm/mipsmtregs.h> -#endif /* CONFIG_MIPS_MT_SMTC */ +#include <asm/setup.h> volatile cpumask_t cpu_callin_map; /* Bitmask of started secondaries */ @@ -65,6 +62,8 @@ EXPORT_SYMBOL(cpu_sibling_map); /* representing cpus for which sibling maps can be computed */ static cpumask_t cpu_sibling_setup_map; +cpumask_t cpu_coherent_mask; + static inline void set_cpu_sibling_map(int cpu) { int i; @@ -83,8 +82,9 @@ static inline void set_cpu_sibling_map(int cpu) } struct plat_smp_ops *mp_ops; +EXPORT_SYMBOL(mp_ops); -__cpuinit void register_smp_ops(struct plat_smp_ops *ops) +void register_smp_ops(struct plat_smp_ops *ops) { if (mp_ops) printk(KERN_WARNING "Overriding previously set SMP ops\n"); @@ -96,17 +96,13 @@ __cpuinit void register_smp_ops(struct plat_smp_ops *ops) * First C code run on the secondary CPUs after being started up by * the master. */ -asmlinkage __cpuinit void start_secondary(void) +asmlinkage void start_secondary(void) { unsigned int cpu; -#ifdef CONFIG_MIPS_MT_SMTC - /* Only do cpu_probe for first TC of CPU */ - if ((read_c0_tcbind() & TCBIND_CURTC) == 0) -#endif /* CONFIG_MIPS_MT_SMTC */ cpu_probe(); cpu_report(); - per_cpu_trap_init(); + per_cpu_trap_init(false); mips_clockevent_init(); mp_ops->init_secondary(); @@ -120,16 +116,25 @@ asmlinkage __cpuinit void start_secondary(void) cpu = smp_processor_id(); cpu_data[cpu].udelay_val = loops_per_jiffy; + cpu_set(cpu, cpu_coherent_mask); notify_cpu_starting(cpu); - mp_ops->smp_finish(); + set_cpu_online(cpu, true); + set_cpu_sibling_map(cpu); cpu_set(cpu, cpu_callin_map); - synchronise_count_slave(); + synchronise_count_slave(cpu); - cpu_idle(); + /* + * irq will be enabled in ->smp_finish(), enabling it too early + * is dangerous. + */ + WARN_ON_ONCE(!irqs_disabled()); + mp_ops->smp_finish(); + + cpu_startup_entry(CPUHP_ONLINE); } /* @@ -138,7 +143,6 @@ asmlinkage __cpuinit void start_secondary(void) void __irq_entry smp_call_function_interrupt(void) { irq_enter(); - generic_smp_call_function_single_interrupt(); generic_smp_call_function_interrupt(); irq_exit(); } @@ -148,7 +152,7 @@ static void stop_this_cpu(void *dummy) /* * Remove this CPU: */ - cpu_clear(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), false); for (;;) { if (cpu_wait) (*cpu_wait)(); /* Wait if available. */ @@ -162,8 +166,6 @@ void smp_send_stop(void) void __init smp_cpus_done(unsigned int max_cpus) { - mp_ops->cpus_done(); - synchronise_count_master(); } /* called from main before smp_init() */ @@ -174,46 +176,22 @@ void __init smp_prepare_cpus(unsigned int max_cpus) mp_ops->prepare_cpus(max_cpus); set_cpu_sibling_map(0); #ifndef CONFIG_HOTPLUG_CPU - init_cpu_present(&cpu_possible_map); + init_cpu_present(cpu_possible_mask); #endif + cpumask_copy(&cpu_coherent_mask, cpu_possible_mask); } /* preload SMP state for boot cpu */ -void __devinit smp_prepare_boot_cpu(void) +void smp_prepare_boot_cpu(void) { set_cpu_possible(0, true); set_cpu_online(0, true); cpu_set(0, cpu_callin_map); } -/* - * Called once for each "cpu_possible(cpu)". Needs to spin up the cpu - * and keep control until "cpu_online(cpu)" is set. Note: cpu is - * physical, not logical. - */ -static struct task_struct *cpu_idle_thread[NR_CPUS]; - -int __cpuinit __cpu_up(unsigned int cpu) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - struct task_struct *idle; - - /* - * Processor goes to start_secondary(), sets online flag - * The following code is purely to make sure - * Linux can schedule processes on this slave. - */ - if (!cpu_idle_thread[cpu]) { - idle = fork_idle(cpu); - cpu_idle_thread[cpu] = idle; - - if (IS_ERR(idle)) - panic(KERN_ERR "Fork failed for CPU %d", cpu); - } else { - idle = cpu_idle_thread[cpu]; - init_idle(idle, cpu); - } - - mp_ops->boot_secondary(cpu, idle); + mp_ops->boot_secondary(cpu, tidle); /* * Trust is futile. We should really have timeouts ... @@ -221,8 +199,7 @@ int __cpuinit __cpu_up(unsigned int cpu) while (!cpu_isset(cpu, cpu_callin_map)) udelay(100); - cpu_set(cpu, cpu_online_map); - + synchronise_count_master(cpu); return 0; } @@ -254,13 +231,10 @@ static void flush_tlb_mm_ipi(void *mm) * o collapses to normal function call on UP kernels * o collapses to normal function call on systems with a single shared * primary cache. - * o CONFIG_MIPS_MT_SMTC currently implies there is only one physical core. */ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info) { -#ifndef CONFIG_MIPS_MT_SMTC smp_call_function(func, info, 1); -#endif } static inline void smp_on_each_tlb(void (*func) (void *info), void *info) @@ -293,13 +267,12 @@ void flush_tlb_mm(struct mm_struct *mm) if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) { smp_on_other_tlbs(flush_tlb_mm_ipi, mm); } else { - cpumask_t mask = cpu_online_map; unsigned int cpu; - cpu_clear(smp_processor_id(), mask); - for_each_cpu_mask(cpu, mask) - if (cpu_context(cpu, mm)) + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) cpu_context(cpu, mm) = 0; + } } local_flush_tlb_mm(mm); @@ -333,13 +306,12 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l smp_on_other_tlbs(flush_tlb_range_ipi, &fd); } else { - cpumask_t mask = cpu_online_map; unsigned int cpu; - cpu_clear(smp_processor_id(), mask); - for_each_cpu_mask(cpu, mask) - if (cpu_context(cpu, mm)) + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, mm)) cpu_context(cpu, mm) = 0; + } } local_flush_tlb_range(vma, start, end); preempt_enable(); @@ -380,13 +352,12 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) smp_on_other_tlbs(flush_tlb_page_ipi, &fd); } else { - cpumask_t mask = cpu_online_map; unsigned int cpu; - cpu_clear(smp_processor_id(), mask); - for_each_cpu_mask(cpu, mask) - if (cpu_context(cpu, vma->vm_mm)) + for_each_online_cpu(cpu) { + if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm)) cpu_context(cpu, vma->vm_mm) = 0; + } } local_flush_tlb_page(vma, page); preempt_enable(); @@ -406,3 +377,63 @@ void flush_tlb_one(unsigned long vaddr) EXPORT_SYMBOL(flush_tlb_page); EXPORT_SYMBOL(flush_tlb_one); + +#if defined(CONFIG_KEXEC) +void (*dump_ipi_function_ptr)(void *) = NULL; +void dump_send_ipi(void (*dump_ipi_callback)(void *)) +{ + int i; + int cpu = smp_processor_id(); + + dump_ipi_function_ptr = dump_ipi_callback; + smp_mb(); + for_each_online_cpu(i) + if (i != cpu) + mp_ops->send_ipi_single(i, SMP_DUMP); + +} +EXPORT_SYMBOL(dump_send_ipi); +#endif + +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST + +static DEFINE_PER_CPU(atomic_t, tick_broadcast_count); +static DEFINE_PER_CPU(struct call_single_data, tick_broadcast_csd); + +void tick_broadcast(const struct cpumask *mask) +{ + atomic_t *count; + struct call_single_data *csd; + int cpu; + + for_each_cpu(cpu, mask) { + count = &per_cpu(tick_broadcast_count, cpu); + csd = &per_cpu(tick_broadcast_csd, cpu); + + if (atomic_inc_return(count) == 1) + smp_call_function_single_async(cpu, csd); + } +} + +static void tick_broadcast_callee(void *info) +{ + int cpu = smp_processor_id(); + tick_receive_broadcast(); + atomic_set(&per_cpu(tick_broadcast_count, cpu), 0); +} + +static int __init tick_broadcast_init(void) +{ + struct call_single_data *csd; + int cpu; + + for (cpu = 0; cpu < NR_CPUS; cpu++) { + csd = &per_cpu(tick_broadcast_csd, cpu); + csd->func = tick_broadcast_callee; + } + + return 0; +} +early_initcall(tick_broadcast_init); + +#endif /* CONFIG_GENERIC_CLOCKEVENTS_BROADCAST */ diff --git a/arch/mips/kernel/smtc-asm.S b/arch/mips/kernel/smtc-asm.S deleted file mode 100644 index 20938a4cb52..00000000000 --- a/arch/mips/kernel/smtc-asm.S +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Assembly Language Functions for MIPS MT SMTC support - */ - -/* - * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set. */ - -#include <asm/regdef.h> -#include <asm/asmmacro.h> -#include <asm/stackframe.h> -#include <asm/irqflags.h> - -/* - * "Software Interrupt" linkage. - * - * This is invoked when an "Interrupt" is sent from one TC to another, - * where the TC to be interrupted is halted, has it's Restart address - * and Status values saved by the "remote control" thread, then modified - * to cause execution to begin here, in kenel mode. This code then - * disguises the TC state as that of an exception and transfers - * control to the general exception or vectored interrupt handler. - */ - .set noreorder - -/* -The __smtc_ipi_vector would use k0 and k1 as temporaries and -1) Set EXL (this is per-VPE, so this can't be done by proxy!) -2) Restore the K/CU and IXMT bits to the pre "exception" state - (EXL means no interrupts and access to the kernel map). -3) Set EPC to be the saved value of TCRestart. -4) Jump to the exception handler entry point passed by the sender. - -CAN WE PROVE THAT WE WON'T DO THIS IF INTS DISABLED?? -*/ - -/* - * Reviled and slandered vision: Set EXL and restore K/CU/IXMT - * state of pre-halt thread, then save everything and call - * thought some function pointer to imaginary_exception, which - * will parse a register value or memory message queue to - * deliver things like interprocessor interrupts. On return - * from that function, jump to the global ret_from_irq code - * to invoke the scheduler and return as appropriate. - */ - -#define PT_PADSLOT4 (PT_R0-8) -#define PT_PADSLOT5 (PT_R0-4) - - .text - .align 5 -FEXPORT(__smtc_ipi_vector) - .set noat - /* Disable thread scheduling to make Status update atomic */ - DMT 27 # dmt k1 - _ehb - /* Set EXL */ - mfc0 k0,CP0_STATUS - ori k0,k0,ST0_EXL - mtc0 k0,CP0_STATUS - _ehb - /* Thread scheduling now inhibited by EXL. Restore TE state. */ - andi k1,k1,VPECONTROL_TE - beqz k1,1f - emt -1: - /* - * The IPI sender has put some information on the anticipated - * kernel stack frame. If we were in user mode, this will be - * built above the saved kernel SP. If we were already in the - * kernel, it will be built above the current CPU SP. - * - * Were we in kernel mode, as indicated by CU0? - */ - sll k1,k0,3 - .set noreorder - bltz k1,2f - move k1,sp - .set reorder - /* - * If previously in user mode, set CU0 and use kernel stack. - */ - li k1,ST0_CU0 - or k1,k1,k0 - mtc0 k1,CP0_STATUS - _ehb - get_saved_sp - /* Interrupting TC will have pre-set values in slots in the new frame */ -2: subu k1,k1,PT_SIZE - /* Load TCStatus Value */ - lw k0,PT_TCSTATUS(k1) - /* Write it to TCStatus to restore CU/KSU/IXMT state */ - mtc0 k0,$2,1 - _ehb - lw k0,PT_EPC(k1) - mtc0 k0,CP0_EPC - /* Save all will redundantly recompute the SP, but use it for now */ - SAVE_ALL - CLI - TRACE_IRQS_OFF - /* Function to be invoked passed stack pad slot 5 */ - lw t0,PT_PADSLOT5(sp) - /* Argument from sender passed in stack pad slot 4 */ - lw a0,PT_PADSLOT4(sp) - LONG_L s0, TI_REGS($28) - LONG_S sp, TI_REGS($28) - PTR_LA ra, ret_from_irq - jr t0 - -/* - * Called from idle loop to provoke processing of queued IPIs - * First IPI message in queue passed as argument. - */ - -LEAF(self_ipi) - /* Before anything else, block interrupts */ - mfc0 t0,CP0_TCSTATUS - ori t1,t0,TCSTATUS_IXMT - mtc0 t1,CP0_TCSTATUS - _ehb - /* We know we're in kernel mode, so prepare stack frame */ - subu t1,sp,PT_SIZE - sw ra,PT_EPC(t1) - sw a0,PT_PADSLOT4(t1) - la t2,ipi_decode - sw t2,PT_PADSLOT5(t1) - /* Save pre-disable value of TCStatus */ - sw t0,PT_TCSTATUS(t1) - j __smtc_ipi_vector - nop -END(self_ipi) diff --git a/arch/mips/kernel/smtc-proc.c b/arch/mips/kernel/smtc-proc.c deleted file mode 100644 index fe256559c99..00000000000 --- a/arch/mips/kernel/smtc-proc.c +++ /dev/null @@ -1,92 +0,0 @@ -/* - * /proc hooks for SMTC kernel - * Copyright (C) 2005 Mips Technologies, Inc - */ - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/cpumask.h> -#include <linux/interrupt.h> - -#include <asm/cpu.h> -#include <asm/processor.h> -#include <asm/atomic.h> -#include <asm/system.h> -#include <asm/hardirq.h> -#include <asm/mmu_context.h> -#include <asm/mipsregs.h> -#include <asm/cacheflush.h> -#include <linux/proc_fs.h> - -#include <asm/smtc_proc.h> - -/* - * /proc diagnostic and statistics hooks - */ - -/* - * Statistics gathered - */ -unsigned long selfipis[NR_CPUS]; - -struct smtc_cpu_proc smtc_cpu_stats[NR_CPUS]; - -static struct proc_dir_entry *smtc_stats; - -atomic_t smtc_fpu_recoveries; - -static int proc_read_smtc(char *page, char **start, off_t off, - int count, int *eof, void *data) -{ - int totalen = 0; - int len; - int i; - extern unsigned long ebase; - - len = sprintf(page, "SMTC Status Word: 0x%08x\n", smtc_status); - totalen += len; - page += len; - len = sprintf(page, "Config7: 0x%08x\n", read_c0_config7()); - totalen += len; - page += len; - len = sprintf(page, "EBASE: 0x%08lx\n", ebase); - totalen += len; - page += len; - len = sprintf(page, "Counter Interrupts taken per CPU (TC)\n"); - totalen += len; - page += len; - for (i=0; i < NR_CPUS; i++) { - len = sprintf(page, "%d: %ld\n", i, smtc_cpu_stats[i].timerints); - totalen += len; - page += len; - } - len = sprintf(page, "Self-IPIs by CPU:\n"); - totalen += len; - page += len; - for(i = 0; i < NR_CPUS; i++) { - len = sprintf(page, "%d: %ld\n", i, smtc_cpu_stats[i].selfipis); - totalen += len; - page += len; - } - len = sprintf(page, "%d Recoveries of \"stolen\" FPU\n", - atomic_read(&smtc_fpu_recoveries)); - totalen += len; - page += len; - - return totalen; -} - -void init_smtc_stats(void) -{ - int i; - - for (i=0; i<NR_CPUS; i++) { - smtc_cpu_stats[i].timerints = 0; - smtc_cpu_stats[i].selfipis = 0; - } - - atomic_set(&smtc_fpu_recoveries, 0); - - smtc_stats = create_proc_read_entry("smtc", 0444, NULL, - proc_read_smtc, NULL); -} diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c deleted file mode 100644 index 39c08254b0f..00000000000 --- a/arch/mips/kernel/smtc.c +++ /dev/null @@ -1,1469 +0,0 @@ -/* - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - * - * Copyright (C) 2004 Mips Technologies, Inc - * Copyright (C) 2008 Kevin D. Kissell - */ - -#include <linux/clockchips.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/cpumask.h> -#include <linux/interrupt.h> -#include <linux/kernel_stat.h> -#include <linux/module.h> -#include <linux/ftrace.h> -#include <linux/slab.h> - -#include <asm/cpu.h> -#include <asm/processor.h> -#include <asm/atomic.h> -#include <asm/system.h> -#include <asm/hardirq.h> -#include <asm/hazards.h> -#include <asm/irq.h> -#include <asm/mmu_context.h> -#include <asm/mipsregs.h> -#include <asm/cacheflush.h> -#include <asm/time.h> -#include <asm/addrspace.h> -#include <asm/smtc.h> -#include <asm/smtc_proc.h> - -/* - * SMTC Kernel needs to manipulate low-level CPU interrupt mask - * in do_IRQ. These are passed in setup_irq_smtc() and stored - * in this table. - */ -unsigned long irq_hwmask[NR_IRQS]; - -#define LOCK_MT_PRA() \ - local_irq_save(flags); \ - mtflags = dmt() - -#define UNLOCK_MT_PRA() \ - emt(mtflags); \ - local_irq_restore(flags) - -#define LOCK_CORE_PRA() \ - local_irq_save(flags); \ - mtflags = dvpe() - -#define UNLOCK_CORE_PRA() \ - evpe(mtflags); \ - local_irq_restore(flags) - -/* - * Data structures purely associated with SMTC parallelism - */ - - -/* - * Table for tracking ASIDs whose lifetime is prolonged. - */ - -asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS]; - -/* - * Number of InterProcessor Interrupt (IPI) message buffers to allocate - */ - -#define IPIBUF_PER_CPU 4 - -struct smtc_ipi_q IPIQ[NR_CPUS]; -static struct smtc_ipi_q freeIPIq; - - -/* Forward declarations */ - -void ipi_decode(struct smtc_ipi *); -static void post_direct_ipi(int cpu, struct smtc_ipi *pipi); -static void setup_cross_vpe_interrupts(unsigned int nvpe); -void init_smtc_stats(void); - -/* Global SMTC Status */ - -unsigned int smtc_status; - -/* Boot command line configuration overrides */ - -static int vpe0limit; -static int ipibuffers; -static int nostlb; -static int asidmask; -unsigned long smtc_asid_mask = 0xff; - -static int __init vpe0tcs(char *str) -{ - get_option(&str, &vpe0limit); - - return 1; -} - -static int __init ipibufs(char *str) -{ - get_option(&str, &ipibuffers); - return 1; -} - -static int __init stlb_disable(char *s) -{ - nostlb = 1; - return 1; -} - -static int __init asidmask_set(char *str) -{ - get_option(&str, &asidmask); - switch (asidmask) { - case 0x1: - case 0x3: - case 0x7: - case 0xf: - case 0x1f: - case 0x3f: - case 0x7f: - case 0xff: - smtc_asid_mask = (unsigned long)asidmask; - break; - default: - printk("ILLEGAL ASID mask 0x%x from command line\n", asidmask); - } - return 1; -} - -__setup("vpe0tcs=", vpe0tcs); -__setup("ipibufs=", ipibufs); -__setup("nostlb", stlb_disable); -__setup("asidmask=", asidmask_set); - -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - -static int hang_trig; - -static int __init hangtrig_enable(char *s) -{ - hang_trig = 1; - return 1; -} - - -__setup("hangtrig", hangtrig_enable); - -#define DEFAULT_BLOCKED_IPI_LIMIT 32 - -static int timerq_limit = DEFAULT_BLOCKED_IPI_LIMIT; - -static int __init tintq(char *str) -{ - get_option(&str, &timerq_limit); - return 1; -} - -__setup("tintq=", tintq); - -static int imstuckcount[2][8]; -/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */ -static int vpemask[2][8] = { - {0, 0, 1, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 0, 0, 1} -}; -int tcnoprog[NR_CPUS]; -static atomic_t idle_hook_initialized = ATOMIC_INIT(0); -static int clock_hang_reported[NR_CPUS]; - -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - -/* - * Configure shared TLB - VPC configuration bit must be set by caller - */ - -static void smtc_configure_tlb(void) -{ - int i, tlbsiz, vpes; - unsigned long mvpconf0; - unsigned long config1val; - - /* Set up ASID preservation table */ - for (vpes=0; vpes<MAX_SMTC_TLBS; vpes++) { - for(i = 0; i < MAX_SMTC_ASIDS; i++) { - smtc_live_asid[vpes][i] = 0; - } - } - mvpconf0 = read_c0_mvpconf0(); - - if ((vpes = ((mvpconf0 & MVPCONF0_PVPE) - >> MVPCONF0_PVPE_SHIFT) + 1) > 1) { - /* If we have multiple VPEs, try to share the TLB */ - if ((mvpconf0 & MVPCONF0_TLBS) && !nostlb) { - /* - * If TLB sizing is programmable, shared TLB - * size is the total available complement. - * Otherwise, we have to take the sum of all - * static VPE TLB entries. - */ - if ((tlbsiz = ((mvpconf0 & MVPCONF0_PTLBE) - >> MVPCONF0_PTLBE_SHIFT)) == 0) { - /* - * If there's more than one VPE, there had better - * be more than one TC, because we need one to bind - * to each VPE in turn to be able to read - * its configuration state! - */ - settc(1); - /* Stop the TC from doing anything foolish */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - /* No need to un-Halt - that happens later anyway */ - for (i=0; i < vpes; i++) { - write_tc_c0_tcbind(i); - /* - * To be 100% sure we're really getting the right - * information, we exit the configuration state - * and do an IHB after each rebinding. - */ - write_c0_mvpcontrol( - read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); - mips_ihb(); - /* - * Only count if the MMU Type indicated is TLB - */ - if (((read_vpe_c0_config() & MIPS_CONF_MT) >> 7) == 1) { - config1val = read_vpe_c0_config1(); - tlbsiz += ((config1val >> 25) & 0x3f) + 1; - } - - /* Put core back in configuration state */ - write_c0_mvpcontrol( - read_c0_mvpcontrol() | MVPCONTROL_VPC ); - mips_ihb(); - } - } - write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB); - ehb(); - - /* - * Setup kernel data structures to use software total, - * rather than read the per-VPE Config1 value. The values - * for "CPU 0" gets copied to all the other CPUs as part - * of their initialization in smtc_cpu_setup(). - */ - - /* MIPS32 limits TLB indices to 64 */ - if (tlbsiz > 64) - tlbsiz = 64; - cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz; - smtc_status |= SMTC_TLB_SHARED; - local_flush_tlb_all(); - - printk("TLB of %d entry pairs shared by %d VPEs\n", - tlbsiz, vpes); - } else { - printk("WARNING: TLB Not Sharable on SMTC Boot!\n"); - } - } -} - - -/* - * Incrementally build the CPU map out of constituent MIPS MT cores, - * using the specified available VPEs and TCs. Plaform code needs - * to ensure that each MIPS MT core invokes this routine on reset, - * one at a time(!). - * - * This version of the build_cpu_map and prepare_cpus routines assumes - * that *all* TCs of a MIPS MT core will be used for Linux, and that - * they will be spread across *all* available VPEs (to minimise the - * loss of efficiency due to exception service serialization). - * An improved version would pick up configuration information and - * possibly leave some TCs/VPEs as "slave" processors. - * - * Use c0_MVPConf0 to find out how many TCs are available, setting up - * cpu_possible_map and the logical/physical mappings. - */ - -int __init smtc_build_cpu_map(int start_cpu_slot) -{ - int i, ntcs; - - /* - * The CPU map isn't actually used for anything at this point, - * so it's not clear what else we should do apart from set - * everything up so that "logical" = "physical". - */ - ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) { - set_cpu_possible(i, true); - __cpu_number_map[i] = i; - __cpu_logical_map[i] = i; - } -#ifdef CONFIG_MIPS_MT_FPAFF - /* Initialize map of CPUs with FPUs */ - cpus_clear(mt_fpu_cpumask); -#endif - - /* One of those TC's is the one booting, and not a secondary... */ - printk("%i available secondary CPU TC(s)\n", i - 1); - - return i; -} - -/* - * Common setup before any secondaries are started - * Make sure all CPU's are in a sensible state before we boot any of the - * secondaries. - * - * For MIPS MT "SMTC" operation, we set up all TCs, spread as evenly - * as possible across the available VPEs. - */ - -static void smtc_tc_setup(int vpe, int tc, int cpu) -{ - settc(tc); - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - write_tc_c0_tcstatus((read_tc_c0_tcstatus() - & ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT)) - | TCSTATUS_A); - /* - * TCContext gets an offset from the base of the IPIQ array - * to be used in low-level code to detect the presence of - * an active IPI queue - */ - write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16); - /* Bind tc to vpe */ - write_tc_c0_tcbind(vpe); - /* In general, all TCs should have the same cpu_data indications */ - memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips)); - /* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */ - if (cpu_data[0].cputype == CPU_34K || - cpu_data[0].cputype == CPU_1004K) - cpu_data[cpu].options &= ~MIPS_CPU_FPU; - cpu_data[cpu].vpe_id = vpe; - cpu_data[cpu].tc_id = tc; - /* Multi-core SMTC hasn't been tested, but be prepared */ - cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff; -} - -/* - * Tweak to get Count registes in as close a sync as possible. - * Value seems good for 34K-class cores. - */ - -#define CP0_SKEW 8 - -void smtc_prepare_cpus(int cpus) -{ - int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu; - unsigned long flags; - unsigned long val; - int nipi; - struct smtc_ipi *pipi; - - /* disable interrupts so we can disable MT */ - local_irq_save(flags); - /* disable MT so we can configure */ - dvpe(); - dmt(); - - spin_lock_init(&freeIPIq.lock); - - /* - * We probably don't have as many VPEs as we do SMP "CPUs", - * but it's possible - and in any case we'll never use more! - */ - for (i=0; i<NR_CPUS; i++) { - IPIQ[i].head = IPIQ[i].tail = NULL; - spin_lock_init(&IPIQ[i].lock); - IPIQ[i].depth = 0; - IPIQ[i].resched_flag = 0; /* No reschedules queued initially */ - } - - /* cpu_data index starts at zero */ - cpu = 0; - cpu_data[cpu].vpe_id = 0; - cpu_data[cpu].tc_id = 0; - cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff; - cpu++; - - /* Report on boot-time options */ - mips_mt_set_cpuoptions(); - if (vpelimit > 0) - printk("Limit of %d VPEs set\n", vpelimit); - if (tclimit > 0) - printk("Limit of %d TCs set\n", tclimit); - if (nostlb) { - printk("Shared TLB Use Inhibited - UNSAFE for Multi-VPE Operation\n"); - } - if (asidmask) - printk("ASID mask value override to 0x%x\n", asidmask); - - /* Temporary */ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - if (hang_trig) - printk("Logic Analyser Trigger on suspected TC hang\n"); -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - - /* Put MVPE's into 'configuration state' */ - write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC ); - - val = read_c0_mvpconf0(); - nvpe = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; - if (vpelimit > 0 && nvpe > vpelimit) - nvpe = vpelimit; - ntc = ((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - if (ntc > NR_CPUS) - ntc = NR_CPUS; - if (tclimit > 0 && ntc > tclimit) - ntc = tclimit; - slop = ntc % nvpe; - for (i = 0; i < nvpe; i++) { - tcpervpe[i] = ntc / nvpe; - if (slop) { - if((slop - i) > 0) tcpervpe[i]++; - } - } - /* Handle command line override for VPE0 */ - if (vpe0limit > ntc) vpe0limit = ntc; - if (vpe0limit > 0) { - int slopslop; - if (vpe0limit < tcpervpe[0]) { - /* Reducing TC count - distribute to others */ - slop = tcpervpe[0] - vpe0limit; - slopslop = slop % (nvpe - 1); - tcpervpe[0] = vpe0limit; - for (i = 1; i < nvpe; i++) { - tcpervpe[i] += slop / (nvpe - 1); - if(slopslop && ((slopslop - (i - 1) > 0))) - tcpervpe[i]++; - } - } else if (vpe0limit > tcpervpe[0]) { - /* Increasing TC count - steal from others */ - slop = vpe0limit - tcpervpe[0]; - slopslop = slop % (nvpe - 1); - tcpervpe[0] = vpe0limit; - for (i = 1; i < nvpe; i++) { - tcpervpe[i] -= slop / (nvpe - 1); - if(slopslop && ((slopslop - (i - 1) > 0))) - tcpervpe[i]--; - } - } - } - - /* Set up shared TLB */ - smtc_configure_tlb(); - - for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) { - if (tcpervpe[vpe] == 0) - continue; - if (vpe != 0) - printk(", "); - printk("VPE %d: TC", vpe); - for (i = 0; i < tcpervpe[vpe]; i++) { - /* - * TC 0 is bound to VPE 0 at reset, - * and is presumably executing this - * code. Leave it alone! - */ - if (tc != 0) { - smtc_tc_setup(vpe, tc, cpu); - cpu++; - } - printk(" %d", tc); - tc++; - } - if (vpe != 0) { - /* - * Allow this VPE to control others. - */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | - VPECONF0_MVP); - - /* - * Clear any stale software interrupts from VPE's Cause - */ - write_vpe_c0_cause(0); - - /* - * Clear ERL/EXL of VPEs other than 0 - * and set restricted interrupt enable/mask. - */ - write_vpe_c0_status((read_vpe_c0_status() - & ~(ST0_BEV | ST0_ERL | ST0_EXL | ST0_IM)) - | (STATUSF_IP0 | STATUSF_IP1 | STATUSF_IP7 - | ST0_IE)); - /* - * set config to be the same as vpe0, - * particularly kseg0 coherency alg - */ - write_vpe_c0_config(read_c0_config()); - /* Clear any pending timer interrupt */ - write_vpe_c0_compare(0); - /* Propagate Config7 */ - write_vpe_c0_config7(read_c0_config7()); - write_vpe_c0_count(read_c0_count() + CP0_SKEW); - ehb(); - } - /* enable multi-threading within VPE */ - write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE); - /* enable the VPE */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); - } - - /* - * Pull any physically present but unused TCs out of circulation. - */ - while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) { - set_cpu_possible(tc, false); - set_cpu_present(tc, false); - tc++; - } - - /* release config state */ - write_c0_mvpcontrol( read_c0_mvpcontrol() & ~ MVPCONTROL_VPC ); - - printk("\n"); - - /* Set up coprocessor affinity CPU mask(s) */ - -#ifdef CONFIG_MIPS_MT_FPAFF - for (tc = 0; tc < ntc; tc++) { - if (cpu_data[tc].options & MIPS_CPU_FPU) - cpu_set(tc, mt_fpu_cpumask); - } -#endif - - /* set up ipi interrupts... */ - - /* If we have multiple VPEs running, set up the cross-VPE interrupt */ - - setup_cross_vpe_interrupts(nvpe); - - /* Set up queue of free IPI "messages". */ - nipi = NR_CPUS * IPIBUF_PER_CPU; - if (ipibuffers > 0) - nipi = ipibuffers; - - pipi = kmalloc(nipi *sizeof(struct smtc_ipi), GFP_KERNEL); - if (pipi == NULL) - panic("kmalloc of IPI message buffers failed\n"); - else - printk("IPI buffer pool of %d buffers\n", nipi); - for (i = 0; i < nipi; i++) { - smtc_ipi_nq(&freeIPIq, pipi); - pipi++; - } - - /* Arm multithreading and enable other VPEs - but all TCs are Halted */ - emt(EMT_ENABLE); - evpe(EVPE_ENABLE); - local_irq_restore(flags); - /* Initialize SMTC /proc statistics/diagnostics */ - init_smtc_stats(); -} - - -/* - * Setup the PC, SP, and GP of a secondary processor and start it - * running! - * smp_bootstrap is the place to resume from - * __KSTK_TOS(idle) is apparently the stack pointer - * (unsigned long)idle->thread_info the gp - * - */ -void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle) -{ - extern u32 kernelsp[NR_CPUS]; - unsigned long flags; - int mtflags; - - LOCK_MT_PRA(); - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - dvpe(); - } - settc(cpu_data[cpu].tc_id); - - /* pc */ - write_tc_c0_tcrestart((unsigned long)&smp_bootstrap); - - /* stack pointer */ - kernelsp[cpu] = __KSTK_TOS(idle); - write_tc_gpr_sp(__KSTK_TOS(idle)); - - /* global pointer */ - write_tc_gpr_gp((unsigned long)task_thread_info(idle)); - - smtc_status |= SMTC_MTC_ACTIVE; - write_tc_c0_tchalt(0); - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - evpe(EVPE_ENABLE); - } - UNLOCK_MT_PRA(); -} - -void smtc_init_secondary(void) -{ - local_irq_enable(); -} - -void smtc_smp_finish(void) -{ - int cpu = smp_processor_id(); - - /* - * Lowest-numbered CPU per VPE starts a clock tick. - * Like per_cpu_trap_init() hack, this assumes that - * SMTC init code assigns TCs consdecutively and - * in ascending order across available VPEs. - */ - if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id)) - write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ); - - printk("TC %d going on-line as CPU %d\n", - cpu_data[smp_processor_id()].tc_id, smp_processor_id()); -} - -void smtc_cpus_done(void) -{ -} - -/* - * Support for SMTC-optimized driver IRQ registration - */ - -/* - * SMTC Kernel needs to manipulate low-level CPU interrupt mask - * in do_IRQ. These are passed in setup_irq_smtc() and stored - * in this table. - */ - -int setup_irq_smtc(unsigned int irq, struct irqaction * new, - unsigned long hwmask) -{ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - unsigned int vpe = current_cpu_data.vpe_id; - - vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1; -#endif - irq_hwmask[irq] = hwmask; - - return setup_irq(irq, new); -} - -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF -/* - * Support for IRQ affinity to TCs - */ - -void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity) -{ - /* - * If a "fast path" cache of quickly decodable affinity state - * is maintained, this is where it gets done, on a call up - * from the platform affinity code. - */ -} - -void smtc_forward_irq(unsigned int irq) -{ - int target; - - /* - * OK wise guy, now figure out how to get the IRQ - * to be serviced on an authorized "CPU". - * - * Ideally, to handle the situation where an IRQ has multiple - * eligible CPUS, we would maintain state per IRQ that would - * allow a fair distribution of service requests. Since the - * expected use model is any-or-only-one, for simplicity - * and efficiency, we just pick the easiest one to find. - */ - - target = cpumask_first(irq_desc[irq].affinity); - - /* - * We depend on the platform code to have correctly processed - * IRQ affinity change requests to ensure that the IRQ affinity - * mask has been purged of bits corresponding to nonexistent and - * offline "CPUs", and to TCs bound to VPEs other than the VPE - * connected to the physical interrupt input for the interrupt - * in question. Otherwise we have a nasty problem with interrupt - * mask management. This is best handled in non-performance-critical - * platform IRQ affinity setting code, to minimize interrupt-time - * checks. - */ - - /* If no one is eligible, service locally */ - if (target >= NR_CPUS) { - do_IRQ_no_affinity(irq); - return; - } - - smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq); -} - -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ - -/* - * IPI model for SMTC is tricky, because interrupts aren't TC-specific. - * Within a VPE one TC can interrupt another by different approaches. - * The easiest to get right would probably be to make all TCs except - * the target IXMT and set a software interrupt, but an IXMT-based - * scheme requires that a handler must run before a new IPI could - * be sent, which would break the "broadcast" loops in MIPS MT. - * A more gonzo approach within a VPE is to halt the TC, extract - * its Restart, Status, and a couple of GPRs, and program the Restart - * address to emulate an interrupt. - * - * Within a VPE, one can be confident that the target TC isn't in - * a critical EXL state when halted, since the write to the Halt - * register could not have issued on the writing thread if the - * halting thread had EXL set. So k0 and k1 of the target TC - * can be used by the injection code. Across VPEs, one can't - * be certain that the target TC isn't in a critical exception - * state. So we try a two-step process of sending a software - * interrupt to the target VPE, which either handles the event - * itself (if it was the target) or injects the event within - * the VPE. - */ - -static void smtc_ipi_qdump(void) -{ - int i; - struct smtc_ipi *temp; - - for (i = 0; i < NR_CPUS ;i++) { - pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n", - i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail, - IPIQ[i].depth); - temp = IPIQ[i].head; - - while (temp != IPIQ[i].tail) { - pr_debug("%d %d %d: ", temp->type, temp->dest, - (int)temp->arg); -#ifdef SMTC_IPI_DEBUG - pr_debug("%u %lu\n", temp->sender, temp->stamp); -#else - pr_debug("\n"); -#endif - temp = temp->flink; - } - } -} - -/* - * The standard atomic.h primitives don't quite do what we want - * here: We need an atomic add-and-return-previous-value (which - * could be done with atomic_add_return and a decrement) and an - * atomic set/zero-and-return-previous-value (which can't really - * be done with the atomic.h primitives). And since this is - * MIPS MT, we can assume that we have LL/SC. - */ -static inline int atomic_postincrement(atomic_t *v) -{ - unsigned long result; - - unsigned long temp; - - __asm__ __volatile__( - "1: ll %0, %2 \n" - " addu %1, %0, 1 \n" - " sc %1, %2 \n" - " beqz %1, 1b \n" - __WEAK_LLSC_MB - : "=&r" (result), "=&r" (temp), "=m" (v->counter) - : "m" (v->counter) - : "memory"); - - return result; -} - -void smtc_send_ipi(int cpu, int type, unsigned int action) -{ - int tcstatus; - struct smtc_ipi *pipi; - unsigned long flags; - int mtflags; - unsigned long tcrestart; - extern void r4k_wait_irqoff(void), __pastwait(void); - int set_resched_flag = (type == LINUX_SMP_IPI && - action == SMP_RESCHEDULE_YOURSELF); - - if (cpu == smp_processor_id()) { - printk("Cannot Send IPI to self!\n"); - return; - } - if (set_resched_flag && IPIQ[cpu].resched_flag != 0) - return; /* There is a reschedule queued already */ - - /* Set up a descriptor, to be delivered either promptly or queued */ - pipi = smtc_ipi_dq(&freeIPIq); - if (pipi == NULL) { - bust_spinlocks(1); - mips_mt_regdump(dvpe()); - panic("IPI Msg. Buffers Depleted\n"); - } - pipi->type = type; - pipi->arg = (void *)action; - pipi->dest = cpu; - if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) { - /* If not on same VPE, enqueue and send cross-VPE interrupt */ - IPIQ[cpu].resched_flag |= set_resched_flag; - smtc_ipi_nq(&IPIQ[cpu], pipi); - LOCK_CORE_PRA(); - settc(cpu_data[cpu].tc_id); - write_vpe_c0_cause(read_vpe_c0_cause() | C_SW1); - UNLOCK_CORE_PRA(); - } else { - /* - * Not sufficient to do a LOCK_MT_PRA (dmt) here, - * since ASID shootdown on the other VPE may - * collide with this operation. - */ - LOCK_CORE_PRA(); - settc(cpu_data[cpu].tc_id); - /* Halt the targeted TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* - * Inspect TCStatus - if IXMT is set, we have to queue - * a message. Otherwise, we set up the "interrupt" - * of the other TC - */ - tcstatus = read_tc_c0_tcstatus(); - - if ((tcstatus & TCSTATUS_IXMT) != 0) { - /* - * If we're in the the irq-off version of the wait - * loop, we need to force exit from the wait and - * do a direct post of the IPI. - */ - if (cpu_wait == r4k_wait_irqoff) { - tcrestart = read_tc_c0_tcrestart(); - if (tcrestart >= (unsigned long)r4k_wait_irqoff - && tcrestart < (unsigned long)__pastwait) { - write_tc_c0_tcrestart(__pastwait); - tcstatus &= ~TCSTATUS_IXMT; - write_tc_c0_tcstatus(tcstatus); - goto postdirect; - } - } - /* - * Otherwise we queue the message for the target TC - * to pick up when he does a local_irq_restore() - */ - write_tc_c0_tchalt(0); - UNLOCK_CORE_PRA(); - IPIQ[cpu].resched_flag |= set_resched_flag; - smtc_ipi_nq(&IPIQ[cpu], pipi); - } else { -postdirect: - post_direct_ipi(cpu, pipi); - write_tc_c0_tchalt(0); - UNLOCK_CORE_PRA(); - } - } -} - -/* - * Send IPI message to Halted TC, TargTC/TargVPE already having been set - */ -static void post_direct_ipi(int cpu, struct smtc_ipi *pipi) -{ - struct pt_regs *kstack; - unsigned long tcstatus; - unsigned long tcrestart; - extern u32 kernelsp[NR_CPUS]; - extern void __smtc_ipi_vector(void); -//printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu); - - /* Extract Status, EPC from halted TC */ - tcstatus = read_tc_c0_tcstatus(); - tcrestart = read_tc_c0_tcrestart(); - /* If TCRestart indicates a WAIT instruction, advance the PC */ - if ((tcrestart & 0x80000000) - && ((*(unsigned int *)tcrestart & 0xfe00003f) == 0x42000020)) { - tcrestart += 4; - } - /* - * Save on TC's future kernel stack - * - * CU bit of Status is indicator that TC was - * already running on a kernel stack... - */ - if (tcstatus & ST0_CU0) { - /* Note that this "- 1" is pointer arithmetic */ - kstack = ((struct pt_regs *)read_tc_gpr_sp()) - 1; - } else { - kstack = ((struct pt_regs *)kernelsp[cpu]) - 1; - } - - kstack->cp0_epc = (long)tcrestart; - /* Save TCStatus */ - kstack->cp0_tcstatus = tcstatus; - /* Pass token of operation to be performed kernel stack pad area */ - kstack->pad0[4] = (unsigned long)pipi; - /* Pass address of function to be called likewise */ - kstack->pad0[5] = (unsigned long)&ipi_decode; - /* Set interrupt exempt and kernel mode */ - tcstatus |= TCSTATUS_IXMT; - tcstatus &= ~TCSTATUS_TKSU; - write_tc_c0_tcstatus(tcstatus); - ehb(); - /* Set TC Restart address to be SMTC IPI vector */ - write_tc_c0_tcrestart(__smtc_ipi_vector); -} - -static void ipi_resched_interrupt(void) -{ - /* Return from interrupt should be enough to cause scheduler check */ -} - -static void ipi_call_interrupt(void) -{ - /* Invoke generic function invocation code in smp.c */ - smp_call_function_interrupt(); -} - -DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device); - -static void __irq_entry smtc_clock_tick_interrupt(void) -{ - unsigned int cpu = smp_processor_id(); - struct clock_event_device *cd; - int irq = MIPS_CPU_IRQ_BASE + 1; - - irq_enter(); - kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq)); - cd = &per_cpu(mips_clockevent_device, cpu); - cd->event_handler(cd); - irq_exit(); -} - -void ipi_decode(struct smtc_ipi *pipi) -{ - void *arg_copy = pipi->arg; - int type_copy = pipi->type; - - smtc_ipi_nq(&freeIPIq, pipi); - - switch (type_copy) { - case SMTC_CLOCK_TICK: - smtc_clock_tick_interrupt(); - break; - - case LINUX_SMP_IPI: - switch ((int)arg_copy) { - case SMP_RESCHEDULE_YOURSELF: - ipi_resched_interrupt(); - break; - case SMP_CALL_FUNCTION: - ipi_call_interrupt(); - break; - default: - printk("Impossible SMTC IPI Argument %p\n", arg_copy); - break; - } - break; -#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF - case IRQ_AFFINITY_IPI: - /* - * Accept a "forwarded" interrupt that was initially - * taken by a TC who doesn't have affinity for the IRQ. - */ - do_IRQ_no_affinity((int)arg_copy); - break; -#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */ - default: - printk("Impossible SMTC IPI Type 0x%x\n", type_copy); - break; - } -} - -/* - * Similar to smtc_ipi_replay(), but invoked from context restore, - * so it reuses the current exception frame rather than set up a - * new one with self_ipi. - */ - -void deferred_smtc_ipi(void) -{ - int cpu = smp_processor_id(); - - /* - * Test is not atomic, but much faster than a dequeue, - * and the vast majority of invocations will have a null queue. - * If irq_disabled when this was called, then any IPIs queued - * after we test last will be taken on the next irq_enable/restore. - * If interrupts were enabled, then any IPIs added after the - * last test will be taken directly. - */ - - while (IPIQ[cpu].head != NULL) { - struct smtc_ipi_q *q = &IPIQ[cpu]; - struct smtc_ipi *pipi; - unsigned long flags; - - /* - * It may be possible we'll come in with interrupts - * already enabled. - */ - local_irq_save(flags); - spin_lock(&q->lock); - pipi = __smtc_ipi_dq(q); - spin_unlock(&q->lock); - if (pipi != NULL) { - if (pipi->type == LINUX_SMP_IPI && - (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) - IPIQ[cpu].resched_flag = 0; - ipi_decode(pipi); - } - /* - * The use of the __raw_local restore isn't - * as obviously necessary here as in smtc_ipi_replay(), - * but it's more efficient, given that we're already - * running down the IPI queue. - */ - __arch_local_irq_restore(flags); - } -} - -/* - * Cross-VPE interrupts in the SMTC prototype use "software interrupts" - * set via cross-VPE MTTR manipulation of the Cause register. It would be - * in some regards preferable to have external logic for "doorbell" hardware - * interrupts. - */ - -static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ; - -static irqreturn_t ipi_interrupt(int irq, void *dev_idm) -{ - int my_vpe = cpu_data[smp_processor_id()].vpe_id; - int my_tc = cpu_data[smp_processor_id()].tc_id; - int cpu; - struct smtc_ipi *pipi; - unsigned long tcstatus; - int sent; - unsigned long flags; - unsigned int mtflags; - unsigned int vpflags; - - /* - * So long as cross-VPE interrupts are done via - * MFTR/MTTR read-modify-writes of Cause, we need - * to stop other VPEs whenever the local VPE does - * anything similar. - */ - local_irq_save(flags); - vpflags = dvpe(); - clear_c0_cause(0x100 << MIPS_CPU_IPI_IRQ); - set_c0_status(0x100 << MIPS_CPU_IPI_IRQ); - irq_enable_hazard(); - evpe(vpflags); - local_irq_restore(flags); - - /* - * Cross-VPE Interrupt handler: Try to directly deliver IPIs - * queued for TCs on this VPE other than the current one. - * Return-from-interrupt should cause us to drain the queue - * for the current TC, so we ought not to have to do it explicitly here. - */ - - for_each_online_cpu(cpu) { - if (cpu_data[cpu].vpe_id != my_vpe) - continue; - - pipi = smtc_ipi_dq(&IPIQ[cpu]); - if (pipi != NULL) { - if (cpu_data[cpu].tc_id != my_tc) { - sent = 0; - LOCK_MT_PRA(); - settc(cpu_data[cpu].tc_id); - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - tcstatus = read_tc_c0_tcstatus(); - if ((tcstatus & TCSTATUS_IXMT) == 0) { - post_direct_ipi(cpu, pipi); - sent = 1; - } - write_tc_c0_tchalt(0); - UNLOCK_MT_PRA(); - if (!sent) { - smtc_ipi_req(&IPIQ[cpu], pipi); - } - } else { - /* - * ipi_decode() should be called - * with interrupts off - */ - local_irq_save(flags); - if (pipi->type == LINUX_SMP_IPI && - (int)pipi->arg == SMP_RESCHEDULE_YOURSELF) - IPIQ[cpu].resched_flag = 0; - ipi_decode(pipi); - local_irq_restore(flags); - } - } - } - - return IRQ_HANDLED; -} - -static void ipi_irq_dispatch(void) -{ - do_IRQ(cpu_ipi_irq); -} - -static struct irqaction irq_ipi = { - .handler = ipi_interrupt, - .flags = IRQF_DISABLED | IRQF_PERCPU, - .name = "SMTC_IPI" -}; - -static void setup_cross_vpe_interrupts(unsigned int nvpe) -{ - if (nvpe < 1) - return; - - if (!cpu_has_vint) - panic("SMTC Kernel requires Vectored Interrupt support"); - - set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch); - - setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ)); - - set_irq_handler(cpu_ipi_irq, handle_percpu_irq); -} - -/* - * SMTC-specific hacks invoked from elsewhere in the kernel. - */ - - /* - * smtc_ipi_replay is called from raw_local_irq_restore - */ - -void smtc_ipi_replay(void) -{ - unsigned int cpu = smp_processor_id(); - - /* - * To the extent that we've ever turned interrupts off, - * we may have accumulated deferred IPIs. This is subtle. - * we should be OK: If we pick up something and dispatch - * it here, that's great. If we see nothing, but concurrent - * with this operation, another TC sends us an IPI, IXMT - * is clear, and we'll handle it as a real pseudo-interrupt - * and not a pseudo-pseudo interrupt. The important thing - * is to do the last check for queued message *after* the - * re-enabling of interrupts. - */ - while (IPIQ[cpu].head != NULL) { - struct smtc_ipi_q *q = &IPIQ[cpu]; - struct smtc_ipi *pipi; - unsigned long flags; - - /* - * It's just possible we'll come in with interrupts - * already enabled. - */ - local_irq_save(flags); - - spin_lock(&q->lock); - pipi = __smtc_ipi_dq(q); - spin_unlock(&q->lock); - /* - ** But use a raw restore here to avoid recursion. - */ - __arch_local_irq_restore(flags); - - if (pipi) { - self_ipi(pipi); - smtc_cpu_stats[cpu].selfipis++; - } - } -} - -EXPORT_SYMBOL(smtc_ipi_replay); - -void smtc_idle_loop_hook(void) -{ -#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG - int im; - int flags; - int mtflags; - int bit; - int vpe; - int tc; - int hook_ntcs; - /* - * printk within DMT-protected regions can deadlock, - * so buffer diagnostic messages for later output. - */ - char *pdb_msg; - char id_ho_db_msg[768]; /* worst-case use should be less than 700 */ - - if (atomic_read(&idle_hook_initialized) == 0) { /* fast test */ - if (atomic_add_return(1, &idle_hook_initialized) == 1) { - int mvpconf0; - /* Tedious stuff to just do once */ - mvpconf0 = read_c0_mvpconf0(); - hook_ntcs = ((mvpconf0 & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1; - if (hook_ntcs > NR_CPUS) - hook_ntcs = NR_CPUS; - for (tc = 0; tc < hook_ntcs; tc++) { - tcnoprog[tc] = 0; - clock_hang_reported[tc] = 0; - } - for (vpe = 0; vpe < 2; vpe++) - for (im = 0; im < 8; im++) - imstuckcount[vpe][im] = 0; - printk("Idle loop test hook initialized for %d TCs\n", hook_ntcs); - atomic_set(&idle_hook_initialized, 1000); - } else { - /* Someone else is initializing in parallel - let 'em finish */ - while (atomic_read(&idle_hook_initialized) < 1000) - ; - } - } - - /* Have we stupidly left IXMT set somewhere? */ - if (read_c0_tcstatus() & 0x400) { - write_c0_tcstatus(read_c0_tcstatus() & ~0x400); - ehb(); - printk("Dangling IXMT in cpu_idle()\n"); - } - - /* Have we stupidly left an IM bit turned off? */ -#define IM_LIMIT 2000 - local_irq_save(flags); - mtflags = dmt(); - pdb_msg = &id_ho_db_msg[0]; - im = read_c0_status(); - vpe = current_cpu_data.vpe_id; - for (bit = 0; bit < 8; bit++) { - /* - * In current prototype, I/O interrupts - * are masked for VPE > 0 - */ - if (vpemask[vpe][bit]) { - if (!(im & (0x100 << bit))) - imstuckcount[vpe][bit]++; - else - imstuckcount[vpe][bit] = 0; - if (imstuckcount[vpe][bit] > IM_LIMIT) { - set_c0_status(0x100 << bit); - ehb(); - imstuckcount[vpe][bit] = 0; - pdb_msg += sprintf(pdb_msg, - "Dangling IM %d fixed for VPE %d\n", bit, - vpe); - } - } - } - - emt(mtflags); - local_irq_restore(flags); - if (pdb_msg != &id_ho_db_msg[0]) - printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg); -#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */ - - smtc_ipi_replay(); -} - -void smtc_soft_dump(void) -{ - int i; - - printk("Counter Interrupts taken per CPU (TC)\n"); - for (i=0; i < NR_CPUS; i++) { - printk("%d: %ld\n", i, smtc_cpu_stats[i].timerints); - } - printk("Self-IPI invocations:\n"); - for (i=0; i < NR_CPUS; i++) { - printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis); - } - smtc_ipi_qdump(); - printk("%d Recoveries of \"stolen\" FPU\n", - atomic_read(&smtc_fpu_recoveries)); -} - - -/* - * TLB management routines special to SMTC - */ - -void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu) -{ - unsigned long flags, mtflags, tcstat, prevhalt, asid; - int tlb, i; - - /* - * It would be nice to be able to use a spinlock here, - * but this is invoked from within TLB flush routines - * that protect themselves with DVPE, so if a lock is - * held by another TC, it'll never be freed. - * - * DVPE/DMT must not be done with interrupts enabled, - * so even so most callers will already have disabled - * them, let's be really careful... - */ - - local_irq_save(flags); - if (smtc_status & SMTC_TLB_SHARED) { - mtflags = dvpe(); - tlb = 0; - } else { - mtflags = dmt(); - tlb = cpu_data[cpu].vpe_id; - } - asid = asid_cache(cpu); - - do { - if (!((asid += ASID_INC) & ASID_MASK) ) { - if (cpu_has_vtag_icache) - flush_icache_all(); - /* Traverse all online CPUs (hack requires contiguous range) */ - for_each_online_cpu(i) { - /* - * We don't need to worry about our own CPU, nor those of - * CPUs who don't share our TLB. - */ - if ((i != smp_processor_id()) && - ((smtc_status & SMTC_TLB_SHARED) || - (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))) { - settc(cpu_data[i].tc_id); - prevhalt = read_tc_c0_tchalt() & TCHALT_H; - if (!prevhalt) { - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - } - tcstat = read_tc_c0_tcstatus(); - smtc_live_asid[tlb][(tcstat & ASID_MASK)] |= (asiduse)(0x1 << i); - if (!prevhalt) - write_tc_c0_tchalt(0); - } - } - if (!asid) /* fix version if needed */ - asid = ASID_FIRST_VERSION; - local_flush_tlb_all(); /* start new asid cycle */ - } - } while (smtc_live_asid[tlb][(asid & ASID_MASK)]); - - /* - * SMTC shares the TLB within VPEs and possibly across all VPEs. - */ - for_each_online_cpu(i) { - if ((smtc_status & SMTC_TLB_SHARED) || - (cpu_data[i].vpe_id == cpu_data[cpu].vpe_id)) - cpu_context(i, mm) = asid_cache(i) = asid; - } - - if (smtc_status & SMTC_TLB_SHARED) - evpe(mtflags); - else - emt(mtflags); - local_irq_restore(flags); -} - -/* - * Invoked from macros defined in mmu_context.h - * which must already have disabled interrupts - * and done a DVPE or DMT as appropriate. - */ - -void smtc_flush_tlb_asid(unsigned long asid) -{ - int entry; - unsigned long ehi; - - entry = read_c0_wired(); - - /* Traverse all non-wired entries */ - while (entry < current_cpu_data.tlbsize) { - write_c0_index(entry); - ehb(); - tlb_read(); - ehb(); - ehi = read_c0_entryhi(); - if ((ehi & ASID_MASK) == asid) { - /* - * Invalidate only entries with specified ASID, - * makiing sure all entries differ. - */ - write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1))); - write_c0_entrylo0(0); - write_c0_entrylo1(0); - mtc0_tlbw_hazard(); - tlb_write_indexed(); - } - entry++; - } - write_c0_index(PARKED_INDEX); - tlbw_use_hazard(); -} - -/* - * Support for single-threading cache flush operations. - */ - -static int halt_state_save[NR_CPUS]; - -/* - * To really, really be sure that nothing is being done - * by other TCs, halt them all. This code assumes that - * a DVPE has already been done, so while their Halted - * state is theoretically architecturally unstable, in - * practice, it's not going to change while we're looking - * at it. - */ - -void smtc_cflush_lockdown(void) -{ - int cpu; - - for_each_online_cpu(cpu) { - if (cpu != smp_processor_id()) { - settc(cpu_data[cpu].tc_id); - halt_state_save[cpu] = read_tc_c0_tchalt(); - write_tc_c0_tchalt(TCHALT_H); - } - } - mips_ihb(); -} - -/* It would be cheating to change the cpu_online states during a flush! */ - -void smtc_cflush_release(void) -{ - int cpu; - - /* - * Start with a hazard barrier to ensure - * that all CACHE ops have played through. - */ - mips_ihb(); - - for_each_online_cpu(cpu) { - if (cpu != smp_processor_id()) { - settc(cpu_data[cpu].tc_id); - write_tc_c0_tchalt(halt_state_save[cpu]); - } - } - mips_ihb(); -} diff --git a/arch/mips/kernel/spinlock_test.c b/arch/mips/kernel/spinlock_test.c index da61134dfc5..39f7ab7b042 100644 --- a/arch/mips/kernel/spinlock_test.c +++ b/arch/mips/kernel/spinlock_test.c @@ -3,7 +3,7 @@ #include <linux/hrtimer.h> #include <linux/fs.h> #include <linux/debugfs.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/spinlock.h> diff --git a/arch/mips/kernel/spram.c b/arch/mips/kernel/spram.c index 1821d12a641..67f2495def1 100644 --- a/arch/mips/kernel/spram.c +++ b/arch/mips/kernel/spram.c @@ -8,14 +8,12 @@ * * Copyright (C) 2007, 2008 MIPS Technologies, Inc. */ -#include <linux/init.h> #include <linux/kernel.h> #include <linux/ptrace.h> #include <linux/stddef.h> #include <asm/fpu.h> #include <asm/mipsregs.h> -#include <asm/system.h> #include <asm/r4kcache.h> #include <asm/hazards.h> @@ -38,7 +36,7 @@ /* * Different semantics to the set_c0_* function built by __BUILD_SET_C0 */ -static __cpuinit unsigned int bis_c0_errctl(unsigned int set) +static unsigned int bis_c0_errctl(unsigned int set) { unsigned int res; res = read_c0_errctl(); @@ -46,7 +44,7 @@ static __cpuinit unsigned int bis_c0_errctl(unsigned int set) return res; } -static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) +static void ispram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -65,7 +63,7 @@ static __cpuinit void ispram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int ispram_load_tag(unsigned int offset) +static unsigned int ispram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -83,7 +81,7 @@ static __cpuinit unsigned int ispram_load_tag(unsigned int offset) return data; } -static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) +static void dspram_store_tag(unsigned int offset, unsigned int data) { unsigned int errctl; @@ -99,7 +97,7 @@ static __cpuinit void dspram_store_tag(unsigned int offset, unsigned int data) } -static __cpuinit unsigned int dspram_load_tag(unsigned int offset) +static unsigned int dspram_load_tag(unsigned int offset) { unsigned int data; unsigned int errctl; @@ -116,7 +114,7 @@ static __cpuinit unsigned int dspram_load_tag(unsigned int offset) return data; } -static __cpuinit void probe_spram(char *type, +static void probe_spram(char *type, unsigned int base, unsigned int (*read)(unsigned int), void (*write)(unsigned int, unsigned int)) @@ -197,16 +195,19 @@ static __cpuinit void probe_spram(char *type, offset += 2 * SPRAM_TAG_STRIDE; } } -void __cpuinit spram_config(void) +void spram_config(void) { - struct cpuinfo_mips *c = ¤t_cpu_data; unsigned int config0; - switch (c->cputype) { + switch (current_cpu_type()) { case CPU_24K: case CPU_34K: case CPU_74K: case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_P5600: config0 = read_c0_config(); /* FIXME: addresses are Malta specific */ if (config0 & (1<<24)) { diff --git a/arch/mips/kernel/stacktrace.c b/arch/mips/kernel/stacktrace.c index d52ff77baf3..1ba775d24d3 100644 --- a/arch/mips/kernel/stacktrace.c +++ b/arch/mips/kernel/stacktrace.c @@ -5,7 +5,7 @@ */ #include <linux/sched.h> #include <linux/stacktrace.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/stacktrace.h> /* diff --git a/arch/mips/kernel/sync-r4k.c b/arch/mips/kernel/sync-r4k.c index 05dd170a83f..2242bdd4370 100644 --- a/arch/mips/kernel/sync-r4k.c +++ b/arch/mips/kernel/sync-r4k.c @@ -6,45 +6,32 @@ * not have done anything significant (but they may have had interrupts * enabled briefly - prom_smp_finish() should not be responsible for enabling * interrupts...) - * - * FIXME: broken for SMTC */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/irqflags.h> #include <linux/cpumask.h> #include <asm/r4k-timer.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/barrier.h> #include <asm/mipsregs.h> -static atomic_t __cpuinitdata count_start_flag = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_start = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_count_stop = ATOMIC_INIT(0); -static atomic_t __cpuinitdata count_reference = ATOMIC_INIT(0); +static atomic_t count_start_flag = ATOMIC_INIT(0); +static atomic_t count_count_start = ATOMIC_INIT(0); +static atomic_t count_count_stop = ATOMIC_INIT(0); +static atomic_t count_reference = ATOMIC_INIT(0); -#define COUNTON 100 +#define COUNTON 100 #define NR_LOOPS 5 -void __cpuinit synchronise_count_master(void) +void synchronise_count_master(int cpu) { int i; unsigned long flags; unsigned int initcount; - int nslaves; - -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC needs to synchronise per VPE, not per CPU - * ignore for now - */ - return; -#endif - printk(KERN_INFO "Synchronize counters across %u CPUs: ", - num_online_cpus()); + printk(KERN_INFO "Synchronize counters for CPU %u: ", cpu); local_irq_save(flags); @@ -52,7 +39,7 @@ void __cpuinit synchronise_count_master(void) * Notify the slaves that it's time to start */ atomic_set(&count_reference, read_c0_count()); - atomic_set(&count_start_flag, 1); + atomic_set(&count_start_flag, cpu); smp_wmb(); /* Count will be initialised to current timer for all CPU's */ @@ -69,10 +56,9 @@ void __cpuinit synchronise_count_master(void) * two CPUs. */ - nslaves = num_online_cpus()-1; for (i = 0; i < NR_LOOPS; i++) { - /* slaves loop on '!= ncpus' */ - while (atomic_read(&count_count_start) != nslaves) + /* slaves loop on '!= 2' */ + while (atomic_read(&count_count_start) != 1) mb(); atomic_set(&count_count_stop, 0); smp_wmb(); @@ -89,7 +75,7 @@ void __cpuinit synchronise_count_master(void) /* * Wait for all slaves to leave the synchronization point: */ - while (atomic_read(&count_count_stop) != nslaves) + while (atomic_read(&count_count_stop) != 1) mb(); atomic_set(&count_count_start, 0); smp_wmb(); @@ -97,6 +83,7 @@ void __cpuinit synchronise_count_master(void) } /* Arrange for an interrupt in a short while */ write_c0_compare(read_c0_count() + COUNTON); + atomic_set(&count_start_flag, 0); local_irq_restore(flags); @@ -108,38 +95,25 @@ void __cpuinit synchronise_count_master(void) printk("done.\n"); } -void __cpuinit synchronise_count_slave(void) +void synchronise_count_slave(int cpu) { int i; - unsigned long flags; unsigned int initcount; - int ncpus; - -#ifdef CONFIG_MIPS_MT_SMTC - /* - * SMTC needs to synchronise per VPE, not per CPU - * ignore for now - */ - return; -#endif - - local_irq_save(flags); /* * Not every cpu is online at the time this gets called, * so we first wait for the master to say everyone is ready */ - while (!atomic_read(&count_start_flag)) + while (atomic_read(&count_start_flag) != cpu) mb(); /* Count will be initialised to next expire for all CPU's */ initcount = atomic_read(&count_reference); - ncpus = num_online_cpus(); for (i = 0; i < NR_LOOPS; i++) { atomic_inc(&count_count_start); - while (atomic_read(&count_count_start) != ncpus) + while (atomic_read(&count_count_start) != 2) mb(); /* @@ -149,12 +123,10 @@ void __cpuinit synchronise_count_slave(void) write_c0_count(initcount); atomic_inc(&count_count_stop); - while (atomic_read(&count_count_stop) != ncpus) + while (atomic_read(&count_count_stop) != 2) mb(); } /* Arrange for an interrupt in a short while */ write_c0_compare(read_c0_count() + COUNTON); - - local_irq_restore(flags); } #undef NR_LOOPS diff --git a/arch/mips/kernel/syscall.c b/arch/mips/kernel/syscall.c index 1dc6edff45e..4a4f9dda565 100644 --- a/arch/mips/kernel/syscall.c +++ b/arch/mips/kernel/syscall.c @@ -10,12 +10,9 @@ #include <linux/capability.h> #include <linux/errno.h> #include <linux/linkage.h> -#include <linux/mm.h> #include <linux/fs.h> #include <linux/smp.h> -#include <linux/mman.h> #include <linux/ptrace.h> -#include <linux/sched.h> #include <linux/string.h> #include <linux/syscalls.h> #include <linux/file.h> @@ -25,11 +22,9 @@ #include <linux/msg.h> #include <linux/shm.h> #include <linux/compiler.h> -#include <linux/module.h> #include <linux/ipc.h> #include <linux/uaccess.h> #include <linux/slab.h> -#include <linux/random.h> #include <linux/elf.h> #include <asm/asm.h> @@ -42,143 +37,23 @@ #include <asm/shmparam.h> #include <asm/sysmips.h> #include <asm/uaccess.h> +#include <asm/switch_to.h> /* * For historic reasons the pipe(2) syscall on MIPS has an unusual calling - * convention. It returns results in registers $v0 / $v1 which means there + * convention. It returns results in registers $v0 / $v1 which means there * is no need for it to do verify the validity of a userspace pointer - * argument. Historically that used to be expensive in Linux. These days + * argument. Historically that used to be expensive in Linux. These days * the performance advantage is negligible. */ -asmlinkage int sysm_pipe(nabi_no_regargs volatile struct pt_regs regs) +asmlinkage int sysm_pipe(void) { int fd[2]; - int error, res; - - error = do_pipe_flags(fd, 0); - if (error) { - res = error; - goto out; - } - regs.regs[3] = fd[1]; - res = fd[0]; -out: - return res; -} - -unsigned long shm_align_mask = PAGE_SIZE - 1; /* Sane caches */ - -EXPORT_SYMBOL(shm_align_mask); - -#define COLOUR_ALIGN(addr,pgoff) \ - ((((addr) + shm_align_mask) & ~shm_align_mask) + \ - (((pgoff) << PAGE_SHIFT) & shm_align_mask)) - -unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, unsigned long flags) -{ - struct vm_area_struct * vmm; - int do_color_align; - unsigned long task_size; - -#ifdef CONFIG_32BIT - task_size = TASK_SIZE; -#else /* Must be CONFIG_64BIT*/ - task_size = test_thread_flag(TIF_32BIT_ADDR) ? TASK_SIZE32 : TASK_SIZE; -#endif - - if (len > task_size) - return -ENOMEM; - - if (flags & MAP_FIXED) { - /* Even MAP_FIXED mappings must reside within task_size. */ - if (task_size - len < addr) - return -EINVAL; - - /* - * We do not accept a shared mapping if it would violate - * cache aliasing constraints. - */ - if ((flags & MAP_SHARED) && - ((addr - (pgoff << PAGE_SHIFT)) & shm_align_mask)) - return -EINVAL; - return addr; - } - - do_color_align = 0; - if (filp || (flags & MAP_SHARED)) - do_color_align = 1; - if (addr) { - if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); - else - addr = PAGE_ALIGN(addr); - vmm = find_vma(current->mm, addr); - if (task_size - len >= addr && - (!vmm || addr + len <= vmm->vm_start)) - return addr; - } - addr = current->mm->mmap_base; - if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); - else - addr = PAGE_ALIGN(addr); - - for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { - /* At this point: (!vmm || addr < vmm->vm_end). */ - if (task_size - len < addr) - return -ENOMEM; - if (!vmm || addr + len <= vmm->vm_start) - return addr; - addr = vmm->vm_end; - if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); - } -} - -void arch_pick_mmap_layout(struct mm_struct *mm) -{ - unsigned long random_factor = 0UL; - - if (current->flags & PF_RANDOMIZE) { - random_factor = get_random_int(); - random_factor = random_factor << PAGE_SHIFT; - if (TASK_IS_32BIT_ADDR) - random_factor &= 0xfffffful; - else - random_factor &= 0xffffffful; - } - - mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; - mm->get_unmapped_area = arch_get_unmapped_area; - mm->unmap_area = arch_unmap_area; -} - -static inline unsigned long brk_rnd(void) -{ - unsigned long rnd = get_random_int(); - - rnd = rnd << PAGE_SHIFT; - /* 8MB for 32bit, 256MB for 64bit */ - if (TASK_IS_32BIT_ADDR) - rnd = rnd & 0x7ffffful; - else - rnd = rnd & 0xffffffful; - - return rnd; -} - -unsigned long arch_randomize_brk(struct mm_struct *mm) -{ - unsigned long base = mm->brk; - unsigned long ret; - - ret = PAGE_ALIGN(base + brk_rnd()); - - if (ret < mm->brk) - return mm->brk; - - return ret; + int error = do_pipe_flags(fd, 0); + if (error) + return error; + current_pt_regs()->regs[3] = fd[1]; + return fd[0]; } SYSCALL_DEFINE6(mips_mmap, unsigned long, addr, unsigned long, len, @@ -208,65 +83,7 @@ SYSCALL_DEFINE6(mips_mmap2, unsigned long, addr, unsigned long, len, } save_static_function(sys_fork); -static int __used noinline -_sys_fork(nabi_no_regargs struct pt_regs regs) -{ - return do_fork(SIGCHLD, regs.regs[29], ®s, 0, NULL, NULL); -} - save_static_function(sys_clone); -static int __used noinline -_sys_clone(nabi_no_regargs struct pt_regs regs) -{ - unsigned long clone_flags; - unsigned long newsp; - int __user *parent_tidptr, *child_tidptr; - - clone_flags = regs.regs[4]; - newsp = regs.regs[5]; - if (!newsp) - newsp = regs.regs[29]; - parent_tidptr = (int __user *) regs.regs[6]; -#ifdef CONFIG_32BIT - /* We need to fetch the fifth argument off the stack. */ - child_tidptr = NULL; - if (clone_flags & (CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)) { - int __user *__user *usp = (int __user *__user *) regs.regs[29]; - if (regs.regs[2] == __NR_syscall) { - if (get_user (child_tidptr, &usp[5])) - return -EFAULT; - } - else if (get_user (child_tidptr, &usp[4])) - return -EFAULT; - } -#else - child_tidptr = (int __user *) regs.regs[8]; -#endif - return do_fork(clone_flags, newsp, ®s, 0, - parent_tidptr, child_tidptr); -} - -/* - * sys_execve() executes a new program. - */ -asmlinkage int sys_execve(nabi_no_regargs struct pt_regs regs) -{ - int error; - char * filename; - - filename = getname((const char __user *) (long)regs.regs[4]); - error = PTR_ERR(filename); - if (IS_ERR(filename)) - goto out; - error = do_execve(filename, - (const char __user *const __user *) (long)regs.regs[5], - (const char __user *const __user *) (long)regs.regs[6], - ®s); - putname(filename); - -out: - return error; -} SYSCALL_DEFINE1(set_thread_area, unsigned long, addr) { @@ -279,10 +96,10 @@ SYSCALL_DEFINE1(set_thread_area, unsigned long, addr) return 0; } -static inline int mips_atomic_set(struct pt_regs *regs, - unsigned long addr, unsigned long new) +static inline int mips_atomic_set(unsigned long addr, unsigned long new) { unsigned long old, tmp; + struct pt_regs *regs; unsigned int err; if (unlikely(addr & 3)) @@ -293,7 +110,7 @@ static inline int mips_atomic_set(struct pt_regs *regs, if (cpu_has_llsc && R10000_LLSC_WAR) { __asm__ __volatile__ ( - " .set mips3 \n" + " .set arch=r4000 \n" " li %[err], 0 \n" "1: ll %[old], (%[addr]) \n" " move %[tmp], %[new] \n" @@ -318,7 +135,7 @@ static inline int mips_atomic_set(struct pt_regs *regs, : "memory"); } else if (cpu_has_llsc) { __asm__ __volatile__ ( - " .set mips3 \n" + " .set arch=r4000 \n" " li %[err], 0 \n" "1: ll %[old], (%[addr]) \n" " move %[tmp], %[new] \n" @@ -363,6 +180,7 @@ static inline int mips_atomic_set(struct pt_regs *regs, if (unlikely(err)) return err; + regs = current_pt_regs(); regs->regs[2] = old; regs->regs[7] = 0; /* No error */ @@ -376,23 +194,14 @@ static inline int mips_atomic_set(struct pt_regs *regs, : "r" (regs)); /* unreached. Honestly. */ - while (1); + unreachable(); } -save_static_function(sys_sysmips); -static int __used noinline -_sys_sysmips(nabi_no_regargs struct pt_regs regs) +SYSCALL_DEFINE3(sysmips, long, cmd, long, arg1, long, arg2) { - long cmd, arg1, arg2, arg3; - - cmd = regs.regs[4]; - arg1 = regs.regs[5]; - arg2 = regs.regs[6]; - arg3 = regs.regs[7]; - switch (cmd) { case MIPS_ATOMIC_SET: - return mips_atomic_set(®s, arg1, arg2); + return mips_atomic_set(arg1, arg2); case MIPS_FIXADE: if (arg1 & ~3) @@ -405,7 +214,7 @@ _sys_sysmips(nabi_no_regargs struct pt_regs regs) if (arg1 & 2) set_thread_flag(TIF_LOGADE); else - clear_thread_flag(TIF_FIXADE); + clear_thread_flag(TIF_LOGADE); return 0; @@ -433,34 +242,3 @@ asmlinkage void bad_stack(void) { do_exit(SIGSEGV); } - -/* - * Do a system call from kernel instead of calling sys_execve so we - * end up with proper pt_regs. - */ -int kernel_execve(const char *filename, - const char *const argv[], - const char *const envp[]) -{ - register unsigned long __a0 asm("$4") = (unsigned long) filename; - register unsigned long __a1 asm("$5") = (unsigned long) argv; - register unsigned long __a2 asm("$6") = (unsigned long) envp; - register unsigned long __a3 asm("$7"); - unsigned long __v0; - - __asm__ volatile (" \n" - " .set noreorder \n" - " li $2, %5 # __NR_execve \n" - " syscall \n" - " move %0, $2 \n" - " .set reorder \n" - : "=&r" (__v0), "=r" (__a3) - : "r" (__a0), "r" (__a1), "r" (__a2), "i" (__NR_execve) - : "$2", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", - "memory"); - - if (__a3 == 0) - return __v0; - - return -__v0; -} diff --git a/arch/mips/kernel/time.c b/arch/mips/kernel/time.c index fb749740551..8d0170969e2 100644 --- a/arch/mips/kernel/time.c +++ b/arch/mips/kernel/time.c @@ -5,8 +5,8 @@ * * Common time service routines for MIPS machines. * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. */ @@ -21,11 +21,11 @@ #include <linux/timex.h> #include <linux/smp.h> #include <linux/spinlock.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/cpu-features.h> +#include <asm/cpu-type.h> #include <asm/div64.h> -#include <asm/smtc_ipi.h> #include <asm/time.h> /* @@ -62,8 +62,8 @@ EXPORT_SYMBOL(perf_irq); * time_init() - it does the following things. * * 1) plat_time_init() - - * a) (optional) set up RTC routines, - * b) (optional) calibrate and set the mips_hpt_frequency + * a) (optional) set up RTC routines, + * b) (optional) calibrate and set the mips_hpt_frequency * (only needed if you intended to use cpu counter as timer interrupt * source) * 2) calculate a couple of cached variables for later usage @@ -75,7 +75,7 @@ unsigned int mips_hpt_frequency; * This function exists in order to cause an error due to a duplicate * definition if platform code should have its own implementation. The hook * to use instead is plat_time_init. plat_time_init does not receive the - * irqaction pointer argument anymore. This is because any function which + * irqaction pointer argument anymore. This is because any function which * initializes an interrupt timer now takes care of its own request_irq rsp. * setup_irq calls and each clock_event_device should use its own * struct irqrequest. @@ -93,7 +93,7 @@ static __init int cpu_has_mfc0_count_bug(void) case CPU_R4000MC: /* * V3.0 is documented as suffering from the mfc0 from count bug. - * Afaik this is the last version of the R4000. Later versions + * Afaik this is the last version of the R4000. Later versions * were marketed as R4400. */ return 1; @@ -102,7 +102,7 @@ static __init int cpu_has_mfc0_count_bug(void) case CPU_R4400SC: case CPU_R4400MC: /* - * The published errata for the R4400 upto 3.0 say the CPU + * The published errata for the R4400 up to 3.0 say the CPU * has the mfc0 from count bug. */ if ((current_cpu_data.processor_id & 0xff) <= 0x30) @@ -121,6 +121,14 @@ void __init time_init(void) { plat_time_init(); - if (!mips_clockevent_init() || !cpu_has_mfc0_count_bug()) + /* + * The use of the R4k timer as a clock event takes precedence; + * if reading the Count register might interfere with the timer + * interrupt, then we don't use the timer as a clock source. + * We may still use the timer as a clock source though if the + * timer interrupt isn't reliable; the interference doesn't + * matter then, because we don't use the interrupt. + */ + if (mips_clockevent_init() != 0 || !cpu_has_mfc0_count_bug()) init_mips_clocksource(); } diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index 8e9fbe75894..51706d6dd5b 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -8,14 +8,19 @@ * Copyright (C) 1998 Ulf Carlsson * Copyright (C) 1999 Silicon Graphics, Inc. * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com - * Copyright (C) 2000, 01 MIPS Technologies, Inc. * Copyright (C) 2002, 2003, 2004, 2005, 2007 Maciej W. Rozycki + * Copyright (C) 2000, 2001, 2012 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2014, Imagination Technologies Ltd. */ #include <linux/bug.h> #include <linux/compiler.h> +#include <linux/context_tracking.h> +#include <linux/cpu_pm.h> +#include <linux/kexec.h> #include <linux/init.h> -#include <linux/mm.h> +#include <linux/kernel.h> #include <linux/module.h> +#include <linux/mm.h> #include <linux/sched.h> #include <linux/smp.h> #include <linux/spinlock.h> @@ -36,16 +41,18 @@ #include <asm/break.h> #include <asm/cop2.h> #include <asm/cpu.h> +#include <asm/cpu-type.h> #include <asm/dsp.h> #include <asm/fpu.h> #include <asm/fpu_emulator.h> +#include <asm/idle.h> #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/module.h> +#include <asm/msa.h> #include <asm/pgtable.h> #include <asm/ptrace.h> #include <asm/sections.h> -#include <asm/system.h> #include <asm/tlbdebug.h> #include <asm/traps.h> #include <asm/uaccess.h> @@ -56,12 +63,11 @@ #include <asm/uasm.h> extern void check_wait(void); -extern asmlinkage void r4k_wait(void); extern asmlinkage void rollback_handle_int(void); extern asmlinkage void handle_int(void); -extern asmlinkage void handle_tlbm(void); -extern asmlinkage void handle_tlbl(void); -extern asmlinkage void handle_tlbs(void); +extern u32 handle_tlbl[]; +extern u32 handle_tlbs[]; +extern u32 handle_tlbm[]; extern asmlinkage void handle_adel(void); extern asmlinkage void handle_ades(void); extern asmlinkage void handle_ibe(void); @@ -74,7 +80,10 @@ extern asmlinkage void handle_ri_rdhwr(void); extern asmlinkage void handle_cpu(void); extern asmlinkage void handle_ov(void); extern asmlinkage void handle_tr(void); +extern asmlinkage void handle_msa_fpe(void); extern asmlinkage void handle_fpe(void); +extern asmlinkage void handle_ftlb(void); +extern asmlinkage void handle_msa(void); extern asmlinkage void handle_mdmx(void); extern asmlinkage void handle_watch(void); extern asmlinkage void handle_mt(void); @@ -82,15 +91,13 @@ extern asmlinkage void handle_dsp(void); extern asmlinkage void handle_mcheck(void); extern asmlinkage void handle_reserved(void); -extern int fpu_emulator_cop1Handler(struct pt_regs *xcp, - struct mips_fpu_struct *ctx, int has_fpu); - void (*board_be_init)(void); int (*board_be_handler)(struct pt_regs *regs, int is_fixup); void (*board_nmi_handler_setup)(void); void (*board_ejtag_handler_setup)(void); void (*board_bind_eic_interrupt)(int irq, int regset); - +void (*board_ebase_setup)(void); +void(*board_cache_error_setup)(void); static void show_raw_backtrace(unsigned long reg29) { @@ -130,6 +137,9 @@ static void show_backtrace(struct task_struct *task, const struct pt_regs *regs) unsigned long ra = regs->regs[31]; unsigned long pc = regs->cp0_epc; + if (!task) + task = current; + if (raw_show_trace || !__kernel_text_address(pc)) { show_raw_backtrace(sp); return; @@ -158,7 +168,7 @@ static void show_stacktrace(struct task_struct *task, i = 0; while ((unsigned long) sp & (PAGE_SIZE - 1)) { if (i && ((i % (64 / field)) == 0)) - printk("\n "); + printk("\n "); if (i > 39) { printk(" ..."); break; @@ -200,19 +210,6 @@ void show_stack(struct task_struct *task, unsigned long *sp) show_stacktrace(task, ®s); } -/* - * The architecture-independent dump_stack generator - */ -void dump_stack(void) -{ - struct pt_regs regs; - - prepare_frametrace(®s); - show_backtrace(current, ®s); -} - -EXPORT_SYMBOL(dump_stack); - static void show_code(unsigned int __user *pc) { long i; @@ -238,7 +235,7 @@ static void __show_regs(const struct pt_regs *regs) unsigned int cause = regs->cp0_cause; int i; - printk("Cpu %d\n", smp_processor_id()); + show_regs_print_info(KERN_DEFAULT); /* * Saved main processor registers @@ -273,9 +270,9 @@ static void __show_regs(const struct pt_regs *regs) printk("ra : %0*lx %pS\n", field, regs->regs[31], (void *) regs->regs[31]); - printk("Status: %08x ", (uint32_t) regs->cp0_status); + printk("Status: %08x ", (uint32_t) regs->cp0_status); - if (current_cpu_data.isa_level == MIPS_CPU_ISA_I) { + if (cpu_has_3kex) { if (regs->cp0_status & ST0_KUO) printk("KUo "); if (regs->cp0_status & ST0_IEO) @@ -288,7 +285,7 @@ static void __show_regs(const struct pt_regs *regs) printk("KUc "); if (regs->cp0_status & ST0_IEC) printk("IEc "); - } else { + } else if (cpu_has_4kex) { if (regs->cp0_status & ST0_KX) printk("KX "); if (regs->cp0_status & ST0_SX) @@ -339,6 +336,7 @@ void show_regs(struct pt_regs *regs) void show_registers(struct pt_regs *regs) { const int field = 2 * sizeof(unsigned long); + mm_segment_t old_fs = get_fs(); __show_regs(regs); print_modules(); @@ -353,9 +351,13 @@ void show_registers(struct pt_regs *regs) printk("*HwTLS: %0*lx\n", field, tls); } + if (!user_mode(regs)) + /* Necessary for getting the correct stack content */ + set_fs(KERNEL_DS); show_stacktrace(current, regs); show_code((unsigned int __user *) regs->cp0_epc); printk("\n"); + set_fs(old_fs); } static int regs_to_trapnr(struct pt_regs *regs) @@ -363,42 +365,42 @@ static int regs_to_trapnr(struct pt_regs *regs) return (regs->cp0_cause >> 2) & 0x1f; } -static DEFINE_SPINLOCK(die_lock); +static DEFINE_RAW_SPINLOCK(die_lock); void __noreturn die(const char *str, struct pt_regs *regs) { static int die_counter; int sig = SIGSEGV; -#ifdef CONFIG_MIPS_MT_SMTC - unsigned long dvpret = dvpe(); -#endif /* CONFIG_MIPS_MT_SMTC */ - notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), SIGSEGV); + oops_enter(); + + if (notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), + SIGSEGV) == NOTIFY_STOP) + sig = 0; console_verbose(); - spin_lock_irq(&die_lock); + raw_spin_lock_irq(&die_lock); bust_spinlocks(1); -#ifdef CONFIG_MIPS_MT_SMTC - mips_mt_regdump(dvpret); -#endif /* CONFIG_MIPS_MT_SMTC */ - - if (notify_die(DIE_OOPS, str, regs, 0, regs_to_trapnr(regs), SIGSEGV) == NOTIFY_STOP) - sig = 0; printk("%s[#%d]:\n", str, ++die_counter); show_registers(regs); - add_taint(TAINT_DIE); - spin_unlock_irq(&die_lock); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + raw_spin_unlock_irq(&die_lock); + + oops_exit(); if (in_interrupt()) panic("Fatal exception in interrupt"); if (panic_on_oops) { - printk(KERN_EMERG "Fatal exception: panic in 5 seconds\n"); + printk(KERN_EMERG "Fatal exception: panic in 5 seconds"); ssleep(5); panic("Fatal exception"); } + if (regs && kexec_should_crash(current)) + crash_kexec(regs); + do_exit(sig); } @@ -426,8 +428,10 @@ asmlinkage void do_be(struct pt_regs *regs) const struct exception_table_entry *fixup = NULL; int data = regs->cp0_cause & 4; int action = MIPS_BE_FATAL; + enum ctx_state prev_state; - /* XXX For now. Fixme, this searches the wrong table ... */ + prev_state = exception_enter(); + /* XXX For now. Fixme, this searches the wrong table ... */ if (data && !user_mode(regs)) fixup = search_dbe_tables(exception_epc(regs)); @@ -439,11 +443,11 @@ asmlinkage void do_be(struct pt_regs *regs) switch (action) { case MIPS_BE_DISCARD: - return; + goto out; case MIPS_BE_FIXUP: if (fixup) { regs->cp0_epc = fixup->nextinsn; - return; + goto out; } break; default: @@ -456,12 +460,15 @@ asmlinkage void do_be(struct pt_regs *regs) printk(KERN_ALERT "%s bus error, epc == %0*lx, ra == %0*lx\n", data ? "Data" : "Instruction", field, regs->cp0_epc, field, regs->regs[31]); - if (notify_die(DIE_OOPS, "bus error", regs, 0, regs_to_trapnr(regs), SIGBUS) - == NOTIFY_STOP) - return; + if (notify_die(DIE_OOPS, "bus error", regs, 0, regs_to_trapnr(regs), + SIGBUS) == NOTIFY_STOP) + goto out; die_if_kernel("Oops", regs); force_sig(SIGBUS, current); + +out: + exception_exit(prev_state); } /* @@ -481,6 +488,12 @@ asmlinkage void do_be(struct pt_regs *regs) #define SYNC 0x0000000f #define RDHWR 0x0000003b +/* microMIPS definitions */ +#define MM_POOL32A_FUNC 0xfc00ffff +#define MM_RDHWR 0x00006b3c +#define MM_RS 0x001f0000 +#define MM_RT 0x03e00000 + /* * The ll_bit is cleared by r*_switch.S */ @@ -504,7 +517,7 @@ static inline int simulate_ll(struct pt_regs *regs, unsigned int opcode) offset >>= 16; vaddr = (unsigned long __user *) - ((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset); + ((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset); if ((unsigned long)vaddr & 3) return SIGBUS; @@ -544,7 +557,7 @@ static inline int simulate_sc(struct pt_regs *regs, unsigned int opcode) offset >>= 16; vaddr = (unsigned long __user *) - ((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset); + ((unsigned long)(regs->regs[(opcode & BASE) >> 21]) + offset); reg = (opcode & RT) >> 16; if ((unsigned long)vaddr & 3) @@ -579,12 +592,12 @@ static int simulate_llsc(struct pt_regs *regs, unsigned int opcode) { if ((opcode & OPCODE) == LL) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return simulate_ll(regs, opcode); } if ((opcode & OPCODE) == SC) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return simulate_sc(regs, opcode); } @@ -595,42 +608,62 @@ static int simulate_llsc(struct pt_regs *regs, unsigned int opcode) * Simulate trapping 'rdhwr' instructions to provide user accessible * registers not implemented in hardware. */ -static int simulate_rdhwr(struct pt_regs *regs, unsigned int opcode) +static int simulate_rdhwr(struct pt_regs *regs, int rd, int rt) { struct thread_info *ti = task_thread_info(current); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, + 1, regs, 0); + switch (rd) { + case 0: /* CPU number */ + regs->regs[rt] = smp_processor_id(); + return 0; + case 1: /* SYNCI length */ + regs->regs[rt] = min(current_cpu_data.dcache.linesz, + current_cpu_data.icache.linesz); + return 0; + case 2: /* Read count register */ + regs->regs[rt] = read_c0_count(); + return 0; + case 3: /* Count register resolution */ + switch (current_cpu_type()) { + case CPU_20KC: + case CPU_25KF: + regs->regs[rt] = 1; + break; + default: + regs->regs[rt] = 2; + } + return 0; + case 29: + regs->regs[rt] = ti->tp_value; + return 0; + default: + return -1; + } +} + +static int simulate_rdhwr_normal(struct pt_regs *regs, unsigned int opcode) +{ if ((opcode & OPCODE) == SPEC3 && (opcode & FUNC) == RDHWR) { int rd = (opcode & RD) >> 11; int rt = (opcode & RT) >> 16; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); - switch (rd) { - case 0: /* CPU number */ - regs->regs[rt] = smp_processor_id(); - return 0; - case 1: /* SYNCI length */ - regs->regs[rt] = min(current_cpu_data.dcache.linesz, - current_cpu_data.icache.linesz); - return 0; - case 2: /* Read count register */ - regs->regs[rt] = read_c0_count(); - return 0; - case 3: /* Count register resolution */ - switch (current_cpu_data.cputype) { - case CPU_20KC: - case CPU_25KF: - regs->regs[rt] = 1; - break; - default: - regs->regs[rt] = 2; - } - return 0; - case 29: - regs->regs[rt] = ti->tp_value; - return 0; - default: - return -1; - } + + simulate_rdhwr(regs, rd, rt); + return 0; + } + + /* Not ours. */ + return -1; +} + +static int simulate_rdhwr_mm(struct pt_regs *regs, unsigned short opcode) +{ + if ((opcode & MM_POOL32A_FUNC) == MM_RDHWR) { + int rd = (opcode & MM_RS) >> 16; + int rt = (opcode & MM_RT) >> 21; + simulate_rdhwr(regs, rd, rt); + return 0; } /* Not ours. */ @@ -641,7 +674,7 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode) { if ((opcode & OPCODE) == SPEC0 && (opcode & FUNC) == SYNC) { perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + 1, regs, 0); return 0; } @@ -650,8 +683,10 @@ static int simulate_sync(struct pt_regs *regs, unsigned int opcode) asmlinkage void do_ov(struct pt_regs *regs) { + enum ctx_state prev_state; siginfo_t info; + prev_state = exception_enter(); die_if_kernel("Integer overflow", regs); info.si_code = FPE_INTOVF; @@ -659,6 +694,33 @@ asmlinkage void do_ov(struct pt_regs *regs) info.si_errno = 0; info.si_addr = (void __user *) regs->cp0_epc; force_sig_info(SIGFPE, &info, current); + exception_exit(prev_state); +} + +int process_fpemu_return(int sig, void __user *fault_addr) +{ + if (sig == SIGSEGV || sig == SIGBUS) { + struct siginfo si = {0}; + si.si_addr = fault_addr; + si.si_signo = sig; + if (sig == SIGSEGV) { + down_read(¤t->mm->mmap_sem); + if (find_vma(current->mm, (unsigned long)fault_addr)) + si.si_code = SEGV_ACCERR; + else + si.si_code = SEGV_MAPERR; + up_read(¤t->mm->mmap_sem); + } else { + si.si_code = BUS_ADRERR; + } + force_sig_info(sig, &si, current); + return 1; + } else if (sig) { + force_sig(sig, current); + return 1; + } else { + return 0; + } } /* @@ -666,15 +728,18 @@ asmlinkage void do_ov(struct pt_regs *regs) */ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) { - siginfo_t info; + enum ctx_state prev_state; + siginfo_t info = {0}; - if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), SIGFPE) - == NOTIFY_STOP) - return; + prev_state = exception_enter(); + if (notify_die(DIE_FP, "FP exception", regs, 0, regs_to_trapnr(regs), + SIGFPE) == NOTIFY_STOP) + goto out; die_if_kernel("FP exception in kernel code", regs); if (fcr31 & FPU_CSR_UNI_X) { int sig; + void __user *fault_addr = NULL; /* * Unimplemented operation exception. If we've got the full @@ -690,7 +755,8 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) lose_fpu(1); /* Run the emulator */ - sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1); + sig = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, + &fault_addr); /* * We can't allow the emulated instruction to leave any of @@ -699,13 +765,12 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) current->thread.fpu.fcr31 &= ~FPU_CSR_ALL_X; /* Restore the hardware register state */ - own_fpu(1); /* Using the FPU again. */ + own_fpu(1); /* Using the FPU again. */ /* If something went wrong, signal */ - if (sig) - force_sig(sig, current); + process_fpemu_return(sig, fault_addr); - return; + goto out; } else if (fcr31 & FPU_CSR_INV_X) info.si_code = FPE_FLTINV; else if (fcr31 & FPU_CSR_DIV_X) @@ -722,6 +787,9 @@ asmlinkage void do_fpe(struct pt_regs *regs, unsigned long fcr31) info.si_errno = 0; info.si_addr = (void __user *) regs->cp0_epc; force_sig_info(SIGFPE, &info, current); + +out: + exception_exit(prev_state); } static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, @@ -735,7 +803,8 @@ static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, return; #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ - if (notify_die(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + if (notify_die(DIE_TRAP, str, regs, code, regs_to_trapnr(regs), + SIGTRAP) == NOTIFY_STOP) return; /* @@ -787,9 +856,38 @@ static void do_trap_or_bp(struct pt_regs *regs, unsigned int code, asmlinkage void do_bp(struct pt_regs *regs) { unsigned int opcode, bcode; - - if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) - goto out_sigsegv; + enum ctx_state prev_state; + unsigned long epc; + u16 instr[2]; + mm_segment_t seg; + + seg = get_fs(); + if (!user_mode(regs)) + set_fs(KERNEL_DS); + + prev_state = exception_enter(); + if (get_isa16_mode(regs->cp0_epc)) { + /* Calculate EPC. */ + epc = exception_epc(regs); + if (cpu_has_mmips) { + if ((__get_user(instr[0], (u16 __user *)msk_isa16_mode(epc)) || + (__get_user(instr[1], (u16 __user *)msk_isa16_mode(epc + 2))))) + goto out_sigsegv; + opcode = (instr[0] << 16) | instr[1]; + } else { + /* MIPS16e mode */ + if (__get_user(instr[0], + (u16 __user *)msk_isa16_mode(epc))) + goto out_sigsegv; + bcode = (instr[0] >> 6) & 0x3f; + do_trap_or_bp(regs, bcode, "Break"); + goto out; + } + } else { + if (__get_user(opcode, + (unsigned int __user *) exception_epc(regs))) + goto out_sigsegv; + } /* * There is the ancient bug in the MIPS assemblers that the break @@ -807,13 +905,15 @@ asmlinkage void do_bp(struct pt_regs *regs) */ switch (bcode) { case BRK_KPROBE_BP: - if (notify_die(DIE_BREAK, "debug", regs, bcode, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) - return; + if (notify_die(DIE_BREAK, "debug", regs, bcode, + regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + goto out; else break; case BRK_KPROBE_SSTEPBP: - if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode, regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) - return; + if (notify_die(DIE_SSTEPBP, "single_step", regs, bcode, + regs_to_trapnr(regs), SIGTRAP) == NOTIFY_STOP) + goto out; else break; default: @@ -821,65 +921,113 @@ asmlinkage void do_bp(struct pt_regs *regs) } do_trap_or_bp(regs, bcode, "Break"); + +out: + set_fs(seg); + exception_exit(prev_state); return; out_sigsegv: force_sig(SIGSEGV, current); + goto out; } asmlinkage void do_tr(struct pt_regs *regs) { - unsigned int opcode, tcode = 0; - - if (__get_user(opcode, (unsigned int __user *) exception_epc(regs))) - goto out_sigsegv; - - /* Immediate versions don't provide a code. */ - if (!(opcode & OPCODE)) - tcode = ((opcode >> 6) & ((1 << 10) - 1)); + u32 opcode, tcode = 0; + enum ctx_state prev_state; + u16 instr[2]; + mm_segment_t seg; + unsigned long epc = msk_isa16_mode(exception_epc(regs)); + + seg = get_fs(); + if (!user_mode(regs)) + set_fs(get_ds()); + + prev_state = exception_enter(); + if (get_isa16_mode(regs->cp0_epc)) { + if (__get_user(instr[0], (u16 __user *)(epc + 0)) || + __get_user(instr[1], (u16 __user *)(epc + 2))) + goto out_sigsegv; + opcode = (instr[0] << 16) | instr[1]; + /* Immediate versions don't provide a code. */ + if (!(opcode & OPCODE)) + tcode = (opcode >> 12) & ((1 << 4) - 1); + } else { + if (__get_user(opcode, (u32 __user *)epc)) + goto out_sigsegv; + /* Immediate versions don't provide a code. */ + if (!(opcode & OPCODE)) + tcode = (opcode >> 6) & ((1 << 10) - 1); + } do_trap_or_bp(regs, tcode, "Trap"); + +out: + set_fs(seg); + exception_exit(prev_state); return; out_sigsegv: force_sig(SIGSEGV, current); + goto out; } asmlinkage void do_ri(struct pt_regs *regs) { unsigned int __user *epc = (unsigned int __user *)exception_epc(regs); unsigned long old_epc = regs->cp0_epc; + unsigned long old31 = regs->regs[31]; + enum ctx_state prev_state; unsigned int opcode = 0; int status = -1; - if (notify_die(DIE_RI, "RI Fault", regs, 0, regs_to_trapnr(regs), SIGILL) - == NOTIFY_STOP) - return; + prev_state = exception_enter(); + if (notify_die(DIE_RI, "RI Fault", regs, 0, regs_to_trapnr(regs), + SIGILL) == NOTIFY_STOP) + goto out; die_if_kernel("Reserved instruction in kernel code", regs); if (unlikely(compute_return_epc(regs) < 0)) - return; + goto out; - if (unlikely(get_user(opcode, epc) < 0)) - status = SIGSEGV; + if (get_isa16_mode(regs->cp0_epc)) { + unsigned short mmop[2] = { 0 }; - if (!cpu_has_llsc && status < 0) - status = simulate_llsc(regs, opcode); + if (unlikely(get_user(mmop[0], epc) < 0)) + status = SIGSEGV; + if (unlikely(get_user(mmop[1], epc) < 0)) + status = SIGSEGV; + opcode = (mmop[0] << 16) | mmop[1]; - if (status < 0) - status = simulate_rdhwr(regs, opcode); + if (status < 0) + status = simulate_rdhwr_mm(regs, opcode); + } else { + if (unlikely(get_user(opcode, epc) < 0)) + status = SIGSEGV; - if (status < 0) - status = simulate_sync(regs, opcode); + if (!cpu_has_llsc && status < 0) + status = simulate_llsc(regs, opcode); + + if (status < 0) + status = simulate_rdhwr_normal(regs, opcode); + + if (status < 0) + status = simulate_sync(regs, opcode); + } if (status < 0) status = SIGILL; if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ + regs->regs[31] = old31; force_sig(status, current); } + +out: + exception_exit(prev_state); } /* @@ -927,99 +1075,229 @@ int cu2_notifier_call_chain(unsigned long val, void *v) } static int default_cu2_call(struct notifier_block *nfb, unsigned long action, - void *data) + void *data) { struct pt_regs *regs = data; - switch (action) { - default: - die_if_kernel("Unhandled kernel unaligned access or invalid " + die_if_kernel("COP2: Unhandled kernel unaligned access or invalid " "instruction", regs); - /* Fall through */ + force_sig(SIGILL, current); - case CU2_EXCEPTION: - force_sig(SIGILL, current); + return NOTIFY_OK; +} + +static int enable_restore_fp_context(int msa) +{ + int err, was_fpu_owner; + + if (!used_math()) { + /* First time FP context user. */ + err = init_fpu(); + if (msa && !err) + enable_msa(); + if (!err) + set_used_math(); + return err; } - return NOTIFY_OK; + /* + * This task has formerly used the FP context. + * + * If this thread has no live MSA vector context then we can simply + * restore the scalar FP context. If it has live MSA vector context + * (that is, it has or may have used MSA since last performing a + * function call) then we'll need to restore the vector context. This + * applies even if we're currently only executing a scalar FP + * instruction. This is because if we were to later execute an MSA + * instruction then we'd either have to: + * + * - Restore the vector context & clobber any registers modified by + * scalar FP instructions between now & then. + * + * or + * + * - Not restore the vector context & lose the most significant bits + * of all vector registers. + * + * Neither of those options is acceptable. We cannot restore the least + * significant bits of the registers now & only restore the most + * significant bits later because the most significant bits of any + * vector registers whose aliased FP register is modified now will have + * been zeroed. We'd have no way to know that when restoring the vector + * context & thus may load an outdated value for the most significant + * bits of a vector register. + */ + if (!msa && !thread_msa_context_live()) + return own_fpu(1); + + /* + * This task is using or has previously used MSA. Thus we require + * that Status.FR == 1. + */ + was_fpu_owner = is_fpu_owner(); + err = own_fpu(0); + if (err) + return err; + + enable_msa(); + write_msa_csr(current->thread.fpu.msacsr); + set_thread_flag(TIF_USEDMSA); + + /* + * If this is the first time that the task is using MSA and it has + * previously used scalar FP in this time slice then we already nave + * FP context which we shouldn't clobber. + */ + if (!test_and_set_thread_flag(TIF_MSA_CTX_LIVE) && was_fpu_owner) + return 0; + + /* We need to restore the vector context. */ + restore_msa(current); + return 0; } asmlinkage void do_cpu(struct pt_regs *regs) { + enum ctx_state prev_state; unsigned int __user *epc; - unsigned long old_epc; + unsigned long old_epc, old31; unsigned int opcode; unsigned int cpid; - int status; + int status, err; unsigned long __maybe_unused flags; - die_if_kernel("do_cpu invoked from kernel context!", regs); - + prev_state = exception_enter(); cpid = (regs->cp0_cause >> CAUSEB_CE) & 3; + if (cpid != 2) + die_if_kernel("do_cpu invoked from kernel context!", regs); + switch (cpid) { case 0: epc = (unsigned int __user *)exception_epc(regs); old_epc = regs->cp0_epc; + old31 = regs->regs[31]; opcode = 0; status = -1; if (unlikely(compute_return_epc(regs) < 0)) - return; + goto out; - if (unlikely(get_user(opcode, epc) < 0)) - status = SIGSEGV; + if (get_isa16_mode(regs->cp0_epc)) { + unsigned short mmop[2] = { 0 }; - if (!cpu_has_llsc && status < 0) - status = simulate_llsc(regs, opcode); + if (unlikely(get_user(mmop[0], epc) < 0)) + status = SIGSEGV; + if (unlikely(get_user(mmop[1], epc) < 0)) + status = SIGSEGV; + opcode = (mmop[0] << 16) | mmop[1]; - if (status < 0) - status = simulate_rdhwr(regs, opcode); + if (status < 0) + status = simulate_rdhwr_mm(regs, opcode); + } else { + if (unlikely(get_user(opcode, epc) < 0)) + status = SIGSEGV; + + if (!cpu_has_llsc && status < 0) + status = simulate_llsc(regs, opcode); + + if (status < 0) + status = simulate_rdhwr_normal(regs, opcode); + } if (status < 0) status = SIGILL; if (unlikely(status > 0)) { regs->cp0_epc = old_epc; /* Undo skip-over. */ + regs->regs[31] = old31; force_sig(status, current); } - return; + goto out; + + case 3: + /* + * Old (MIPS I and MIPS II) processors will set this code + * for COP1X opcode instructions that replaced the original + * COP3 space. We don't limit COP1 space instructions in + * the emulator according to the CPU ISA, so we want to + * treat COP1X instructions consistently regardless of which + * code the CPU chose. Therefore we redirect this trap to + * the FP emulator too. + * + * Then some newer FPU-less processors use this code + * erroneously too, so they are covered by this choice + * as well. + */ + if (raw_cpu_has_fpu) + break; + /* Fall through. */ case 1: - if (used_math()) /* Using the FPU again. */ - own_fpu(1); - else { /* First time FPU user. */ - init_fpu(); - set_used_math(); - } + err = enable_restore_fp_context(0); - if (!raw_cpu_has_fpu) { + if (!raw_cpu_has_fpu || err) { int sig; + void __user *fault_addr = NULL; sig = fpu_emulator_cop1Handler(regs, - ¤t->thread.fpu, 0); - if (sig) - force_sig(sig, current); - else + ¤t->thread.fpu, + 0, &fault_addr); + if (!process_fpemu_return(sig, fault_addr) && !err) mt_ase_fp_affinity(); } - return; + goto out; case 2: raw_notifier_call_chain(&cu2_chain, CU2_EXCEPTION, regs); - return; - - case 3: - break; + goto out; } force_sig(SIGILL, current); + +out: + exception_exit(prev_state); +} + +asmlinkage void do_msa_fpe(struct pt_regs *regs) +{ + enum ctx_state prev_state; + + prev_state = exception_enter(); + die_if_kernel("do_msa_fpe invoked from kernel context!", regs); + force_sig(SIGFPE, current); + exception_exit(prev_state); +} + +asmlinkage void do_msa(struct pt_regs *regs) +{ + enum ctx_state prev_state; + int err; + + prev_state = exception_enter(); + + if (!cpu_has_msa || test_thread_flag(TIF_32BIT_FPREGS)) { + force_sig(SIGILL, current); + goto out; + } + + die_if_kernel("do_msa invoked from kernel context!", regs); + + err = enable_restore_fp_context(1); + if (err) + force_sig(SIGILL, current); +out: + exception_exit(prev_state); } asmlinkage void do_mdmx(struct pt_regs *regs) { + enum ctx_state prev_state; + + prev_state = exception_enter(); force_sig(SIGILL, current); + exception_exit(prev_state); } /* @@ -1027,8 +1305,10 @@ asmlinkage void do_mdmx(struct pt_regs *regs) */ asmlinkage void do_watch(struct pt_regs *regs) { + enum ctx_state prev_state; u32 cause; + prev_state = exception_enter(); /* * Clear WP (bit 22) bit of cause register so we don't loop * forever. @@ -1050,17 +1330,20 @@ asmlinkage void do_watch(struct pt_regs *regs) mips_clear_watch_registers(); local_irq_enable(); } + exception_exit(prev_state); } asmlinkage void do_mcheck(struct pt_regs *regs) { const int field = 2 * sizeof(unsigned long); int multi_match = regs->cp0_status & ST0_TS; + enum ctx_state prev_state; + prev_state = exception_enter(); show_regs(regs); if (multi_match) { - printk("Index : %0x\n", read_c0_index()); + printk("Index : %0x\n", read_c0_index()); printk("Pagemask: %0x\n", read_c0_pagemask()); printk("EntryHi : %0*lx\n", field, read_c0_entryhi()); printk("EntryLo0: %0*lx\n", field, read_c0_entrylo0()); @@ -1103,7 +1386,7 @@ asmlinkage void do_mt(struct pt_regs *regs) printk(KERN_DEBUG "YIELD Scheduler Exception\n"); break; case 5: - printk(KERN_DEBUG "Gating Storage Schedulier Exception\n"); + printk(KERN_DEBUG "Gating Storage Scheduler Exception\n"); break; default: printk(KERN_DEBUG "*** UNKNOWN THREAD EXCEPTION %d ***\n", @@ -1119,7 +1402,7 @@ asmlinkage void do_mt(struct pt_regs *regs) asmlinkage void do_dsp(struct pt_regs *regs) { if (cpu_has_dsp) - panic("Unexpected DSP exception\n"); + panic("Unexpected DSP exception"); force_sig(SIGILL, current); } @@ -1127,7 +1410,7 @@ asmlinkage void do_dsp(struct pt_regs *regs) asmlinkage void do_reserved(struct pt_regs *regs) { /* - * Game over - no way to handle this if it ever occurs. Most probably + * Game over - no way to handle this if it ever occurs. Most probably * caused by a new unknown cpu type or after another deadly * hard/software error. */ @@ -1162,6 +1445,10 @@ static inline void parity_protection_init(void) case CPU_34K: case CPU_74K: case CPU_1004K: + case CPU_1074K: + case CPU_INTERAPTIV: + case CPU_PROAPTIV: + case CPU_P5600: { #define ERRCTL_PE 0x80000000 #define ERRCTL_L2P 0x00800000 @@ -1217,6 +1504,8 @@ static inline void parity_protection_init(void) break; case CPU_5KC: + case CPU_5KE: + case CPU_LOONGSON1: write_c0_ecc(0x80000000); back_to_back_c0_hazard(); /* Set the PE bit (bit 31) in the c0_errctl register. */ @@ -1249,14 +1538,27 @@ asmlinkage void cache_parity_error(void) printk("Decoded c0_cacheerr: %s cache fault in %s reference.\n", reg_val & (1<<30) ? "secondary" : "primary", reg_val & (1<<31) ? "data" : "insn"); - printk("Error bits: %s%s%s%s%s%s%s\n", - reg_val & (1<<29) ? "ED " : "", - reg_val & (1<<28) ? "ET " : "", - reg_val & (1<<26) ? "EE " : "", - reg_val & (1<<25) ? "EB " : "", - reg_val & (1<<24) ? "EI " : "", - reg_val & (1<<23) ? "E1 " : "", - reg_val & (1<<22) ? "E0 " : ""); + if (cpu_has_mips_r2 && + ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS)) { + pr_err("Error bits: %s%s%s%s%s%s%s%s\n", + reg_val & (1<<29) ? "ED " : "", + reg_val & (1<<28) ? "ET " : "", + reg_val & (1<<27) ? "ES " : "", + reg_val & (1<<26) ? "EE " : "", + reg_val & (1<<25) ? "EB " : "", + reg_val & (1<<24) ? "EI " : "", + reg_val & (1<<23) ? "E1 " : "", + reg_val & (1<<22) ? "E0 " : ""); + } else { + pr_err("Error bits: %s%s%s%s%s%s%s\n", + reg_val & (1<<29) ? "ED " : "", + reg_val & (1<<28) ? "ET " : "", + reg_val & (1<<26) ? "EE " : "", + reg_val & (1<<25) ? "EB " : "", + reg_val & (1<<24) ? "EI " : "", + reg_val & (1<<23) ? "E1 " : "", + reg_val & (1<<22) ? "E0 " : ""); + } printk("IDX: 0x%08x\n", reg_val & ((1<<22)-1)); #if defined(CONFIG_CPU_MIPS32) || defined(CONFIG_CPU_MIPS64) @@ -1270,6 +1572,34 @@ asmlinkage void cache_parity_error(void) panic("Can't handle the cache error!"); } +asmlinkage void do_ftlb(void) +{ + const int field = 2 * sizeof(unsigned long); + unsigned int reg_val; + + /* For the moment, report the problem and hang. */ + if (cpu_has_mips_r2 && + ((current_cpu_data.processor_id & 0xff0000) == PRID_COMP_MIPS)) { + pr_err("FTLB error exception, cp0_ecc=0x%08x:\n", + read_c0_ecc()); + pr_err("cp0_errorepc == %0*lx\n", field, read_c0_errorepc()); + reg_val = read_c0_cacheerr(); + pr_err("c0_cacheerr == %08x\n", reg_val); + + if ((reg_val & 0xc0000000) == 0xc0000000) { + pr_err("Decoded c0_cacheerr: FTLB parity error\n"); + } else { + pr_err("Decoded c0_cacheerr: %s cache fault in %s reference.\n", + reg_val & (1<<30) ? "secondary" : "primary", + reg_val & (1<<31) ? "data" : "insn"); + } + } else { + pr_err("FTLB error exception\n"); + } + /* Just print the cacheerr bits for now */ + cache_parity_error(); +} + /* * SDBBP EJTAG debug exception handler. * We skip the instruction and return to the next instruction. @@ -1277,7 +1607,7 @@ asmlinkage void cache_parity_error(void) void ejtag_exception_handler(struct pt_regs *regs) { const int field = 2 * sizeof(unsigned long); - unsigned long depc, old_epc; + unsigned long depc, old_epc, old_ra; unsigned int debug; printk(KERN_DEBUG "SDBBP EJTAG debug exception - not handled yet, just ignored!\n"); @@ -1292,10 +1622,12 @@ void ejtag_exception_handler(struct pt_regs *regs) * calculation. */ old_epc = regs->cp0_epc; + old_ra = regs->regs[31]; regs->cp0_epc = depc; - __compute_return_epc(regs); + compute_return_epc(regs); depc = regs->cp0_epc; regs->cp0_epc = old_epc; + regs->regs[31] = old_ra; } else depc += 4; write_c0_depc(depc); @@ -1308,12 +1640,25 @@ void ejtag_exception_handler(struct pt_regs *regs) /* * NMI exception handler. + * No lock; only written during early bootup by CPU 0. */ -NORET_TYPE void ATTRIB_NORET nmi_exception_handler(struct pt_regs *regs) +static RAW_NOTIFIER_HEAD(nmi_chain); + +int register_nmi_notifier(struct notifier_block *nb) { + return raw_notifier_chain_register(&nmi_chain, nb); +} + +void __noreturn nmi_exception_handler(struct pt_regs *regs) +{ + char str[100]; + + raw_notifier_call_chain(&nmi_chain, 0, regs); bust_spinlocks(1); - printk("NMI taken!!!!\n"); - die("NMI", regs); + snprintf(str, 100, "CPU%d NMI taken, CP0_EPC=%lx\n", + smp_processor_id(), regs->cp0_epc); + regs->cp0_epc = read_c0_errorepc(); + die(str, regs); } #define VECTORSPACING 0x100 /* for EI/VI mode */ @@ -1325,11 +1670,27 @@ unsigned long vi_handlers[64]; void __init *set_except_vector(int n, void *addr) { unsigned long handler = (unsigned long) addr; - unsigned long old_handler = exception_handlers[n]; + unsigned long old_handler; + +#ifdef CONFIG_CPU_MICROMIPS + /* + * Only the TLB handlers are cache aligned with an even + * address. All other handlers are on an odd address and + * require no modification. Otherwise, MIPS32 mode will + * be entered when handling any TLB exceptions. That + * would be bad...since we must stay in microMIPS mode. + */ + if (!(handler & 0x1)) + handler |= 1; +#endif + old_handler = xchg(&exception_handlers[n], handler); - exception_handlers[n] = handler; if (n == 0 && cpu_has_divec) { +#ifdef CONFIG_CPU_MICROMIPS + unsigned long jump_mask = ~((1 << 27) - 1); +#else unsigned long jump_mask = ~((1 << 28) - 1); +#endif u32 *buf = (u32 *)(ebase + 0x200); unsigned int k0 = 26; if ((handler & jump_mask) == ((ebase + 0x200) & jump_mask)) { @@ -1345,7 +1706,7 @@ void __init *set_except_vector(int n, void *addr) return (void *)old_handler; } -static asmlinkage void do_default_vi(void) +static void do_default_vi(void) { show_regs(get_irq_regs()); panic("Caught unexpected vectored interrupt."); @@ -1356,7 +1717,7 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) unsigned long handler; unsigned long old_handler = vi_handlers[n]; int srssets = current_cpu_data.srsets; - u32 *w; + u16 *h; unsigned char *b; BUG_ON(!cpu_has_veic && !cpu_has_vint); @@ -1366,7 +1727,7 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) srs = 0; } else handler = (unsigned long) addr; - vi_handlers[n] = (unsigned long) addr; + vi_handlers[n] = handler; b = (unsigned char *)(ebase + 0x200 + n*VECTORSPACING); @@ -1385,26 +1746,21 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) if (srs == 0) { /* * If no shadow set is selected then use the default handler - * that does normal register saving and a standard interrupt exit + * that does normal register saving and standard interrupt exit */ - extern char except_vec_vi, except_vec_vi_lui; extern char except_vec_vi_ori, except_vec_vi_end; extern char rollback_except_vec_vi; - char *vec_start = (cpu_wait == r4k_wait) ? + char *vec_start = using_rollback_handler() ? &rollback_except_vec_vi : &except_vec_vi; -#ifdef CONFIG_MIPS_MT_SMTC - /* - * We need to provide the SMTC vectored interrupt handler - * not only with the address of the handler, but with the - * Status.IM bit to be masked before going there. - */ - extern char except_vec_vi_mori; - const int mori_offset = &except_vec_vi_mori - vec_start; -#endif /* CONFIG_MIPS_MT_SMTC */ - const int handler_len = &except_vec_vi_end - vec_start; +#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_BIG_ENDIAN) + const int lui_offset = &except_vec_vi_lui - vec_start + 2; + const int ori_offset = &except_vec_vi_ori - vec_start + 2; +#else const int lui_offset = &except_vec_vi_lui - vec_start; const int ori_offset = &except_vec_vi_ori - vec_start; +#endif + const int handler_len = &except_vec_vi_end - vec_start; if (handler_len > VECTORSPACING) { /* @@ -1414,30 +1770,38 @@ static void *set_vi_srs_handler(int n, vi_handler_t addr, int srs) panic("VECTORSPACING too small"); } - memcpy(b, vec_start, handler_len); -#ifdef CONFIG_MIPS_MT_SMTC - BUG_ON(n > 7); /* Vector index %d exceeds SMTC maximum. */ - - w = (u32 *)(b + mori_offset); - *w = (*w & 0xffff0000) | (0x100 << n); -#endif /* CONFIG_MIPS_MT_SMTC */ - w = (u32 *)(b + lui_offset); - *w = (*w & 0xffff0000) | (((u32)handler >> 16) & 0xffff); - w = (u32 *)(b + ori_offset); - *w = (*w & 0xffff0000) | ((u32)handler & 0xffff); + set_handler(((unsigned long)b - ebase), vec_start, +#ifdef CONFIG_CPU_MICROMIPS + (handler_len - 1)); +#else + handler_len); +#endif + h = (u16 *)(b + lui_offset); + *h = (handler >> 16) & 0xffff; + h = (u16 *)(b + ori_offset); + *h = (handler & 0xffff); local_flush_icache_range((unsigned long)b, (unsigned long)(b+handler_len)); } else { /* - * In other cases jump directly to the interrupt handler - * - * It is the handlers responsibility to save registers if required - * (eg hi/lo) and return from the exception using "eret" + * In other cases jump directly to the interrupt handler. It + * is the handler's responsibility to save registers if required + * (eg hi/lo) and return from the exception using "eret". */ - w = (u32 *)b; - *w++ = 0x08000000 | (((u32)handler >> 2) & 0x03fffff); /* j handler */ - *w = 0; + u32 insn; + + h = (u16 *)b; + /* j handler */ +#ifdef CONFIG_CPU_MICROMIPS + insn = 0xd4000000 | (((u32)handler & 0x07ffffff) >> 1); +#else + insn = 0x08000000 | (((u32)handler & 0x0fffffff) >> 2); +#endif + h[0] = (insn >> 16) & 0xffff; + h[1] = insn & 0xffff; + h[2] = 0; + h[3] = 0; local_flush_icache_range((unsigned long)b, (unsigned long)(b+8)); } @@ -1450,14 +1814,13 @@ void *set_vi_handler(int n, vi_handler_t addr) return set_vi_srs_handler(n, addr, 0); } -extern void cpu_cache_init(void); extern void tlb_init(void); -extern void flush_tlb_handlers(void); /* * Timer interrupt */ int cp0_compare_irq; +EXPORT_SYMBOL_GPL(cp0_compare_irq); int cp0_compare_irq_shift; /* @@ -1466,7 +1829,7 @@ int cp0_compare_irq_shift; int cp0_perfcount_irq; EXPORT_SYMBOL_GPL(cp0_perfcount_irq); -static int __cpuinitdata noulri; +static int noulri; static int __init ulri_disable(char *s) { @@ -1477,42 +1840,32 @@ static int __init ulri_disable(char *s) } __setup("noulri", ulri_disable); -void __cpuinit per_cpu_trap_init(void) +/* configure STATUS register */ +static void configure_status(void) { - unsigned int cpu = smp_processor_id(); - unsigned int status_set = ST0_CU0; - unsigned int hwrena = cpu_hwrena_impl_bits; -#ifdef CONFIG_MIPS_MT_SMTC - int secondaryTC = 0; - int bootTC = (cpu == 0); - - /* - * Only do per_cpu_trap_init() for first TC of Each VPE. - * Note that this hack assumes that the SMTC init code - * assigns TCs consecutively and in ascending order. - */ - - if (((read_c0_tcbind() & TCBIND_CURTC) != 0) && - ((read_c0_tcbind() & TCBIND_CURVPE) == cpu_data[cpu - 1].vpe_id)) - secondaryTC = 1; -#endif /* CONFIG_MIPS_MT_SMTC */ - /* * Disable coprocessors and select 32-bit or 64-bit addressing * and the 16/32 or 32/32 FPR register model. Reset the BEV * flag that some firmware may have left set and the TS bit (for * IP27). Set XX for ISA IV code to work. */ + unsigned int status_set = ST0_CU0; #ifdef CONFIG_64BIT status_set |= ST0_FR|ST0_KX|ST0_SX|ST0_UX; #endif - if (current_cpu_data.isa_level == MIPS_CPU_ISA_IV) + if (current_cpu_data.isa_level & MIPS_CPU_ISA_IV) status_set |= ST0_XX; if (cpu_has_dsp) status_set |= ST0_MX; change_c0_status(ST0_CU|ST0_MX|ST0_RE|ST0_FR|ST0_BEV|ST0_TS|ST0_KX|ST0_SX|ST0_UX, status_set); +} + +/* configure HWRENA register */ +static void configure_hwrena(void) +{ + unsigned int hwrena = cpu_hwrena_impl_bits; if (cpu_has_mips_r2) hwrena |= 0x0000000f; @@ -1522,11 +1875,10 @@ void __cpuinit per_cpu_trap_init(void) if (hwrena) write_c0_hwrena(hwrena); +} -#ifdef CONFIG_MIPS_MT_SMTC - if (!secondaryTC) { -#endif /* CONFIG_MIPS_MT_SMTC */ - +static void configure_exception_vector(void) +{ if (cpu_has_veic || cpu_has_vint) { unsigned long sr = set_c0_status(ST0_BEV); write_c0_ebase(ebase); @@ -1542,6 +1894,16 @@ void __cpuinit per_cpu_trap_init(void) } else set_c0_cause(CAUSEF_IV); } +} + +void per_cpu_trap_init(bool is_boot_cpu) +{ + unsigned int cpu = smp_processor_id(); + + configure_status(); + configure_hwrena(); + + configure_exception_vector(); /* * Before R2 both interrupt numbers were fixed to 7, so on R2 only: @@ -1557,47 +1919,37 @@ void __cpuinit per_cpu_trap_init(void) cp0_perfcount_irq = -1; } else { cp0_compare_irq = CP0_LEGACY_COMPARE_IRQ; - cp0_compare_irq_shift = cp0_compare_irq; + cp0_compare_irq_shift = CP0_LEGACY_PERFCNT_IRQ; cp0_perfcount_irq = -1; } -#ifdef CONFIG_MIPS_MT_SMTC - } -#endif /* CONFIG_MIPS_MT_SMTC */ - - cpu_data[cpu].asid_cache = ASID_FIRST_VERSION; - TLBMISS_HANDLER_SETUP(); + if (!cpu_data[cpu].asid_cache) + cpu_data[cpu].asid_cache = ASID_FIRST_VERSION; atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; BUG_ON(current->mm); enter_lazy_tlb(&init_mm, current); -#ifdef CONFIG_MIPS_MT_SMTC - if (bootTC) { -#endif /* CONFIG_MIPS_MT_SMTC */ - cpu_cache_init(); + /* Boot CPU's cache setup in setup_arch(). */ + if (!is_boot_cpu) + cpu_cache_init(); tlb_init(); -#ifdef CONFIG_MIPS_MT_SMTC - } else if (!secondaryTC) { - /* - * First TC in non-boot VPE must do subset of tlb_init() - * for MMU countrol registers. - */ - write_c0_pagemask(PM_DEFAULT_MASK); - write_c0_wired(0); - } -#endif /* CONFIG_MIPS_MT_SMTC */ + TLBMISS_HANDLER_SETUP(); } /* Install CPU exception handler */ -void __init set_handler(unsigned long offset, void *addr, unsigned long size) +void set_handler(unsigned long offset, void *addr, unsigned long size) { +#ifdef CONFIG_CPU_MICROMIPS + memcpy((void *)(ebase + offset), ((unsigned char *)addr - 1), size); +#else memcpy((void *)(ebase + offset), addr, size); +#endif local_flush_icache_range(ebase + offset, ebase + offset + size); } -static char panic_null_cerr[] __cpuinitdata = +static char panic_null_cerr[] = "Trying to set NULL cache error exception handler"; /* @@ -1605,7 +1957,7 @@ static char panic_null_cerr[] __cpuinitdata = * This is suitable only for the cache error exception which is the only * exception handler that is being run uncached. */ -void __cpuinit set_uncached_handler(unsigned long offset, void *addr, +void set_uncached_handler(unsigned long offset, void *addr, unsigned long size) { unsigned long uncached_ebase = CKSEG1ADDR(ebase); @@ -1627,17 +1979,16 @@ __setup("rdhwr_noopt", set_rdhwr_noopt); void __init trap_init(void) { - extern char except_vec3_generic, except_vec3_r4000; + extern char except_vec3_generic; extern char except_vec4; + extern char except_vec3_r4000; unsigned long i; - int rollback; check_wait(); - rollback = (cpu_wait == r4k_wait); #if defined(CONFIG_KGDB) if (kgdb_early_setup) - return; /* Already done */ + return; /* Already done */ #endif if (cpu_has_veic || cpu_has_vint) { @@ -1645,12 +1996,28 @@ void __init trap_init(void) ebase = (unsigned long) __alloc_bootmem(size, 1 << fls(size), 0); } else { - ebase = CKSEG0; +#ifdef CONFIG_KVM_GUEST +#define KVM_GUEST_KSEG0 0x40000000 + ebase = KVM_GUEST_KSEG0; +#else + ebase = CKSEG0; +#endif if (cpu_has_mips_r2) ebase += (read_c0_ebase() & 0x3ffff000); } - per_cpu_trap_init(); + if (cpu_has_mmips) { + unsigned int config3 = read_c0_config3(); + + if (IS_ENABLED(CONFIG_CPU_MICROMIPS)) + write_c0_config3(config3 | MIPS_CONF3_ISA_OE); + else + write_c0_config3(config3 & ~MIPS_CONF3_ISA_OE); + } + + if (board_ebase_setup) + board_ebase_setup(); + per_cpu_trap_init(true); /* * Copy the generic exception handlers to their final destination. @@ -1703,7 +2070,8 @@ void __init trap_init(void) if (board_be_init) board_be_init(); - set_except_vector(0, rollback ? rollback_handle_int : handle_int); + set_except_vector(0, using_rollback_handler() ? rollback_handle_int + : handle_int); set_except_vector(1, handle_tlbm); set_except_vector(2, handle_tlbl); set_except_vector(3, handle_tlbs); @@ -1722,6 +2090,7 @@ void __init trap_init(void) set_except_vector(11, handle_cpu); set_except_vector(12, handle_ov); set_except_vector(13, handle_tr); + set_except_vector(14, handle_msa_fpe); if (current_cpu_type() == CPU_R6000 || current_cpu_type() == CPU_R6000A) { @@ -1729,7 +2098,7 @@ void __init trap_init(void) * The R6000 is the only R-series CPU that features a machine * check exception (similar to the R4000 cache error) and * unaligned ldc1/sdc1 exception. The handlers have not been - * written yet. Well, anyway there is no R6000 machine on the + * written yet. Well, anyway there is no R6000 machine on the * current list of targets for Linux/MIPS. * (Duh, crap, there is someone with a triple R6k machine) */ @@ -1744,6 +2113,8 @@ void __init trap_init(void) if (cpu_has_fpu && !cpu_has_nofpuex) set_except_vector(15, handle_fpe); + set_except_vector(16, handle_ftlb); + set_except_vector(21, handle_msa); set_except_vector(22, handle_mdmx); if (cpu_has_mcheck) @@ -1754,18 +2125,49 @@ void __init trap_init(void) set_except_vector(26, handle_dsp); + if (board_cache_error_setup) + board_cache_error_setup(); + if (cpu_has_vce) /* Special exception: R4[04]00 uses also the divec space. */ - memcpy((void *)(ebase + 0x180), &except_vec3_r4000, 0x100); + set_handler(0x180, &except_vec3_r4000, 0x100); else if (cpu_has_4kex) - memcpy((void *)(ebase + 0x180), &except_vec3_generic, 0x80); + set_handler(0x180, &except_vec3_generic, 0x80); else - memcpy((void *)(ebase + 0x080), &except_vec3_generic, 0x80); + set_handler(0x080, &except_vec3_generic, 0x80); local_flush_icache_range(ebase, ebase + 0x400); - flush_tlb_handlers(); sort_extable(__start___dbe_table, __stop___dbe_table); cu2_notifier(default_cu2_call, 0x80000000); /* Run last */ } + +static int trap_pm_notifier(struct notifier_block *self, unsigned long cmd, + void *v) +{ + switch (cmd) { + case CPU_PM_ENTER_FAILED: + case CPU_PM_EXIT: + configure_status(); + configure_hwrena(); + configure_exception_vector(); + + /* Restore register with CPU number for TLB handlers */ + TLBMISS_HANDLER_RESTORE(); + + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block trap_pm_notifier_block = { + .notifier_call = trap_pm_notifier, +}; + +static int __init trap_pm_init(void) +{ + return cpu_pm_register_notifier(&trap_pm_notifier_block); +} +arch_initcall(trap_pm_init); diff --git a/arch/mips/kernel/unaligned.c b/arch/mips/kernel/unaligned.c index cfea1adfa15..2b3517214d6 100644 --- a/arch/mips/kernel/unaligned.c +++ b/arch/mips/kernel/unaligned.c @@ -7,6 +7,7 @@ * * Copyright (C) 1996, 1998, 1999, 2002 by Ralf Baechle * Copyright (C) 1999 Silicon Graphics, Inc. + * Copyright (C) 2014 Imagination Technologies Ltd. * * This file contains exception handler for address error exception with the * special capability to execute faulting instructions in software. The @@ -21,11 +22,11 @@ * * For now I enable fixing of address errors by default to make life easier. * I however intend to disable this somewhen in the future when the alignment - * problems with user programs have been fixed. For programmers this is the + * problems with user programs have been fixed. For programmers this is the * right way to go. * * Fixing address errors is a per process option. The option is inherited - * across fork(2) and execve(2) calls. If you really want to use the + * across fork(2) and execve(2) calls. If you really want to use the * option in your user programs - I discourage the use of the software * emulation strongly - use the following code in your userland stuff: * @@ -43,37 +44,37 @@ * #include <sys/sysmips.h> * * struct foo { - * unsigned char bar[8]; + * unsigned char bar[8]; * }; * * main(int argc, char *argv[]) * { - * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7}; - * unsigned int *p = (unsigned int *) (x.bar + 3); - * int i; + * struct foo x = {0, 1, 2, 3, 4, 5, 6, 7}; + * unsigned int *p = (unsigned int *) (x.bar + 3); + * int i; * - * if (argc > 1) - * sysmips(MIPS_FIXADE, atoi(argv[1])); + * if (argc > 1) + * sysmips(MIPS_FIXADE, atoi(argv[1])); * - * printf("*p = %08lx\n", *p); + * printf("*p = %08lx\n", *p); * - * *p = 0xdeadface; + * *p = 0xdeadface; * - * for(i = 0; i <= 7; i++) - * printf("%02x ", x.bar[i]); - * printf("\n"); + * for(i = 0; i <= 7; i++) + * printf("%02x ", x.bar[i]); + * printf("\n"); * } * * Coprocessor loads are not supported; I think this case is unimportant * in the practice. * * TODO: Handle ndc (attempted store to doubleword in uncached memory) - * exception for the R6000. - * A store crossing a page boundary might be executed only partially. - * Undo the partial store in this case. + * exception for the R6000. + * A store crossing a page boundary might be executed only partially. + * Undo the partial store in this case. */ +#include <linux/context_tracking.h> #include <linux/mm.h> -#include <linux/module.h> #include <linux/signal.h> #include <linux/smp.h> #include <linux/sched.h> @@ -84,11 +85,14 @@ #include <asm/branch.h> #include <asm/byteorder.h> #include <asm/cop2.h> +#include <asm/fpu.h> +#include <asm/fpu_emulator.h> #include <asm/inst.h> #include <asm/uaccess.h> -#include <asm/system.h> +#include <asm/fpu.h> +#include <asm/fpu_emulator.h> -#define STR(x) __STR(x) +#define STR(x) __STR(x) #define __STR(x) #x enum { @@ -104,15 +108,336 @@ static u32 unaligned_action; #endif extern void show_registers(struct pt_regs *regs); +#ifdef __BIG_ENDIAN +#define LoadHW(addr, value, res) \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"user_lb("%0", "0(%2)")"\n" \ + "2:\t"user_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_lwl("%0", "(%2)")"\n" \ + "2:\t"user_lwr("%0", "3(%2)")"\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadHWU(addr, value, res) \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"user_lbu("%0", "0(%2)")"\n" \ + "2:\t"user_lbu("$1", "1(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadWU(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_lwl("%0", "(%2)")"\n" \ + "2:\t"user_lwr("%0", "3(%2)")"\n\t" \ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadDW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, (%2)\n" \ + "2:\tldr\t%0, 7(%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define StoreHW(addr, value, res) \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"user_sb("%1", "1(%2)")"\n" \ + "srl\t$1, %1, 0x8\n" \ + "2:\t"user_sb("$1", "0(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); + +#define StoreW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_swl("%1", "(%2)")"\n" \ + "2:\t"user_swr("%1", "3(%2)")"\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); + +#define StoreDW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1,(%2)\n" \ + "2:\tsdr\t%1, 7(%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); +#endif + +#ifdef __LITTLE_ENDIAN +#define LoadHW(addr, value, res) \ + __asm__ __volatile__ (".set\tnoat\n" \ + "1:\t"user_lb("%0", "1(%2)")"\n" \ + "2:\t"user_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\t.set\tat\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_lwl("%0", "3(%2)")"\n" \ + "2:\t"user_lwr("%0", "(%2)")"\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadHWU(addr, value, res) \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"user_lbu("%0", "1(%2)")"\n" \ + "2:\t"user_lbu("$1", "0(%2)")"\n\t" \ + "sll\t%0, 0x8\n\t" \ + "or\t%0, $1\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".set\tat\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadWU(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_lwl("%0", "3(%2)")"\n" \ + "2:\t"user_lwr("%0", "(%2)")"\n\t" \ + "dsll\t%0, %0, 32\n\t" \ + "dsrl\t%0, %0, 32\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define LoadDW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\tldl\t%0, 7(%2)\n" \ + "2:\tldr\t%0, (%2)\n\t" \ + "li\t%1, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + "\t.section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%1, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=&r" (value), "=r" (res) \ + : "r" (addr), "i" (-EFAULT)); + +#define StoreHW(addr, value, res) \ + __asm__ __volatile__ ( \ + ".set\tnoat\n" \ + "1:\t"user_sb("%1", "0(%2)")"\n" \ + "srl\t$1,%1, 0x8\n" \ + "2:\t"user_sb("$1", "1(%2)")"\n" \ + ".set\tat\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); + +#define StoreW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\t"user_swl("%1", "3(%2)")"\n" \ + "2:\t"user_swr("%1", "(%2)")"\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); + +#define StoreDW(addr, value, res) \ + __asm__ __volatile__ ( \ + "1:\tsdl\t%1, 7(%2)\n" \ + "2:\tsdr\t%1, (%2)\n\t" \ + "li\t%0, 0\n" \ + "3:\n\t" \ + ".insn\n\t" \ + ".section\t.fixup,\"ax\"\n\t" \ + "4:\tli\t%0, %3\n\t" \ + "j\t3b\n\t" \ + ".previous\n\t" \ + ".section\t__ex_table,\"a\"\n\t" \ + STR(PTR)"\t1b, 4b\n\t" \ + STR(PTR)"\t2b, 4b\n\t" \ + ".previous" \ + : "=r" (res) \ + : "r" (value), "r" (addr), "i" (-EFAULT)); +#endif + static void emulate_load_store_insn(struct pt_regs *regs, void __user *addr, unsigned int __user *pc) { union mips_instruction insn; unsigned long value; unsigned int res; + unsigned long origpc; + unsigned long orig31; + void __user *fault_addr = NULL; +#ifdef CONFIG_EVA + mm_segment_t seg; +#endif + origpc = (unsigned long)pc; + orig31 = regs->regs[31]; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, - 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); /* * This load never faults. @@ -120,22 +445,22 @@ static void emulate_load_store_insn(struct pt_regs *regs, __get_user(insn.word, pc); switch (insn.i_format.opcode) { - /* - * These are instructions that a compiler doesn't generate. We - * can assume therefore that the code is MIPS-aware and - * really buggy. Emulating these instructions would break the - * semantics anyway. - */ + /* + * These are instructions that a compiler doesn't generate. We + * can assume therefore that the code is MIPS-aware and + * really buggy. Emulating these instructions would break the + * semantics anyway. + */ case ll_op: case lld_op: case sc_op: case scd_op: - /* - * For these instructions the only way to create an address - * error is an attempted access to kernel/supervisor address - * space. - */ + /* + * For these instructions the only way to create an address + * error is an attempted access to kernel/supervisor address + * space. + */ case ldl_op: case ldr_op: case lwl_op: @@ -149,36 +474,97 @@ static void emulate_load_store_insn(struct pt_regs *regs, case sb_op: goto sigbus; - /* - * The remaining opcodes are the ones that are really of interest. - */ + /* + * The remaining opcodes are the ones that are really of + * interest. + */ +#ifdef CONFIG_EVA + case spec3_op: + /* + * we can land here only from kernel accessing user memory, + * so we need to "switch" the address limit to user space, so + * address check can work properly. + */ + seg = get_fs(); + set_fs(USER_DS); + switch (insn.spec3_format.func) { + case lhe_op: + if (!access_ok(VERIFY_READ, addr, 2)) { + set_fs(seg); + goto sigbus; + } + LoadHW(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lwe_op: + if (!access_ok(VERIFY_READ, addr, 4)) { + set_fs(seg); + goto sigbus; + } + LoadW(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case lhue_op: + if (!access_ok(VERIFY_READ, addr, 2)) { + set_fs(seg); + goto sigbus; + } + LoadHWU(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + compute_return_epc(regs); + regs->regs[insn.spec3_format.rt] = value; + break; + case she_op: + if (!access_ok(VERIFY_WRITE, addr, 2)) { + set_fs(seg); + goto sigbus; + } + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreHW(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + break; + case swe_op: + if (!access_ok(VERIFY_WRITE, addr, 4)) { + set_fs(seg); + goto sigbus; + } + compute_return_epc(regs); + value = regs->regs[insn.spec3_format.rt]; + StoreW(addr, value, res); + if (res) { + set_fs(seg); + goto fault; + } + break; + default: + set_fs(seg); + goto sigill; + } + set_fs(seg); + break; +#endif case lh_op: if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; - __asm__ __volatile__ (".set\tnoat\n" -#ifdef __BIG_ENDIAN - "1:\tlb\t%0, 0(%2)\n" - "2:\tlbu\t$1, 1(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tlb\t%0, 1(%2)\n" - "2:\tlbu\t$1, 0(%2)\n\t" -#endif - "sll\t%0, 0x8\n\t" - "or\t%0, $1\n\t" - "li\t%1, 0\n" - "3:\t.set\tat\n\t" - ".section\t.fixup,\"ax\"\n\t" - "4:\tli\t%1, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=&r" (value), "=r" (res) - : "r" (addr), "i" (-EFAULT)); + LoadHW(addr, value, res); if (res) goto fault; compute_return_epc(regs); @@ -189,26 +575,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - "1:\tlwl\t%0, (%2)\n" - "2:\tlwr\t%0, 3(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tlwl\t%0, 3(%2)\n" - "2:\tlwr\t%0, (%2)\n\t" -#endif - "li\t%1, 0\n" - "3:\t.section\t.fixup,\"ax\"\n\t" - "4:\tli\t%1, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=&r" (value), "=r" (res) - : "r" (addr), "i" (-EFAULT)); + LoadW(addr, value, res); if (res) goto fault; compute_return_epc(regs); @@ -219,30 +586,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_READ, addr, 2)) goto sigbus; - __asm__ __volatile__ ( - ".set\tnoat\n" -#ifdef __BIG_ENDIAN - "1:\tlbu\t%0, 0(%2)\n" - "2:\tlbu\t$1, 1(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tlbu\t%0, 1(%2)\n" - "2:\tlbu\t$1, 0(%2)\n\t" -#endif - "sll\t%0, 0x8\n\t" - "or\t%0, $1\n\t" - "li\t%1, 0\n" - "3:\t.set\tat\n\t" - ".section\t.fixup,\"ax\"\n\t" - "4:\tli\t%1, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=&r" (value), "=r" (res) - : "r" (addr), "i" (-EFAULT)); + LoadHWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); @@ -261,28 +605,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_READ, addr, 4)) goto sigbus; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - "1:\tlwl\t%0, (%2)\n" - "2:\tlwr\t%0, 3(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tlwl\t%0, 3(%2)\n" - "2:\tlwr\t%0, (%2)\n\t" -#endif - "dsll\t%0, %0, 32\n\t" - "dsrl\t%0, %0, 32\n\t" - "li\t%1, 0\n" - "3:\t.section\t.fixup,\"ax\"\n\t" - "4:\tli\t%1, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=&r" (value), "=r" (res) - : "r" (addr), "i" (-EFAULT)); + LoadWU(addr, value, res); if (res) goto fault; compute_return_epc(regs); @@ -305,26 +628,7 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_READ, addr, 8)) goto sigbus; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - "1:\tldl\t%0, (%2)\n" - "2:\tldr\t%0, 7(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tldl\t%0, 7(%2)\n" - "2:\tldr\t%0, (%2)\n\t" -#endif - "li\t%1, 0\n" - "3:\t.section\t.fixup,\"ax\"\n\t" - "4:\tli\t%1, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=&r" (value), "=r" (res) - : "r" (addr), "i" (-EFAULT)); + LoadDW(addr, value, res); if (res) goto fault; compute_return_epc(regs); @@ -339,68 +643,22 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_WRITE, addr, 2)) goto sigbus; + compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - ".set\tnoat\n" - "1:\tsb\t%1, 1(%2)\n\t" - "srl\t$1, %1, 0x8\n" - "2:\tsb\t$1, 0(%2)\n\t" - ".set\tat\n\t" -#endif -#ifdef __LITTLE_ENDIAN - ".set\tnoat\n" - "1:\tsb\t%1, 0(%2)\n\t" - "srl\t$1,%1, 0x8\n" - "2:\tsb\t$1, 1(%2)\n\t" - ".set\tat\n\t" -#endif - "li\t%0, 0\n" - "3:\n\t" - ".section\t.fixup,\"ax\"\n\t" - "4:\tli\t%0, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=r" (res) - : "r" (value), "r" (addr), "i" (-EFAULT)); + StoreHW(addr, value, res); if (res) goto fault; - compute_return_epc(regs); break; case sw_op: if (!access_ok(VERIFY_WRITE, addr, 4)) goto sigbus; + compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - "1:\tswl\t%1,(%2)\n" - "2:\tswr\t%1, 3(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tswl\t%1, 3(%2)\n" - "2:\tswr\t%1, (%2)\n\t" -#endif - "li\t%0, 0\n" - "3:\n\t" - ".section\t.fixup,\"ax\"\n\t" - "4:\tli\t%0, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=r" (res) - : "r" (value), "r" (addr), "i" (-EFAULT)); + StoreW(addr, value, res); if (res) goto fault; - compute_return_epc(regs); break; case sd_op: @@ -415,31 +673,11 @@ static void emulate_load_store_insn(struct pt_regs *regs, if (!access_ok(VERIFY_WRITE, addr, 8)) goto sigbus; + compute_return_epc(regs); value = regs->regs[insn.i_format.rt]; - __asm__ __volatile__ ( -#ifdef __BIG_ENDIAN - "1:\tsdl\t%1,(%2)\n" - "2:\tsdr\t%1, 7(%2)\n\t" -#endif -#ifdef __LITTLE_ENDIAN - "1:\tsdl\t%1, 7(%2)\n" - "2:\tsdr\t%1, (%2)\n\t" -#endif - "li\t%0, 0\n" - "3:\n\t" - ".section\t.fixup,\"ax\"\n\t" - "4:\tli\t%0, %3\n\t" - "j\t3b\n\t" - ".previous\n\t" - ".section\t__ex_table,\"a\"\n\t" - STR(PTR)"\t1b, 4b\n\t" - STR(PTR)"\t2b, 4b\n\t" - ".previous" - : "=r" (res) - : "r" (value), "r" (addr), "i" (-EFAULT)); + StoreDW(addr, value, res); if (res) goto fault; - compute_return_epc(regs); break; #endif /* CONFIG_64BIT */ @@ -450,10 +688,21 @@ static void emulate_load_store_insn(struct pt_regs *regs, case ldc1_op: case swc1_op: case sdc1_op: - /* - * I herewith declare: this does not happen. So send SIGBUS. - */ - goto sigbus; + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + BUG_ON(!is_fpu_owner()); + + lose_fpu(1); /* Save FPU state for the emulator. */ + res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, + &fault_addr); + own_fpu(1); /* Restore FPU state. */ + + /* Signal if something went wrong. */ + process_fpemu_return(res, fault_addr); + + if (res == 0) + break; + return; /* * COP2 is available to implementor for application specific use. @@ -491,6 +740,9 @@ static void emulate_load_store_insn(struct pt_regs *regs, return; fault: + /* roll back jump/branch */ + regs->cp0_epc = origpc; + regs->regs[31] = orig31; /* Did we have an exception handler installed? */ if (fixup_exception(regs)) return; @@ -507,36 +759,950 @@ sigbus: return; sigill: - die_if_kernel("Unhandled kernel unaligned access or invalid instruction", regs); + die_if_kernel + ("Unhandled kernel unaligned access or invalid instruction", regs); + force_sig(SIGILL, current); +} + +/* Recode table from 16-bit register notation to 32-bit GPR. */ +const int reg16to32[] = { 16, 17, 2, 3, 4, 5, 6, 7 }; + +/* Recode table from 16-bit STORE register notation to 32-bit GPR. */ +const int reg16to32st[] = { 0, 17, 2, 3, 4, 5, 6, 7 }; + +static void emulate_load_store_microMIPS(struct pt_regs *regs, + void __user *addr) +{ + unsigned long value; + unsigned int res; + int i; + unsigned int reg = 0, rvar; + unsigned long orig31; + u16 __user *pc16; + u16 halfword; + unsigned int word; + unsigned long origpc, contpc; + union mips_instruction insn; + struct mm_decoded_insn mminsn; + void __user *fault_addr = NULL; + + origpc = regs->cp0_epc; + orig31 = regs->regs[31]; + + mminsn.micro_mips_mode = 1; + + /* + * This load never faults. + */ + pc16 = (unsigned short __user *)msk_isa16_mode(regs->cp0_epc); + __get_user(halfword, pc16); + pc16++; + contpc = regs->cp0_epc + 2; + word = ((unsigned int)halfword << 16); + mminsn.pc_inc = 2; + + if (!mm_insn_16bit(halfword)) { + __get_user(halfword, pc16); + pc16++; + contpc = regs->cp0_epc + 4; + mminsn.pc_inc = 4; + word |= halfword; + } + mminsn.insn = word; + + if (get_user(halfword, pc16)) + goto fault; + mminsn.next_pc_inc = 2; + word = ((unsigned int)halfword << 16); + + if (!mm_insn_16bit(halfword)) { + pc16++; + if (get_user(halfword, pc16)) + goto fault; + mminsn.next_pc_inc = 4; + word |= halfword; + } + mminsn.next_insn = word; + + insn = (union mips_instruction)(mminsn.insn); + if (mm_isBranchInstr(regs, mminsn, &contpc)) + insn = (union mips_instruction)(mminsn.next_insn); + + /* Parse instruction to find what to do */ + + switch (insn.mm_i_format.opcode) { + + case mm_pool32a_op: + switch (insn.mm_x_format.func) { + case mm_lwxs_op: + reg = insn.mm_x_format.rd; + goto loadW; + } + + goto sigbus; + + case mm_pool32b_op: + switch (insn.mm_m_format.func) { + case mm_lwp_func: + reg = insn.mm_m_format.rd; + if (reg == 31) + goto sigbus; + + if (!access_ok(VERIFY_READ, addr, 8)) + goto sigbus; + + LoadW(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + addr += 4; + LoadW(addr, value, res); + if (res) + goto fault; + regs->regs[reg + 1] = value; + goto success; + + case mm_swp_func: + reg = insn.mm_m_format.rd; + if (reg == 31) + goto sigbus; + + if (!access_ok(VERIFY_WRITE, addr, 8)) + goto sigbus; + + value = regs->regs[reg]; + StoreW(addr, value, res); + if (res) + goto fault; + addr += 4; + value = regs->regs[reg + 1]; + StoreW(addr, value, res); + if (res) + goto fault; + goto success; + + case mm_ldp_func: +#ifdef CONFIG_64BIT + reg = insn.mm_m_format.rd; + if (reg == 31) + goto sigbus; + + if (!access_ok(VERIFY_READ, addr, 16)) + goto sigbus; + + LoadDW(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + addr += 8; + LoadDW(addr, value, res); + if (res) + goto fault; + regs->regs[reg + 1] = value; + goto success; +#endif /* CONFIG_64BIT */ + + goto sigill; + + case mm_sdp_func: +#ifdef CONFIG_64BIT + reg = insn.mm_m_format.rd; + if (reg == 31) + goto sigbus; + + if (!access_ok(VERIFY_WRITE, addr, 16)) + goto sigbus; + + value = regs->regs[reg]; + StoreDW(addr, value, res); + if (res) + goto fault; + addr += 8; + value = regs->regs[reg + 1]; + StoreDW(addr, value, res); + if (res) + goto fault; + goto success; +#endif /* CONFIG_64BIT */ + + goto sigill; + + case mm_lwm32_func: + reg = insn.mm_m_format.rd; + rvar = reg & 0xf; + if ((rvar > 9) || !reg) + goto sigill; + if (reg & 0x10) { + if (!access_ok + (VERIFY_READ, addr, 4 * (rvar + 1))) + goto sigbus; + } else { + if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + goto sigbus; + } + if (rvar == 9) + rvar = 8; + for (i = 16; rvar; rvar--, i++) { + LoadW(addr, value, res); + if (res) + goto fault; + addr += 4; + regs->regs[i] = value; + } + if ((reg & 0xf) == 9) { + LoadW(addr, value, res); + if (res) + goto fault; + addr += 4; + regs->regs[30] = value; + } + if (reg & 0x10) { + LoadW(addr, value, res); + if (res) + goto fault; + regs->regs[31] = value; + } + goto success; + + case mm_swm32_func: + reg = insn.mm_m_format.rd; + rvar = reg & 0xf; + if ((rvar > 9) || !reg) + goto sigill; + if (reg & 0x10) { + if (!access_ok + (VERIFY_WRITE, addr, 4 * (rvar + 1))) + goto sigbus; + } else { + if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + goto sigbus; + } + if (rvar == 9) + rvar = 8; + for (i = 16; rvar; rvar--, i++) { + value = regs->regs[i]; + StoreW(addr, value, res); + if (res) + goto fault; + addr += 4; + } + if ((reg & 0xf) == 9) { + value = regs->regs[30]; + StoreW(addr, value, res); + if (res) + goto fault; + addr += 4; + } + if (reg & 0x10) { + value = regs->regs[31]; + StoreW(addr, value, res); + if (res) + goto fault; + } + goto success; + + case mm_ldm_func: +#ifdef CONFIG_64BIT + reg = insn.mm_m_format.rd; + rvar = reg & 0xf; + if ((rvar > 9) || !reg) + goto sigill; + if (reg & 0x10) { + if (!access_ok + (VERIFY_READ, addr, 8 * (rvar + 1))) + goto sigbus; + } else { + if (!access_ok(VERIFY_READ, addr, 8 * rvar)) + goto sigbus; + } + if (rvar == 9) + rvar = 8; + + for (i = 16; rvar; rvar--, i++) { + LoadDW(addr, value, res); + if (res) + goto fault; + addr += 4; + regs->regs[i] = value; + } + if ((reg & 0xf) == 9) { + LoadDW(addr, value, res); + if (res) + goto fault; + addr += 8; + regs->regs[30] = value; + } + if (reg & 0x10) { + LoadDW(addr, value, res); + if (res) + goto fault; + regs->regs[31] = value; + } + goto success; +#endif /* CONFIG_64BIT */ + + goto sigill; + + case mm_sdm_func: +#ifdef CONFIG_64BIT + reg = insn.mm_m_format.rd; + rvar = reg & 0xf; + if ((rvar > 9) || !reg) + goto sigill; + if (reg & 0x10) { + if (!access_ok + (VERIFY_WRITE, addr, 8 * (rvar + 1))) + goto sigbus; + } else { + if (!access_ok(VERIFY_WRITE, addr, 8 * rvar)) + goto sigbus; + } + if (rvar == 9) + rvar = 8; + + for (i = 16; rvar; rvar--, i++) { + value = regs->regs[i]; + StoreDW(addr, value, res); + if (res) + goto fault; + addr += 8; + } + if ((reg & 0xf) == 9) { + value = regs->regs[30]; + StoreDW(addr, value, res); + if (res) + goto fault; + addr += 8; + } + if (reg & 0x10) { + value = regs->regs[31]; + StoreDW(addr, value, res); + if (res) + goto fault; + } + goto success; +#endif /* CONFIG_64BIT */ + + goto sigill; + + /* LWC2, SWC2, LDC2, SDC2 are not serviced */ + } + + goto sigbus; + + case mm_pool32c_op: + switch (insn.mm_m_format.func) { + case mm_lwu_func: + reg = insn.mm_m_format.rd; + goto loadWU; + } + + /* LL,SC,LLD,SCD are not serviced */ + goto sigbus; + + case mm_pool32f_op: + switch (insn.mm_x_format.func) { + case mm_lwxc1_func: + case mm_swxc1_func: + case mm_ldxc1_func: + case mm_sdxc1_func: + goto fpu_emul; + } + + goto sigbus; + + case mm_ldc132_op: + case mm_sdc132_op: + case mm_lwc132_op: + case mm_swc132_op: +fpu_emul: + /* roll back jump/branch */ + regs->cp0_epc = origpc; + regs->regs[31] = orig31; + + die_if_kernel("Unaligned FP access in kernel code", regs); + BUG_ON(!used_math()); + BUG_ON(!is_fpu_owner()); + + lose_fpu(1); /* save the FPU state for the emulator */ + res = fpu_emulator_cop1Handler(regs, ¤t->thread.fpu, 1, + &fault_addr); + own_fpu(1); /* restore FPU state */ + + /* If something went wrong, signal */ + process_fpemu_return(res, fault_addr); + + if (res == 0) + goto success; + return; + + case mm_lh32_op: + reg = insn.mm_i_format.rt; + goto loadHW; + + case mm_lhu32_op: + reg = insn.mm_i_format.rt; + goto loadHWU; + + case mm_lw32_op: + reg = insn.mm_i_format.rt; + goto loadW; + + case mm_sh32_op: + reg = insn.mm_i_format.rt; + goto storeHW; + + case mm_sw32_op: + reg = insn.mm_i_format.rt; + goto storeW; + + case mm_ld32_op: + reg = insn.mm_i_format.rt; + goto loadDW; + + case mm_sd32_op: + reg = insn.mm_i_format.rt; + goto storeDW; + + case mm_pool16c_op: + switch (insn.mm16_m_format.func) { + case mm_lwm16_op: + reg = insn.mm16_m_format.rlist; + rvar = reg + 1; + if (!access_ok(VERIFY_READ, addr, 4 * rvar)) + goto sigbus; + + for (i = 16; rvar; rvar--, i++) { + LoadW(addr, value, res); + if (res) + goto fault; + addr += 4; + regs->regs[i] = value; + } + LoadW(addr, value, res); + if (res) + goto fault; + regs->regs[31] = value; + + goto success; + + case mm_swm16_op: + reg = insn.mm16_m_format.rlist; + rvar = reg + 1; + if (!access_ok(VERIFY_WRITE, addr, 4 * rvar)) + goto sigbus; + + for (i = 16; rvar; rvar--, i++) { + value = regs->regs[i]; + StoreW(addr, value, res); + if (res) + goto fault; + addr += 4; + } + value = regs->regs[31]; + StoreW(addr, value, res); + if (res) + goto fault; + + goto success; + + } + + goto sigbus; + + case mm_lhu16_op: + reg = reg16to32[insn.mm16_rb_format.rt]; + goto loadHWU; + + case mm_lw16_op: + reg = reg16to32[insn.mm16_rb_format.rt]; + goto loadW; + + case mm_sh16_op: + reg = reg16to32st[insn.mm16_rb_format.rt]; + goto storeHW; + + case mm_sw16_op: + reg = reg16to32st[insn.mm16_rb_format.rt]; + goto storeW; + + case mm_lwsp16_op: + reg = insn.mm16_r5_format.rt; + goto loadW; + + case mm_swsp16_op: + reg = insn.mm16_r5_format.rt; + goto storeW; + + case mm_lwgp16_op: + reg = reg16to32[insn.mm16_r3_format.rt]; + goto loadW; + + default: + goto sigill; + } + +loadHW: + if (!access_ok(VERIFY_READ, addr, 2)) + goto sigbus; + + LoadHW(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + goto success; + +loadHWU: + if (!access_ok(VERIFY_READ, addr, 2)) + goto sigbus; + + LoadHWU(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + goto success; + +loadW: + if (!access_ok(VERIFY_READ, addr, 4)) + goto sigbus; + + LoadW(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + goto success; + +loadWU: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_READ, addr, 4)) + goto sigbus; + + LoadWU(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + goto success; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + +loadDW: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_READ, addr, 8)) + goto sigbus; + + LoadDW(addr, value, res); + if (res) + goto fault; + regs->regs[reg] = value; + goto success; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + +storeHW: + if (!access_ok(VERIFY_WRITE, addr, 2)) + goto sigbus; + + value = regs->regs[reg]; + StoreHW(addr, value, res); + if (res) + goto fault; + goto success; + +storeW: + if (!access_ok(VERIFY_WRITE, addr, 4)) + goto sigbus; + + value = regs->regs[reg]; + StoreW(addr, value, res); + if (res) + goto fault; + goto success; + +storeDW: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_WRITE, addr, 8)) + goto sigbus; + + value = regs->regs[reg]; + StoreDW(addr, value, res); + if (res) + goto fault; + goto success; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + +success: + regs->cp0_epc = contpc; /* advance or branch */ + +#ifdef CONFIG_DEBUG_FS + unaligned_instructions++; +#endif + return; + +fault: + /* roll back jump/branch */ + regs->cp0_epc = origpc; + regs->regs[31] = orig31; + /* Did we have an exception handler installed? */ + if (fixup_exception(regs)) + return; + + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGSEGV, current); + + return; + +sigbus: + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGBUS, current); + + return; + +sigill: + die_if_kernel + ("Unhandled kernel unaligned access or invalid instruction", regs); + force_sig(SIGILL, current); +} + +static void emulate_load_store_MIPS16e(struct pt_regs *regs, void __user * addr) +{ + unsigned long value; + unsigned int res; + int reg; + unsigned long orig31; + u16 __user *pc16; + unsigned long origpc; + union mips16e_instruction mips16inst, oldinst; + + origpc = regs->cp0_epc; + orig31 = regs->regs[31]; + pc16 = (unsigned short __user *)msk_isa16_mode(origpc); + /* + * This load never faults. + */ + __get_user(mips16inst.full, pc16); + oldinst = mips16inst; + + /* skip EXTEND instruction */ + if (mips16inst.ri.opcode == MIPS16e_extend_op) { + pc16++; + __get_user(mips16inst.full, pc16); + } else if (delay_slot(regs)) { + /* skip jump instructions */ + /* JAL/JALX are 32 bits but have OPCODE in first short int */ + if (mips16inst.ri.opcode == MIPS16e_jal_op) + pc16++; + pc16++; + if (get_user(mips16inst.full, pc16)) + goto sigbus; + } + + switch (mips16inst.ri.opcode) { + case MIPS16e_i64_op: /* I64 or RI64 instruction */ + switch (mips16inst.i64.func) { /* I64/RI64 func field check */ + case MIPS16e_ldpc_func: + case MIPS16e_ldsp_func: + reg = reg16to32[mips16inst.ri64.ry]; + goto loadDW; + + case MIPS16e_sdsp_func: + reg = reg16to32[mips16inst.ri64.ry]; + goto writeDW; + + case MIPS16e_sdrasp_func: + reg = 29; /* GPRSP */ + goto writeDW; + } + + goto sigbus; + + case MIPS16e_swsp_op: + case MIPS16e_lwpc_op: + case MIPS16e_lwsp_op: + reg = reg16to32[mips16inst.ri.rx]; + break; + + case MIPS16e_i8_op: + if (mips16inst.i8.func != MIPS16e_swrasp_func) + goto sigbus; + reg = 29; /* GPRSP */ + break; + + default: + reg = reg16to32[mips16inst.rri.ry]; + break; + } + + switch (mips16inst.ri.opcode) { + + case MIPS16e_lb_op: + case MIPS16e_lbu_op: + case MIPS16e_sb_op: + goto sigbus; + + case MIPS16e_lh_op: + if (!access_ok(VERIFY_READ, addr, 2)) + goto sigbus; + + LoadHW(addr, value, res); + if (res) + goto fault; + MIPS16e_compute_return_epc(regs, &oldinst); + regs->regs[reg] = value; + break; + + case MIPS16e_lhu_op: + if (!access_ok(VERIFY_READ, addr, 2)) + goto sigbus; + + LoadHWU(addr, value, res); + if (res) + goto fault; + MIPS16e_compute_return_epc(regs, &oldinst); + regs->regs[reg] = value; + break; + + case MIPS16e_lw_op: + case MIPS16e_lwpc_op: + case MIPS16e_lwsp_op: + if (!access_ok(VERIFY_READ, addr, 4)) + goto sigbus; + + LoadW(addr, value, res); + if (res) + goto fault; + MIPS16e_compute_return_epc(regs, &oldinst); + regs->regs[reg] = value; + break; + + case MIPS16e_lwu_op: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_READ, addr, 4)) + goto sigbus; + + LoadWU(addr, value, res); + if (res) + goto fault; + MIPS16e_compute_return_epc(regs, &oldinst); + regs->regs[reg] = value; + break; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + + case MIPS16e_ld_op: +loadDW: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_READ, addr, 8)) + goto sigbus; + + LoadDW(addr, value, res); + if (res) + goto fault; + MIPS16e_compute_return_epc(regs, &oldinst); + regs->regs[reg] = value; + break; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + + case MIPS16e_sh_op: + if (!access_ok(VERIFY_WRITE, addr, 2)) + goto sigbus; + + MIPS16e_compute_return_epc(regs, &oldinst); + value = regs->regs[reg]; + StoreHW(addr, value, res); + if (res) + goto fault; + break; + + case MIPS16e_sw_op: + case MIPS16e_swsp_op: + case MIPS16e_i8_op: /* actually - MIPS16e_swrasp_func */ + if (!access_ok(VERIFY_WRITE, addr, 4)) + goto sigbus; + + MIPS16e_compute_return_epc(regs, &oldinst); + value = regs->regs[reg]; + StoreW(addr, value, res); + if (res) + goto fault; + break; + + case MIPS16e_sd_op: +writeDW: +#ifdef CONFIG_64BIT + /* + * A 32-bit kernel might be running on a 64-bit processor. But + * if we're on a 32-bit processor and an i-cache incoherency + * or race makes us see a 64-bit instruction here the sdl/sdr + * would blow up, so for now we don't handle unaligned 64-bit + * instructions on 32-bit kernels. + */ + if (!access_ok(VERIFY_WRITE, addr, 8)) + goto sigbus; + + MIPS16e_compute_return_epc(regs, &oldinst); + value = regs->regs[reg]; + StoreDW(addr, value, res); + if (res) + goto fault; + break; +#endif /* CONFIG_64BIT */ + + /* Cannot handle 64-bit instructions in 32-bit kernel */ + goto sigill; + + default: + /* + * Pheeee... We encountered an yet unknown instruction or + * cache coherence problem. Die sucker, die ... + */ + goto sigill; + } + +#ifdef CONFIG_DEBUG_FS + unaligned_instructions++; +#endif + + return; + +fault: + /* roll back jump/branch */ + regs->cp0_epc = origpc; + regs->regs[31] = orig31; + /* Did we have an exception handler installed? */ + if (fixup_exception(regs)) + return; + + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGSEGV, current); + + return; + +sigbus: + die_if_kernel("Unhandled kernel unaligned access", regs); + force_sig(SIGBUS, current); + + return; + +sigill: + die_if_kernel + ("Unhandled kernel unaligned access or invalid instruction", regs); force_sig(SIGILL, current); } asmlinkage void do_ade(struct pt_regs *regs) { + enum ctx_state prev_state; unsigned int __user *pc; mm_segment_t seg; + prev_state = exception_enter(); perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, - 1, 0, regs, regs->cp0_badvaddr); + 1, regs, regs->cp0_badvaddr); /* * Did we catch a fault trying to load an instruction? - * Or are we running in MIPS16 mode? */ - if ((regs->cp0_badvaddr == regs->cp0_epc) || (regs->cp0_epc & 0x1)) + if (regs->cp0_badvaddr == regs->cp0_epc) goto sigbus; - pc = (unsigned int __user *) exception_epc(regs); if (user_mode(regs) && !test_thread_flag(TIF_FIXADE)) goto sigbus; if (unaligned_action == UNALIGNED_ACTION_SIGNAL) goto sigbus; - else if (unaligned_action == UNALIGNED_ACTION_SHOW) - show_registers(regs); /* * Do branch emulation only if we didn't forward the exception. * This is all so but ugly ... */ + + /* + * Are we running in microMIPS mode? + */ + if (get_isa16_mode(regs->cp0_epc)) { + /* + * Did we catch a fault trying to load an instruction in + * 16-bit mode? + */ + if (regs->cp0_badvaddr == msk_isa16_mode(regs->cp0_epc)) + goto sigbus; + if (unaligned_action == UNALIGNED_ACTION_SHOW) + show_registers(regs); + + if (cpu_has_mmips) { + seg = get_fs(); + if (!user_mode(regs)) + set_fs(KERNEL_DS); + emulate_load_store_microMIPS(regs, + (void __user *)regs->cp0_badvaddr); + set_fs(seg); + + return; + } + + if (cpu_has_mips16) { + seg = get_fs(); + if (!user_mode(regs)) + set_fs(KERNEL_DS); + emulate_load_store_MIPS16e(regs, + (void __user *)regs->cp0_badvaddr); + set_fs(seg); + + return; + } + + goto sigbus; + } + + if (unaligned_action == UNALIGNED_ACTION_SHOW) + show_registers(regs); + pc = (unsigned int __user *)exception_epc(regs); + seg = get_fs(); if (!user_mode(regs)) set_fs(KERNEL_DS); @@ -552,6 +1718,7 @@ sigbus: /* * XXX On return from the signal handler we should advance the epc */ + exception_exit(prev_state); } #ifdef CONFIG_DEBUG_FS diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c index e5cdfd603f8..0f1af58b036 100644 --- a/arch/mips/kernel/vdso.c +++ b/arch/mips/kernel/vdso.c @@ -88,8 +88,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) ret = install_special_mapping(mm, addr, PAGE_SIZE, VM_READ|VM_EXEC| - VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC| - VM_ALWAYSDUMP, + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, &vdso_page); if (ret) diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index f25df73db92..3b46f7ce9ca 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -1,5 +1,14 @@ #include <asm/asm-offsets.h> -#include <asm/page.h> +#include <asm/thread_info.h> + +#define PAGE_SIZE _PAGE_SIZE + +/* + * Put .bss..swapper_pg_dir as the first thing in .bss. This will + * ensure that it has .bss alignment (64K). + */ +#define BSS_FIRST_SECTIONS *(.bss..swapper_pg_dir) + #include <asm-generic/vmlinux.lds.h> #undef mips @@ -13,12 +22,12 @@ PHDRS { #ifdef CONFIG_32BIT #ifdef CONFIG_CPU_LITTLE_ENDIAN - jiffies = jiffies_64; + jiffies = jiffies_64; #else - jiffies = jiffies_64 + 4; + jiffies = jiffies_64 + 4; #endif #else - jiffies = jiffies_64; + jiffies = jiffies_64; #endif SECTIONS @@ -65,15 +74,17 @@ SECTIONS NOTES :text :note .dummy : { *(.dummy) } :text + _sdata = .; /* Start of data section */ RODATA /* writeable */ .data : { /* Data */ . = . + DATAOFFSET; /* for CONFIG_MAPPED_KERNEL */ - INIT_TASK_DATA(PAGE_SIZE) + INIT_TASK_DATA(THREAD_SIZE) NOSAVE_DATA CACHELINE_ALIGNED_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT) + READ_MOSTLY_DATA(1 << CONFIG_MIPS_L1_CACHE_SHIFT) DATA_DATA CONSTRUCTORS } @@ -98,6 +109,13 @@ SECTIONS INIT_TEXT_SECTION(PAGE_SIZE) INIT_DATA_SECTION(16) + . = ALIGN(4); + .mips.machines.init : AT(ADDR(.mips.machines.init) - LOAD_OFFSET) { + __mips_machines_start = .; + *(.mips.machines.init) + __mips_machines_end = .; + } + /* .exit.text is discarded at runtime, not link time, to deal with * references from .rodata */ @@ -108,12 +126,22 @@ SECTIONS EXIT_DATA } - PERCPU(PAGE_SIZE) - . = ALIGN(PAGE_SIZE); + PERCPU_SECTION(1 << CONFIG_MIPS_L1_CACHE_SHIFT) + /* + * Align to 64K in attempt to eliminate holes before the + * .bss..swapper_pg_dir section at the start of .bss. This + * also satisfies PAGE_SIZE alignment as the largest page size + * allowed is 64K. + */ + . = ALIGN(0x10000); __init_end = .; /* freed after init ends here */ - BSS_SECTION(0, 0, 0) + /* + * Force .bss to 64K alignment so that .bss..swapper_pg_dir + * gets that alignment. .sbss should be empty, so there will be + * no holes after __init_end. */ + BSS_SECTION(0, 0x10000, 0) _end = . ; @@ -151,5 +179,6 @@ SECTIONS *(.options) *(.pdr) *(.reginfo) + *(.eh_frame) } } diff --git a/arch/mips/kernel/vpe-cmp.c b/arch/mips/kernel/vpe-cmp.c new file mode 100644 index 00000000000..9268ebc0f61 --- /dev/null +++ b/arch/mips/kernel/vpe-cmp.c @@ -0,0 +1,180 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/export.h> + +#include <asm/vpe.h> + +static int major; + +void cleanup_tc(struct tc *tc) +{ + +} + +static ssize_t store_kill(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + struct vpe_notifications *notifier; + + list_for_each_entry(notifier, &vpe->notify, list) + notifier->stop(aprp_cpu_index()); + + release_progmem(vpe->load_addr); + vpe->state = VPE_STATE_UNUSED; + + return len; +} +static DEVICE_ATTR(kill, S_IWUSR, NULL, store_kill); + +static ssize_t ntcs_show(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + + return sprintf(buf, "%d\n", vpe->ntcs); +} + +static ssize_t ntcs_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + unsigned long new; + int ret; + + ret = kstrtoul(buf, 0, &new); + if (ret < 0) + return ret; + + /* APRP can only reserve one TC in a VPE and no more. */ + if (new != 1) + return -EINVAL; + + vpe->ntcs = new; + + return len; +} +static DEVICE_ATTR_RW(ntcs); + +static struct attribute *vpe_attrs[] = { + &dev_attr_kill.attr, + &dev_attr_ntcs.attr, + NULL, +}; +ATTRIBUTE_GROUPS(vpe); + +static void vpe_device_release(struct device *cd) +{ + kfree(cd); +} + +static struct class vpe_class = { + .name = "vpe", + .owner = THIS_MODULE, + .dev_release = vpe_device_release, + .dev_groups = vpe_groups, +}; + +static struct device vpe_device; + +int __init vpe_module_init(void) +{ + struct vpe *v = NULL; + struct tc *t; + int err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (num_possible_cpus() - aprp_cpu_index() < 1) { + pr_warn("No VPEs reserved for AP/SP, not initialize VPE loader\n" + "Pass maxcpus=<n> argument as kernel argument\n"); + return -ENODEV; + } + + major = register_chrdev(0, VPE_MODULE_NAME, &vpe_fops); + if (major < 0) { + pr_warn("VPE loader: unable to register character device\n"); + return major; + } + + err = class_register(&vpe_class); + if (err) { + pr_err("vpe_class registration failed\n"); + goto out_chrdev; + } + + device_initialize(&vpe_device); + vpe_device.class = &vpe_class, + vpe_device.parent = NULL, + dev_set_name(&vpe_device, "vpe_sp"); + vpe_device.devt = MKDEV(major, VPE_MODULE_MINOR); + err = device_add(&vpe_device); + if (err) { + pr_err("Adding vpe_device failed\n"); + goto out_class; + } + + t = alloc_tc(aprp_cpu_index()); + if (!t) { + pr_warn("VPE: unable to allocate TC\n"); + err = -ENOMEM; + goto out_dev; + } + + /* VPE */ + v = alloc_vpe(aprp_cpu_index()); + if (v == NULL) { + pr_warn("VPE: unable to allocate VPE\n"); + kfree(t); + err = -ENOMEM; + goto out_dev; + } + + v->ntcs = 1; + + /* add the tc to the list of this vpe's tc's. */ + list_add(&t->tc, &v->tc); + + /* TC */ + t->pvpe = v; /* set the parent vpe */ + + return 0; + +out_dev: + device_del(&vpe_device); + +out_class: + class_unregister(&vpe_class); + +out_chrdev: + unregister_chrdev(major, VPE_MODULE_NAME); + + return err; +} + +void __exit vpe_module_exit(void) +{ + struct vpe *v, *n; + + device_del(&vpe_device); + class_unregister(&vpe_class); + unregister_chrdev(major, VPE_MODULE_NAME); + + /* No locking needed here */ + list_for_each_entry_safe(v, n, &vpecontrol.vpe_list, list) + if (v->state != VPE_STATE_UNUSED) + release_vpe(v); +} diff --git a/arch/mips/kernel/vpe-mt.c b/arch/mips/kernel/vpe-mt.c new file mode 100644 index 00000000000..2e003b11a09 --- /dev/null +++ b/arch/mips/kernel/vpe-mt.c @@ -0,0 +1,521 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. + */ +#include <linux/kernel.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/export.h> + +#include <asm/mipsregs.h> +#include <asm/mipsmtregs.h> +#include <asm/mips_mt.h> +#include <asm/vpe.h> + +static int major; + +/* The number of TCs and VPEs physically available on the core */ +static int hw_tcs, hw_vpes; + +/* We are prepared so configure and start the VPE... */ +int vpe_run(struct vpe *v) +{ + unsigned long flags, val, dmt_flag; + struct vpe_notifications *notifier; + unsigned int vpeflags; + struct tc *t; + + /* check we are the Master VPE */ + local_irq_save(flags); + val = read_c0_vpeconf0(); + if (!(val & VPECONF0_MVP)) { + pr_warn("VPE loader: only Master VPE's are able to config MT\n"); + local_irq_restore(flags); + + return -1; + } + + dmt_flag = dmt(); + vpeflags = dvpe(); + + if (list_empty(&v->tc)) { + evpe(vpeflags); + emt(dmt_flag); + local_irq_restore(flags); + + pr_warn("VPE loader: No TC's associated with VPE %d\n", + v->minor); + + return -ENOEXEC; + } + + t = list_first_entry(&v->tc, struct tc, tc); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(t->index); + + /* should check it is halted, and not activated */ + if ((read_tc_c0_tcstatus() & TCSTATUS_A) || + !(read_tc_c0_tchalt() & TCHALT_H)) { + evpe(vpeflags); + emt(dmt_flag); + local_irq_restore(flags); + + pr_warn("VPE loader: TC %d is already active!\n", + t->index); + + return -ENOEXEC; + } + + /* + * Write the address we want it to start running from in the TCPC + * register. + */ + write_tc_c0_tcrestart((unsigned long)v->__start); + write_tc_c0_tccontext((unsigned long)0); + + /* + * Mark the TC as activated, not interrupt exempt and not dynamically + * allocatable + */ + val = read_tc_c0_tcstatus(); + val = (val & ~(TCSTATUS_DA | TCSTATUS_IXMT)) | TCSTATUS_A; + write_tc_c0_tcstatus(val); + + write_tc_c0_tchalt(read_tc_c0_tchalt() & ~TCHALT_H); + + /* + * The sde-kit passes 'memsize' to __start in $a3, so set something + * here... Or set $a3 to zero and define DFLT_STACK_SIZE and + * DFLT_HEAP_SIZE when you compile your program + */ + mttgpr(6, v->ntcs); + mttgpr(7, physical_memsize); + + /* set up VPE1 */ + /* + * bind the TC to VPE 1 as late as possible so we only have the final + * VPE registers to set up, and so an EJTAG probe can trigger on it + */ + write_tc_c0_tcbind((read_tc_c0_tcbind() & ~TCBIND_CURVPE) | 1); + + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~(VPECONF0_VPA)); + + back_to_back_c0_hazard(); + + /* Set up the XTC bit in vpeconf0 to point at our tc */ + write_vpe_c0_vpeconf0((read_vpe_c0_vpeconf0() & ~(VPECONF0_XTC)) + | (t->index << VPECONF0_XTC_SHIFT)); + + back_to_back_c0_hazard(); + + /* enable this VPE */ + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); + + /* clear out any left overs from a previous program */ + write_vpe_c0_status(0); + write_vpe_c0_cause(0); + + /* take system out of configuration state */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + + /* + * SMVP kernels manage VPE enable independently, but uniprocessor + * kernels need to turn it on, even if that wasn't the pre-dvpe() state. + */ +#ifdef CONFIG_SMP + evpe(vpeflags); +#else + evpe(EVPE_ENABLE); +#endif + emt(dmt_flag); + local_irq_restore(flags); + + list_for_each_entry(notifier, &v->notify, list) + notifier->start(VPE_MODULE_MINOR); + + return 0; +} + +void cleanup_tc(struct tc *tc) +{ + unsigned long flags; + unsigned int mtflags, vpflags; + int tmp; + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(tc->index); + tmp = read_tc_c0_tcstatus(); + + /* mark not allocated and not dynamically allocatable */ + tmp &= ~(TCSTATUS_A | TCSTATUS_DA); + tmp |= TCSTATUS_IXMT; /* interrupt exempt */ + write_tc_c0_tcstatus(tmp); + + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); +} + +/* module wrapper entry points */ +/* give me a vpe */ +void *vpe_alloc(void) +{ + int i; + struct vpe *v; + + /* find a vpe */ + for (i = 1; i < MAX_VPES; i++) { + v = get_vpe(i); + if (v != NULL) { + v->state = VPE_STATE_INUSE; + return v; + } + } + return NULL; +} +EXPORT_SYMBOL(vpe_alloc); + +/* start running from here */ +int vpe_start(void *vpe, unsigned long start) +{ + struct vpe *v = vpe; + + v->__start = start; + return vpe_run(v); +} +EXPORT_SYMBOL(vpe_start); + +/* halt it for now */ +int vpe_stop(void *vpe) +{ + struct vpe *v = vpe; + struct tc *t; + unsigned int evpe_flags; + + evpe_flags = dvpe(); + + t = list_entry(v->tc.next, struct tc, tc); + if (t != NULL) { + settc(t->index); + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); + } + + evpe(evpe_flags); + + return 0; +} +EXPORT_SYMBOL(vpe_stop); + +/* I've done with it thank you */ +int vpe_free(void *vpe) +{ + struct vpe *v = vpe; + struct tc *t; + unsigned int evpe_flags; + + t = list_entry(v->tc.next, struct tc, tc); + if (t == NULL) + return -ENOEXEC; + + evpe_flags = dvpe(); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + settc(t->index); + write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); + + /* halt the TC */ + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + /* mark the TC unallocated */ + write_tc_c0_tcstatus(read_tc_c0_tcstatus() & ~TCSTATUS_A); + + v->state = VPE_STATE_UNUSED; + + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(evpe_flags); + + return 0; +} +EXPORT_SYMBOL(vpe_free); + +static ssize_t store_kill(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + struct vpe_notifications *notifier; + + list_for_each_entry(notifier, &vpe->notify, list) + notifier->stop(aprp_cpu_index()); + + release_progmem(vpe->load_addr); + cleanup_tc(get_tc(aprp_cpu_index())); + vpe_stop(vpe); + vpe_free(vpe); + + return len; +} +static DEVICE_ATTR(kill, S_IWUSR, NULL, store_kill); + +static ssize_t ntcs_show(struct device *cd, struct device_attribute *attr, + char *buf) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + + return sprintf(buf, "%d\n", vpe->ntcs); +} + +static ssize_t ntcs_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct vpe *vpe = get_vpe(aprp_cpu_index()); + unsigned long new; + int ret; + + ret = kstrtoul(buf, 0, &new); + if (ret < 0) + return ret; + + if (new == 0 || new > (hw_tcs - aprp_cpu_index())) + return -EINVAL; + + vpe->ntcs = new; + + return len; +} +static DEVICE_ATTR_RW(ntcs); + +static struct attribute *vpe_attrs[] = { + &dev_attr_kill.attr, + &dev_attr_ntcs.attr, + NULL, +}; +ATTRIBUTE_GROUPS(vpe); + +static void vpe_device_release(struct device *cd) +{ + kfree(cd); +} + +static struct class vpe_class = { + .name = "vpe", + .owner = THIS_MODULE, + .dev_release = vpe_device_release, + .dev_groups = vpe_groups, +}; + +static struct device vpe_device; + +int __init vpe_module_init(void) +{ + unsigned int mtflags, vpflags; + unsigned long flags, val; + struct vpe *v = NULL; + struct tc *t; + int tc, err; + + if (!cpu_has_mipsmt) { + pr_warn("VPE loader: not a MIPS MT capable processor\n"); + return -ENODEV; + } + + if (vpelimit == 0) { + pr_warn("No VPEs reserved for AP/SP, not initialize VPE loader\n" + "Pass maxvpes=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + if (aprp_cpu_index() == 0) { + pr_warn("No TCs reserved for AP/SP, not initialize VPE loader\n" + "Pass maxtcs=<n> argument as kernel argument\n"); + + return -ENODEV; + } + + major = register_chrdev(0, VPE_MODULE_NAME, &vpe_fops); + if (major < 0) { + pr_warn("VPE loader: unable to register character device\n"); + return major; + } + + err = class_register(&vpe_class); + if (err) { + pr_err("vpe_class registration failed\n"); + goto out_chrdev; + } + + device_initialize(&vpe_device); + vpe_device.class = &vpe_class, + vpe_device.parent = NULL, + dev_set_name(&vpe_device, "vpe1"); + vpe_device.devt = MKDEV(major, VPE_MODULE_MINOR); + err = device_add(&vpe_device); + if (err) { + pr_err("Adding vpe_device failed\n"); + goto out_class; + } + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + + /* Put MVPE's into 'configuration state' */ + set_c0_mvpcontrol(MVPCONTROL_VPC); + + val = read_c0_mvpconf0(); + hw_tcs = (val & MVPCONF0_PTC) + 1; + hw_vpes = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; + + for (tc = aprp_cpu_index(); tc < hw_tcs; tc++) { + /* + * Must re-enable multithreading temporarily or in case we + * reschedule send IPIs or similar we might hang. + */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); + t = alloc_tc(tc); + if (!t) { + err = -ENOMEM; + goto out_dev; + } + + local_irq_save(flags); + mtflags = dmt(); + vpflags = dvpe(); + set_c0_mvpcontrol(MVPCONTROL_VPC); + + /* VPE's */ + if (tc < hw_tcs) { + settc(tc); + + v = alloc_vpe(tc); + if (v == NULL) { + pr_warn("VPE: unable to allocate VPE\n"); + goto out_reenable; + } + + v->ntcs = hw_tcs - aprp_cpu_index(); + + /* add the tc to the list of this vpe's tc's. */ + list_add(&t->tc, &v->tc); + + /* deactivate all but vpe0 */ + if (tc >= aprp_cpu_index()) { + unsigned long tmp = read_vpe_c0_vpeconf0(); + + tmp &= ~VPECONF0_VPA; + + /* master VPE */ + tmp |= VPECONF0_MVP; + write_vpe_c0_vpeconf0(tmp); + } + + /* disable multi-threading with TC's */ + write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() & + ~VPECONTROL_TE); + + if (tc >= vpelimit) { + /* + * Set config to be the same as vpe0, + * particularly kseg0 coherency alg + */ + write_vpe_c0_config(read_c0_config()); + } + } + + /* TC's */ + t->pvpe = v; /* set the parent vpe */ + + if (tc >= aprp_cpu_index()) { + unsigned long tmp; + + settc(tc); + + /* + * A TC that is bound to any other VPE gets bound to + * VPE0, ideally I'd like to make it homeless but it + * doesn't appear to let me bind a TC to a non-existent + * VPE. Which is perfectly reasonable. + * + * The (un)bound state is visible to an EJTAG probe so + * may notify GDB... + */ + tmp = read_tc_c0_tcbind(); + if (tmp & TCBIND_CURVPE) { + /* tc is bound >vpe0 */ + write_tc_c0_tcbind(tmp & ~TCBIND_CURVPE); + + t->pvpe = get_vpe(0); /* set the parent vpe */ + } + + /* halt the TC */ + write_tc_c0_tchalt(TCHALT_H); + mips_ihb(); + + tmp = read_tc_c0_tcstatus(); + + /* mark not activated and not dynamically allocatable */ + tmp &= ~(TCSTATUS_A | TCSTATUS_DA); + tmp |= TCSTATUS_IXMT; /* interrupt exempt */ + write_tc_c0_tcstatus(tmp); + } + } + +out_reenable: + /* release config state */ + clear_c0_mvpcontrol(MVPCONTROL_VPC); + + evpe(vpflags); + emt(mtflags); + local_irq_restore(flags); + + return 0; + +out_dev: + device_del(&vpe_device); + +out_class: + class_unregister(&vpe_class); + +out_chrdev: + unregister_chrdev(major, VPE_MODULE_NAME); + + return err; +} + +void __exit vpe_module_exit(void) +{ + struct vpe *v, *n; + + device_del(&vpe_device); + class_unregister(&vpe_class); + unregister_chrdev(major, VPE_MODULE_NAME); + + /* No locking needed here */ + list_for_each_entry_safe(v, n, &vpecontrol.vpe_list, list) { + if (v->state != VPE_STATE_UNUSED) + release_vpe(v); + } +} diff --git a/arch/mips/kernel/vpe.c b/arch/mips/kernel/vpe.c index 3eb3cde2f66..11da314565c 100644 --- a/arch/mips/kernel/vpe.c +++ b/arch/mips/kernel/vpe.c @@ -1,38 +1,22 @@ /* - * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. - * - * This program is free software; you can distribute it and/or modify it - * under the terms of the GNU General Public License (Version 2) as - * published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. - */ - -/* - * VPE support module + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. * - * Provides support for loading a MIPS SP program on VPE1. - * The SP enviroment is rather simple, no tlb's. It needs to be relocatable - * (or partially linked). You should initialise your stack in the startup - * code. This loader looks for the symbol __start and sets up - * execution to resume from there. The MIPS SDE kit contains suitable examples. + * Copyright (C) 2004, 2005 MIPS Technologies, Inc. All rights reserved. + * Copyright (C) 2013 Imagination Technologies Ltd. * - * To load and run, simply cat a SP 'program file' to /dev/vpe1. - * i.e cat spapp >/dev/vpe1. + * VPE spport module for loading a MIPS SP program into VPE1. The SP + * environment is rather simple since there are no TLBs. It needs + * to be relocatable (or partiall linked). Initialize your stack in + * the startup-code. The loader looks for the symbol __start and sets + * up the execution to resume from there. To load and run, simply do + * a cat SP 'binary' to the /dev/vpe1 device. */ #include <linux/kernel.h> #include <linux/device.h> -#include <linux/module.h> #include <linux/fs.h> #include <linux/init.h> -#include <asm/uaccess.h> #include <linux/slab.h> #include <linux/list.h> #include <linux/vmalloc.h> @@ -46,15 +30,10 @@ #include <asm/mipsregs.h> #include <asm/mipsmtregs.h> #include <asm/cacheflush.h> -#include <asm/atomic.h> -#include <asm/cpu.h> +#include <linux/atomic.h> #include <asm/mips_mt.h> #include <asm/processor.h> -#include <asm/system.h> #include <asm/vpe.h> -#include <asm/kspd.h> - -typedef void *vpe_handle; #ifndef ARCH_SHF_SMALL #define ARCH_SHF_SMALL 0 @@ -63,101 +42,15 @@ typedef void *vpe_handle; /* If this is set, the section belongs in the init part of the module */ #define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1)) -/* - * The number of TCs and VPEs physically available on the core - */ -static int hw_tcs, hw_vpes; -static char module_name[] = "vpe"; -static int major; -static const int minor = 1; /* fixed for now */ - -#ifdef CONFIG_MIPS_APSP_KSPD -static struct kspd_notifications kspd_events; -static int kspd_events_reqd; -#endif - -/* grab the likely amount of memory we will need. */ -#ifdef CONFIG_MIPS_VPE_LOADER_TOM -#define P_SIZE (2 * 1024 * 1024) -#else -/* add an overhead to the max kmalloc size for non-striped symbols/etc */ -#define P_SIZE (256 * 1024) -#endif - -extern unsigned long physical_memsize; - -#define MAX_VPES 16 -#define VPE_PATH_MAX 256 - -enum vpe_state { - VPE_STATE_UNUSED = 0, - VPE_STATE_INUSE, - VPE_STATE_RUNNING -}; - -enum tc_state { - TC_STATE_UNUSED = 0, - TC_STATE_INUSE, - TC_STATE_RUNNING, - TC_STATE_DYNAMIC -}; - -struct vpe { - enum vpe_state state; - - /* (device) minor associated with this vpe */ - int minor; - - /* elfloader stuff */ - void *load_addr; - unsigned long len; - char *pbuffer; - unsigned long plen; - unsigned int uid, gid; - char cwd[VPE_PATH_MAX]; - - unsigned long __start; - - /* tc's associated with this vpe */ - struct list_head tc; - - /* The list of vpe's */ - struct list_head list; - - /* shared symbol address */ - void *shared_ptr; - - /* the list of who wants to know when something major happens */ - struct list_head notify; - - unsigned int ntcs; -}; - -struct tc { - enum tc_state state; - int index; - - struct vpe *pvpe; /* parent VPE */ - struct list_head tc; /* The list of TC's with this VPE */ - struct list_head list; /* The global list of tc's */ -}; - -struct { - spinlock_t vpe_list_lock; - struct list_head vpe_list; /* Virtual processing elements */ - spinlock_t tc_list_lock; - struct list_head tc_list; /* Thread contexts */ -} vpecontrol = { - .vpe_list_lock = SPIN_LOCK_UNLOCKED, +struct vpe_control vpecontrol = { + .vpe_list_lock = __SPIN_LOCK_UNLOCKED(vpe_list_lock), .vpe_list = LIST_HEAD_INIT(vpecontrol.vpe_list), - .tc_list_lock = SPIN_LOCK_UNLOCKED, + .tc_list_lock = __SPIN_LOCK_UNLOCKED(tc_list_lock), .tc_list = LIST_HEAD_INIT(vpecontrol.tc_list) }; -static void release_progmem(void *ptr); - /* get the vpe associated with this minor */ -static struct vpe *get_vpe(int minor) +struct vpe *get_vpe(int minor) { struct vpe *res, *v; @@ -167,7 +60,7 @@ static struct vpe *get_vpe(int minor) res = NULL; spin_lock(&vpecontrol.vpe_list_lock); list_for_each_entry(v, &vpecontrol.vpe_list, list) { - if (v->minor == minor) { + if (v->minor == VPE_MODULE_MINOR) { res = v; break; } @@ -178,7 +71,7 @@ static struct vpe *get_vpe(int minor) } /* get the vpe associated with this minor */ -static struct tc *get_tc(int index) +struct tc *get_tc(int index) { struct tc *res, *t; @@ -192,16 +85,17 @@ static struct tc *get_tc(int index) } spin_unlock(&vpecontrol.tc_list_lock); - return NULL; + return res; } /* allocate a vpe and associate it with this minor (or index) */ -static struct vpe *alloc_vpe(int minor) +struct vpe *alloc_vpe(int minor) { struct vpe *v; - if ((v = kzalloc(sizeof(struct vpe), GFP_KERNEL)) == NULL) - return NULL; + v = kzalloc(sizeof(struct vpe), GFP_KERNEL); + if (v == NULL) + goto out; INIT_LIST_HEAD(&v->tc); spin_lock(&vpecontrol.vpe_list_lock); @@ -209,17 +103,19 @@ static struct vpe *alloc_vpe(int minor) spin_unlock(&vpecontrol.vpe_list_lock); INIT_LIST_HEAD(&v->notify); - v->minor = minor; + v->minor = VPE_MODULE_MINOR; +out: return v; } /* allocate a tc. At startup only tc0 is running, all other can be halted. */ -static struct tc *alloc_tc(int index) +struct tc *alloc_tc(int index) { struct tc *tc; - if ((tc = kzalloc(sizeof(struct tc), GFP_KERNEL)) == NULL) + tc = kzalloc(sizeof(struct tc), GFP_KERNEL); + if (tc == NULL) goto out; INIT_LIST_HEAD(&tc->tc); @@ -234,7 +130,7 @@ out: } /* clean up and free everything */ -static void release_vpe(struct vpe *v) +void release_vpe(struct vpe *v) { list_del(&v->list); if (v->load_addr) @@ -242,28 +138,8 @@ static void release_vpe(struct vpe *v) kfree(v); } -static void __maybe_unused dump_mtregs(void) -{ - unsigned long val; - - val = read_c0_config3(); - printk("config3 0x%lx MT %ld\n", val, - (val & CONFIG3_MT) >> CONFIG3_MT_SHIFT); - - val = read_c0_mvpcontrol(); - printk("MVPControl 0x%lx, STLB %ld VPC %ld EVP %ld\n", val, - (val & MVPCONTROL_STLB) >> MVPCONTROL_STLB_SHIFT, - (val & MVPCONTROL_VPC) >> MVPCONTROL_VPC_SHIFT, - (val & MVPCONTROL_EVP)); - - val = read_c0_mvpconf0(); - printk("mvpconf0 0x%lx, PVPE %ld PTC %ld M %ld\n", val, - (val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT, - val & MVPCONF0_PTC, (val & MVPCONF0_M) >> MVPCONF0_M_SHIFT); -} - -/* Find some VPE program space */ -static void *alloc_progmem(unsigned long len) +/* Find some VPE program space */ +void *alloc_progmem(unsigned long len) { void *addr; @@ -282,7 +158,7 @@ static void *alloc_progmem(unsigned long len) return addr; } -static void release_progmem(void *ptr) +void release_progmem(void *ptr) { #ifndef CONFIG_MIPS_VPE_LOADER_TOM kfree(ptr); @@ -290,7 +166,7 @@ static void release_progmem(void *ptr) } /* Update size with this section: return offset. */ -static long get_offset(unsigned long *size, Elf_Shdr * sechdr) +static long get_offset(unsigned long *size, Elf_Shdr *sechdr) { long ret; @@ -300,11 +176,11 @@ static long get_offset(unsigned long *size, Elf_Shdr * sechdr) } /* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld - might -- code, read-only data, read-write data, small data. Tally + might -- code, read-only data, read-write data, small data. Tally sizes, and place the offsets into sh_entsize fields: high bit means it belongs in init. */ -static void layout_sections(struct module *mod, const Elf_Ehdr * hdr, - Elf_Shdr * sechdrs, const char *secstrings) +static void layout_sections(struct module *mod, const Elf_Ehdr *hdr, + Elf_Shdr *sechdrs, const char *secstrings) { static unsigned long const masks[][2] = { /* NOTE: all executable code must be the first section @@ -324,7 +200,6 @@ static void layout_sections(struct module *mod, const Elf_Ehdr * hdr, for (i = 0; i < hdr->e_shnum; ++i) { Elf_Shdr *s = &sechdrs[i]; - // || strncmp(secstrings + s->sh_name, ".init", 5) == 0) if ((s->sh_flags & masks[m][0]) != masks[m][0] || (s->sh_flags & masks[m][1]) || s->sh_entsize != ~0UL) @@ -339,7 +214,6 @@ static void layout_sections(struct module *mod, const Elf_Ehdr * hdr, } } - /* from module-elf32.c, but subverted a little */ struct mips_hi16 { @@ -362,20 +236,18 @@ static int apply_r_mips_gprel16(struct module *me, uint32_t *location, { int rel; - if( !(*location & 0xffff) ) { + if (!(*location & 0xffff)) { rel = (int)v - gp_addr; - } - else { + } else { /* .sbss + gp(relative) + offset */ /* kludge! */ rel = (int)(short)((int)v + gp_offs + (int)(short)(*location & 0xffff) - gp_addr); } - if( (rel > 32768) || (rel < -32768) ) { - printk(KERN_DEBUG "VPE loader: apply_r_mips_gprel16: " - "relative address 0x%x out of range of gp register\n", - rel); + if ((rel > 32768) || (rel < -32768)) { + pr_debug("VPE loader: apply_r_mips_gprel16: relative address 0x%x out of range of gp register\n", + rel); return -ENOEXEC; } @@ -389,12 +261,12 @@ static int apply_r_mips_pc16(struct module *me, uint32_t *location, { int rel; rel = (((unsigned int)v - (unsigned int)location)); - rel >>= 2; // because the offset is in _instructions_ not bytes. - rel -= 1; // and one instruction less due to the branch delay slot. + rel >>= 2; /* because the offset is in _instructions_ not bytes. */ + rel -= 1; /* and one instruction less due to the branch delay slot. */ - if( (rel > 32768) || (rel < -32768) ) { - printk(KERN_DEBUG "VPE loader: " - "apply_r_mips_pc16: relative address out of range 0x%x\n", rel); + if ((rel > 32768) || (rel < -32768)) { + pr_debug("VPE loader: apply_r_mips_pc16: relative address out of range 0x%x\n", + rel); return -ENOEXEC; } @@ -415,8 +287,7 @@ static int apply_r_mips_26(struct module *me, uint32_t *location, Elf32_Addr v) { if (v % 4) { - printk(KERN_DEBUG "VPE loader: apply_r_mips_26 " - " unaligned relocation\n"); + pr_debug("VPE loader: apply_r_mips_26: unaligned relocation\n"); return -ENOEXEC; } @@ -447,7 +318,7 @@ static int apply_r_mips_hi16(struct module *me, uint32_t *location, * the carry we need to add. Save the information, and let LO16 do the * actual relocation. */ - n = kmalloc(sizeof *n, GFP_KERNEL); + n = kmalloc(sizeof(*n), GFP_KERNEL); if (!n) return -ENOMEM; @@ -466,7 +337,7 @@ static int apply_r_mips_lo16(struct module *me, uint32_t *location, Elf32_Addr val, vallo; struct mips_hi16 *l, *next; - /* Sign extend the addend we extract from the lo insn. */ + /* Sign extend the addend we extract from the lo insn. */ vallo = ((insnlo & 0xffff) ^ 0x8000) - 0x8000; if (mips_hi16_list != NULL) { @@ -478,10 +349,8 @@ static int apply_r_mips_lo16(struct module *me, uint32_t *location, /* * The value for the HI16 had best be the same. */ - if (v != l->value) { - printk(KERN_DEBUG "VPE loader: " - "apply_r_mips_lo16/hi16: \t" - "inconsistent value information\n"); + if (v != l->value) { + pr_debug("VPE loader: apply_r_mips_lo16/hi16: inconsistent value information\n"); goto out_free; } @@ -513,7 +382,7 @@ static int apply_r_mips_lo16(struct module *me, uint32_t *location, } /* - * Ok, we're done with the HI16 relocs. Now deal with the LO16. + * Ok, we're done with the HI16 relocs. Now deal with the LO16. */ val = v + vallo; insnlo = (insnlo & ~0xffff) | (val & 0xffff); @@ -577,20 +446,19 @@ static int apply_relocations(Elf32_Shdr *sechdrs, + ELF32_R_SYM(r_info); if (!sym->st_value) { - printk(KERN_DEBUG "%s: undefined weak symbol %s\n", - me->name, strtab + sym->st_name); + pr_debug("%s: undefined weak symbol %s\n", + me->name, strtab + sym->st_name); /* just print the warning, dont barf */ } v = sym->st_value; res = reloc_handlers[ELF32_R_TYPE(r_info)](me, location, v); - if( res ) { + if (res) { char *r = rstrs[ELF32_R_TYPE(r_info)]; - printk(KERN_WARNING "VPE loader: .text+0x%x " - "relocation type %s for symbol \"%s\" failed\n", - rel[i].r_offset, r ? r : "UNKNOWN", - strtab + sym->st_name); + pr_warn("VPE loader: .text+0x%x relocation type %s for symbol \"%s\" failed\n", + rel[i].r_offset, r ? r : "UNKNOWN", + strtab + sym->st_name); return res; } } @@ -605,10 +473,8 @@ static inline void save_gp_address(unsigned int secbase, unsigned int rel) } /* end module-elf32.c */ - - /* Change all symbols so that sh_value encodes the pointer directly. */ -static void simplify_symbols(Elf_Shdr * sechdrs, +static void simplify_symbols(Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, const char *secstrings, @@ -649,18 +515,16 @@ static void simplify_symbols(Elf_Shdr * sechdrs, break; case SHN_MIPS_SCOMMON: - printk(KERN_DEBUG "simplify_symbols: ignoring SHN_MIPS_SCOMMON " - "symbol <%s> st_shndx %d\n", strtab + sym[i].st_name, - sym[i].st_shndx); - // .sbss section + pr_debug("simplify_symbols: ignoring SHN_MIPS_SCOMMON symbol <%s> st_shndx %d\n", + strtab + sym[i].st_name, sym[i].st_shndx); + /* .sbss section */ break; default: secbase = sechdrs[sym[i].st_shndx].sh_addr; - if (strncmp(strtab + sym[i].st_name, "_gp", 3) == 0) { + if (strncmp(strtab + sym[i].st_name, "_gp", 3) == 0) save_gp_address(secbase, sym[i].st_value); - } sym[i].st_value += secbase; break; @@ -669,151 +533,21 @@ static void simplify_symbols(Elf_Shdr * sechdrs, } #ifdef DEBUG_ELFLOADER -static void dump_elfsymbols(Elf_Shdr * sechdrs, unsigned int symindex, +static void dump_elfsymbols(Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, struct module *mod) { Elf_Sym *sym = (void *)sechdrs[symindex].sh_addr; unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); - printk(KERN_DEBUG "dump_elfsymbols: n %d\n", n); + pr_debug("dump_elfsymbols: n %d\n", n); for (i = 1; i < n; i++) { - printk(KERN_DEBUG " i %d name <%s> 0x%x\n", i, - strtab + sym[i].st_name, sym[i].st_value); + pr_debug(" i %d name <%s> 0x%x\n", i, strtab + sym[i].st_name, + sym[i].st_value); } } #endif -/* We are prepared so configure and start the VPE... */ -static int vpe_run(struct vpe * v) -{ - unsigned long flags, val, dmt_flag; - struct vpe_notifications *n; - unsigned int vpeflags; - struct tc *t; - - /* check we are the Master VPE */ - local_irq_save(flags); - val = read_c0_vpeconf0(); - if (!(val & VPECONF0_MVP)) { - printk(KERN_WARNING - "VPE loader: only Master VPE's are allowed to configure MT\n"); - local_irq_restore(flags); - - return -1; - } - - dmt_flag = dmt(); - vpeflags = dvpe(); - - if (!list_empty(&v->tc)) { - if ((t = list_entry(v->tc.next, struct tc, tc)) == NULL) { - evpe(vpeflags); - emt(dmt_flag); - local_irq_restore(flags); - - printk(KERN_WARNING - "VPE loader: TC %d is already in use.\n", - t->index); - return -ENOEXEC; - } - } else { - evpe(vpeflags); - emt(dmt_flag); - local_irq_restore(flags); - - printk(KERN_WARNING - "VPE loader: No TC's associated with VPE %d\n", - v->minor); - - return -ENOEXEC; - } - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(t->index); - - /* should check it is halted, and not activated */ - if ((read_tc_c0_tcstatus() & TCSTATUS_A) || !(read_tc_c0_tchalt() & TCHALT_H)) { - evpe(vpeflags); - emt(dmt_flag); - local_irq_restore(flags); - - printk(KERN_WARNING "VPE loader: TC %d is already active!\n", - t->index); - - return -ENOEXEC; - } - - /* Write the address we want it to start running from in the TCPC register. */ - write_tc_c0_tcrestart((unsigned long)v->__start); - write_tc_c0_tccontext((unsigned long)0); - - /* - * Mark the TC as activated, not interrupt exempt and not dynamically - * allocatable - */ - val = read_tc_c0_tcstatus(); - val = (val & ~(TCSTATUS_DA | TCSTATUS_IXMT)) | TCSTATUS_A; - write_tc_c0_tcstatus(val); - - write_tc_c0_tchalt(read_tc_c0_tchalt() & ~TCHALT_H); - - /* - * The sde-kit passes 'memsize' to __start in $a3, so set something - * here... Or set $a3 to zero and define DFLT_STACK_SIZE and - * DFLT_HEAP_SIZE when you compile your program - */ - mttgpr(6, v->ntcs); - mttgpr(7, physical_memsize); - - /* set up VPE1 */ - /* - * bind the TC to VPE 1 as late as possible so we only have the final - * VPE registers to set up, and so an EJTAG probe can trigger on it - */ - write_tc_c0_tcbind((read_tc_c0_tcbind() & ~TCBIND_CURVPE) | 1); - - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~(VPECONF0_VPA)); - - back_to_back_c0_hazard(); - - /* Set up the XTC bit in vpeconf0 to point at our tc */ - write_vpe_c0_vpeconf0( (read_vpe_c0_vpeconf0() & ~(VPECONF0_XTC)) - | (t->index << VPECONF0_XTC_SHIFT)); - - back_to_back_c0_hazard(); - - /* enable this VPE */ - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_VPA); - - /* clear out any left overs from a previous program */ - write_vpe_c0_status(0); - write_vpe_c0_cause(0); - - /* take system out of configuration state */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - - /* - * SMTC/SMVP kernels manage VPE enable independently, - * but uniprocessor kernels need to turn it on, even - * if that wasn't the pre-dvpe() state. - */ -#ifdef CONFIG_SMP - evpe(vpeflags); -#else - evpe(EVPE_ENABLE); -#endif - emt(dmt_flag); - local_irq_restore(flags); - - list_for_each_entry(n, &v->notify, list) - n->start(minor); - - return 0; -} - -static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, +static int find_vpe_symbols(struct vpe *v, Elf_Shdr *sechdrs, unsigned int symindex, const char *strtab, struct module *mod) { @@ -821,16 +555,14 @@ static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, unsigned int i, n = sechdrs[symindex].sh_size / sizeof(Elf_Sym); for (i = 1; i < n; i++) { - if (strcmp(strtab + sym[i].st_name, "__start") == 0) { + if (strcmp(strtab + sym[i].st_name, "__start") == 0) v->__start = sym[i].st_value; - } - if (strcmp(strtab + sym[i].st_name, "vpe_shared") == 0) { + if (strcmp(strtab + sym[i].st_name, "vpe_shared") == 0) v->shared_ptr = (void *)sym[i].st_value; - } } - if ( (v->__start == 0) || (v->shared_ptr == NULL)) + if ((v->__start == 0) || (v->shared_ptr == NULL)) return -1; return 0; @@ -841,14 +573,14 @@ static int find_vpe_symbols(struct vpe * v, Elf_Shdr * sechdrs, * contents of the program (p)buffer performing relocatations/etc, free's it * when finished. */ -static int vpe_elfload(struct vpe * v) +static int vpe_elfload(struct vpe *v) { Elf_Ehdr *hdr; Elf_Shdr *sechdrs; long err = 0; char *secstrings, *strtab = NULL; unsigned int len, i, symindex = 0, strindex = 0, relocate = 0; - struct module mod; // so we can re-use the relocations code + struct module mod; /* so we can re-use the relocations code */ memset(&mod, 0, sizeof(struct module)); strcpy(mod.name, "VPE loader"); @@ -862,8 +594,7 @@ static int vpe_elfload(struct vpe * v) || (hdr->e_type != ET_REL && hdr->e_type != ET_EXEC) || !elf_check_arch(hdr) || hdr->e_shentsize != sizeof(*sechdrs)) { - printk(KERN_WARNING - "VPE loader: program wrong arch or weird elf version\n"); + pr_warn("VPE loader: program wrong arch or weird elf version\n"); return -ENOEXEC; } @@ -872,8 +603,7 @@ static int vpe_elfload(struct vpe * v) relocate = 1; if (len < hdr->e_shoff + hdr->e_shnum * sizeof(Elf_Shdr)) { - printk(KERN_ERR "VPE loader: program length %u truncated\n", - len); + pr_err("VPE loader: program length %u truncated\n", len); return -ENOEXEC; } @@ -888,22 +618,24 @@ static int vpe_elfload(struct vpe * v) if (relocate) { for (i = 1; i < hdr->e_shnum; i++) { - if (sechdrs[i].sh_type != SHT_NOBITS - && len < sechdrs[i].sh_offset + sechdrs[i].sh_size) { - printk(KERN_ERR "VPE program length %u truncated\n", + if ((sechdrs[i].sh_type != SHT_NOBITS) && + (len < sechdrs[i].sh_offset + sechdrs[i].sh_size)) { + pr_err("VPE program length %u truncated\n", len); return -ENOEXEC; } /* Mark all sections sh_addr with their address in the temporary image. */ - sechdrs[i].sh_addr = (size_t) hdr + sechdrs[i].sh_offset; + sechdrs[i].sh_addr = (size_t) hdr + + sechdrs[i].sh_offset; /* Internal symbols and strings. */ if (sechdrs[i].sh_type == SHT_SYMTAB) { symindex = i; strindex = sechdrs[i].sh_link; - strtab = (char *)hdr + sechdrs[strindex].sh_offset; + strtab = (char *)hdr + + sechdrs[strindex].sh_offset; } } layout_sections(&mod, hdr, sechdrs, secstrings); @@ -930,39 +662,41 @@ static int vpe_elfload(struct vpe * v) /* Update sh_addr to point to copy in image. */ sechdrs[i].sh_addr = (unsigned long)dest; - printk(KERN_DEBUG " section sh_name %s sh_addr 0x%x\n", - secstrings + sechdrs[i].sh_name, sechdrs[i].sh_addr); + pr_debug(" section sh_name %s sh_addr 0x%x\n", + secstrings + sechdrs[i].sh_name, + sechdrs[i].sh_addr); } - /* Fix up syms, so that st_value is a pointer to location. */ - simplify_symbols(sechdrs, symindex, strtab, secstrings, - hdr->e_shnum, &mod); + /* Fix up syms, so that st_value is a pointer to location. */ + simplify_symbols(sechdrs, symindex, strtab, secstrings, + hdr->e_shnum, &mod); - /* Now do relocations. */ - for (i = 1; i < hdr->e_shnum; i++) { - const char *strtab = (char *)sechdrs[strindex].sh_addr; - unsigned int info = sechdrs[i].sh_info; + /* Now do relocations. */ + for (i = 1; i < hdr->e_shnum; i++) { + const char *strtab = (char *)sechdrs[strindex].sh_addr; + unsigned int info = sechdrs[i].sh_info; - /* Not a valid relocation section? */ - if (info >= hdr->e_shnum) - continue; + /* Not a valid relocation section? */ + if (info >= hdr->e_shnum) + continue; - /* Don't bother with non-allocated sections */ - if (!(sechdrs[info].sh_flags & SHF_ALLOC)) - continue; + /* Don't bother with non-allocated sections */ + if (!(sechdrs[info].sh_flags & SHF_ALLOC)) + continue; - if (sechdrs[i].sh_type == SHT_REL) - err = apply_relocations(sechdrs, strtab, symindex, i, - &mod); - else if (sechdrs[i].sh_type == SHT_RELA) - err = apply_relocate_add(sechdrs, strtab, symindex, i, - &mod); - if (err < 0) - return err; + if (sechdrs[i].sh_type == SHT_REL) + err = apply_relocations(sechdrs, strtab, + symindex, i, &mod); + else if (sechdrs[i].sh_type == SHT_RELA) + err = apply_relocate_add(sechdrs, strtab, + symindex, i, &mod); + if (err < 0) + return err; - } - } else { - struct elf_phdr *phdr = (struct elf_phdr *) ((char *)hdr + hdr->e_phoff); + } + } else { + struct elf_phdr *phdr = (struct elf_phdr *) + ((char *)hdr + hdr->e_phoff); for (i = 0; i < hdr->e_phnum; i++) { if (phdr->p_type == PT_LOAD) { @@ -976,16 +710,20 @@ static int vpe_elfload(struct vpe * v) } for (i = 0; i < hdr->e_shnum; i++) { - /* Internal symbols and strings. */ - if (sechdrs[i].sh_type == SHT_SYMTAB) { - symindex = i; - strindex = sechdrs[i].sh_link; - strtab = (char *)hdr + sechdrs[strindex].sh_offset; - - /* mark the symtab's address for when we try to find the - magic symbols */ - sechdrs[i].sh_addr = (size_t) hdr + sechdrs[i].sh_offset; - } + /* Internal symbols and strings. */ + if (sechdrs[i].sh_type == SHT_SYMTAB) { + symindex = i; + strindex = sechdrs[i].sh_link; + strtab = (char *)hdr + + sechdrs[strindex].sh_offset; + + /* + * mark symtab's address for when we try + * to find the magic symbols + */ + sechdrs[i].sh_addr = (size_t) hdr + + sechdrs[i].sh_offset; + } } } @@ -995,53 +733,19 @@ static int vpe_elfload(struct vpe * v) if ((find_vpe_symbols(v, sechdrs, symindex, strtab, &mod)) < 0) { if (v->__start == 0) { - printk(KERN_WARNING "VPE loader: program does not contain " - "a __start symbol\n"); + pr_warn("VPE loader: program does not contain a __start symbol\n"); return -ENOEXEC; } if (v->shared_ptr == NULL) - printk(KERN_WARNING "VPE loader: " - "program does not contain vpe_shared symbol.\n" - " Unable to use AMVP (AP/SP) facilities.\n"); + pr_warn("VPE loader: program does not contain vpe_shared symbol.\n" + " Unable to use AMVP (AP/SP) facilities.\n"); } - printk(" elf loaded\n"); + pr_info(" elf loaded\n"); return 0; } -static void cleanup_tc(struct tc *tc) -{ - unsigned long flags; - unsigned int mtflags, vpflags; - int tmp; - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(tc->index); - tmp = read_tc_c0_tcstatus(); - - /* mark not allocated and not dynamically allocatable */ - tmp &= ~(TCSTATUS_A | TCSTATUS_DA); - tmp |= TCSTATUS_IXMT; /* interrupt exempt */ - write_tc_c0_tcstatus(tmp); - - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* bind it to anything other than VPE1 */ -// write_tc_c0_tcbind(read_tc_c0_tcbind() & ~TCBIND_CURVPE); // | TCBIND_CURVPE - - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); -} - static int getcwd(char *buff, int size) { mm_segment_t old_fs; @@ -1057,60 +761,53 @@ static int getcwd(char *buff, int size) return ret; } -/* checks VPE is unused and gets ready to load program */ +/* checks VPE is unused and gets ready to load program */ static int vpe_open(struct inode *inode, struct file *filp) { enum vpe_state state; - struct vpe_notifications *not; + struct vpe_notifications *notifier; struct vpe *v; int ret; - if (minor != iminor(inode)) { + if (VPE_MODULE_MINOR != iminor(inode)) { /* assume only 1 device at the moment. */ - pr_warning("VPE loader: only vpe1 is supported\n"); + pr_warn("VPE loader: only vpe1 is supported\n"); return -ENODEV; } - if ((v = get_vpe(tclimit)) == NULL) { - pr_warning("VPE loader: unable to get vpe\n"); + v = get_vpe(aprp_cpu_index()); + if (v == NULL) { + pr_warn("VPE loader: unable to get vpe\n"); return -ENODEV; } state = xchg(&v->state, VPE_STATE_INUSE); if (state != VPE_STATE_UNUSED) { - printk(KERN_DEBUG "VPE loader: tc in use dumping regs\n"); + pr_debug("VPE loader: tc in use dumping regs\n"); - list_for_each_entry(not, &v->notify, list) { - not->stop(tclimit); - } + list_for_each_entry(notifier, &v->notify, list) + notifier->stop(aprp_cpu_index()); release_progmem(v->load_addr); - cleanup_tc(get_tc(tclimit)); + cleanup_tc(get_tc(aprp_cpu_index())); } /* this of-course trashes what was there before... */ v->pbuffer = vmalloc(P_SIZE); + if (!v->pbuffer) { + pr_warn("VPE loader: unable to allocate memory\n"); + return -ENOMEM; + } v->plen = P_SIZE; v->load_addr = NULL; v->len = 0; - v->uid = filp->f_cred->fsuid; - v->gid = filp->f_cred->fsgid; - -#ifdef CONFIG_MIPS_APSP_KSPD - /* get kspd to tell us when a syscall_exit happens */ - if (!kspd_events_reqd) { - kspd_notify(&kspd_events); - kspd_events_reqd++; - } -#endif - v->cwd[0] = 0; ret = getcwd(v->cwd, VPE_PATH_MAX); if (ret < 0) - printk(KERN_WARNING "VPE loader: open, getcwd returned %d\n", ret); + pr_warn("VPE loader: open, getcwd returned %d\n", ret); v->shared_ptr = NULL; v->__start = 0; @@ -1124,20 +821,20 @@ static int vpe_release(struct inode *inode, struct file *filp) Elf_Ehdr *hdr; int ret = 0; - v = get_vpe(tclimit); + v = get_vpe(aprp_cpu_index()); if (v == NULL) return -ENODEV; hdr = (Elf_Ehdr *) v->pbuffer; if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) == 0) { - if (vpe_elfload(v) >= 0) { + if ((vpe_elfload(v) >= 0) && vpe_run) { vpe_run(v); } else { - printk(KERN_WARNING "VPE loader: ELF load failed.\n"); + pr_warn("VPE loader: ELF load failed.\n"); ret = -ENOEXEC; } } else { - printk(KERN_WARNING "VPE loader: only elf files are supported\n"); + pr_warn("VPE loader: only elf files are supported\n"); ret = -ENOEXEC; } @@ -1149,34 +846,28 @@ static int vpe_release(struct inode *inode, struct file *filp) if (ret < 0) v->shared_ptr = NULL; - // cleanup any temp buffers - if (v->pbuffer) - vfree(v->pbuffer); + vfree(v->pbuffer); v->plen = 0; + return ret; } -static ssize_t vpe_write(struct file *file, const char __user * buffer, - size_t count, loff_t * ppos) +static ssize_t vpe_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) { size_t ret = count; struct vpe *v; - if (iminor(file->f_path.dentry->d_inode) != minor) + if (iminor(file_inode(file)) != VPE_MODULE_MINOR) return -ENODEV; - v = get_vpe(tclimit); + v = get_vpe(aprp_cpu_index()); + if (v == NULL) return -ENODEV; - if (v->pbuffer == NULL) { - printk(KERN_ERR "VPE loader: no buffer for program\n"); - return -ENOMEM; - } - if ((count + v->len) > v->plen) { - printk(KERN_WARNING - "VPE loader: elf size too big. Perhaps strip uneeded symbols\n"); + pr_warn("VPE loader: elf size too big. Perhaps strip uneeded symbols\n"); return -ENOMEM; } @@ -1188,7 +879,7 @@ static ssize_t vpe_write(struct file *file, const char __user * buffer, return ret; } -static const struct file_operations vpe_fops = { +const struct file_operations vpe_fops = { .owner = THIS_MODULE, .open = vpe_open, .release = vpe_release, @@ -1196,427 +887,40 @@ static const struct file_operations vpe_fops = { .llseek = noop_llseek, }; -/* module wrapper entry points */ -/* give me a vpe */ -vpe_handle vpe_alloc(void) -{ - int i; - struct vpe *v; - - /* find a vpe */ - for (i = 1; i < MAX_VPES; i++) { - if ((v = get_vpe(i)) != NULL) { - v->state = VPE_STATE_INUSE; - return v; - } - } - return NULL; -} - -EXPORT_SYMBOL(vpe_alloc); - -/* start running from here */ -int vpe_start(vpe_handle vpe, unsigned long start) -{ - struct vpe *v = vpe; - - v->__start = start; - return vpe_run(v); -} - -EXPORT_SYMBOL(vpe_start); - -/* halt it for now */ -int vpe_stop(vpe_handle vpe) -{ - struct vpe *v = vpe; - struct tc *t; - unsigned int evpe_flags; - - evpe_flags = dvpe(); - - if ((t = list_entry(v->tc.next, struct tc, tc)) != NULL) { - - settc(t->index); - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); - } - - evpe(evpe_flags); - - return 0; -} - -EXPORT_SYMBOL(vpe_stop); - -/* I've done with it thank you */ -int vpe_free(vpe_handle vpe) -{ - struct vpe *v = vpe; - struct tc *t; - unsigned int evpe_flags; - - if ((t = list_entry(v->tc.next, struct tc, tc)) == NULL) { - return -ENOEXEC; - } - - evpe_flags = dvpe(); - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - settc(t->index); - write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() & ~VPECONF0_VPA); - - /* halt the TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - /* mark the TC unallocated */ - write_tc_c0_tcstatus(read_tc_c0_tcstatus() & ~TCSTATUS_A); - - v->state = VPE_STATE_UNUSED; - - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(evpe_flags); - - return 0; -} - -EXPORT_SYMBOL(vpe_free); - void *vpe_get_shared(int index) { - struct vpe *v; + struct vpe *v = get_vpe(index); - if ((v = get_vpe(index)) == NULL) + if (v == NULL) return NULL; return v->shared_ptr; } - EXPORT_SYMBOL(vpe_get_shared); -int vpe_getuid(int index) -{ - struct vpe *v; - - if ((v = get_vpe(index)) == NULL) - return -1; - - return v->uid; -} - -EXPORT_SYMBOL(vpe_getuid); - -int vpe_getgid(int index) -{ - struct vpe *v; - - if ((v = get_vpe(index)) == NULL) - return -1; - - return v->gid; -} - -EXPORT_SYMBOL(vpe_getgid); - int vpe_notify(int index, struct vpe_notifications *notify) { - struct vpe *v; + struct vpe *v = get_vpe(index); - if ((v = get_vpe(index)) == NULL) + if (v == NULL) return -1; list_add(¬ify->list, &v->notify); return 0; } - EXPORT_SYMBOL(vpe_notify); char *vpe_getcwd(int index) { - struct vpe *v; + struct vpe *v = get_vpe(index); - if ((v = get_vpe(index)) == NULL) + if (v == NULL) return NULL; return v->cwd; } - EXPORT_SYMBOL(vpe_getcwd); -#ifdef CONFIG_MIPS_APSP_KSPD -static void kspd_sp_exit( int sp_id) -{ - cleanup_tc(get_tc(sp_id)); -} -#endif - -static ssize_t store_kill(struct device *dev, struct device_attribute *attr, - const char *buf, size_t len) -{ - struct vpe *vpe = get_vpe(tclimit); - struct vpe_notifications *not; - - list_for_each_entry(not, &vpe->notify, list) { - not->stop(tclimit); - } - - release_progmem(vpe->load_addr); - cleanup_tc(get_tc(tclimit)); - vpe_stop(vpe); - vpe_free(vpe); - - return len; -} - -static ssize_t show_ntcs(struct device *cd, struct device_attribute *attr, - char *buf) -{ - struct vpe *vpe = get_vpe(tclimit); - - return sprintf(buf, "%d\n", vpe->ntcs); -} - -static ssize_t store_ntcs(struct device *dev, struct device_attribute *attr, - const char *buf, size_t len) -{ - struct vpe *vpe = get_vpe(tclimit); - unsigned long new; - char *endp; - - new = simple_strtoul(buf, &endp, 0); - if (endp == buf) - goto out_einval; - - if (new == 0 || new > (hw_tcs - tclimit)) - goto out_einval; - - vpe->ntcs = new; - - return len; - -out_einval: - return -EINVAL; -} - -static struct device_attribute vpe_class_attributes[] = { - __ATTR(kill, S_IWUSR, NULL, store_kill), - __ATTR(ntcs, S_IRUGO | S_IWUSR, show_ntcs, store_ntcs), - {} -}; - -static void vpe_device_release(struct device *cd) -{ - kfree(cd); -} - -struct class vpe_class = { - .name = "vpe", - .owner = THIS_MODULE, - .dev_release = vpe_device_release, - .dev_attrs = vpe_class_attributes, -}; - -struct device vpe_device; - -static int __init vpe_module_init(void) -{ - unsigned int mtflags, vpflags; - unsigned long flags, val; - struct vpe *v = NULL; - struct tc *t; - int tc, err; - - if (!cpu_has_mipsmt) { - printk("VPE loader: not a MIPS MT capable processor\n"); - return -ENODEV; - } - - if (vpelimit == 0) { - printk(KERN_WARNING "No VPEs reserved for AP/SP, not " - "initializing VPE loader.\nPass maxvpes=<n> argument as " - "kernel argument\n"); - - return -ENODEV; - } - - if (tclimit == 0) { - printk(KERN_WARNING "No TCs reserved for AP/SP, not " - "initializing VPE loader.\nPass maxtcs=<n> argument as " - "kernel argument\n"); - - return -ENODEV; - } - - major = register_chrdev(0, module_name, &vpe_fops); - if (major < 0) { - printk("VPE loader: unable to register character device\n"); - return major; - } - - err = class_register(&vpe_class); - if (err) { - printk(KERN_ERR "vpe_class registration failed\n"); - goto out_chrdev; - } - - device_initialize(&vpe_device); - vpe_device.class = &vpe_class, - vpe_device.parent = NULL, - dev_set_name(&vpe_device, "vpe1"); - vpe_device.devt = MKDEV(major, minor); - err = device_add(&vpe_device); - if (err) { - printk(KERN_ERR "Adding vpe_device failed\n"); - goto out_class; - } - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - - /* Put MVPE's into 'configuration state' */ - set_c0_mvpcontrol(MVPCONTROL_VPC); - - /* dump_mtregs(); */ - - val = read_c0_mvpconf0(); - hw_tcs = (val & MVPCONF0_PTC) + 1; - hw_vpes = ((val & MVPCONF0_PVPE) >> MVPCONF0_PVPE_SHIFT) + 1; - - for (tc = tclimit; tc < hw_tcs; tc++) { - /* - * Must re-enable multithreading temporarily or in case we - * reschedule send IPIs or similar we might hang. - */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); - t = alloc_tc(tc); - if (!t) { - err = -ENOMEM; - goto out; - } - - local_irq_save(flags); - mtflags = dmt(); - vpflags = dvpe(); - set_c0_mvpcontrol(MVPCONTROL_VPC); - - /* VPE's */ - if (tc < hw_tcs) { - settc(tc); - - if ((v = alloc_vpe(tc)) == NULL) { - printk(KERN_WARNING "VPE: unable to allocate VPE\n"); - - goto out_reenable; - } - - v->ntcs = hw_tcs - tclimit; - - /* add the tc to the list of this vpe's tc's. */ - list_add(&t->tc, &v->tc); - - /* deactivate all but vpe0 */ - if (tc >= tclimit) { - unsigned long tmp = read_vpe_c0_vpeconf0(); - - tmp &= ~VPECONF0_VPA; - - /* master VPE */ - tmp |= VPECONF0_MVP; - write_vpe_c0_vpeconf0(tmp); - } - - /* disable multi-threading with TC's */ - write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() & ~VPECONTROL_TE); - - if (tc >= vpelimit) { - /* - * Set config to be the same as vpe0, - * particularly kseg0 coherency alg - */ - write_vpe_c0_config(read_c0_config()); - } - } - - /* TC's */ - t->pvpe = v; /* set the parent vpe */ - - if (tc >= tclimit) { - unsigned long tmp; - - settc(tc); - - /* Any TC that is bound to VPE0 gets left as is - in case - we are running SMTC on VPE0. A TC that is bound to any - other VPE gets bound to VPE0, ideally I'd like to make - it homeless but it doesn't appear to let me bind a TC - to a non-existent VPE. Which is perfectly reasonable. - - The (un)bound state is visible to an EJTAG probe so may - notify GDB... - */ - - if (((tmp = read_tc_c0_tcbind()) & TCBIND_CURVPE)) { - /* tc is bound >vpe0 */ - write_tc_c0_tcbind(tmp & ~TCBIND_CURVPE); - - t->pvpe = get_vpe(0); /* set the parent vpe */ - } - - /* halt the TC */ - write_tc_c0_tchalt(TCHALT_H); - mips_ihb(); - - tmp = read_tc_c0_tcstatus(); - - /* mark not activated and not dynamically allocatable */ - tmp &= ~(TCSTATUS_A | TCSTATUS_DA); - tmp |= TCSTATUS_IXMT; /* interrupt exempt */ - write_tc_c0_tcstatus(tmp); - } - } - -out_reenable: - /* release config state */ - clear_c0_mvpcontrol(MVPCONTROL_VPC); - - evpe(vpflags); - emt(mtflags); - local_irq_restore(flags); - -#ifdef CONFIG_MIPS_APSP_KSPD - kspd_events.kspd_sp_exit = kspd_sp_exit; -#endif - return 0; - -out_class: - class_unregister(&vpe_class); -out_chrdev: - unregister_chrdev(major, module_name); - -out: - return err; -} - -static void __exit vpe_module_exit(void) -{ - struct vpe *v, *n; - - device_del(&vpe_device); - unregister_chrdev(major, module_name); - - /* No locking needed here */ - list_for_each_entry_safe(v, n, &vpecontrol.vpe_list, list) { - if (v->state != VPE_STATE_UNUSED) - release_vpe(v); - } -} - module_init(vpe_module_init); module_exit(vpe_module_exit); MODULE_DESCRIPTION("MIPS VPE Loader"); diff --git a/arch/mips/kernel/watch.c b/arch/mips/kernel/watch.c index c1540696803..2a03abb5bd2 100644 --- a/arch/mips/kernel/watch.c +++ b/arch/mips/kernel/watch.c @@ -12,7 +12,7 @@ #include <asm/watch.h> /* - * Install the watch registers for the current thread. A maximum of + * Install the watch registers for the current thread. A maximum of * four registers are installed although the machine may have more. */ void mips_install_watch_registers(void) @@ -72,7 +72,7 @@ void mips_read_watch_registers(void) } /* - * Disable all watch registers. Although only four registers are + * Disable all watch registers. Although only four registers are * installed, all are cleared to eliminate the possibility of endless * looping in the watch handler. */ @@ -100,7 +100,7 @@ void mips_clear_watch_registers(void) } } -__cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) +void mips_probe_watch_registers(struct cpuinfo_mips *c) { unsigned int t; @@ -111,6 +111,7 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) * disable the register. */ write_c0_watchlo0(7); + back_to_back_c0_hazard(); t = read_c0_watchlo0(); write_c0_watchlo0(0); c->watch_reg_masks[0] = t & 7; @@ -121,12 +122,14 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) c->watch_reg_use_cnt = 1; t = read_c0_watchhi0(); write_c0_watchhi0(t | 0xff8); + back_to_back_c0_hazard(); t = read_c0_watchhi0(); c->watch_reg_masks[0] |= (t & 0xff8); if ((t & 0x80000000) == 0) return; write_c0_watchlo1(7); + back_to_back_c0_hazard(); t = read_c0_watchlo1(); write_c0_watchlo1(0); c->watch_reg_masks[1] = t & 7; @@ -135,12 +138,14 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) c->watch_reg_use_cnt = 2; t = read_c0_watchhi1(); write_c0_watchhi1(t | 0xff8); + back_to_back_c0_hazard(); t = read_c0_watchhi1(); c->watch_reg_masks[1] |= (t & 0xff8); if ((t & 0x80000000) == 0) return; write_c0_watchlo2(7); + back_to_back_c0_hazard(); t = read_c0_watchlo2(); write_c0_watchlo2(0); c->watch_reg_masks[2] = t & 7; @@ -149,12 +154,14 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) c->watch_reg_use_cnt = 3; t = read_c0_watchhi2(); write_c0_watchhi2(t | 0xff8); + back_to_back_c0_hazard(); t = read_c0_watchhi2(); c->watch_reg_masks[2] |= (t & 0xff8); if ((t & 0x80000000) == 0) return; write_c0_watchlo3(7); + back_to_back_c0_hazard(); t = read_c0_watchlo3(); write_c0_watchlo3(0); c->watch_reg_masks[3] = t & 7; @@ -163,6 +170,7 @@ __cpuinit void mips_probe_watch_registers(struct cpuinfo_mips *c) c->watch_reg_use_cnt = 4; t = read_c0_watchhi3(); write_c0_watchhi3(t | 0xff8); + back_to_back_c0_hazard(); t = read_c0_watchhi3(); c->watch_reg_masks[3] |= (t & 0xff8); if ((t & 0x80000000) == 0) |