diff options
Diffstat (limited to 'arch/arm/vfp')
| -rw-r--r-- | arch/arm/vfp/entry.S | 30 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpdouble.c | 2 | ||||
| -rw-r--r-- | arch/arm/vfp/vfphw.S | 61 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpmodule.c | 182 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpsingle.c | 2 |
5 files changed, 216 insertions, 61 deletions
diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S index 323ce1a62bb..fe6ca574d09 100644 --- a/arch/arm/vfp/entry.S +++ b/arch/arm/vfp/entry.S @@ -8,9 +8,12 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include <linux/init.h> +#include <linux/linkage.h> #include <asm/thread_info.h> #include <asm/vfpmacros.h> -#include "../kernel/entry-header.S" +#include <asm/assembler.h> +#include <asm/asm-offsets.h> @ VFP entry point. @ @@ -19,15 +22,10 @@ @ r9 = normal "successful" return address @ r10 = this threads thread_info structure @ lr = unrecognised instruction return address -@ IRQs disabled. +@ IRQs enabled. @ ENTRY(do_vfp) -#ifdef CONFIG_PREEMPT_COUNT - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - add r11, r4, #1 @ increment it - str r11, [r10, #TI_PREEMPT] -#endif - enable_irq + inc_preempt_count r10, r4 ldr r4, .LCvfp ldr r11, [r10, #TI_CPU] @ CPU number add r10, r10, #TI_VFPSTATE @ r10 = workspace @@ -35,12 +33,7 @@ ENTRY(do_vfp) ENDPROC(do_vfp) ENTRY(vfp_null_entry) -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, lr ENDPROC(vfp_null_entry) @@ -53,14 +46,9 @@ ENDPROC(vfp_null_entry) __INIT ENTRY(vfp_testing_entry) -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 ldr r0, VFP_arch_address - str r5, [r0] @ known non-zero value + str r0, [r0] @ set to non-zero value mov pc, r9 @ we have handled the fault ENDPROC(vfp_testing_entry) diff --git a/arch/arm/vfp/vfpdouble.c b/arch/arm/vfp/vfpdouble.c index 6cac43bd1d8..423f56dd402 100644 --- a/arch/arm/vfp/vfpdouble.c +++ b/arch/arm/vfp/vfpdouble.c @@ -866,6 +866,8 @@ vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, ch vdp.sign = vfp_sign_negate(vdp.sign); vfp_double_unpack(&vdn, vfp_get_double(dd)); + if (vdn.exponent == 0 && vdn.significand) + vfp_double_normalise_denormal(&vdn); if (negate & NEG_SUBTRACT) vdn.sign = vfp_sign_negate(vdn.sign); diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index 6ff903e5019..be807625ed8 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -14,19 +14,25 @@ * r10 points at the start of the private FP workspace in the thread structure * sp points to a struct pt_regs (as defined in include/asm/proc/ptrace.h) */ +#include <linux/init.h> +#include <linux/linkage.h> #include <asm/thread_info.h> #include <asm/vfpmacros.h> -#include "../kernel/entry-header.S" +#include <linux/kern_levels.h> +#include <asm/assembler.h> +#include <asm/asm-offsets.h> .macro DBGSTR, str #ifdef DEBUG stmfd sp!, {r0-r3, ip, lr} - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz "<7>VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -34,12 +40,14 @@ #ifdef DEBUG stmfd sp!, {r0-r3, ip, lr} mov r1, \arg - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz "<7>VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -49,12 +57,14 @@ mov r3, \arg3 mov r2, \arg2 mov r1, \arg1 - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz "<7>VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -71,6 +81,11 @@ ENTRY(vfp_support_entry) DBGSTR3 "instr %08x pc %08x state %p", r0, r2, r10 + ldr r3, [sp, #S_PSR] @ Neither lazy restore nor FP exceptions + and r3, r3, #MODE_MASK @ are supported in kernel mode + teq r3, #USR_MODE + bne vfp_kmode_exception @ Returns through lr + VFPFMRX r1, FPEXC @ Is the VFP enabled? DBGSTR1 "fpexc %08x", r1 tst r1, #FPEXC_EN @@ -167,12 +182,7 @@ vfp_hw_state_valid: @ else it's one 32-bit instruction, so @ always subtract 4 from the following @ instruction address. -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, r9 @ we think we have handled things @@ -191,12 +201,7 @@ look_for_VFP_exceptions: @ not recognised by VFP DBGSTR "not VFP" -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, lr process_exception: diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 111691c3a26..2f37e1d6cb4 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -18,8 +18,13 @@ #include <linux/sched.h> #include <linux/smp.h> #include <linux/init.h> +#include <linux/uaccess.h> +#include <linux/user.h> +#include <linux/export.h> +#include <asm/cp15.h> #include <asm/cputype.h> +#include <asm/system_info.h> #include <asm/thread_notify.h> #include <asm/vfp.h> @@ -237,11 +242,11 @@ static void vfp_panic(char *reason, u32 inst) { int i; - printk(KERN_ERR "VFP: Error: %s\n", reason); - printk(KERN_ERR "VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n", + pr_err("VFP: Error: %s\n", reason); + pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n", fmrx(FPEXC), fmrx(FPSCR), inst); for (i = 0; i < 32; i += 2) - printk(KERN_ERR "VFP: s%2u: 0x%08x s%2u: 0x%08x\n", + pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n", i, vfp_get_float(i), i+1, vfp_get_float(i+1)); } @@ -448,7 +453,7 @@ static int vfp_pm_suspend(void) /* if vfp is on, then save state for resumption */ if (fpexc & FPEXC_EN) { - printk(KERN_DEBUG "%s: saving vfp state\n", __func__); + pr_debug("%s: saving vfp state\n", __func__); vfp_save_state(&ti->vfpstate, fpexc); /* disable, just in case */ @@ -537,6 +542,93 @@ void vfp_flush_hwstate(struct thread_info *thread) } /* + * Save the current VFP state into the provided structures and prepare + * for entry into a new function (signal handler). + */ +int vfp_preserve_user_clear_hwstate(struct user_vfp __user *ufp, + struct user_vfp_exc __user *ufp_exc) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; + int err = 0; + + /* Ensure that the saved hwstate is up-to-date. */ + vfp_sync_hwstate(thread); + + /* + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. + */ + err |= __copy_to_user(&ufp->fpregs, &hwstate->fpregs, + sizeof(hwstate->fpregs)); + /* + * Copy the status and control register. + */ + __put_user_error(hwstate->fpscr, &ufp->fpscr, err); + + /* + * Copy the exception registers. + */ + __put_user_error(hwstate->fpexc, &ufp_exc->fpexc, err); + __put_user_error(hwstate->fpinst, &ufp_exc->fpinst, err); + __put_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err); + + if (err) + return -EFAULT; + + /* Ensure that VFP is disabled. */ + vfp_flush_hwstate(thread); + + /* + * As per the PCS, clear the length and stride bits for function + * entry. + */ + hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK); + return 0; +} + +/* Sanitise and restore the current VFP state from the provided structures. */ +int vfp_restore_user_hwstate(struct user_vfp __user *ufp, + struct user_vfp_exc __user *ufp_exc) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *hwstate = &thread->vfpstate.hard; + unsigned long fpexc; + int err = 0; + + /* Disable VFP to avoid corrupting the new thread state. */ + vfp_flush_hwstate(thread); + + /* + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. + */ + err |= __copy_from_user(&hwstate->fpregs, &ufp->fpregs, + sizeof(hwstate->fpregs)); + /* + * Copy the status and control register. + */ + __get_user_error(hwstate->fpscr, &ufp->fpscr, err); + + /* + * Sanitise and restore the exception registers. + */ + __get_user_error(fpexc, &ufp_exc->fpexc, err); + + /* Ensure the VFP is enabled. */ + fpexc |= FPEXC_EN; + + /* Ensure FPINST2 is invalid and the exception flag is cleared. */ + fpexc &= ~(FPEXC_EX | FPEXC_FP2V); + hwstate->fpexc = fpexc; + + __get_user_error(hwstate->fpinst, &ufp_exc->fpinst, err); + __get_user_error(hwstate->fpinst2, &ufp_exc->fpinst2, err); + + return err ? -EFAULT : 0; +} + +/* * VFP hardware can lose all context when a CPU goes offline. * As we will be running in SMP mode with CPU hotplug, we will save the * hardware state at every thread switch. We clear our held state when @@ -550,13 +642,79 @@ void vfp_flush_hwstate(struct thread_info *thread) static int vfp_hotplug(struct notifier_block *b, unsigned long action, void *hcpu) { - if (action == CPU_DYING || action == CPU_DYING_FROZEN) { - vfp_force_reload((long)hcpu, current_thread_info()); - } else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) + if (action == CPU_DYING || action == CPU_DYING_FROZEN) + vfp_current_hw_state[(long)hcpu] = NULL; + else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) vfp_enable(NULL); return NOTIFY_OK; } +void vfp_kmode_exception(void) +{ + /* + * If we reach this point, a floating point exception has been raised + * while running in kernel mode. If the NEON/VFP unit was enabled at the + * time, it means a VFP instruction has been issued that requires + * software assistance to complete, something which is not currently + * supported in kernel mode. + * If the NEON/VFP unit was disabled, and the location pointed to below + * is properly preceded by a call to kernel_neon_begin(), something has + * caused the task to be scheduled out and back in again. In this case, + * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should + * be helpful in localizing the problem. + */ + if (fmrx(FPEXC) & FPEXC_EN) + pr_crit("BUG: unsupported FP instruction in kernel mode\n"); + else + pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); +} + +#ifdef CONFIG_KERNEL_MODE_NEON + +/* + * Kernel-side NEON support functions + */ +void kernel_neon_begin(void) +{ + struct thread_info *thread = current_thread_info(); + unsigned int cpu; + u32 fpexc; + + /* + * Kernel mode NEON is only allowed outside of interrupt context + * with preemption disabled. This will make sure that the kernel + * mode NEON register contents never need to be preserved. + */ + BUG_ON(in_interrupt()); + cpu = get_cpu(); + + fpexc = fmrx(FPEXC) | FPEXC_EN; + fmxr(FPEXC, fpexc); + + /* + * Save the userland NEON/VFP state. Under UP, + * the owner could be a task other than 'current' + */ + if (vfp_state_in_hw(cpu, thread)) + vfp_save_state(&thread->vfpstate, fpexc); +#ifndef CONFIG_SMP + else if (vfp_current_hw_state[cpu] != NULL) + vfp_save_state(vfp_current_hw_state[cpu], fpexc); +#endif + vfp_current_hw_state[cpu] = NULL; +} +EXPORT_SYMBOL(kernel_neon_begin); + +void kernel_neon_end(void) +{ + /* Disable the NEON/VFP unit. */ + fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); + put_cpu(); +} +EXPORT_SYMBOL(kernel_neon_end); + +#endif /* CONFIG_KERNEL_MODE_NEON */ + /* * VFP support code initialisation. */ @@ -579,16 +737,16 @@ static int __init vfp_init(void) barrier(); vfp_vector = vfp_null_entry; - printk(KERN_INFO "VFP support v0.3: "); + pr_info("VFP support v0.3: "); if (VFP_arch) - printk("not present\n"); + pr_cont("not present\n"); else if (vfpsid & FPSID_NODOUBLE) { - printk("no double precision support\n"); + pr_cont("no double precision support\n"); } else { hotcpu_notifier(vfp_hotplug, 0); VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT; /* Extract the architecture version */ - printk("implementor %02x architecture %d part %02x variant %x rev %x\n", + pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n", (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT, (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT, (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT, @@ -640,4 +798,4 @@ static int __init vfp_init(void) return 0; } -late_initcall(vfp_init); +core_initcall(vfp_init); diff --git a/arch/arm/vfp/vfpsingle.c b/arch/arm/vfp/vfpsingle.c index b252631b406..4f96c1617aa 100644 --- a/arch/arm/vfp/vfpsingle.c +++ b/arch/arm/vfp/vfpsingle.c @@ -915,6 +915,8 @@ vfp_single_multiply_accumulate(int sd, int sn, s32 m, u32 fpscr, u32 negate, cha v = vfp_get_float(sd); pr_debug("VFP: s%u = %08x\n", sd, v); vfp_single_unpack(&vsn, v); + if (vsn.exponent == 0 && vsn.significand) + vfp_single_normalise_denormal(&vsn); if (negate & NEG_SUBTRACT) vsn.sign = vfp_sign_negate(vsn.sign); |