diff options
Diffstat (limited to 'arch/arm/vfp')
| -rw-r--r-- | arch/arm/vfp/entry.S | 30 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpdouble.c | 2 | ||||
| -rw-r--r-- | arch/arm/vfp/vfphw.S | 60 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpmodule.c | 77 | ||||
| -rw-r--r-- | arch/arm/vfp/vfpsingle.c | 2 |
5 files changed, 117 insertions, 54 deletions
diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S index 323ce1a62bb..fe6ca574d09 100644 --- a/arch/arm/vfp/entry.S +++ b/arch/arm/vfp/entry.S @@ -8,9 +8,12 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include <linux/init.h> +#include <linux/linkage.h> #include <asm/thread_info.h> #include <asm/vfpmacros.h> -#include "../kernel/entry-header.S" +#include <asm/assembler.h> +#include <asm/asm-offsets.h> @ VFP entry point. @ @@ -19,15 +22,10 @@ @ r9 = normal "successful" return address @ r10 = this threads thread_info structure @ lr = unrecognised instruction return address -@ IRQs disabled. +@ IRQs enabled. @ ENTRY(do_vfp) -#ifdef CONFIG_PREEMPT_COUNT - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - add r11, r4, #1 @ increment it - str r11, [r10, #TI_PREEMPT] -#endif - enable_irq + inc_preempt_count r10, r4 ldr r4, .LCvfp ldr r11, [r10, #TI_CPU] @ CPU number add r10, r10, #TI_VFPSTATE @ r10 = workspace @@ -35,12 +33,7 @@ ENTRY(do_vfp) ENDPROC(do_vfp) ENTRY(vfp_null_entry) -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, lr ENDPROC(vfp_null_entry) @@ -53,14 +46,9 @@ ENDPROC(vfp_null_entry) __INIT ENTRY(vfp_testing_entry) -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 ldr r0, VFP_arch_address - str r5, [r0] @ known non-zero value + str r0, [r0] @ set to non-zero value mov pc, r9 @ we have handled the fault ENDPROC(vfp_testing_entry) diff --git a/arch/arm/vfp/vfpdouble.c b/arch/arm/vfp/vfpdouble.c index 6cac43bd1d8..423f56dd402 100644 --- a/arch/arm/vfp/vfpdouble.c +++ b/arch/arm/vfp/vfpdouble.c @@ -866,6 +866,8 @@ vfp_double_multiply_accumulate(int dd, int dn, int dm, u32 fpscr, u32 negate, ch vdp.sign = vfp_sign_negate(vdp.sign); vfp_double_unpack(&vdn, vfp_get_double(dd)); + if (vdn.exponent == 0 && vdn.significand) + vfp_double_normalise_denormal(&vdn); if (negate & NEG_SUBTRACT) vdn.sign = vfp_sign_negate(vdn.sign); diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index dd5e56f95f3..be807625ed8 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -14,20 +14,25 @@ * r10 points at the start of the private FP workspace in the thread structure * sp points to a struct pt_regs (as defined in include/asm/proc/ptrace.h) */ +#include <linux/init.h> +#include <linux/linkage.h> #include <asm/thread_info.h> #include <asm/vfpmacros.h> #include <linux/kern_levels.h> -#include "../kernel/entry-header.S" +#include <asm/assembler.h> +#include <asm/asm-offsets.h> .macro DBGSTR, str #ifdef DEBUG stmfd sp!, {r0-r3, ip, lr} - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz KERN_DEBUG "VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -35,12 +40,14 @@ #ifdef DEBUG stmfd sp!, {r0-r3, ip, lr} mov r1, \arg - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz KERN_DEBUG "VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -50,12 +57,14 @@ mov r3, \arg3 mov r2, \arg2 mov r1, \arg1 - add r0, pc, #4 + ldr r0, =1f bl printk - b 1f - .asciz KERN_DEBUG "VFP: \str\n" - .balign 4 -1: ldmfd sp!, {r0-r3, ip, lr} + ldmfd sp!, {r0-r3, ip, lr} + + .pushsection .rodata, "a" +1: .ascii KERN_DEBUG "VFP: \str\n" + .byte 0 + .previous #endif .endm @@ -72,6 +81,11 @@ ENTRY(vfp_support_entry) DBGSTR3 "instr %08x pc %08x state %p", r0, r2, r10 + ldr r3, [sp, #S_PSR] @ Neither lazy restore nor FP exceptions + and r3, r3, #MODE_MASK @ are supported in kernel mode + teq r3, #USR_MODE + bne vfp_kmode_exception @ Returns through lr + VFPFMRX r1, FPEXC @ Is the VFP enabled? DBGSTR1 "fpexc %08x", r1 tst r1, #FPEXC_EN @@ -168,12 +182,7 @@ vfp_hw_state_valid: @ else it's one 32-bit instruction, so @ always subtract 4 from the following @ instruction address. -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, r9 @ we think we have handled things @@ -192,12 +201,7 @@ look_for_VFP_exceptions: @ not recognised by VFP DBGSTR "not VFP" -#ifdef CONFIG_PREEMPT_COUNT - get_thread_info r10 - ldr r4, [r10, #TI_PREEMPT] @ get preempt count - sub r11, r4, #1 @ decrement it - str r11, [r10, #TI_PREEMPT] -#endif + dec_preempt_count_ti r10, r4 mov pc, lr process_exception: diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 3b44e0dd0a9..2f37e1d6cb4 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -20,6 +20,7 @@ #include <linux/init.h> #include <linux/uaccess.h> #include <linux/user.h> +#include <linux/export.h> #include <asm/cp15.h> #include <asm/cputype.h> @@ -413,7 +414,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) * If there isn't a second FP instruction, exit now. Note that * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1. */ - if (fpexc ^ (FPEXC_EX | FPEXC_FP2V)) + if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) goto exit; /* @@ -641,13 +642,79 @@ int vfp_restore_user_hwstate(struct user_vfp __user *ufp, static int vfp_hotplug(struct notifier_block *b, unsigned long action, void *hcpu) { - if (action == CPU_DYING || action == CPU_DYING_FROZEN) { - vfp_force_reload((long)hcpu, current_thread_info()); - } else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) + if (action == CPU_DYING || action == CPU_DYING_FROZEN) + vfp_current_hw_state[(long)hcpu] = NULL; + else if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) vfp_enable(NULL); return NOTIFY_OK; } +void vfp_kmode_exception(void) +{ + /* + * If we reach this point, a floating point exception has been raised + * while running in kernel mode. If the NEON/VFP unit was enabled at the + * time, it means a VFP instruction has been issued that requires + * software assistance to complete, something which is not currently + * supported in kernel mode. + * If the NEON/VFP unit was disabled, and the location pointed to below + * is properly preceded by a call to kernel_neon_begin(), something has + * caused the task to be scheduled out and back in again. In this case, + * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should + * be helpful in localizing the problem. + */ + if (fmrx(FPEXC) & FPEXC_EN) + pr_crit("BUG: unsupported FP instruction in kernel mode\n"); + else + pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n"); +} + +#ifdef CONFIG_KERNEL_MODE_NEON + +/* + * Kernel-side NEON support functions + */ +void kernel_neon_begin(void) +{ + struct thread_info *thread = current_thread_info(); + unsigned int cpu; + u32 fpexc; + + /* + * Kernel mode NEON is only allowed outside of interrupt context + * with preemption disabled. This will make sure that the kernel + * mode NEON register contents never need to be preserved. + */ + BUG_ON(in_interrupt()); + cpu = get_cpu(); + + fpexc = fmrx(FPEXC) | FPEXC_EN; + fmxr(FPEXC, fpexc); + + /* + * Save the userland NEON/VFP state. Under UP, + * the owner could be a task other than 'current' + */ + if (vfp_state_in_hw(cpu, thread)) + vfp_save_state(&thread->vfpstate, fpexc); +#ifndef CONFIG_SMP + else if (vfp_current_hw_state[cpu] != NULL) + vfp_save_state(vfp_current_hw_state[cpu], fpexc); +#endif + vfp_current_hw_state[cpu] = NULL; +} +EXPORT_SYMBOL(kernel_neon_begin); + +void kernel_neon_end(void) +{ + /* Disable the NEON/VFP unit. */ + fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); + put_cpu(); +} +EXPORT_SYMBOL(kernel_neon_end); + +#endif /* CONFIG_KERNEL_MODE_NEON */ + /* * VFP support code initialisation. */ @@ -731,4 +798,4 @@ static int __init vfp_init(void) return 0; } -late_initcall(vfp_init); +core_initcall(vfp_init); diff --git a/arch/arm/vfp/vfpsingle.c b/arch/arm/vfp/vfpsingle.c index b252631b406..4f96c1617aa 100644 --- a/arch/arm/vfp/vfpsingle.c +++ b/arch/arm/vfp/vfpsingle.c @@ -915,6 +915,8 @@ vfp_single_multiply_accumulate(int sd, int sn, s32 m, u32 fpscr, u32 negate, cha v = vfp_get_float(sd); pr_debug("VFP: s%u = %08x\n", sd, v); vfp_single_unpack(&vsn, v); + if (vsn.exponent == 0 && vsn.significand) + vfp_single_normalise_denormal(&vsn); if (negate & NEG_SUBTRACT) vsn.sign = vfp_sign_negate(vsn.sign); |