diff options
Diffstat (limited to 'arch/xtensa/kernel/entry.S')
| -rw-r--r-- | arch/xtensa/kernel/entry.S | 1636 |
1 files changed, 774 insertions, 862 deletions
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index c64a01f71de..ef7f4990722 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -7,15 +7,16 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (C) 2004-2005 by Tensilica Inc. + * Copyright (C) 2004 - 2008 by Tensilica Inc. * * Chris Zankel <chris@zankel.net> * */ #include <linux/linkage.h> -#include <asm/offsets.h> +#include <asm/asm-offsets.h> #include <asm/processor.h> +#include <asm/coprocessor.h> #include <asm/thread_info.h> #include <asm/uaccess.h> #include <asm/unistd.h> @@ -24,14 +25,12 @@ #include <asm/pgtable.h> #include <asm/page.h> #include <asm/signal.h> -#include <xtensa/coreasm.h> +#include <asm/tlbflush.h> +#include <variant/tie-asm.h> /* Unimplemented features. */ -#undef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION #undef KERNEL_STACK_OVERFLOW_CHECK -#undef PREEMPTIBLE_KERNEL -#undef ALLOCA_EXCEPTION_IN_IRAM /* Not well tested. * @@ -91,9 +90,9 @@ * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original value in depc - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave1: a3 + * excsave1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -104,15 +103,14 @@ * excsave has been restored, and * stack pointer (a1) has been set. * - * Note: _user_exception might be at an odd adress. Don't use call0..call12 + * Note: _user_exception might be at an odd address. Don't use call0..call12 */ ENTRY(user_exception) - /* Save a2, a3, and depc, restore excsave_1 and set SP. */ + /* Save a1, a2, a3, and set SP. */ - xsr a3, EXCSAVE_1 - rsr a0, DEPC + rsr a0, depc s32i a1, a2, PT_AREG1 s32i a0, a2, PT_AREG2 s32i a3, a2, PT_AREG3 @@ -124,15 +122,21 @@ _user_exception: /* Save SAR and turn off single stepping */ movi a2, 0 - rsr a3, SAR - wsr a2, ICOUNTLEVEL + rsr a3, sar + xsr a2, icountlevel s32i a3, a1, PT_SAR + s32i a2, a1, PT_ICOUNTLEVEL + +#if XCHAL_HAVE_THREADPTR + rur a2, threadptr + s32i a2, a1, PT_THREADPTR +#endif /* Rotate ws so that the current windowbase is at bit0. */ /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ - rsr a2, WINDOWBASE - rsr a3, WINDOWSTART + rsr a2, windowbase + rsr a3, windowstart ssr a2 s32i a2, a1, PT_WINDOWBASE s32i a3, a1, PT_WINDOWSTART @@ -168,7 +172,7 @@ _user_exception: * We have to save all registers up to the first '1' from * the right, except the current frame (bit 0). * Assume a2 is: 001001000110001 - * All regiser frames starting from the top fiel to the marked '1' + * All register frames starting from the top field to the marked '1' * must be saved. */ @@ -203,32 +207,21 @@ _user_exception: /* WINDOWBASE still in SAR! */ - rsr a2, SAR # original WINDOWBASE + rsr a2, sar # original WINDOWBASE movi a3, 1 ssl a2 sll a3, a3 - wsr a3, WINDOWSTART # set corresponding WINDOWSTART bit - wsr a2, WINDOWBASE # and WINDOWSTART + wsr a3, windowstart # set corresponding WINDOWSTART bit + wsr a2, windowbase # and WINDOWSTART rsync /* We are back to the original stack pointer (a1) */ -2: -#if XCHAL_EXTRA_SA_SIZE - - /* For user exceptions, save the extra state into the user's TCB. - * Note: We must assume that xchal_extra_store_funcbody destroys a2..a15 - */ - - GET_CURRENT(a2,a1) - addi a2, a2, THREAD_CP_SAVE - xchal_extra_store_funcbody -#endif - - /* Now, jump to the common exception handler. */ +2: /* Now, jump to the common exception handler. */ j common_exception +ENDPROC(user_exception) /* * First-level exit handler for kernel exceptions @@ -242,9 +235,9 @@ _user_exception: * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -255,15 +248,14 @@ _user_exception: * excsave has been restored, and * stack pointer (a1) has been set. * - * Note: _kernel_exception might be at an odd adress. Don't use call0..call12 + * Note: _kernel_exception might be at an odd address. Don't use call0..call12 */ ENTRY(kernel_exception) - /* Save a0, a2, a3, DEPC and set SP. */ + /* Save a1, a2, a3, and set SP. */ - xsr a3, EXCSAVE_1 # restore a3, excsave_1 - rsr a0, DEPC # get a2 + rsr a0, depc # get a2 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_AREG2 s32i a3, a2, PT_AREG3 @@ -275,15 +267,16 @@ _kernel_exception: /* Save SAR and turn off single stepping */ movi a2, 0 - rsr a3, SAR - wsr a2, ICOUNTLEVEL + rsr a3, sar + xsr a2, icountlevel s32i a3, a1, PT_SAR + s32i a2, a1, PT_ICOUNTLEVEL /* Rotate ws so that the current windowbase is at bit0. */ /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ - rsr a2, WINDOWBASE # don't need to save these, we only - rsr a3, WINDOWSTART # need shifted windowstart: windowmask + rsr a2, windowbase # don't need to save these, we only + rsr a3, windowstart # need shifted windowstart: windowmask ssr a2 slli a2, a3, 32-WSBITS src a2, a3, a2 @@ -330,24 +323,26 @@ _kernel_exception: common_exception: - /* Save EXCVADDR, DEBUGCAUSE, and PC, and clear LCOUNT */ + /* Save some registers, disable loops and clear the syscall flag. */ - rsr a2, DEBUGCAUSE - rsr a3, EPC_1 + rsr a2, debugcause + rsr a3, epc1 s32i a2, a1, PT_DEBUGCAUSE s32i a3, a1, PT_PC - rsr a3, EXCVADDR + movi a2, -1 + rsr a3, excvaddr + s32i a2, a1, PT_SYSCALL movi a2, 0 s32i a3, a1, PT_EXCVADDR - xsr a2, LCOUNT + xsr a2, lcount s32i a2, a1, PT_LCOUNT /* It is now save to restore the EXC_TABLE_FIXUP variable. */ - rsr a0, EXCCAUSE + rsr a0, exccause movi a3, 0 - rsr a2, EXCSAVE_1 + rsr a2, excsave1 s32i a0, a1, PT_EXCCAUSE s32i a3, a2, EXC_TABLE_FIXUP @@ -355,34 +350,62 @@ common_exception: * so we can allow exceptions and interrupts (*) again. * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X) * - * (*) We only allow interrupts if PS.INTLEVEL was not set to 1 before - * (interrupts disabled) and if this exception is not an interrupt. + * (*) We only allow interrupts if they were previously enabled and + * we're not handling an IRQ */ - rsr a3, PS - addi a0, a0, -4 - movi a2, 1 - extui a3, a3, 0, 1 # a3 = PS.INTLEVEL[0] - moveqz a3, a2, a0 # a3 = 1 iff interrupt exception - movi a2, PS_WOE_MASK + rsr a3, ps + addi a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT + movi a2, LOCKLEVEL + extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH + # a3 = PS.INTLEVEL + moveqz a3, a2, a0 # a3 = LOCKLEVEL iff interrupt + movi a2, 1 << PS_WOE_BIT or a3, a3, a2 - rsr a0, EXCCAUSE - xsr a3, PS + rsr a0, exccause + xsr a3, ps s32i a3, a1, PT_PS # save ps - /* Save LBEG, LEND */ + /* Save lbeg, lend */ - rsr a2, LBEG - rsr a3, LEND + rsr a2, lbeg + rsr a3, lend s32i a2, a1, PT_LBEG s32i a3, a1, PT_LEND + /* Save SCOMPARE1 */ + +#if XCHAL_HAVE_S32C1I + rsr a2, scompare1 + s32i a2, a1, PT_SCOMPARE1 +#endif + + /* Save optional registers. */ + + save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT + +#ifdef CONFIG_TRACE_IRQFLAGS + l32i a4, a1, PT_DEPC + /* Double exception means we came here with an exception + * while PS.EXCM was set, i.e. interrupts disabled. + */ + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f + l32i a4, a1, PT_EXCCAUSE + bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f + /* We came here with an interrupt means interrupts were enabled + * and we've just disabled them. + */ + movi a4, trace_hardirqs_off + callx4 a4 +1: +#endif + /* Go to second-level dispatcher. Set up parameters to pass to the * exception handler and call the exception handler. */ - movi a4, exc_table + rsr a4, excsave1 mov a6, a1 # pass stack frame mov a7, a0 # pass EXCCAUSE addx4 a4, a0, a4 @@ -393,110 +416,113 @@ common_exception: callx4 a4 /* Jump here for exception exit */ - + .global common_exception_return common_exception_return: +1: + rsil a2, LOCKLEVEL + /* Jump if we are returning from kernel exceptions. */ -1: l32i a3, a1, PT_PS - _bbsi.l a3, PS_UM_SHIFT, 2f - j kernel_exception_exit + l32i a3, a1, PT_PS + GET_THREAD_INFO(a2, a1) + l32i a4, a2, TI_FLAGS + _bbci.l a3, PS_UM_BIT, 6f /* Specific to a user exception exit: * We need to check some flags for signal handling and rescheduling, * and have to restore WB and WS, extra states, and all registers * in the register file that were in use in the user task. + * Note that we don't disable interrupts here. */ -2: wsr a3, PS /* disable interrupts */ - - /* Check for signals (keep interrupts disabled while we read TI_FLAGS) - * Note: PS.INTLEVEL = 0, PS.EXCM = 1 - */ - - GET_THREAD_INFO(a2,a1) - l32i a4, a2, TI_FLAGS - - /* Enable interrupts again. - * Note: When we get here, we certainly have handled any interrupts. - * (Hint: There is only one user exception frame on stack) - */ - - movi a3, PS_WOE_MASK - _bbsi.l a4, TIF_NEED_RESCHED, 3f - _bbci.l a4, TIF_SIGPENDING, 4f + _bbsi.l a4, TIF_NOTIFY_RESUME, 2f + _bbci.l a4, TIF_SIGPENDING, 5f -#ifndef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION - l32i a4, a1, PT_DEPC +2: l32i a4, a1, PT_DEPC bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f -#endif - /* Reenable interrupts and call do_signal() */ + /* Call do_signal() */ - wsr a3, PS - movi a4, do_signal # int do_signal(struct pt_regs*, sigset_t*) + rsil a2, 0 + movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*) mov a6, a1 - movi a7, 0 callx4 a4 j 1b -3: /* Reenable interrupts and reschedule */ +3: /* Reschedule */ - wsr a3, PS + rsil a2, 0 movi a4, schedule # void schedule (void) callx4 a4 j 1b - /* Restore the state of the task and return from the exception. */ - - - /* If we are returning from a user exception, and the process - * to run next has PT_SINGLESTEP set, we want to setup - * ICOUNT and ICOUNTLEVEL to step one instruction. - * PT_SINGLESTEP is set by sys_ptrace (ptrace.c) - */ - -4: /* a2 holds GET_CURRENT(a2,a1) */ +#ifdef CONFIG_PREEMPT +6: + _bbci.l a4, TIF_NEED_RESCHED, 4f - l32i a3, a2, TI_TASK - l32i a3, a3, TASK_PTRACE - bbci.l a3, PT_SINGLESTEP_BIT, 1f # jump if single-step flag is not set + /* Check current_thread_info->preempt_count */ - movi a3, -2 # PT_SINGLESTEP flag is set, - movi a4, 1 # icountlevel of 1 means it won't - wsr a3, ICOUNT # start counting until after rfe - wsr a4, ICOUNTLEVEL # so setup icount & icountlevel. - isync + l32i a4, a2, TI_PRE_COUNT + bnez a4, 4f + movi a4, preempt_schedule_irq + callx4 a4 + j 1b +#endif +5: +#ifdef CONFIG_DEBUG_TLB_SANITY + l32i a4, a1, PT_DEPC + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f + movi a4, check_tlb_sanity + callx4 a4 +#endif +6: +4: +#ifdef CONFIG_TRACE_IRQFLAGS + l32i a4, a1, PT_DEPC + /* Double exception means we came here with an exception + * while PS.EXCM was set, i.e. interrupts disabled. + */ + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f + l32i a4, a1, PT_EXCCAUSE + bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f + /* We came here with an interrupt means interrupts were enabled + * and we'll reenable them on return. + */ + movi a4, trace_hardirqs_on + callx4 a4 1: +#endif + /* Restore optional registers. */ -#if XCHAL_EXTRA_SA_SIZE + load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT - /* For user exceptions, restore the extra state from the user's TCB. */ + /* Restore SCOMPARE1 */ - /* Note: a2 still contains GET_CURRENT(a2,a1) */ - addi a2, a2, THREAD_CP_SAVE - xchal_extra_load_funcbody +#if XCHAL_HAVE_S32C1I + l32i a2, a1, PT_SCOMPARE1 + wsr a2, scompare1 +#endif + wsr a3, ps /* disable interrupts */ - /* We must assume that xchal_extra_store_funcbody destroys - * registers a2..a15. FIXME, this list can eventually be - * reduced once real register requirements of the macro are - * finalized. */ + _bbci.l a3, PS_UM_BIT, kernel_exception_exit -#endif /* XCHAL_EXTRA_SA_SIZE */ +user_exception_exit: + /* Restore the state of the task and return from the exception. */ /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */ l32i a2, a1, PT_WINDOWBASE l32i a3, a1, PT_WINDOWSTART - wsr a1, DEPC # use DEPC as temp storage - wsr a3, WINDOWSTART # restore WINDOWSTART + wsr a1, depc # use DEPC as temp storage + wsr a3, windowstart # restore WINDOWSTART ssr a2 # preserve user's WB in the SAR - wsr a2, WINDOWBASE # switch to user's saved WB + wsr a2, windowbase # switch to user's saved WB rsync - rsr a1, DEPC # restore stack pointer + rsr a1, depc # restore stack pointer l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9) rotw -1 # we restore a4..a7 _bltui a6, 16, 1f # only have to restore current window? @@ -522,8 +548,8 @@ common_exception_return: /* Clear unrestored registers (don't leak anything to user-land */ -1: rsr a0, WINDOWBASE - rsr a3, SAR +1: rsr a0, windowbase + rsr a3, sar sub a3, a0, a3 beqz a3, 2f extui a3, a3, 0, WBBITS @@ -542,6 +568,11 @@ common_exception_return: * (if we have restored WSBITS-1 frames). */ +#if XCHAL_HAVE_THREADPTR + l32i a3, a1, PT_THREADPTR + wur a3, threadptr +#endif + 2: j common_exception_exit /* This is the kernel exception exit. @@ -551,33 +582,6 @@ common_exception_return: kernel_exception_exit: - /* Disable interrupts (a3 holds PT_PS) */ - - wsr a3, PS - -#ifdef PREEMPTIBLE_KERNEL - -#ifdef CONFIG_PREEMPT - - /* - * Note: We've just returned from a call4, so we have - * at least 4 addt'l regs. - */ - - /* Check current_thread_info->preempt_count */ - - GET_THREAD_INFO(a2) - l32i a3, a2, TI_PREEMPT - bnez a3, 1f - - l32i a2, a2, TI_FLAGS - -1: - -#endif - -#endif - /* Check if we have to do a movsp. * * We only have to do a movsp if the previous window-frame has @@ -607,7 +611,7 @@ kernel_exception_exit: /* Test WINDOWSTART now. If spilled, do the movsp */ - rsr a3, WINDOWSTART + rsr a3, windowstart addi a0, a3, -1 and a3, a3, a0 _bnez a3, common_exception_exit @@ -633,6 +637,8 @@ kernel_exception_exit: common_exception_exit: + /* Restore address registers. */ + _bbsi.l a2, 1, 1f l32i a4, a1, PT_AREG4 l32i a5, a1, PT_AREG5 @@ -653,17 +659,24 @@ common_exception_exit: 1: l32i a2, a1, PT_PC l32i a3, a1, PT_SAR - wsr a2, EPC_1 - wsr a3, SAR + wsr a2, epc1 + wsr a3, sar /* Restore LBEG, LEND, LCOUNT */ l32i a2, a1, PT_LBEG l32i a3, a1, PT_LEND - wsr a2, LBEG + wsr a2, lbeg l32i a2, a1, PT_LCOUNT - wsr a3, LEND - wsr a2, LCOUNT + wsr a3, lend + wsr a2, lcount + + /* We control single stepping through the ICOUNTLEVEL register. */ + + l32i a2, a1, PT_ICOUNTLEVEL + movi a3, -2 + wsr a2, icountlevel + wsr a3, icount /* Check if it was double exception. */ @@ -678,11 +691,13 @@ common_exception_exit: l32i a1, a1, PT_AREG1 rfe -1: wsr a0, DEPC +1: wsr a0, depc l32i a0, a1, PT_AREG0 l32i a1, a1, PT_AREG1 rfde +ENDPROC(kernel_exception) + /* * Debug exception handler. * @@ -693,48 +708,48 @@ common_exception_exit: ENTRY(debug_exception) - rsr a0, EPS + XCHAL_DEBUGLEVEL - bbsi.l a0, PS_EXCM_SHIFT, 1f # exception mode + rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL + bbsi.l a0, PS_EXCM_BIT, 1f # exception mode - /* Set EPC_1 and EXCCAUSE */ + /* Set EPC1 and EXCCAUSE */ - wsr a2, DEPC # save a2 temporarily - rsr a2, EPC + XCHAL_DEBUGLEVEL - wsr a2, EPC_1 + wsr a2, depc # save a2 temporarily + rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL + wsr a2, epc1 movi a2, EXCCAUSE_MAPPED_DEBUG - wsr a2, EXCCAUSE + wsr a2, exccause /* Restore PS to the value before the debug exc but with PS.EXCM set.*/ - movi a2, 1 << PS_EXCM_SHIFT + movi a2, 1 << PS_EXCM_BIT or a2, a0, a2 movi a0, debug_exception # restore a3, debug jump vector - wsr a2, PS - xsr a0, EXCSAVE + XCHAL_DEBUGLEVEL + wsr a2, ps + xsr a0, SREG_EXCSAVE + XCHAL_DEBUGLEVEL /* Switch to kernel/user stack, restore jump vector, and save a0 */ - bbsi.l a2, PS_UM_SHIFT, 2f # jump if user mode + bbsi.l a2, PS_UM_BIT, 2f # jump if user mode addi a2, a1, -16-PT_SIZE # assume kernel stack s32i a0, a2, PT_AREG0 movi a0, 0 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_DEPC # mark it as a regular exception - xsr a0, DEPC + xsr a0, depc s32i a3, a2, PT_AREG3 s32i a0, a2, PT_AREG2 mov a1, a2 j _kernel_exception -2: rsr a2, EXCSAVE_1 +2: rsr a2, excsave1 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer s32i a0, a2, PT_AREG0 movi a0, 0 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_DEPC - xsr a0, DEPC + xsr a0, depc s32i a3, a2, PT_AREG3 s32i a0, a2, PT_AREG2 mov a1, a2 @@ -743,6 +758,7 @@ ENTRY(debug_exception) /* Debug exception while in exception mode. */ 1: j 1b // FIXME!! +ENDPROC(debug_exception) /* * We get here in case of an unrecoverable exception. @@ -774,12 +790,12 @@ ENTRY(unrecoverable_exception) movi a0, 1 movi a1, 0 - wsr a0, WINDOWSTART - wsr a1, WINDOWBASE + wsr a0, windowstart + wsr a1, windowbase rsync - movi a1, PS_WOE_MASK | 1 - wsr a1, PS + movi a1, (1 << PS_WOE_BIT) | LOCKLEVEL + wsr a1, ps rsync movi a1, init_task @@ -793,6 +809,7 @@ ENTRY(unrecoverable_exception) 1: j 1b +ENDPROC(unrecoverable_exception) /* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */ @@ -801,176 +818,64 @@ ENTRY(unrecoverable_exception) * * The ALLOCA handler is entered when user code executes the MOVSP * instruction and the caller's frame is not in the register file. - * In this case, the caller frame's a0..a3 are on the stack just - * below sp (a1), and this handler moves them. * - * For "MOVSP <ar>,<as>" without destination register a1, this routine - * simply moves the value from <as> to <ar> without moving the save area. + * This algorithm was taken from the Ross Morley's RTOS Porting Layer: + * + * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S + * + * It leverages the existing window spill/fill routines and their support for + * double exceptions. The 'movsp' instruction will only cause an exception if + * the next window needs to be loaded. In fact this ALLOCA exception may be + * replaced at some point by changing the hardware to do a underflow exception + * of the proper size instead. + * + * This algorithm simply backs out the register changes started by the user + * excpetion handler, makes it appear that we have started a window underflow + * by rotating the window back and then setting the old window base (OWB) in + * the 'ps' register with the rolled back window base. The 'movsp' instruction + * will be re-executed and this time since the next window frames is in the + * active AR registers it won't cause an exception. + * + * If the WindowUnderflow code gets a TLB miss the page will get mapped + * the the partial windeowUnderflow will be handeled in the double exception + * handler. * * Entry condition: * * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception */ -#if XCHAL_HAVE_BE -#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4 -#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4 -#else -#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4 -#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4 -#endif - ENTRY(fast_alloca) + rsr a0, windowbase + rotw -1 + rsr a2, ps + extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH + xor a3, a3, a4 + l32i a4, a6, PT_AREG0 + l32i a1, a6, PT_DEPC + rsr a6, depc + wsr a1, depc + slli a3, a3, PS_OWB_SHIFT + xor a2, a2, a3 + wsr a2, ps + rsync - /* We shouldn't be in a double exception. */ - - l32i a0, a2, PT_DEPC - _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double - - rsr a0, DEPC # get a2 - s32i a4, a2, PT_AREG4 # save a4 and - s32i a0, a2, PT_AREG2 # a2 to stack - - /* Exit critical section. */ - - movi a0, 0 - s32i a0, a3, EXC_TABLE_FIXUP - - /* Restore a3, excsave_1 */ - - xsr a3, EXCSAVE_1 # make sure excsave_1 is valid for dbl. - rsr a4, EPC_1 # get exception address - s32i a3, a2, PT_AREG3 # save a3 to stack - -#ifdef ALLOCA_EXCEPTION_IN_IRAM -#error iram not supported -#else - /* Note: l8ui not allowed in IRAM/IROM!! */ - l8ui a0, a4, 1 # read as(src) from MOVSP instruction -#endif - movi a3, .Lmovsp_src - _EXTUI_MOVSP_SRC(a0) # extract source register number - addx8 a3, a0, a3 - jx a3 - -.Lunhandled_double: - wsr a0, EXCSAVE_1 - movi a0, unrecoverable_exception - callx0 a0 - - .align 8 -.Lmovsp_src: - l32i a3, a2, PT_AREG0; _j 1f; .align 8 - mov a3, a1; _j 1f; .align 8 - l32i a3, a2, PT_AREG2; _j 1f; .align 8 - l32i a3, a2, PT_AREG3; _j 1f; .align 8 - l32i a3, a2, PT_AREG4; _j 1f; .align 8 - mov a3, a5; _j 1f; .align 8 - mov a3, a6; _j 1f; .align 8 - mov a3, a7; _j 1f; .align 8 - mov a3, a8; _j 1f; .align 8 - mov a3, a9; _j 1f; .align 8 - mov a3, a10; _j 1f; .align 8 - mov a3, a11; _j 1f; .align 8 - mov a3, a12; _j 1f; .align 8 - mov a3, a13; _j 1f; .align 8 - mov a3, a14; _j 1f; .align 8 - mov a3, a15; _j 1f; .align 8 - -1: - -#ifdef ALLOCA_EXCEPTION_IN_IRAM -#error iram not supported -#else - l8ui a0, a4, 0 # read ar(dst) from MOVSP instruction -#endif - addi a4, a4, 3 # step over movsp - _EXTUI_MOVSP_DST(a0) # extract destination register - wsr a4, EPC_1 # save new epc_1 - - _bnei a0, 1, 1f # no 'movsp a1, ax': jump - - /* Move the save area. This implies the use of the L32E - * and S32E instructions, because this move must be done with - * the user's PS.RING privilege levels, not with ring 0 - * (kernel's) privileges currently active with PS.EXCM - * set. Note that we have stil registered a fixup routine with the - * double exception vector in case a double exception occurs. - */ - - /* a0,a4:avail a1:old user stack a2:exc. stack a3:new user stack. */ - - l32e a0, a1, -16 - l32e a4, a1, -12 - s32e a0, a3, -16 - s32e a4, a3, -12 - l32e a0, a1, -8 - l32e a4, a1, -4 - s32e a0, a3, -8 - s32e a4, a3, -4 - - /* Restore stack-pointer and all the other saved registers. */ - - mov a1, a3 - - l32i a4, a2, PT_AREG4 - l32i a3, a2, PT_AREG3 - l32i a0, a2, PT_AREG0 - l32i a2, a2, PT_AREG2 - rfe - - /* MOVSP <at>,<as> was invoked with <at> != a1. - * Because the stack pointer is not being modified, - * we should be able to just modify the pointer - * without moving any save area. - * The processor only traps these occurrences if the - * caller window isn't live, so unfortunately we can't - * use this as an alternate trap mechanism. - * So we just do the move. This requires that we - * resolve the destination register, not just the source, - * so there's some extra work. - * (PERHAPS NOT REALLY NEEDED, BUT CLEANER...) - */ - - /* a0 dst-reg, a1 user-stack, a2 stack, a3 value of src reg. */ - -1: movi a4, .Lmovsp_dst - addx8 a4, a0, a4 - jx a4 - - .align 8 -.Lmovsp_dst: - s32i a3, a2, PT_AREG0; _j 1f; .align 8 - mov a1, a3; _j 1f; .align 8 - s32i a3, a2, PT_AREG2; _j 1f; .align 8 - s32i a3, a2, PT_AREG3; _j 1f; .align 8 - s32i a3, a2, PT_AREG4; _j 1f; .align 8 - mov a5, a3; _j 1f; .align 8 - mov a6, a3; _j 1f; .align 8 - mov a7, a3; _j 1f; .align 8 - mov a8, a3; _j 1f; .align 8 - mov a9, a3; _j 1f; .align 8 - mov a10, a3; _j 1f; .align 8 - mov a11, a3; _j 1f; .align 8 - mov a12, a3; _j 1f; .align 8 - mov a13, a3; _j 1f; .align 8 - mov a14, a3; _j 1f; .align 8 - mov a15, a3; _j 1f; .align 8 - -1: l32i a4, a2, PT_AREG4 - l32i a3, a2, PT_AREG3 - l32i a0, a2, PT_AREG0 - l32i a2, a2, PT_AREG2 - rfe - + _bbci.l a4, 31, 4f + rotw -1 + _bbci.l a8, 30, 8f + rotw -1 + j _WindowUnderflow12 +8: j _WindowUnderflow8 +4: j _WindowUnderflow4 +ENDPROC(fast_alloca) /* * fast system calls. @@ -986,81 +891,80 @@ ENTRY(fast_alloca) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table */ ENTRY(fast_syscall_kernel) /* Skip syscall. */ - rsr a0, EPC_1 + rsr a0, epc1 addi a0, a0, 3 - wsr a0, EPC_1 + wsr a0, epc1 l32i a0, a2, PT_DEPC bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable - rsr a0, DEPC # get syscall-nr + rsr a0, depc # get syscall-nr _beqz a0, fast_syscall_spill_registers - - addi a0, a0, -__NR_sysxtensa - _beqz a0, fast_syscall_sysxtensa + _beqi a0, __NR_xtensa, fast_syscall_xtensa j kernel_exception +ENDPROC(fast_syscall_kernel) ENTRY(fast_syscall_user) /* Skip syscall. */ - rsr a0, EPC_1 + rsr a0, epc1 addi a0, a0, 3 - wsr a0, EPC_1 + wsr a0, epc1 l32i a0, a2, PT_DEPC bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable - rsr a0, DEPC # get syscall-nr + rsr a0, depc # get syscall-nr _beqz a0, fast_syscall_spill_registers - - addi a0, a0, -__NR_sysxtensa - _beqz a0, fast_syscall_sysxtensa + _beqi a0, __NR_xtensa, fast_syscall_xtensa j user_exception -ENTRY(fast_syscall_unrecoverable) +ENDPROC(fast_syscall_user) - /* Restore all states. */ +ENTRY(fast_syscall_unrecoverable) - l32i a0, a2, PT_AREG0 # restore a0 - xsr a2, DEPC # restore a2, depc - rsr a3, EXCSAVE_1 + /* Restore all states. */ - wsr a0, EXCSAVE_1 - movi a0, unrecoverable_exception - callx0 a0 + l32i a0, a2, PT_AREG0 # restore a0 + xsr a2, depc # restore a2, depc + wsr a0, excsave1 + movi a0, unrecoverable_exception + callx0 a0 +ENDPROC(fast_syscall_unrecoverable) /* * sysxtensa syscall handler * - * int sysxtensa (XTENSA_ATOMIC_SET, ptr, val, unused); - * int sysxtensa (XTENSA_ATOMIC_ADD, ptr, val, unused); - * int sysxtensa (XTENSA_ATOMIC_EXG_ADD, ptr, val, unused); - * int sysxtensa (XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval); - * a2 a6 a3 a4 a5 + * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused); + * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused); + * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused); + * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval); + * a2 a6 a3 a4 a5 * * Entry condition: * - * a0: trashed, original value saved on stack (PT_AREG0) + * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0) * a1: a1 - * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a2: new stack pointer, original in a0 and DEPC + * a3: a3 + * a4..a15: unchanged * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1072,7 +976,7 @@ ENTRY(fast_syscall_unrecoverable) * TRY adds an entry to the __ex_table fixup table for the immediately * following instruction. * - * CATCH catches any exception that occurred at one of the preceeding TRY + * CATCH catches any exception that occurred at one of the preceding TRY * statements and continues from there * * Usage TRY l32i a0, a1, 0 @@ -1091,60 +995,61 @@ ENTRY(fast_syscall_unrecoverable) #define CATCH \ 67: -ENTRY(fast_syscall_sysxtensa) - - _beqz a6, 1f - _blti a6, SYSXTENSA_COUNT, 2f - -1: j user_exception - -2: xsr a3, EXCSAVE_1 # restore a3, excsave1 - s32i a7, a2, PT_AREG7 +ENTRY(fast_syscall_xtensa) + s32i a7, a2, PT_AREG7 # we need an additional register movi a7, 4 # sizeof(unsigned int) - verify_area a3, a7, a0, a2, .Leac + access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp - _beqi a6, SYSXTENSA_ATOMIC_SET, .Lset - _beqi a6, SYSXTENSA_ATOMIC_EXG_ADD, .Lexg - _beqi a6, SYSXTENSA_ATOMIC_ADD, .Ladd + addi a6, a6, -1 # assuming SYS_XTENSA_ATOMIC_SET = 1 + _bgeui a6, SYS_XTENSA_COUNT - 1, .Lill + _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP - 1, .Lnswp - /* Fall through for SYSXTENSA_ATOMIC_CMP_SWP */ + /* Fall through for ATOMIC_CMP_SWP. */ .Lswp: /* Atomic compare and swap */ -TRY l32i a7, a3, 0 # read old value - bne a7, a4, 1f # same as old value? jump - s32i a5, a3, 0 # different, modify value - movi a7, 1 # and return 1 - j .Lret - -1: movi a7, 0 # same values: return 0 - j .Lret - -.Ladd: /* Atomic add */ -.Lexg: /* Atomic (exchange) add */ +TRY l32i a0, a3, 0 # read old value + bne a0, a4, 1f # same as old value? jump +TRY s32i a5, a3, 0 # different, modify value + l32i a7, a2, PT_AREG7 # restore a7 + l32i a0, a2, PT_AREG0 # restore a0 + movi a2, 1 # and return 1 + addi a6, a6, 1 # restore a6 (really necessary?) + rfe -TRY l32i a7, a3, 0 - add a4, a4, a7 - s32i a4, a3, 0 - j .Lret +1: l32i a7, a2, PT_AREG7 # restore a7 + l32i a0, a2, PT_AREG0 # restore a0 + movi a2, 0 # return 0 (note that we cannot set + addi a6, a6, 1 # restore a6 (really necessary?) + rfe -.Lset: /* Atomic set */ +.Lnswp: /* Atomic set, add, and exg_add. */ -TRY l32i a7, a3, 0 # read old value as return value - s32i a4, a3, 0 # write new value +TRY l32i a7, a3, 0 # orig + add a0, a4, a7 # + arg + moveqz a0, a4, a6 # set +TRY s32i a0, a3, 0 # write new value -.Lret: mov a0, a2 + mov a0, a2 mov a2, a7 - l32i a7, a0, PT_AREG7 - l32i a3, a0, PT_AREG3 - l32i a0, a0, PT_AREG0 + l32i a7, a0, PT_AREG7 # restore a7 + l32i a0, a0, PT_AREG0 # restore a0 + addi a6, a6, 1 # restore a6 (really necessary?) rfe CATCH -.Leac: movi a7, -EFAULT - j .Lret +.Leac: l32i a7, a2, PT_AREG7 # restore a7 + l32i a0, a2, PT_AREG0 # restore a0 + movi a2, -EFAULT + rfe +.Lill: l32i a7, a2, PT_AREG0 # restore a7 + l32i a0, a2, PT_AREG0 # restore a0 + movi a2, -EINVAL + rfe + +ENDPROC(fast_syscall_xtensa) /* fast_syscall_spill_registers. @@ -1154,231 +1059,79 @@ CATCH * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler. - * Note: We don't need to save a2 in depc (return value) */ ENTRY(fast_syscall_spill_registers) /* Register a FIXUP handler (pass current wb as a parameter) */ + xsr a3, excsave1 movi a0, fast_syscall_spill_registers_fixup s32i a0, a3, EXC_TABLE_FIXUP - rsr a0, WINDOWBASE + rsr a0, windowbase s32i a0, a3, EXC_TABLE_PARAM + xsr a3, excsave1 # restore a3 and excsave_1 - /* Save a3 and SAR on stack. */ + /* Save a3, a4 and SAR on stack. */ - rsr a0, SAR - xsr a3, EXCSAVE_1 # restore a3 and excsave_1 - s32i a0, a2, PT_AREG4 # store SAR to PT_AREG4 + rsr a0, sar s32i a3, a2, PT_AREG3 + s32i a0, a2, PT_SAR - /* The spill routine might clobber a7, a11, and a15. */ - - s32i a7, a2, PT_AREG5 - s32i a11, a2, PT_AREG6 - s32i a15, a2, PT_AREG7 - - call0 _spill_registers # destroys a3, DEPC, and SAR - - /* Advance PC, restore registers and SAR, and return from exception. */ - - l32i a3, a2, PT_AREG4 - l32i a0, a2, PT_AREG0 - wsr a3, SAR - l32i a3, a2, PT_AREG3 - - /* Restore clobbered registers. */ - - l32i a7, a2, PT_AREG5 - l32i a11, a2, PT_AREG6 - l32i a15, a2, PT_AREG7 - - movi a2, 0 - rfe - -/* Fixup handler. - * - * We get here if the spill routine causes an exception, e.g. tlb miss. - * We basically restore WINDOWBASE and WINDOWSTART to the condition when - * we entered the spill routine and jump to the user exception handler. - * - * a0: value of depc, original value in depc - * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE - * a3: exctable, original value in excsave1 - */ - -fast_syscall_spill_registers_fixup: - - rsr a2, WINDOWBASE # get current windowbase (a2 is saved) - xsr a0, DEPC # restore depc and a0 - ssl a2 # set shift (32 - WB) - - /* We need to make sure the current registers (a0-a3) are preserved. - * To do this, we simply set the bit for the current window frame - * in WS, so that the exception handlers save them to the task stack. - */ - - rsr a3, EXCSAVE_1 # get spill-mask - slli a2, a3, 1 # shift left by one - - slli a3, a2, 32-WSBITS - src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy...... - wsr a2, WINDOWSTART # set corrected windowstart - - movi a3, exc_table - l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2 - l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task) - - /* Return to the original (user task) WINDOWBASE. - * We leave the following frame behind: - * a0, a1, a2 same - * a3: trashed (saved in excsave_1) - * depc: depc (we have to return to that address) - * excsave_1: a3 - */ - - wsr a3, WINDOWBASE - rsync - - /* We are now in the original frame when we entered _spill_registers: - * a0: return address - * a1: used, stack pointer - * a2: kernel stack pointer - * a3: available, saved in EXCSAVE_1 - * depc: exception address - * excsave: a3 - * Note: This frame might be the same as above. - */ - -#ifdef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION - /* Restore registers we precautiously saved. - * We have the value of the 'right' a3 - */ - - l32i a7, a2, PT_AREG5 - l32i a11, a2, PT_AREG6 - l32i a15, a2, PT_AREG7 -#endif + /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */ - /* Setup stack pointer. */ - - addi a2, a2, -PT_USER_SIZE - s32i a0, a2, PT_AREG0 - - /* Make sure we return to this fixup handler. */ - - movi a3, fast_syscall_spill_registers_fixup_return - s32i a3, a2, PT_DEPC # setup depc - - /* Jump to the exception handler. */ - - movi a3, exc_table - rsr a0, EXCCAUSE - addx4 a0, a0, a3 # find entry in table - l32i a0, a0, EXC_TABLE_FAST_USER # load handler - jx a0 - -fast_syscall_spill_registers_fixup_return: - - /* When we return here, all registers have been restored (a2: DEPC) */ - - wsr a2, DEPC # exception address - - /* Restore fixup handler. */ - - xsr a3, EXCSAVE_1 - movi a2, fast_syscall_spill_registers_fixup - s32i a2, a3, EXC_TABLE_FIXUP - rsr a2, WINDOWBASE - s32i a2, a3, EXC_TABLE_PARAM - l32i a2, a3, EXC_TABLE_KSTK - -#ifdef SIGNAL_HANDLING_IN_DOUBLE_EXCEPTION - /* Save registers again that might be clobbered. */ - - s32i a7, a2, PT_AREG5 - s32i a11, a2, PT_AREG6 - s32i a15, a2, PT_AREG7 -#endif - - /* Load WB at the time the exception occurred. */ - - rsr a3, SAR # WB is still in SAR - neg a3, a3 - wsr a3, WINDOWBASE - rsync - - /* Restore a3 and return. */ - - movi a3, exc_table - xsr a3, EXCSAVE_1 - - rfde - - -/* - * spill all registers. - * - * This is not a real function. The following conditions must be met: - * - * - must be called with call0. - * - uses DEPC, a3 and SAR. - * - the last 'valid' register of each frame are clobbered. - * - the caller must have registered a fixup handler - * (or be inside a critical section) - * - PS_EXCM must be set (PS_WOE cleared?) - */ - -ENTRY(_spill_registers) + s32i a4, a2, PT_AREG4 + s32i a7, a2, PT_AREG7 + s32i a8, a2, PT_AREG8 + s32i a11, a2, PT_AREG11 + s32i a12, a2, PT_AREG12 + s32i a15, a2, PT_AREG15 /* * Rotate ws so that the current windowbase is at bit 0. * Assume ws = xxxwww1yy (www1 current window frame). - * Rotate ws right so that a2 = yyxxxwww1. + * Rotate ws right so that a4 = yyxxxwww1. */ - wsr a2, DEPC # preserve a2 - rsr a2, WINDOWBASE - rsr a3, WINDOWSTART - ssr a2 # holds WB - slli a2, a3, WSBITS - or a3, a3, a2 # a2 = xxxwww1yyxxxwww1yy - srl a3, a3 + rsr a0, windowbase + rsr a3, windowstart # a3 = xxxwww1yy + ssr a0 # holds WB + slli a0, a3, WSBITS + or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy + srl a3, a3 # a3 = 00xxxwww1yyxxxwww1 /* We are done if there are no more than the current register frame. */ - extui a3, a3, 1, WSBITS-2 # a3 = 0yyxxxwww - movi a2, (1 << (WSBITS-1)) + extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww + movi a0, (1 << (WSBITS-1)) _beqz a3, .Lnospill # only one active frame? jump /* We want 1 at the top, so that we return to the current windowbase */ - or a3, a3, a2 # 1yyxxxwww + or a3, a3, a0 # 1yyxxxwww /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */ - wsr a3, WINDOWSTART # save shifted windowstart - neg a2, a3 - and a3, a2, a3 # first bit set from right: 000010000 + wsr a3, windowstart # save shifted windowstart + neg a0, a3 + and a3, a0, a3 # first bit set from right: 000010000 - ffs_ws a2, a3 # a2: shifts to skip empty frames + ffs_ws a0, a3 # a0: shifts to skip empty frames movi a3, WSBITS - sub a2, a3, a2 # WSBITS-a2:number of 0-bits from right - ssr a2 # save in SAR for later. + sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right + ssr a0 # save in SAR for later. - rsr a3, WINDOWBASE - add a3, a3, a2 - rsr a2, DEPC # restore a2 - wsr a3, WINDOWBASE + rsr a3, windowbase + add a3, a3, a0 + wsr a3, windowbase rsync - rsr a3, WINDOWSTART + rsr a3, windowstart srl a3, a3 # shift windowstart /* WB is now just one frame below the oldest frame in the register @@ -1389,19 +1142,6 @@ ENTRY(_spill_registers) * we have to save 4,8. or 12 registers. */ - _bbsi.l a3, 1, .Lc4 - _bbsi.l a3, 2, .Lc8 - - /* Special case: we have a call12-frame starting at a4. */ - - _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first) - - s32e a4, a1, -16 # a1 is valid with an empty spill area - l32e a4, a5, -12 - s32e a8, a4, -48 - mov a8, a4 - l32e a4, a1, -16 - j .Lc12c .Lloop: _bbsi.l a3, 1, .Lc4 _bbci.l a3, 2, .Lc12 @@ -1415,22 +1155,10 @@ ENTRY(_spill_registers) s32e a9, a4, -28 s32e a10, a4, -24 s32e a11, a4, -20 - srli a11, a3, 2 # shift windowbase by 2 rotw 2 _bnei a3, 1, .Lloop - -.Lexit: /* Done. Do the final rotation, set WS, and return. */ - - rotw 1 - rsr a3, WINDOWBASE - ssl a3 - movi a3, 1 - sll a3, a3 - wsr a3, WINDOWSTART - -.Lnospill: - jx a0 + j .Lexit .Lc4: s32e a4, a9, -16 s32e a5, a9, -12 @@ -1446,11 +1174,11 @@ ENTRY(_spill_registers) /* 12-register frame (call12) */ - l32e a2, a5, -12 - s32e a8, a2, -48 - mov a8, a2 + l32e a0, a5, -12 + s32e a8, a0, -48 + mov a8, a0 -.Lc12c: s32e a9, a8, -44 + s32e a9, a8, -44 s32e a10, a8, -40 s32e a11, a8, -36 s32e a12, a8, -32 @@ -1470,61 +1198,209 @@ ENTRY(_spill_registers) */ rotw 1 - mov a5, a13 + mov a4, a13 rotw -1 - s32e a4, a9, -16 - s32e a5, a9, -12 - s32e a6, a9, -8 - s32e a7, a9, -4 + s32e a4, a8, -16 + s32e a5, a8, -12 + s32e a6, a8, -8 + s32e a7, a8, -4 rotw 3 _beqi a3, 1, .Lexit j .Lloop -.Linvalid_mask: +.Lexit: - /* We get here because of an unrecoverable error in the window - * registers. If we are in user space, we kill the application, - * however, this condition is unrecoverable in kernel space. - */ + /* Done. Do the final rotation and set WS */ + + rotw 1 + rsr a3, windowbase + ssl a3 + movi a3, 1 + sll a3, a3 + wsr a3, windowstart +.Lnospill: + + /* Advance PC, restore registers and SAR, and return from exception. */ + + l32i a3, a2, PT_SAR + l32i a0, a2, PT_AREG0 + wsr a3, sar + l32i a3, a2, PT_AREG3 + + /* Restore clobbered registers. */ + + l32i a4, a2, PT_AREG4 + l32i a7, a2, PT_AREG7 + l32i a8, a2, PT_AREG8 + l32i a11, a2, PT_AREG11 + l32i a12, a2, PT_AREG12 + l32i a15, a2, PT_AREG15 - rsr a0, PS - _bbci.l a0, PS_UM_SHIFT, 1f + movi a2, 0 + rfe + +.Linvalid_mask: - /* User space: Setup a dummy frame and kill application. + /* We get here because of an unrecoverable error in the window + * registers, so set up a dummy frame and kill the user application. * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer. */ movi a0, 1 movi a1, 0 - wsr a0, WINDOWSTART - wsr a1, WINDOWBASE + wsr a0, windowstart + wsr a1, windowbase rsync movi a0, 0 - movi a3, exc_table + rsr a3, excsave1 l32i a1, a3, EXC_TABLE_KSTK - wsr a3, EXCSAVE_1 - movi a4, PS_WOE_MASK | 1 - wsr a4, PS + movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL + wsr a4, ps rsync movi a6, SIGSEGV movi a4, do_exit callx4 a4 -1: /* Kernel space: PANIC! */ + /* shouldn't return, so panic */ - wsr a0, EXCSAVE_1 + wsr a0, excsave1 movi a0, unrecoverable_exception callx0 a0 # should not return 1: j 1b + +ENDPROC(fast_syscall_spill_registers) + +/* Fixup handler. + * + * We get here if the spill routine causes an exception, e.g. tlb miss. + * We basically restore WINDOWBASE and WINDOWSTART to the condition when + * we entered the spill routine and jump to the user exception handler. + * + * Note that we only need to restore the bits in windowstart that have not + * been spilled yet by the _spill_register routine. Luckily, a3 contains a + * rotated windowstart with only those bits set for frames that haven't been + * spilled yet. Because a3 is rotated such that bit 0 represents the register + * frame for the current windowbase - 1, we need to rotate a3 left by the + * value of the current windowbase + 1 and move it to windowstart. + * + * a0: value of depc, original value in depc + * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE + * a3: exctable, original value in excsave1 + */ + +ENTRY(fast_syscall_spill_registers_fixup) + + rsr a2, windowbase # get current windowbase (a2 is saved) + xsr a0, depc # restore depc and a0 + ssl a2 # set shift (32 - WB) + + /* We need to make sure the current registers (a0-a3) are preserved. + * To do this, we simply set the bit for the current window frame + * in WS, so that the exception handlers save them to the task stack. + * + * Note: we use a3 to set the windowbase, so we take a special care + * of it, saving it in the original _spill_registers frame across + * the exception handler call. + */ + + xsr a3, excsave1 # get spill-mask + slli a3, a3, 1 # shift left by one + addi a3, a3, 1 # set the bit for the current window frame + + slli a2, a3, 32-WSBITS + src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy...... + wsr a2, windowstart # set corrected windowstart + + srli a3, a3, 1 + rsr a2, excsave1 + l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2 + xsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3 + l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task) + xsr a2, excsave1 + + /* Return to the original (user task) WINDOWBASE. + * We leave the following frame behind: + * a0, a1, a2 same + * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE) + * depc: depc (we have to return to that address) + * excsave_1: exctable + */ + + wsr a3, windowbase + rsync + + /* We are now in the original frame when we entered _spill_registers: + * a0: return address + * a1: used, stack pointer + * a2: kernel stack pointer + * a3: available + * depc: exception address + * excsave: exctable + * Note: This frame might be the same as above. + */ + + /* Setup stack pointer. */ + + addi a2, a2, -PT_USER_SIZE + s32i a0, a2, PT_AREG0 + + /* Make sure we return to this fixup handler. */ + + movi a3, fast_syscall_spill_registers_fixup_return + s32i a3, a2, PT_DEPC # setup depc + + /* Jump to the exception handler. */ + + rsr a3, excsave1 + rsr a0, exccause + addx4 a0, a0, a3 # find entry in table + l32i a0, a0, EXC_TABLE_FAST_USER # load handler + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + jx a0 + +ENDPROC(fast_syscall_spill_registers_fixup) + +ENTRY(fast_syscall_spill_registers_fixup_return) + + /* When we return here, all registers have been restored (a2: DEPC) */ + + wsr a2, depc # exception address + + /* Restore fixup handler. */ + + rsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE + movi a3, fast_syscall_spill_registers_fixup + s32i a3, a2, EXC_TABLE_FIXUP + rsr a3, windowbase + s32i a3, a2, EXC_TABLE_PARAM + l32i a2, a2, EXC_TABLE_KSTK + + /* Load WB at the time the exception occurred. */ + + rsr a3, sar # WB is still in SAR + neg a3, a3 + wsr a3, windowbase + rsync + + rsr a3, excsave1 + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + + rfde + +ENDPROC(fast_syscall_spill_registers_fixup_return) + +#ifdef CONFIG_MMU /* * We should never get here. Bail out! */ @@ -1535,6 +1411,8 @@ ENTRY(fast_second_level_miss_double_kernel) callx0 a0 # should not return 1: j 1b +ENDPROC(fast_second_level_miss_double_kernel) + /* First-level entry handler for user, kernel, and double 2nd-level * TLB miss exceptions. Note that for now, user and kernel miss * exceptions share the same entry point and are handled identically. @@ -1547,9 +1425,9 @@ ENTRY(fast_second_level_miss_double_kernel) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1557,9 +1435,10 @@ ENTRY(fast_second_level_miss_double_kernel) ENTRY(fast_second_level_miss) - /* Save a1. Note: we don't expect a double exception. */ + /* Save a1 and a3. Note: we don't expect a double exception. */ s32i a1, a2, PT_AREG1 + s32i a3, a2, PT_AREG3 /* We need to map the page of PTEs for the user task. Find * the pointer to that page. Also, it's possible for tsk->mm @@ -1581,10 +1460,9 @@ ENTRY(fast_second_level_miss) l32i a0, a1, TASK_MM # tsk->mm beqz a0, 9f -8: rsr a1, EXCVADDR # fault address - _PGD_OFFSET(a0, a1, a1) +8: rsr a3, excvaddr # fault address + _PGD_OFFSET(a0, a3, a1) l32i a0, a0, 0 # read pmdval - //beqi a0, _PAGE_USER, 2f beqz a0, 2f /* Read ptevaddr and convert to top of page-table page. @@ -1597,28 +1475,42 @@ ENTRY(fast_second_level_miss) * The messy computation for 'pteval' above really simplifies * into the following: * - * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_KERNEL + * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY */ - movi a1, -PAGE_OFFSET + movi a1, (-PAGE_OFFSET) & 0xffffffff add a0, a0, a1 # pmdval - PAGE_OFFSET extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK xor a0, a0, a1 - - movi a1, PAGE_DIRECTORY + movi a1, _PAGE_DIRECTORY or a0, a0, a1 # ... | PAGE_DIRECTORY - rsr a1, PTEVADDR + /* + * We utilize all three wired-ways (7-9) to hold pmd translations. + * Memory regions are mapped to the DTLBs according to bits 28 and 29. + * This allows to map the three most common regions to three different + * DTLBs: + * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000) + * 2 -> way 8 shared libaries (2000.0000) + * 3 -> way 0 stack (3000.0000) + */ + + extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3 + rsr a1, ptevaddr + addx2 a3, a3, a3 # -> 0,3,6,9 srli a1, a1, PAGE_SHIFT + extui a3, a3, 2, 2 # -> 0,0,1,2 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK - addi a1, a1, DTLB_WAY_PGTABLE # ... + way_number + addi a3, a3, DTLB_WAY_PGD + add a1, a1, a3 # ... + way_number - wdtlb a0, a1 +3: wdtlb a0, a1 dsync /* Exit critical section. */ +4: rsr a3, excsave1 movi a0, 0 s32i a0, a3, EXC_TABLE_FIXUP @@ -1626,38 +1518,104 @@ ENTRY(fast_second_level_miss) l32i a0, a2, PT_AREG0 l32i a1, a2, PT_AREG1 + l32i a3, a2, PT_AREG3 l32i a2, a2, PT_DEPC - xsr a3, EXCSAVE_1 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f /* Restore excsave1 and return. */ - rsr a2, DEPC + rsr a2, depc rfe /* Return from double exception. */ -1: xsr a2, DEPC +1: xsr a2, depc esync rfde 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0 j 8b +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +2: /* Special case for cache aliasing. + * We (should) only get here if a clear_user_page, copy_user_page + * or the aliased cache flush functions got preemptively interrupted + * by another task. Re-establish temporary mapping to the + * TLBTEMP_BASE areas. + */ + + /* We shouldn't be in a double exception */ + + l32i a0, a2, PT_DEPC + bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f + + /* Make sure the exception originated in the special functions */ + + movi a0, __tlbtemp_mapping_start + rsr a3, epc1 + bltu a3, a0, 2f + movi a0, __tlbtemp_mapping_end + bgeu a3, a0, 2f + + /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */ + + movi a3, TLBTEMP_BASE_1 + rsr a0, excvaddr + bltu a0, a3, 2f + + addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT)) + bgeu a1, a3, 2f + + /* Check if we have to restore an ITLB mapping. */ + + movi a1, __tlbtemp_mapping_itlb + rsr a3, epc1 + sub a3, a3, a1 + + /* Calculate VPN */ + + movi a1, PAGE_MASK + and a1, a1, a0 + + /* Jump for ITLB entry */ + + bgez a3, 1f + + /* We can use up to two TLBTEMP areas, one for src and one for dst. */ + + extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1 + add a1, a3, a1 + + /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */ + + mov a0, a6 + movnez a0, a7, a3 + j 3b + + /* ITLB entry. We only use dst in a6. */ + +1: witlb a6, a1 + isync + j 4b + + +#endif // DCACHE_WAY_SIZE > PAGE_SIZE + + 2: /* Invalid PGD, default exception handling */ - rsr a1, DEPC - xsr a3, EXCSAVE_1 + rsr a1, depc s32i a1, a2, PT_AREG2 - s32i a3, a2, PT_AREG3 mov a1, a2 - rsr a2, PS - bbsi.l a2, PS_UM_SHIFT, 1f + rsr a2, ps + bbsi.l a2, PS_UM_BIT, 1f j _kernel_exception 1: j _user_exception +ENDPROC(fast_second_level_miss) /* * StoreProhibitedException @@ -1669,9 +1627,9 @@ ENTRY(fast_second_level_miss) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1679,64 +1637,64 @@ ENTRY(fast_second_level_miss) ENTRY(fast_store_prohibited) - /* Save a1 and a4. */ + /* Save a1 and a3. */ s32i a1, a2, PT_AREG1 - s32i a4, a2, PT_AREG4 + s32i a3, a2, PT_AREG3 GET_CURRENT(a1,a2) l32i a0, a1, TASK_MM # tsk->mm beqz a0, 9f -8: rsr a1, EXCVADDR # fault address - _PGD_OFFSET(a0, a1, a4) +8: rsr a1, excvaddr # fault address + _PGD_OFFSET(a0, a1, a3) l32i a0, a0, 0 - //beqi a0, _PAGE_USER, 2f # FIXME use _PAGE_INVALID beqz a0, 2f - _PTE_OFFSET(a0, a1, a4) - l32i a4, a0, 0 # read pteval - movi a1, _PAGE_VALID | _PAGE_RW - bnall a4, a1, 2f + /* + * Note that we test _PAGE_WRITABLE_BIT only if PTE is present + * and is not PAGE_NONE. See pgtable.h for possible PTE layouts. + */ + + _PTE_OFFSET(a0, a1, a3) + l32i a3, a0, 0 # read pteval + movi a1, _PAGE_CA_INVALID + ball a3, a1, 2f + bbci.l a3, _PAGE_WRITABLE_BIT, 2f - movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_WRENABLE - or a4, a4, a1 - rsr a1, EXCVADDR - s32i a4, a0, 0 + movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE + or a3, a3, a1 + rsr a1, excvaddr + s32i a3, a0, 0 /* We need to flush the cache if we have page coloring. */ #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK dhwb a0, 0 #endif pdtlb a0, a1 - beqz a0, 1f - idtlb a0 // FIXME do we need this? - wdtlb a4, a0 -1: + wdtlb a3, a0 /* Exit critical section. */ movi a0, 0 + rsr a3, excsave1 s32i a0, a3, EXC_TABLE_FIXUP /* Restore the working registers, and return. */ - l32i a4, a2, PT_AREG4 + l32i a3, a2, PT_AREG3 l32i a1, a2, PT_AREG1 l32i a0, a2, PT_AREG0 l32i a2, a2, PT_DEPC - /* Restore excsave1 and a3. */ - - xsr a3, EXCSAVE_1 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f - rsr a2, DEPC + rsr a2, depc rfe /* Double exception. Restore FIXUP handler and return. */ -1: xsr a2, DEPC +1: xsr a2, depc esync rfde @@ -1745,166 +1703,111 @@ ENTRY(fast_store_prohibited) 2: /* If there was a problem, handle fault in C */ - rsr a4, DEPC # still holds a2 - xsr a3, EXCSAVE_1 - s32i a4, a2, PT_AREG2 - s32i a3, a2, PT_AREG3 - l32i a4, a2, PT_AREG4 + rsr a3, depc # still holds a2 + s32i a3, a2, PT_AREG2 mov a1, a2 - rsr a2, PS - bbsi.l a2, PS_UM_SHIFT, 1f + rsr a2, ps + bbsi.l a2, PS_UM_BIT, 1f j _kernel_exception 1: j _user_exception +ENDPROC(fast_store_prohibited) -#if XCHAL_EXTRA_SA_SIZE - -#warning fast_coprocessor untested +#endif /* CONFIG_MMU */ /* - * Entry condition: + * System Calls. * - * a0: trashed, original value saved on stack (PT_AREG0) - * a1: a1 - * a2: new stack pointer, original in DEPC - * a3: dispatch table - * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 - * - * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC - * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception + * void system_call (struct pt_regs* regs, int exccause) + * a2 a3 */ -ENTRY(fast_coprocessor_double) - wsr a0, EXCSAVE_1 - movi a0, unrecoverable_exception - callx0 a0 - -ENTRY(fast_coprocessor) - - /* Fatal if we are in a double exception. */ - - l32i a0, a2, PT_DEPC - _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_coprocessor_double - - /* Save some registers a1, a3, a4, SAR */ - - xsr a3, EXCSAVE_1 - s32i a3, a2, PT_AREG3 - rsr a3, SAR - s32i a4, a2, PT_AREG4 - s32i a1, a2, PT_AREG1 - s32i a5, a1, PT_AREG5 - s32i a3, a2, PT_SAR - mov a1, a2 - - /* Currently, the HAL macros only guarantee saving a0 and a1. - * These can and will be refined in the future, but for now, - * just save the remaining registers of a2...a15. - */ - s32i a6, a1, PT_AREG6 - s32i a7, a1, PT_AREG7 - s32i a8, a1, PT_AREG8 - s32i a9, a1, PT_AREG9 - s32i a10, a1, PT_AREG10 - s32i a11, a1, PT_AREG11 - s32i a12, a1, PT_AREG12 - s32i a13, a1, PT_AREG13 - s32i a14, a1, PT_AREG14 - s32i a15, a1, PT_AREG15 - - /* Find coprocessor number. Subtract first CP EXCCAUSE from EXCCAUSE */ - - rsr a0, EXCCAUSE - addi a3, a0, -XCHAL_EXCCAUSE_COPROCESSOR0_DISABLED +ENTRY(system_call) - /* Set corresponding CPENABLE bit */ + entry a1, 32 - movi a4, 1 - ssl a3 # SAR: 32 - coprocessor_number - rsr a5, CPENABLE - sll a4, a4 - or a4, a5, a4 - wsr a4, CPENABLE - rsync - movi a5, coprocessor_info # list of owner and offset into cp_save - addx8 a0, a4, a5 # entry for CP + /* regs->syscall = regs->areg[2] */ - bne a4, a5, .Lload # bit wasn't set before, cp not in use + l32i a3, a2, PT_AREG2 + mov a6, a2 + movi a4, do_syscall_trace_enter + s32i a3, a2, PT_SYSCALL + callx4 a4 - /* Now compare the current task with the owner of the coprocessor. - * If they are the same, there is no reason to save or restore any - * coprocessor state. Having already enabled the coprocessor, - * branch ahead to return. - */ - GET_CURRENT(a5,a1) - l32i a4, a0, COPROCESSOR_INFO_OWNER # a4: current owner for this CP - beq a4, a5, .Ldone + /* syscall = sys_call_table[syscall_nr] */ - /* Find location to dump current coprocessor state: - * task_struct->task_cp_save_offset + coprocessor_offset[coprocessor] - * - * Note: a0 pointer to the entry in the coprocessor owner table, - * a3 coprocessor number, - * a4 current owner of coprocessor. - */ - l32i a5, a0, COPROCESSOR_INFO_OFFSET - addi a2, a4, THREAD_CP_SAVE - add a2, a2, a5 + movi a4, sys_call_table; + movi a5, __NR_syscall_count + movi a6, -ENOSYS + bgeu a3, a5, 1f - /* Store current coprocessor states. (a5 still has CP number) */ + addx4 a4, a3, a4 + l32i a4, a4, 0 + movi a5, sys_ni_syscall; + beq a4, a5, 1f - xchal_cpi_store_funcbody + /* Load args: arg0 - arg5 are passed via regs. */ - /* The macro might have destroyed a3 (coprocessor number), but - * SAR still has 32 - coprocessor_number! - */ - movi a3, 32 - rsr a4, SAR - sub a3, a3, a4 + l32i a6, a2, PT_AREG6 + l32i a7, a2, PT_AREG3 + l32i a8, a2, PT_AREG4 + l32i a9, a2, PT_AREG5 + l32i a10, a2, PT_AREG8 + l32i a11, a2, PT_AREG9 -.Lload: /* A new task now owns the corpocessors. Save its TCB pointer into - * the coprocessor owner table. - * - * Note: a0 pointer to the entry in the coprocessor owner table, - * a3 coprocessor number. - */ - GET_CURRENT(a4,a1) - s32i a4, a0, 0 + /* Pass one additional argument to the syscall: pt_regs (on stack) */ + s32i a2, a1, 0 - /* Find location from where to restore the current coprocessor state.*/ + callx4 a4 - l32i a5, a0, COPROCESSOR_INFO_OFFSET - addi a2, a4, THREAD_CP_SAVE - add a2, a2, a4 +1: /* regs->areg[2] = return_value */ - xchal_cpi_load_funcbody + s32i a6, a2, PT_AREG2 + movi a4, do_syscall_trace_leave + mov a6, a2 + callx4 a4 + retw - /* We must assume that the xchal_cpi_store_funcbody macro destroyed - * registers a2..a15. - */ +ENDPROC(system_call) -.Ldone: l32i a15, a1, PT_AREG15 - l32i a14, a1, PT_AREG14 - l32i a13, a1, PT_AREG13 - l32i a12, a1, PT_AREG12 - l32i a11, a1, PT_AREG11 - l32i a10, a1, PT_AREG10 - l32i a9, a1, PT_AREG9 - l32i a8, a1, PT_AREG8 - l32i a7, a1, PT_AREG7 - l32i a6, a1, PT_AREG6 - l32i a5, a1, PT_AREG5 - l32i a4, a1, PT_AREG4 - l32i a3, a1, PT_AREG3 - l32i a2, a1, PT_AREG2 - l32i a0, a1, PT_AREG0 - l32i a1, a1, PT_AREG1 - - rfe +/* + * Spill live registers on the kernel stack macro. + * + * Entry condition: ps.woe is set, ps.excm is cleared + * Exit condition: windowstart has single bit set + * May clobber: a12, a13 + */ + .macro spill_registers_kernel -#endif /* XCHAL_EXTRA_SA_SIZE */ +#if XCHAL_NUM_AREGS > 16 + call12 1f + _j 2f + retw + .align 4 +1: + _entry a1, 48 + addi a12, a0, 3 +#if XCHAL_NUM_AREGS > 32 + .rept (XCHAL_NUM_AREGS - 32) / 12 + _entry a1, 48 + mov a12, a0 + .endr +#endif + _entry a1, 48 +#if XCHAL_NUM_AREGS % 12 == 0 + mov a8, a8 +#elif XCHAL_NUM_AREGS % 12 == 4 + mov a12, a12 +#elif XCHAL_NUM_AREGS % 12 == 8 + mov a4, a4 +#endif + retw +2: +#else + mov a12, a12 +#endif + .endm /* * Task switch. @@ -1917,20 +1820,35 @@ ENTRY(_switch_to) entry a1, 16 - mov a4, a3 # preserve a3 + mov a10, a2 # preserve 'prev' (a2) + mov a11, a3 # and 'next' (a3) - s32i a0, a2, THREAD_RA # save return address - s32i a1, a2, THREAD_SP # save stack pointer + l32i a4, a2, TASK_THREAD_INFO + l32i a5, a3, TASK_THREAD_INFO - /* Disable ints while we manipulate the stack pointer; spill regs. */ + save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER - movi a5, PS_EXCM_MASK | LOCKLEVEL - xsr a5, PS - rsr a3, EXCSAVE_1 + s32i a0, a10, THREAD_RA # save return address + s32i a1, a10, THREAD_SP # save stack pointer + + /* Disable ints while we manipulate the stack pointer. */ + + rsil a14, LOCKLEVEL + rsr a3, excsave1 rsync s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */ - call0 _spill_registers + /* Switch CPENABLE */ + +#if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS) + l32i a3, a5, THREAD_CPENABLE + xsr a3, cpenable + s32i a3, a4, THREAD_CPENABLE +#endif + + /* Flush register file. */ + + spill_registers_kernel /* Set kernel stack (and leave critical section) * Note: It's save to set it here. The stack will not be overwritten @@ -1938,23 +1856,26 @@ ENTRY(_switch_to) * we return from kernel space. */ - l32i a0, a4, TASK_THREAD_INFO - rsr a3, EXCSAVE_1 # exc_table - movi a1, 0 - addi a0, a0, PT_REGS_OFFSET - s32i a1, a3, EXC_TABLE_FIXUP - s32i a0, a3, EXC_TABLE_KSTK + rsr a3, excsave1 # exc_table + movi a6, 0 + addi a7, a5, PT_REGS_OFFSET + s32i a6, a3, EXC_TABLE_FIXUP + s32i a7, a3, EXC_TABLE_KSTK + + /* restore context of the task 'next' */ - /* restore context of the task that 'next' addresses */ + l32i a0, a11, THREAD_RA # restore return address + l32i a1, a11, THREAD_SP # restore stack pointer - l32i a0, a4, THREAD_RA /* restore return address */ - l32i a1, a4, THREAD_SP /* restore stack pointer */ + load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER - wsr a5, PS + wsr a14, ps + mov a2, a10 # return 'prev' rsync retw +ENDPROC(_switch_to) ENTRY(ret_from_fork) @@ -1964,33 +1885,24 @@ ENTRY(ret_from_fork) movi a4, schedule_tail callx4 a4 - movi a4, do_syscall_trace + movi a4, do_syscall_trace_leave + mov a6, a1 callx4 a4 j common_exception_return - +ENDPROC(ret_from_fork) /* - * Table of syscalls + * Kernel thread creation helper + * On entry, set up by copy_thread: a2 = thread_fn, a3 = thread_fn arg + * left from _switch_to: a6 = prev */ +ENTRY(ret_from_kernel_thread) -.data -.align 4 -.global sys_call_table -sys_call_table: - -#define SYSCALL(call, narg) .word call -#include "syscalls.h" - -/* - * Number of arguments of each syscall - */ - -.global sys_narg_table -sys_narg_table: - -#undef SYSCALL -#define SYSCALL(call, narg) .byte narg -#include "syscalls.h" + call4 schedule_tail + mov a6, a3 + callx4 a2 + j common_exception_return +ENDPROC(ret_from_kernel_thread) |