diff options
Diffstat (limited to 'arch/xtensa/kernel/entry.S')
| -rw-r--r-- | arch/xtensa/kernel/entry.S | 1150 |
1 files changed, 549 insertions, 601 deletions
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index 5fd01f6aaf3..ef7f4990722 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -7,7 +7,7 @@ * License. See the file "COPYING" in the main directory of this archive * for more details. * - * Copyright (C) 2004-2007 by Tensilica Inc. + * Copyright (C) 2004 - 2008 by Tensilica Inc. * * Chris Zankel <chris@zankel.net> * @@ -31,8 +31,6 @@ /* Unimplemented features. */ #undef KERNEL_STACK_OVERFLOW_CHECK -#undef PREEMPTIBLE_KERNEL -#undef ALLOCA_EXCEPTION_IN_IRAM /* Not well tested. * @@ -92,9 +90,9 @@ * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original value in depc - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave1: a3 + * excsave1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -110,10 +108,9 @@ ENTRY(user_exception) - /* Save a2, a3, and depc, restore excsave_1 and set SP. */ + /* Save a1, a2, a3, and set SP. */ - xsr a3, EXCSAVE_1 - rsr a0, DEPC + rsr a0, depc s32i a1, a2, PT_AREG1 s32i a0, a2, PT_AREG2 s32i a3, a2, PT_AREG3 @@ -125,16 +122,21 @@ _user_exception: /* Save SAR and turn off single stepping */ movi a2, 0 - rsr a3, SAR - xsr a2, ICOUNTLEVEL + rsr a3, sar + xsr a2, icountlevel s32i a3, a1, PT_SAR s32i a2, a1, PT_ICOUNTLEVEL +#if XCHAL_HAVE_THREADPTR + rur a2, threadptr + s32i a2, a1, PT_THREADPTR +#endif + /* Rotate ws so that the current windowbase is at bit0. */ /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ - rsr a2, WINDOWBASE - rsr a3, WINDOWSTART + rsr a2, windowbase + rsr a3, windowstart ssr a2 s32i a2, a1, PT_WINDOWBASE s32i a3, a1, PT_WINDOWSTART @@ -205,12 +207,12 @@ _user_exception: /* WINDOWBASE still in SAR! */ - rsr a2, SAR # original WINDOWBASE + rsr a2, sar # original WINDOWBASE movi a3, 1 ssl a2 sll a3, a3 - wsr a3, WINDOWSTART # set corresponding WINDOWSTART bit - wsr a2, WINDOWBASE # and WINDOWSTART + wsr a3, windowstart # set corresponding WINDOWSTART bit + wsr a2, windowbase # and WINDOWSTART rsync /* We are back to the original stack pointer (a1) */ @@ -219,6 +221,7 @@ _user_exception: j common_exception +ENDPROC(user_exception) /* * First-level exit handler for kernel exceptions @@ -232,9 +235,9 @@ _user_exception: * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -250,10 +253,9 @@ _user_exception: ENTRY(kernel_exception) - /* Save a0, a2, a3, DEPC and set SP. */ + /* Save a1, a2, a3, and set SP. */ - xsr a3, EXCSAVE_1 # restore a3, excsave_1 - rsr a0, DEPC # get a2 + rsr a0, depc # get a2 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_AREG2 s32i a3, a2, PT_AREG3 @@ -265,16 +267,16 @@ _kernel_exception: /* Save SAR and turn off single stepping */ movi a2, 0 - rsr a3, SAR - xsr a2, ICOUNTLEVEL + rsr a3, sar + xsr a2, icountlevel s32i a3, a1, PT_SAR s32i a2, a1, PT_ICOUNTLEVEL /* Rotate ws so that the current windowbase is at bit0. */ /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */ - rsr a2, WINDOWBASE # don't need to save these, we only - rsr a3, WINDOWSTART # need shifted windowstart: windowmask + rsr a2, windowbase # don't need to save these, we only + rsr a3, windowstart # need shifted windowstart: windowmask ssr a2 slli a2, a3, 32-WSBITS src a2, a3, a2 @@ -323,24 +325,24 @@ common_exception: /* Save some registers, disable loops and clear the syscall flag. */ - rsr a2, DEBUGCAUSE - rsr a3, EPC_1 + rsr a2, debugcause + rsr a3, epc1 s32i a2, a1, PT_DEBUGCAUSE s32i a3, a1, PT_PC movi a2, -1 - rsr a3, EXCVADDR + rsr a3, excvaddr s32i a2, a1, PT_SYSCALL movi a2, 0 s32i a3, a1, PT_EXCVADDR - xsr a2, LCOUNT + xsr a2, lcount s32i a2, a1, PT_LCOUNT /* It is now save to restore the EXC_TABLE_FIXUP variable. */ - rsr a0, EXCCAUSE + rsr a0, exccause movi a3, 0 - rsr a2, EXCSAVE_1 + rsr a2, excsave1 s32i a0, a1, PT_EXCCAUSE s32i a3, a2, EXC_TABLE_FIXUP @@ -348,38 +350,62 @@ common_exception: * so we can allow exceptions and interrupts (*) again. * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X) * - * (*) We only allow interrupts if PS.INTLEVEL was not set to 1 before - * (interrupts disabled) and if this exception is not an interrupt. + * (*) We only allow interrupts if they were previously enabled and + * we're not handling an IRQ */ - rsr a3, PS - addi a0, a0, -4 - movi a2, 1 - extui a3, a3, 0, 1 # a3 = PS.INTLEVEL[0] - moveqz a3, a2, a0 # a3 = 1 iff interrupt exception + rsr a3, ps + addi a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT + movi a2, LOCKLEVEL + extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH + # a3 = PS.INTLEVEL + moveqz a3, a2, a0 # a3 = LOCKLEVEL iff interrupt movi a2, 1 << PS_WOE_BIT or a3, a3, a2 - rsr a0, EXCCAUSE - xsr a3, PS + rsr a0, exccause + xsr a3, ps s32i a3, a1, PT_PS # save ps - /* Save LBEG, LEND */ + /* Save lbeg, lend */ - rsr a2, LBEG - rsr a3, LEND + rsr a2, lbeg + rsr a3, lend s32i a2, a1, PT_LBEG s32i a3, a1, PT_LEND + /* Save SCOMPARE1 */ + +#if XCHAL_HAVE_S32C1I + rsr a2, scompare1 + s32i a2, a1, PT_SCOMPARE1 +#endif + /* Save optional registers. */ save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT +#ifdef CONFIG_TRACE_IRQFLAGS + l32i a4, a1, PT_DEPC + /* Double exception means we came here with an exception + * while PS.EXCM was set, i.e. interrupts disabled. + */ + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f + l32i a4, a1, PT_EXCCAUSE + bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f + /* We came here with an interrupt means interrupts were enabled + * and we've just disabled them. + */ + movi a4, trace_hardirqs_off + callx4 a4 +1: +#endif + /* Go to second-level dispatcher. Set up parameters to pass to the * exception handler and call the exception handler. */ - movi a4, exc_table + rsr a4, excsave1 mov a6, a1 # pass stack frame mov a7, a0 # pass EXCCAUSE addx4 a4, a0, a4 @@ -390,13 +416,18 @@ common_exception: callx4 a4 /* Jump here for exception exit */ - + .global common_exception_return common_exception_return: +1: + rsil a2, LOCKLEVEL + /* Jump if we are returning from kernel exceptions. */ -1: l32i a3, a1, PT_PS - _bbci.l a3, PS_UM_BIT, 4f + l32i a3, a1, PT_PS + GET_THREAD_INFO(a2, a1) + l32i a4, a2, TI_FLAGS + _bbci.l a3, PS_UM_BIT, 6f /* Specific to a user exception exit: * We need to check some flags for signal handling and rescheduling, @@ -405,34 +436,76 @@ common_exception_return: * Note that we don't disable interrupts here. */ - GET_THREAD_INFO(a2,a1) - l32i a4, a2, TI_FLAGS - _bbsi.l a4, TIF_NEED_RESCHED, 3f - _bbci.l a4, TIF_SIGPENDING, 4f + _bbsi.l a4, TIF_NOTIFY_RESUME, 2f + _bbci.l a4, TIF_SIGPENDING, 5f - l32i a4, a1, PT_DEPC +2: l32i a4, a1, PT_DEPC bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f /* Call do_signal() */ - movi a4, do_signal # int do_signal(struct pt_regs*, sigset_t*) + rsil a2, 0 + movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*) mov a6, a1 - movi a7, 0 callx4 a4 j 1b 3: /* Reschedule */ + rsil a2, 0 movi a4, schedule # void schedule (void) callx4 a4 j 1b -4: /* Restore optional registers. */ +#ifdef CONFIG_PREEMPT +6: + _bbci.l a4, TIF_NEED_RESCHED, 4f + + /* Check current_thread_info->preempt_count */ + + l32i a4, a2, TI_PRE_COUNT + bnez a4, 4f + movi a4, preempt_schedule_irq + callx4 a4 + j 1b +#endif + +5: +#ifdef CONFIG_DEBUG_TLB_SANITY + l32i a4, a1, PT_DEPC + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f + movi a4, check_tlb_sanity + callx4 a4 +#endif +6: +4: +#ifdef CONFIG_TRACE_IRQFLAGS + l32i a4, a1, PT_DEPC + /* Double exception means we came here with an exception + * while PS.EXCM was set, i.e. interrupts disabled. + */ + bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f + l32i a4, a1, PT_EXCCAUSE + bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f + /* We came here with an interrupt means interrupts were enabled + * and we'll reenable them on return. + */ + movi a4, trace_hardirqs_on + callx4 a4 +1: +#endif + /* Restore optional registers. */ load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT - wsr a3, PS /* disable interrupts */ + /* Restore SCOMPARE1 */ + +#if XCHAL_HAVE_S32C1I + l32i a2, a1, PT_SCOMPARE1 + wsr a2, scompare1 +#endif + wsr a3, ps /* disable interrupts */ _bbci.l a3, PS_UM_BIT, kernel_exception_exit @@ -444,12 +517,12 @@ user_exception_exit: l32i a2, a1, PT_WINDOWBASE l32i a3, a1, PT_WINDOWSTART - wsr a1, DEPC # use DEPC as temp storage - wsr a3, WINDOWSTART # restore WINDOWSTART + wsr a1, depc # use DEPC as temp storage + wsr a3, windowstart # restore WINDOWSTART ssr a2 # preserve user's WB in the SAR - wsr a2, WINDOWBASE # switch to user's saved WB + wsr a2, windowbase # switch to user's saved WB rsync - rsr a1, DEPC # restore stack pointer + rsr a1, depc # restore stack pointer l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9) rotw -1 # we restore a4..a7 _bltui a6, 16, 1f # only have to restore current window? @@ -475,8 +548,8 @@ user_exception_exit: /* Clear unrestored registers (don't leak anything to user-land */ -1: rsr a0, WINDOWBASE - rsr a3, SAR +1: rsr a0, windowbase + rsr a3, sar sub a3, a0, a3 beqz a3, 2f extui a3, a3, 0, WBBITS @@ -495,6 +568,11 @@ user_exception_exit: * (if we have restored WSBITS-1 frames). */ +#if XCHAL_HAVE_THREADPTR + l32i a3, a1, PT_THREADPTR + wur a3, threadptr +#endif + 2: j common_exception_exit /* This is the kernel exception exit. @@ -504,29 +582,6 @@ user_exception_exit: kernel_exception_exit: -#ifdef PREEMPTIBLE_KERNEL - -#ifdef CONFIG_PREEMPT - - /* - * Note: We've just returned from a call4, so we have - * at least 4 addt'l regs. - */ - - /* Check current_thread_info->preempt_count */ - - GET_THREAD_INFO(a2) - l32i a3, a2, TI_PREEMPT - bnez a3, 1f - - l32i a2, a2, TI_FLAGS - -1: - -#endif - -#endif - /* Check if we have to do a movsp. * * We only have to do a movsp if the previous window-frame has @@ -556,7 +611,7 @@ kernel_exception_exit: /* Test WINDOWSTART now. If spilled, do the movsp */ - rsr a3, WINDOWSTART + rsr a3, windowstart addi a0, a3, -1 and a3, a3, a0 _bnez a3, common_exception_exit @@ -604,24 +659,24 @@ common_exception_exit: 1: l32i a2, a1, PT_PC l32i a3, a1, PT_SAR - wsr a2, EPC_1 - wsr a3, SAR + wsr a2, epc1 + wsr a3, sar /* Restore LBEG, LEND, LCOUNT */ l32i a2, a1, PT_LBEG l32i a3, a1, PT_LEND - wsr a2, LBEG + wsr a2, lbeg l32i a2, a1, PT_LCOUNT - wsr a3, LEND - wsr a2, LCOUNT + wsr a3, lend + wsr a2, lcount /* We control single stepping through the ICOUNTLEVEL register. */ l32i a2, a1, PT_ICOUNTLEVEL movi a3, -2 - wsr a2, ICOUNTLEVEL - wsr a3, ICOUNT + wsr a2, icountlevel + wsr a3, icount /* Check if it was double exception. */ @@ -636,11 +691,13 @@ common_exception_exit: l32i a1, a1, PT_AREG1 rfe -1: wsr a0, DEPC +1: wsr a0, depc l32i a0, a1, PT_AREG0 l32i a1, a1, PT_AREG1 rfde +ENDPROC(kernel_exception) + /* * Debug exception handler. * @@ -651,25 +708,25 @@ common_exception_exit: ENTRY(debug_exception) - rsr a0, EPS + XCHAL_DEBUGLEVEL + rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL bbsi.l a0, PS_EXCM_BIT, 1f # exception mode - /* Set EPC_1 and EXCCAUSE */ + /* Set EPC1 and EXCCAUSE */ - wsr a2, DEPC # save a2 temporarily - rsr a2, EPC + XCHAL_DEBUGLEVEL - wsr a2, EPC_1 + wsr a2, depc # save a2 temporarily + rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL + wsr a2, epc1 movi a2, EXCCAUSE_MAPPED_DEBUG - wsr a2, EXCCAUSE + wsr a2, exccause /* Restore PS to the value before the debug exc but with PS.EXCM set.*/ movi a2, 1 << PS_EXCM_BIT or a2, a0, a2 movi a0, debug_exception # restore a3, debug jump vector - wsr a2, PS - xsr a0, EXCSAVE + XCHAL_DEBUGLEVEL + wsr a2, ps + xsr a0, SREG_EXCSAVE + XCHAL_DEBUGLEVEL /* Switch to kernel/user stack, restore jump vector, and save a0 */ @@ -680,19 +737,19 @@ ENTRY(debug_exception) movi a0, 0 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_DEPC # mark it as a regular exception - xsr a0, DEPC + xsr a0, depc s32i a3, a2, PT_AREG3 s32i a0, a2, PT_AREG2 mov a1, a2 j _kernel_exception -2: rsr a2, EXCSAVE_1 +2: rsr a2, excsave1 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer s32i a0, a2, PT_AREG0 movi a0, 0 s32i a1, a2, PT_AREG1 s32i a0, a2, PT_DEPC - xsr a0, DEPC + xsr a0, depc s32i a3, a2, PT_AREG3 s32i a0, a2, PT_AREG2 mov a1, a2 @@ -701,6 +758,7 @@ ENTRY(debug_exception) /* Debug exception while in exception mode. */ 1: j 1b // FIXME!! +ENDPROC(debug_exception) /* * We get here in case of an unrecoverable exception. @@ -732,12 +790,12 @@ ENTRY(unrecoverable_exception) movi a0, 1 movi a1, 0 - wsr a0, WINDOWSTART - wsr a1, WINDOWBASE + wsr a0, windowstart + wsr a1, windowbase rsync - movi a1, (1 << PS_WOE_BIT) | 1 - wsr a1, PS + movi a1, (1 << PS_WOE_BIT) | LOCKLEVEL + wsr a1, ps rsync movi a1, init_task @@ -751,6 +809,7 @@ ENTRY(unrecoverable_exception) 1: j 1b +ENDPROC(unrecoverable_exception) /* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */ @@ -759,176 +818,64 @@ ENTRY(unrecoverable_exception) * * The ALLOCA handler is entered when user code executes the MOVSP * instruction and the caller's frame is not in the register file. - * In this case, the caller frame's a0..a3 are on the stack just - * below sp (a1), and this handler moves them. * - * For "MOVSP <ar>,<as>" without destination register a1, this routine - * simply moves the value from <as> to <ar> without moving the save area. + * This algorithm was taken from the Ross Morley's RTOS Porting Layer: + * + * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S + * + * It leverages the existing window spill/fill routines and their support for + * double exceptions. The 'movsp' instruction will only cause an exception if + * the next window needs to be loaded. In fact this ALLOCA exception may be + * replaced at some point by changing the hardware to do a underflow exception + * of the proper size instead. + * + * This algorithm simply backs out the register changes started by the user + * excpetion handler, makes it appear that we have started a window underflow + * by rotating the window back and then setting the old window base (OWB) in + * the 'ps' register with the rolled back window base. The 'movsp' instruction + * will be re-executed and this time since the next window frames is in the + * active AR registers it won't cause an exception. + * + * If the WindowUnderflow code gets a TLB miss the page will get mapped + * the the partial windeowUnderflow will be handeled in the double exception + * handler. * * Entry condition: * * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception */ -#if XCHAL_HAVE_BE -#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4 -#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4 -#else -#define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4 -#define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4 -#endif - ENTRY(fast_alloca) + rsr a0, windowbase + rotw -1 + rsr a2, ps + extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH + xor a3, a3, a4 + l32i a4, a6, PT_AREG0 + l32i a1, a6, PT_DEPC + rsr a6, depc + wsr a1, depc + slli a3, a3, PS_OWB_SHIFT + xor a2, a2, a3 + wsr a2, ps + rsync - /* We shouldn't be in a double exception. */ - - l32i a0, a2, PT_DEPC - _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double - - rsr a0, DEPC # get a2 - s32i a4, a2, PT_AREG4 # save a4 and - s32i a0, a2, PT_AREG2 # a2 to stack - - /* Exit critical section. */ - - movi a0, 0 - s32i a0, a3, EXC_TABLE_FIXUP - - /* Restore a3, excsave_1 */ - - xsr a3, EXCSAVE_1 # make sure excsave_1 is valid for dbl. - rsr a4, EPC_1 # get exception address - s32i a3, a2, PT_AREG3 # save a3 to stack - -#ifdef ALLOCA_EXCEPTION_IN_IRAM -#error iram not supported -#else - /* Note: l8ui not allowed in IRAM/IROM!! */ - l8ui a0, a4, 1 # read as(src) from MOVSP instruction -#endif - movi a3, .Lmovsp_src - _EXTUI_MOVSP_SRC(a0) # extract source register number - addx8 a3, a0, a3 - jx a3 - -.Lunhandled_double: - wsr a0, EXCSAVE_1 - movi a0, unrecoverable_exception - callx0 a0 - - .align 8 -.Lmovsp_src: - l32i a3, a2, PT_AREG0; _j 1f; .align 8 - mov a3, a1; _j 1f; .align 8 - l32i a3, a2, PT_AREG2; _j 1f; .align 8 - l32i a3, a2, PT_AREG3; _j 1f; .align 8 - l32i a3, a2, PT_AREG4; _j 1f; .align 8 - mov a3, a5; _j 1f; .align 8 - mov a3, a6; _j 1f; .align 8 - mov a3, a7; _j 1f; .align 8 - mov a3, a8; _j 1f; .align 8 - mov a3, a9; _j 1f; .align 8 - mov a3, a10; _j 1f; .align 8 - mov a3, a11; _j 1f; .align 8 - mov a3, a12; _j 1f; .align 8 - mov a3, a13; _j 1f; .align 8 - mov a3, a14; _j 1f; .align 8 - mov a3, a15; _j 1f; .align 8 - -1: - -#ifdef ALLOCA_EXCEPTION_IN_IRAM -#error iram not supported -#else - l8ui a0, a4, 0 # read ar(dst) from MOVSP instruction -#endif - addi a4, a4, 3 # step over movsp - _EXTUI_MOVSP_DST(a0) # extract destination register - wsr a4, EPC_1 # save new epc_1 - - _bnei a0, 1, 1f # no 'movsp a1, ax': jump - - /* Move the save area. This implies the use of the L32E - * and S32E instructions, because this move must be done with - * the user's PS.RING privilege levels, not with ring 0 - * (kernel's) privileges currently active with PS.EXCM - * set. Note that we have stil registered a fixup routine with the - * double exception vector in case a double exception occurs. - */ - - /* a0,a4:avail a1:old user stack a2:exc. stack a3:new user stack. */ - - l32e a0, a1, -16 - l32e a4, a1, -12 - s32e a0, a3, -16 - s32e a4, a3, -12 - l32e a0, a1, -8 - l32e a4, a1, -4 - s32e a0, a3, -8 - s32e a4, a3, -4 - - /* Restore stack-pointer and all the other saved registers. */ - - mov a1, a3 - - l32i a4, a2, PT_AREG4 - l32i a3, a2, PT_AREG3 - l32i a0, a2, PT_AREG0 - l32i a2, a2, PT_AREG2 - rfe - - /* MOVSP <at>,<as> was invoked with <at> != a1. - * Because the stack pointer is not being modified, - * we should be able to just modify the pointer - * without moving any save area. - * The processor only traps these occurrences if the - * caller window isn't live, so unfortunately we can't - * use this as an alternate trap mechanism. - * So we just do the move. This requires that we - * resolve the destination register, not just the source, - * so there's some extra work. - * (PERHAPS NOT REALLY NEEDED, BUT CLEANER...) - */ - - /* a0 dst-reg, a1 user-stack, a2 stack, a3 value of src reg. */ - -1: movi a4, .Lmovsp_dst - addx8 a4, a0, a4 - jx a4 - - .align 8 -.Lmovsp_dst: - s32i a3, a2, PT_AREG0; _j 1f; .align 8 - mov a1, a3; _j 1f; .align 8 - s32i a3, a2, PT_AREG2; _j 1f; .align 8 - s32i a3, a2, PT_AREG3; _j 1f; .align 8 - s32i a3, a2, PT_AREG4; _j 1f; .align 8 - mov a5, a3; _j 1f; .align 8 - mov a6, a3; _j 1f; .align 8 - mov a7, a3; _j 1f; .align 8 - mov a8, a3; _j 1f; .align 8 - mov a9, a3; _j 1f; .align 8 - mov a10, a3; _j 1f; .align 8 - mov a11, a3; _j 1f; .align 8 - mov a12, a3; _j 1f; .align 8 - mov a13, a3; _j 1f; .align 8 - mov a14, a3; _j 1f; .align 8 - mov a15, a3; _j 1f; .align 8 - -1: l32i a4, a2, PT_AREG4 - l32i a3, a2, PT_AREG3 - l32i a0, a2, PT_AREG0 - l32i a2, a2, PT_AREG2 - rfe - + _bbci.l a4, 31, 4f + rotw -1 + _bbci.l a8, 30, 8f + rotw -1 + j _WindowUnderflow12 +8: j _WindowUnderflow8 +4: j _WindowUnderflow4 +ENDPROC(fast_alloca) /* * fast system calls. @@ -944,58 +891,61 @@ ENTRY(fast_alloca) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table */ ENTRY(fast_syscall_kernel) /* Skip syscall. */ - rsr a0, EPC_1 + rsr a0, epc1 addi a0, a0, 3 - wsr a0, EPC_1 + wsr a0, epc1 l32i a0, a2, PT_DEPC bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable - rsr a0, DEPC # get syscall-nr + rsr a0, depc # get syscall-nr _beqz a0, fast_syscall_spill_registers _beqi a0, __NR_xtensa, fast_syscall_xtensa j kernel_exception +ENDPROC(fast_syscall_kernel) + ENTRY(fast_syscall_user) /* Skip syscall. */ - rsr a0, EPC_1 + rsr a0, epc1 addi a0, a0, 3 - wsr a0, EPC_1 + wsr a0, epc1 l32i a0, a2, PT_DEPC bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable - rsr a0, DEPC # get syscall-nr + rsr a0, depc # get syscall-nr _beqz a0, fast_syscall_spill_registers _beqi a0, __NR_xtensa, fast_syscall_xtensa j user_exception -ENTRY(fast_syscall_unrecoverable) +ENDPROC(fast_syscall_user) - /* Restore all states. */ +ENTRY(fast_syscall_unrecoverable) - l32i a0, a2, PT_AREG0 # restore a0 - xsr a2, DEPC # restore a2, depc - rsr a3, EXCSAVE_1 + /* Restore all states. */ - wsr a0, EXCSAVE_1 - movi a0, unrecoverable_exception - callx0 a0 + l32i a0, a2, PT_AREG0 # restore a0 + xsr a2, depc # restore a2, depc + wsr a0, excsave1 + movi a0, unrecoverable_exception + callx0 a0 +ENDPROC(fast_syscall_unrecoverable) /* * sysxtensa syscall handler @@ -1011,10 +961,10 @@ ENTRY(fast_syscall_unrecoverable) * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in a0 and DEPC - * a3: dispatch table, original in excsave_1 + * a3: a3 * a4..a15: unchanged * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1026,7 +976,7 @@ ENTRY(fast_syscall_unrecoverable) * TRY adds an entry to the __ex_table fixup table for the immediately * following instruction. * - * CATCH catches any exception that occurred at one of the preceeding TRY + * CATCH catches any exception that occurred at one of the preceding TRY * statements and continues from there * * Usage TRY l32i a0, a1, 0 @@ -1047,8 +997,6 @@ ENTRY(fast_syscall_unrecoverable) ENTRY(fast_syscall_xtensa) - xsr a3, EXCSAVE_1 # restore a3, excsave1 - s32i a7, a2, PT_AREG7 # we need an additional register movi a7, 4 # sizeof(unsigned int) access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp @@ -1101,7 +1049,7 @@ CATCH movi a2, -EINVAL rfe - +ENDPROC(fast_syscall_xtensa) /* fast_syscall_spill_registers. @@ -1111,9 +1059,9 @@ CATCH * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler. */ @@ -1122,201 +1070,68 @@ ENTRY(fast_syscall_spill_registers) /* Register a FIXUP handler (pass current wb as a parameter) */ + xsr a3, excsave1 movi a0, fast_syscall_spill_registers_fixup s32i a0, a3, EXC_TABLE_FIXUP - rsr a0, WINDOWBASE + rsr a0, windowbase s32i a0, a3, EXC_TABLE_PARAM + xsr a3, excsave1 # restore a3 and excsave_1 - /* Save a3 and SAR on stack. */ + /* Save a3, a4 and SAR on stack. */ - rsr a0, SAR - xsr a3, EXCSAVE_1 # restore a3 and excsave_1 + rsr a0, sar s32i a3, a2, PT_AREG3 - s32i a4, a2, PT_AREG4 - s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5 + s32i a0, a2, PT_SAR - /* The spill routine might clobber a7, a11, and a15. */ + /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */ + s32i a4, a2, PT_AREG4 s32i a7, a2, PT_AREG7 + s32i a8, a2, PT_AREG8 s32i a11, a2, PT_AREG11 + s32i a12, a2, PT_AREG12 s32i a15, a2, PT_AREG15 - call0 _spill_registers # destroys a3, a4, and SAR - - /* Advance PC, restore registers and SAR, and return from exception. */ - - l32i a3, a2, PT_AREG5 - l32i a4, a2, PT_AREG4 - l32i a0, a2, PT_AREG0 - wsr a3, SAR - l32i a3, a2, PT_AREG3 - - /* Restore clobbered registers. */ - - l32i a7, a2, PT_AREG7 - l32i a11, a2, PT_AREG11 - l32i a15, a2, PT_AREG15 - - movi a2, 0 - rfe - -/* Fixup handler. - * - * We get here if the spill routine causes an exception, e.g. tlb miss. - * We basically restore WINDOWBASE and WINDOWSTART to the condition when - * we entered the spill routine and jump to the user exception handler. - * - * a0: value of depc, original value in depc - * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE - * a3: exctable, original value in excsave1 - */ - -fast_syscall_spill_registers_fixup: - - rsr a2, WINDOWBASE # get current windowbase (a2 is saved) - xsr a0, DEPC # restore depc and a0 - ssl a2 # set shift (32 - WB) - - /* We need to make sure the current registers (a0-a3) are preserved. - * To do this, we simply set the bit for the current window frame - * in WS, so that the exception handlers save them to the task stack. - */ - - rsr a3, EXCSAVE_1 # get spill-mask - slli a2, a3, 1 # shift left by one - - slli a3, a2, 32-WSBITS - src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy...... - wsr a2, WINDOWSTART # set corrected windowstart - - movi a3, exc_table - l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2 - l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task) - - /* Return to the original (user task) WINDOWBASE. - * We leave the following frame behind: - * a0, a1, a2 same - * a3: trashed (saved in excsave_1) - * depc: depc (we have to return to that address) - * excsave_1: a3 - */ - - wsr a3, WINDOWBASE - rsync - - /* We are now in the original frame when we entered _spill_registers: - * a0: return address - * a1: used, stack pointer - * a2: kernel stack pointer - * a3: available, saved in EXCSAVE_1 - * depc: exception address - * excsave: a3 - * Note: This frame might be the same as above. - */ - - /* Setup stack pointer. */ - - addi a2, a2, -PT_USER_SIZE - s32i a0, a2, PT_AREG0 - - /* Make sure we return to this fixup handler. */ - - movi a3, fast_syscall_spill_registers_fixup_return - s32i a3, a2, PT_DEPC # setup depc - - /* Jump to the exception handler. */ - - movi a3, exc_table - rsr a0, EXCCAUSE - addx4 a0, a0, a3 # find entry in table - l32i a0, a0, EXC_TABLE_FAST_USER # load handler - jx a0 - -fast_syscall_spill_registers_fixup_return: - - /* When we return here, all registers have been restored (a2: DEPC) */ - - wsr a2, DEPC # exception address - - /* Restore fixup handler. */ - - xsr a3, EXCSAVE_1 - movi a2, fast_syscall_spill_registers_fixup - s32i a2, a3, EXC_TABLE_FIXUP - rsr a2, WINDOWBASE - s32i a2, a3, EXC_TABLE_PARAM - l32i a2, a3, EXC_TABLE_KSTK - - /* Load WB at the time the exception occurred. */ - - rsr a3, SAR # WB is still in SAR - neg a3, a3 - wsr a3, WINDOWBASE - rsync - - /* Restore a3 and return. */ - - movi a3, exc_table - xsr a3, EXCSAVE_1 - - rfde - - -/* - * spill all registers. - * - * This is not a real function. The following conditions must be met: - * - * - must be called with call0. - * - uses a3, a4 and SAR. - * - the last 'valid' register of each frame are clobbered. - * - the caller must have registered a fixup handler - * (or be inside a critical section) - * - PS_EXCM must be set (PS_WOE cleared?) - */ - -ENTRY(_spill_registers) - /* * Rotate ws so that the current windowbase is at bit 0. * Assume ws = xxxwww1yy (www1 current window frame). * Rotate ws right so that a4 = yyxxxwww1. */ - rsr a4, WINDOWBASE - rsr a3, WINDOWSTART # a3 = xxxwww1yy - ssr a4 # holds WB - slli a4, a3, WSBITS - or a3, a3, a4 # a3 = xxxwww1yyxxxwww1yy + rsr a0, windowbase + rsr a3, windowstart # a3 = xxxwww1yy + ssr a0 # holds WB + slli a0, a3, WSBITS + or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy srl a3, a3 # a3 = 00xxxwww1yyxxxwww1 /* We are done if there are no more than the current register frame. */ extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww - movi a4, (1 << (WSBITS-1)) + movi a0, (1 << (WSBITS-1)) _beqz a3, .Lnospill # only one active frame? jump /* We want 1 at the top, so that we return to the current windowbase */ - or a3, a3, a4 # 1yyxxxwww + or a3, a3, a0 # 1yyxxxwww /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */ - wsr a3, WINDOWSTART # save shifted windowstart - neg a4, a3 - and a3, a4, a3 # first bit set from right: 000010000 + wsr a3, windowstart # save shifted windowstart + neg a0, a3 + and a3, a0, a3 # first bit set from right: 000010000 - ffs_ws a4, a3 # a4: shifts to skip empty frames + ffs_ws a0, a3 # a0: shifts to skip empty frames movi a3, WSBITS - sub a4, a3, a4 # WSBITS-a4:number of 0-bits from right - ssr a4 # save in SAR for later. + sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right + ssr a0 # save in SAR for later. - rsr a3, WINDOWBASE - add a3, a3, a4 - wsr a3, WINDOWBASE + rsr a3, windowbase + add a3, a3, a0 + wsr a3, windowbase rsync - rsr a3, WINDOWSTART + rsr a3, windowstart srl a3, a3 # shift windowstart /* WB is now just one frame below the oldest frame in the register @@ -1327,22 +1142,6 @@ ENTRY(_spill_registers) * we have to save 4,8. or 12 registers. */ - _bbsi.l a3, 1, .Lc4 - _bbsi.l a3, 2, .Lc8 - - /* Special case: we have a call12-frame starting at a4. */ - - _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first) - - s32e a4, a1, -16 # a1 is valid with an empty spill area - l32e a4, a5, -12 - s32e a8, a4, -48 - mov a8, a4 - l32e a4, a1, -16 - j .Lc12c - -.Lnospill: - ret .Lloop: _bbsi.l a3, 1, .Lc4 _bbci.l a3, 2, .Lc12 @@ -1356,20 +1155,10 @@ ENTRY(_spill_registers) s32e a9, a4, -28 s32e a10, a4, -24 s32e a11, a4, -20 - srli a11, a3, 2 # shift windowbase by 2 rotw 2 _bnei a3, 1, .Lloop - -.Lexit: /* Done. Do the final rotation, set WS, and return. */ - - rotw 1 - rsr a3, WINDOWBASE - ssl a3 - movi a3, 1 - sll a3, a3 - wsr a3, WINDOWSTART - ret + j .Lexit .Lc4: s32e a4, a9, -16 s32e a5, a9, -12 @@ -1385,11 +1174,11 @@ ENTRY(_spill_registers) /* 12-register frame (call12) */ - l32e a2, a5, -12 - s32e a8, a2, -48 - mov a8, a2 + l32e a0, a5, -12 + s32e a8, a0, -48 + mov a8, a0 -.Lc12c: s32e a9, a8, -44 + s32e a9, a8, -44 s32e a10, a8, -40 s32e a11, a8, -36 s32e a12, a8, -32 @@ -1409,61 +1198,208 @@ ENTRY(_spill_registers) */ rotw 1 - mov a5, a13 + mov a4, a13 rotw -1 - s32e a4, a9, -16 - s32e a5, a9, -12 - s32e a6, a9, -8 - s32e a7, a9, -4 + s32e a4, a8, -16 + s32e a5, a8, -12 + s32e a6, a8, -8 + s32e a7, a8, -4 rotw 3 _beqi a3, 1, .Lexit j .Lloop -.Linvalid_mask: +.Lexit: - /* We get here because of an unrecoverable error in the window - * registers. If we are in user space, we kill the application, - * however, this condition is unrecoverable in kernel space. - */ + /* Done. Do the final rotation and set WS */ - rsr a0, PS - _bbci.l a0, PS_UM_BIT, 1f + rotw 1 + rsr a3, windowbase + ssl a3 + movi a3, 1 + sll a3, a3 + wsr a3, windowstart +.Lnospill: - /* User space: Setup a dummy frame and kill application. + /* Advance PC, restore registers and SAR, and return from exception. */ + + l32i a3, a2, PT_SAR + l32i a0, a2, PT_AREG0 + wsr a3, sar + l32i a3, a2, PT_AREG3 + + /* Restore clobbered registers. */ + + l32i a4, a2, PT_AREG4 + l32i a7, a2, PT_AREG7 + l32i a8, a2, PT_AREG8 + l32i a11, a2, PT_AREG11 + l32i a12, a2, PT_AREG12 + l32i a15, a2, PT_AREG15 + + movi a2, 0 + rfe + +.Linvalid_mask: + + /* We get here because of an unrecoverable error in the window + * registers, so set up a dummy frame and kill the user application. * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer. */ movi a0, 1 movi a1, 0 - wsr a0, WINDOWSTART - wsr a1, WINDOWBASE + wsr a0, windowstart + wsr a1, windowbase rsync movi a0, 0 - movi a3, exc_table + rsr a3, excsave1 l32i a1, a3, EXC_TABLE_KSTK - wsr a3, EXCSAVE_1 - movi a4, (1 << PS_WOE_BIT) | 1 - wsr a4, PS + movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL + wsr a4, ps rsync movi a6, SIGSEGV movi a4, do_exit callx4 a4 -1: /* Kernel space: PANIC! */ + /* shouldn't return, so panic */ - wsr a0, EXCSAVE_1 + wsr a0, excsave1 movi a0, unrecoverable_exception callx0 a0 # should not return 1: j 1b + +ENDPROC(fast_syscall_spill_registers) + +/* Fixup handler. + * + * We get here if the spill routine causes an exception, e.g. tlb miss. + * We basically restore WINDOWBASE and WINDOWSTART to the condition when + * we entered the spill routine and jump to the user exception handler. + * + * Note that we only need to restore the bits in windowstart that have not + * been spilled yet by the _spill_register routine. Luckily, a3 contains a + * rotated windowstart with only those bits set for frames that haven't been + * spilled yet. Because a3 is rotated such that bit 0 represents the register + * frame for the current windowbase - 1, we need to rotate a3 left by the + * value of the current windowbase + 1 and move it to windowstart. + * + * a0: value of depc, original value in depc + * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE + * a3: exctable, original value in excsave1 + */ + +ENTRY(fast_syscall_spill_registers_fixup) + + rsr a2, windowbase # get current windowbase (a2 is saved) + xsr a0, depc # restore depc and a0 + ssl a2 # set shift (32 - WB) + + /* We need to make sure the current registers (a0-a3) are preserved. + * To do this, we simply set the bit for the current window frame + * in WS, so that the exception handlers save them to the task stack. + * + * Note: we use a3 to set the windowbase, so we take a special care + * of it, saving it in the original _spill_registers frame across + * the exception handler call. + */ + + xsr a3, excsave1 # get spill-mask + slli a3, a3, 1 # shift left by one + addi a3, a3, 1 # set the bit for the current window frame + + slli a2, a3, 32-WSBITS + src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy...... + wsr a2, windowstart # set corrected windowstart + + srli a3, a3, 1 + rsr a2, excsave1 + l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2 + xsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3 + l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task) + xsr a2, excsave1 + + /* Return to the original (user task) WINDOWBASE. + * We leave the following frame behind: + * a0, a1, a2 same + * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE) + * depc: depc (we have to return to that address) + * excsave_1: exctable + */ + + wsr a3, windowbase + rsync + + /* We are now in the original frame when we entered _spill_registers: + * a0: return address + * a1: used, stack pointer + * a2: kernel stack pointer + * a3: available + * depc: exception address + * excsave: exctable + * Note: This frame might be the same as above. + */ + + /* Setup stack pointer. */ + + addi a2, a2, -PT_USER_SIZE + s32i a0, a2, PT_AREG0 + + /* Make sure we return to this fixup handler. */ + + movi a3, fast_syscall_spill_registers_fixup_return + s32i a3, a2, PT_DEPC # setup depc + + /* Jump to the exception handler. */ + + rsr a3, excsave1 + rsr a0, exccause + addx4 a0, a0, a3 # find entry in table + l32i a0, a0, EXC_TABLE_FAST_USER # load handler + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + jx a0 + +ENDPROC(fast_syscall_spill_registers_fixup) + +ENTRY(fast_syscall_spill_registers_fixup_return) + + /* When we return here, all registers have been restored (a2: DEPC) */ + + wsr a2, depc # exception address + + /* Restore fixup handler. */ + + rsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE + movi a3, fast_syscall_spill_registers_fixup + s32i a3, a2, EXC_TABLE_FIXUP + rsr a3, windowbase + s32i a3, a2, EXC_TABLE_PARAM + l32i a2, a2, EXC_TABLE_KSTK + + /* Load WB at the time the exception occurred. */ + + rsr a3, sar # WB is still in SAR + neg a3, a3 + wsr a3, windowbase + rsync + + rsr a3, excsave1 + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + + rfde + +ENDPROC(fast_syscall_spill_registers_fixup_return) + #ifdef CONFIG_MMU /* * We should never get here. Bail out! @@ -1475,6 +1411,8 @@ ENTRY(fast_second_level_miss_double_kernel) callx0 a0 # should not return 1: j 1b +ENDPROC(fast_second_level_miss_double_kernel) + /* First-level entry handler for user, kernel, and double 2nd-level * TLB miss exceptions. Note that for now, user and kernel miss * exceptions share the same entry point and are handled identically. @@ -1487,9 +1425,9 @@ ENTRY(fast_second_level_miss_double_kernel) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1497,9 +1435,10 @@ ENTRY(fast_second_level_miss_double_kernel) ENTRY(fast_second_level_miss) - /* Save a1. Note: we don't expect a double exception. */ + /* Save a1 and a3. Note: we don't expect a double exception. */ s32i a1, a2, PT_AREG1 + s32i a3, a2, PT_AREG3 /* We need to map the page of PTEs for the user task. Find * the pointer to that page. Also, it's possible for tsk->mm @@ -1521,10 +1460,7 @@ ENTRY(fast_second_level_miss) l32i a0, a1, TASK_MM # tsk->mm beqz a0, 9f - - /* We deliberately destroy a3 that holds the exception table. */ - -8: rsr a3, EXCVADDR # fault address +8: rsr a3, excvaddr # fault address _PGD_OFFSET(a0, a3, a1) l32i a0, a0, 0 # read pmdval beqz a0, 2f @@ -1542,7 +1478,7 @@ ENTRY(fast_second_level_miss) * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY */ - movi a1, -PAGE_OFFSET + movi a1, (-PAGE_OFFSET) & 0xffffffff add a0, a0, a1 # pmdval - PAGE_OFFSET extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK xor a0, a0, a1 @@ -1561,7 +1497,7 @@ ENTRY(fast_second_level_miss) */ extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3 - rsr a1, PTEVADDR + rsr a1, ptevaddr addx2 a3, a3, a3 # -> 0,3,6,9 srli a1, a1, PAGE_SHIFT extui a3, a3, 2, 2 # -> 0,0,1,2 @@ -1574,7 +1510,7 @@ ENTRY(fast_second_level_miss) /* Exit critical section. */ -4: movi a3, exc_table # restore a3 +4: rsr a3, excsave1 movi a0, 0 s32i a0, a3, EXC_TABLE_FIXUP @@ -1582,19 +1518,19 @@ ENTRY(fast_second_level_miss) l32i a0, a2, PT_AREG0 l32i a1, a2, PT_AREG1 + l32i a3, a2, PT_AREG3 l32i a2, a2, PT_DEPC - xsr a3, EXCSAVE_1 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f /* Restore excsave1 and return. */ - rsr a2, DEPC + rsr a2, depc rfe /* Return from double exception. */ -1: xsr a2, DEPC +1: xsr a2, depc esync rfde @@ -1618,7 +1554,7 @@ ENTRY(fast_second_level_miss) /* Make sure the exception originated in the special functions */ movi a0, __tlbtemp_mapping_start - rsr a3, EPC_1 + rsr a3, epc1 bltu a3, a0, 2f movi a0, __tlbtemp_mapping_end bgeu a3, a0, 2f @@ -1626,7 +1562,7 @@ ENTRY(fast_second_level_miss) /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */ movi a3, TLBTEMP_BASE_1 - rsr a0, EXCVADDR + rsr a0, excvaddr bltu a0, a3, 2f addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT)) @@ -1635,7 +1571,7 @@ ENTRY(fast_second_level_miss) /* Check if we have to restore an ITLB mapping. */ movi a1, __tlbtemp_mapping_itlb - rsr a3, EPC_1 + rsr a3, epc1 sub a3, a3, a1 /* Calculate VPN */ @@ -1670,18 +1606,16 @@ ENTRY(fast_second_level_miss) 2: /* Invalid PGD, default exception handling */ - movi a3, exc_table - rsr a1, DEPC - xsr a3, EXCSAVE_1 + rsr a1, depc s32i a1, a2, PT_AREG2 - s32i a3, a2, PT_AREG3 mov a1, a2 - rsr a2, PS + rsr a2, ps bbsi.l a2, PS_UM_BIT, 1f j _kernel_exception 1: j _user_exception +ENDPROC(fast_second_level_miss) /* * StoreProhibitedException @@ -1693,9 +1627,9 @@ ENTRY(fast_second_level_miss) * a0: trashed, original value saved on stack (PT_AREG0) * a1: a1 * a2: new stack pointer, original in DEPC - * a3: dispatch table + * a3: a3 * depc: a2, original value saved on stack (PT_DEPC) - * excsave_1: a3 + * excsave_1: dispatch table * * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception @@ -1703,61 +1637,64 @@ ENTRY(fast_second_level_miss) ENTRY(fast_store_prohibited) - /* Save a1 and a4. */ + /* Save a1 and a3. */ s32i a1, a2, PT_AREG1 - s32i a4, a2, PT_AREG4 + s32i a3, a2, PT_AREG3 GET_CURRENT(a1,a2) l32i a0, a1, TASK_MM # tsk->mm beqz a0, 9f -8: rsr a1, EXCVADDR # fault address - _PGD_OFFSET(a0, a1, a4) +8: rsr a1, excvaddr # fault address + _PGD_OFFSET(a0, a1, a3) l32i a0, a0, 0 beqz a0, 2f - /* Note that we assume _PAGE_WRITABLE_BIT is only set if pte is valid.*/ + /* + * Note that we test _PAGE_WRITABLE_BIT only if PTE is present + * and is not PAGE_NONE. See pgtable.h for possible PTE layouts. + */ - _PTE_OFFSET(a0, a1, a4) - l32i a4, a0, 0 # read pteval - bbci.l a4, _PAGE_WRITABLE_BIT, 2f + _PTE_OFFSET(a0, a1, a3) + l32i a3, a0, 0 # read pteval + movi a1, _PAGE_CA_INVALID + ball a3, a1, 2f + bbci.l a3, _PAGE_WRITABLE_BIT, 2f movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE - or a4, a4, a1 - rsr a1, EXCVADDR - s32i a4, a0, 0 + or a3, a3, a1 + rsr a1, excvaddr + s32i a3, a0, 0 /* We need to flush the cache if we have page coloring. */ #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK dhwb a0, 0 #endif pdtlb a0, a1 - wdtlb a4, a0 + wdtlb a3, a0 /* Exit critical section. */ movi a0, 0 + rsr a3, excsave1 s32i a0, a3, EXC_TABLE_FIXUP /* Restore the working registers, and return. */ - l32i a4, a2, PT_AREG4 + l32i a3, a2, PT_AREG3 l32i a1, a2, PT_AREG1 l32i a0, a2, PT_AREG0 l32i a2, a2, PT_DEPC - /* Restore excsave1 and a3. */ - - xsr a3, EXCSAVE_1 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f - rsr a2, DEPC + rsr a2, depc rfe /* Double exception. Restore FIXUP handler and return. */ -1: xsr a2, DEPC +1: xsr a2, depc esync rfde @@ -1766,17 +1703,17 @@ ENTRY(fast_store_prohibited) 2: /* If there was a problem, handle fault in C */ - rsr a4, DEPC # still holds a2 - xsr a3, EXCSAVE_1 - s32i a4, a2, PT_AREG2 - s32i a3, a2, PT_AREG3 - l32i a4, a2, PT_AREG4 + rsr a3, depc # still holds a2 + s32i a3, a2, PT_AREG2 mov a1, a2 - rsr a2, PS + rsr a2, ps bbsi.l a2, PS_UM_BIT, 1f j _kernel_exception 1: j _user_exception + +ENDPROC(fast_store_prohibited) + #endif /* CONFIG_MMU */ /* @@ -1787,6 +1724,7 @@ ENTRY(fast_store_prohibited) */ ENTRY(system_call) + entry a1, 32 /* regs->syscall = regs->areg[2] */ @@ -1831,50 +1769,45 @@ ENTRY(system_call) callx4 a4 retw +ENDPROC(system_call) /* - * Create a kernel thread + * Spill live registers on the kernel stack macro. * - * int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) - * a2 a2 a3 a4 + * Entry condition: ps.woe is set, ps.excm is cleared + * Exit condition: windowstart has single bit set + * May clobber: a12, a13 */ + .macro spill_registers_kernel -ENTRY(kernel_thread) - entry a1, 16 - - mov a5, a2 # preserve fn over syscall - mov a7, a3 # preserve args over syscall - - movi a3, _CLONE_VM | _CLONE_UNTRACED - movi a2, __NR_clone - or a6, a4, a3 # arg0: flags - mov a3, a1 # arg1: sp - syscall - - beq a3, a1, 1f # branch if parent - mov a6, a7 # args - callx4 a5 # fn(args) - - movi a2, __NR_exit - syscall # return value of fn(args) still in a6 - -1: retw - -/* - * Do a system call from kernel instead of calling sys_execve, so we end up - * with proper pt_regs. - * - * int kernel_execve(const char *fname, char *const argv[], charg *const envp[]) - * a2 a2 a3 a4 - */ - -ENTRY(kernel_execve) - entry a1, 16 - mov a6, a2 # arg0 is in a6 - movi a2, __NR_execve - syscall - +#if XCHAL_NUM_AREGS > 16 + call12 1f + _j 2f + retw + .align 4 +1: + _entry a1, 48 + addi a12, a0, 3 +#if XCHAL_NUM_AREGS > 32 + .rept (XCHAL_NUM_AREGS - 32) / 12 + _entry a1, 48 + mov a12, a0 + .endr +#endif + _entry a1, 48 +#if XCHAL_NUM_AREGS % 12 == 0 + mov a8, a8 +#elif XCHAL_NUM_AREGS % 12 == 4 + mov a12, a12 +#elif XCHAL_NUM_AREGS % 12 == 8 + mov a4, a4 +#endif retw +2: +#else + mov a12, a12 +#endif + .endm /* * Task switch. @@ -1887,22 +1820,21 @@ ENTRY(_switch_to) entry a1, 16 - mov a12, a2 # preserve 'prev' (a2) - mov a13, a3 # and 'next' (a3) + mov a10, a2 # preserve 'prev' (a2) + mov a11, a3 # and 'next' (a3) l32i a4, a2, TASK_THREAD_INFO l32i a5, a3, TASK_THREAD_INFO - save_xtregs_user a4 a6 a8 a9 a10 a11 THREAD_XTREGS_USER + save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER - s32i a0, a12, THREAD_RA # save return address - s32i a1, a12, THREAD_SP # save stack pointer + s32i a0, a10, THREAD_RA # save return address + s32i a1, a10, THREAD_SP # save stack pointer /* Disable ints while we manipulate the stack pointer. */ - movi a14, (1 << PS_EXCM_BIT) | LOCKLEVEL - xsr a14, PS - rsr a3, EXCSAVE_1 + rsil a14, LOCKLEVEL + rsr a3, excsave1 rsync s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */ @@ -1910,13 +1842,13 @@ ENTRY(_switch_to) #if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS) l32i a3, a5, THREAD_CPENABLE - xsr a3, CPENABLE + xsr a3, cpenable s32i a3, a4, THREAD_CPENABLE #endif /* Flush register file. */ - call0 _spill_registers # destroys a3, a4, and SAR + spill_registers_kernel /* Set kernel stack (and leave critical section) * Note: It's save to set it here. The stack will not be overwritten @@ -1924,25 +1856,26 @@ ENTRY(_switch_to) * we return from kernel space. */ - rsr a3, EXCSAVE_1 # exc_table + rsr a3, excsave1 # exc_table movi a6, 0 addi a7, a5, PT_REGS_OFFSET s32i a6, a3, EXC_TABLE_FIXUP s32i a7, a3, EXC_TABLE_KSTK - /* restore context of the task that 'next' addresses */ + /* restore context of the task 'next' */ - l32i a0, a13, THREAD_RA # restore return address - l32i a1, a13, THREAD_SP # restore stack pointer + l32i a0, a11, THREAD_RA # restore return address + l32i a1, a11, THREAD_SP # restore stack pointer - load_xtregs_user a5 a6 a8 a9 a10 a11 THREAD_XTREGS_USER + load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER - wsr a14, PS - mov a2, a12 # return 'prev' + wsr a14, ps + mov a2, a10 # return 'prev' rsync retw +ENDPROC(_switch_to) ENTRY(ret_from_fork) @@ -1958,3 +1891,18 @@ ENTRY(ret_from_fork) j common_exception_return +ENDPROC(ret_from_fork) + +/* + * Kernel thread creation helper + * On entry, set up by copy_thread: a2 = thread_fn, a3 = thread_fn arg + * left from _switch_to: a6 = prev + */ +ENTRY(ret_from_kernel_thread) + + call4 schedule_tail + mov a6, a3 + callx4 a2 + j common_exception_return + +ENDPROC(ret_from_kernel_thread) |