diff options
Diffstat (limited to 'arch/xtensa/kernel')
| -rw-r--r-- | arch/xtensa/kernel/Makefile | 1 | ||||
| -rw-r--r-- | arch/xtensa/kernel/entry.S | 438 | ||||
| -rw-r--r-- | arch/xtensa/kernel/head.S | 181 | ||||
| -rw-r--r-- | arch/xtensa/kernel/irq.c | 191 | ||||
| -rw-r--r-- | arch/xtensa/kernel/mxhead.S | 85 | ||||
| -rw-r--r-- | arch/xtensa/kernel/setup.c | 235 | ||||
| -rw-r--r-- | arch/xtensa/kernel/signal.c | 2 | ||||
| -rw-r--r-- | arch/xtensa/kernel/smp.c | 607 | ||||
| -rw-r--r-- | arch/xtensa/kernel/time.c | 62 | ||||
| -rw-r--r-- | arch/xtensa/kernel/traps.c | 56 | ||||
| -rw-r--r-- | arch/xtensa/kernel/vectors.S | 160 | ||||
| -rw-r--r-- | arch/xtensa/kernel/vmlinux.lds.S | 30 | ||||
| -rw-r--r-- | arch/xtensa/kernel/xtensa_ksyms.c | 9 |
13 files changed, 1505 insertions, 552 deletions
diff --git a/arch/xtensa/kernel/Makefile b/arch/xtensa/kernel/Makefile index f90265ec1cc..18d962a8c0c 100644 --- a/arch/xtensa/kernel/Makefile +++ b/arch/xtensa/kernel/Makefile @@ -12,6 +12,7 @@ obj-$(CONFIG_KGDB) += xtensa-stub.o obj-$(CONFIG_PCI) += pci.o obj-$(CONFIG_MODULES) += xtensa_ksyms.o module.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o +obj-$(CONFIG_SMP) += smp.o mxhead.o AFLAGS_head.o += -mtext-section-literals diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S index de1dfa18d0a..ef7f4990722 100644 --- a/arch/xtensa/kernel/entry.S +++ b/arch/xtensa/kernel/entry.S @@ -1081,185 +1081,53 @@ ENTRY(fast_syscall_spill_registers) rsr a0, sar s32i a3, a2, PT_AREG3 - s32i a4, a2, PT_AREG4 - s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5 + s32i a0, a2, PT_SAR - /* The spill routine might clobber a7, a11, and a15. */ + /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */ + s32i a4, a2, PT_AREG4 s32i a7, a2, PT_AREG7 + s32i a8, a2, PT_AREG8 s32i a11, a2, PT_AREG11 + s32i a12, a2, PT_AREG12 s32i a15, a2, PT_AREG15 - call0 _spill_registers # destroys a3, a4, and SAR - - /* Advance PC, restore registers and SAR, and return from exception. */ - - l32i a3, a2, PT_AREG5 - l32i a4, a2, PT_AREG4 - l32i a0, a2, PT_AREG0 - wsr a3, sar - l32i a3, a2, PT_AREG3 - - /* Restore clobbered registers. */ - - l32i a7, a2, PT_AREG7 - l32i a11, a2, PT_AREG11 - l32i a15, a2, PT_AREG15 - - movi a2, 0 - rfe - -ENDPROC(fast_syscall_spill_registers) - -/* Fixup handler. - * - * We get here if the spill routine causes an exception, e.g. tlb miss. - * We basically restore WINDOWBASE and WINDOWSTART to the condition when - * we entered the spill routine and jump to the user exception handler. - * - * a0: value of depc, original value in depc - * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE - * a3: exctable, original value in excsave1 - */ - -fast_syscall_spill_registers_fixup: - - rsr a2, windowbase # get current windowbase (a2 is saved) - xsr a0, depc # restore depc and a0 - ssl a2 # set shift (32 - WB) - - /* We need to make sure the current registers (a0-a3) are preserved. - * To do this, we simply set the bit for the current window frame - * in WS, so that the exception handlers save them to the task stack. - */ - - xsr a3, excsave1 # get spill-mask - slli a2, a3, 1 # shift left by one - - slli a3, a2, 32-WSBITS - src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy...... - wsr a2, windowstart # set corrected windowstart - - rsr a3, excsave1 - l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2 - l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task) - - /* Return to the original (user task) WINDOWBASE. - * We leave the following frame behind: - * a0, a1, a2 same - * a3: trashed (saved in excsave_1) - * depc: depc (we have to return to that address) - * excsave_1: a3 - */ - - wsr a3, windowbase - rsync - - /* We are now in the original frame when we entered _spill_registers: - * a0: return address - * a1: used, stack pointer - * a2: kernel stack pointer - * a3: available, saved in EXCSAVE_1 - * depc: exception address - * excsave: a3 - * Note: This frame might be the same as above. - */ - - /* Setup stack pointer. */ - - addi a2, a2, -PT_USER_SIZE - s32i a0, a2, PT_AREG0 - - /* Make sure we return to this fixup handler. */ - - movi a3, fast_syscall_spill_registers_fixup_return - s32i a3, a2, PT_DEPC # setup depc - - /* Jump to the exception handler. */ - - rsr a3, excsave1 - rsr a0, exccause - addx4 a0, a0, a3 # find entry in table - l32i a0, a0, EXC_TABLE_FAST_USER # load handler - jx a0 - -fast_syscall_spill_registers_fixup_return: - - /* When we return here, all registers have been restored (a2: DEPC) */ - - wsr a2, depc # exception address - - /* Restore fixup handler. */ - - xsr a3, excsave1 - movi a2, fast_syscall_spill_registers_fixup - s32i a2, a3, EXC_TABLE_FIXUP - s32i a0, a3, EXC_TABLE_DOUBLE_SAVE - rsr a2, windowbase - s32i a2, a3, EXC_TABLE_PARAM - l32i a2, a3, EXC_TABLE_KSTK - - /* Load WB at the time the exception occurred. */ - - rsr a3, sar # WB is still in SAR - neg a3, a3 - wsr a3, windowbase - rsync - - rfde - - -/* - * spill all registers. - * - * This is not a real function. The following conditions must be met: - * - * - must be called with call0. - * - uses a3, a4 and SAR. - * - the last 'valid' register of each frame are clobbered. - * - the caller must have registered a fixup handler - * (or be inside a critical section) - * - PS_EXCM must be set (PS_WOE cleared?) - */ - -ENTRY(_spill_registers) - /* * Rotate ws so that the current windowbase is at bit 0. * Assume ws = xxxwww1yy (www1 current window frame). * Rotate ws right so that a4 = yyxxxwww1. */ - rsr a4, windowbase + rsr a0, windowbase rsr a3, windowstart # a3 = xxxwww1yy - ssr a4 # holds WB - slli a4, a3, WSBITS - or a3, a3, a4 # a3 = xxxwww1yyxxxwww1yy + ssr a0 # holds WB + slli a0, a3, WSBITS + or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy srl a3, a3 # a3 = 00xxxwww1yyxxxwww1 /* We are done if there are no more than the current register frame. */ extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww - movi a4, (1 << (WSBITS-1)) + movi a0, (1 << (WSBITS-1)) _beqz a3, .Lnospill # only one active frame? jump /* We want 1 at the top, so that we return to the current windowbase */ - or a3, a3, a4 # 1yyxxxwww + or a3, a3, a0 # 1yyxxxwww /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */ wsr a3, windowstart # save shifted windowstart - neg a4, a3 - and a3, a4, a3 # first bit set from right: 000010000 + neg a0, a3 + and a3, a0, a3 # first bit set from right: 000010000 - ffs_ws a4, a3 # a4: shifts to skip empty frames + ffs_ws a0, a3 # a0: shifts to skip empty frames movi a3, WSBITS - sub a4, a3, a4 # WSBITS-a4:number of 0-bits from right - ssr a4 # save in SAR for later. + sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right + ssr a0 # save in SAR for later. rsr a3, windowbase - add a3, a3, a4 + add a3, a3, a0 wsr a3, windowbase rsync @@ -1274,22 +1142,6 @@ ENTRY(_spill_registers) * we have to save 4,8. or 12 registers. */ - _bbsi.l a3, 1, .Lc4 - _bbsi.l a3, 2, .Lc8 - - /* Special case: we have a call12-frame starting at a4. */ - - _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first) - - s32e a4, a1, -16 # a1 is valid with an empty spill area - l32e a4, a5, -12 - s32e a8, a4, -48 - mov a8, a4 - l32e a4, a1, -16 - j .Lc12c - -.Lnospill: - ret .Lloop: _bbsi.l a3, 1, .Lc4 _bbci.l a3, 2, .Lc12 @@ -1303,20 +1155,10 @@ ENTRY(_spill_registers) s32e a9, a4, -28 s32e a10, a4, -24 s32e a11, a4, -20 - srli a11, a3, 2 # shift windowbase by 2 rotw 2 _bnei a3, 1, .Lloop - -.Lexit: /* Done. Do the final rotation, set WS, and return. */ - - rotw 1 - rsr a3, windowbase - ssl a3 - movi a3, 1 - sll a3, a3 - wsr a3, windowstart - ret + j .Lexit .Lc4: s32e a4, a9, -16 s32e a5, a9, -12 @@ -1332,11 +1174,11 @@ ENTRY(_spill_registers) /* 12-register frame (call12) */ - l32e a2, a5, -12 - s32e a8, a2, -48 - mov a8, a2 + l32e a0, a5, -12 + s32e a8, a0, -48 + mov a8, a0 -.Lc12c: s32e a9, a8, -44 + s32e a9, a8, -44 s32e a10, a8, -40 s32e a11, a8, -36 s32e a12, a8, -32 @@ -1356,30 +1198,54 @@ ENTRY(_spill_registers) */ rotw 1 - mov a5, a13 + mov a4, a13 rotw -1 - s32e a4, a9, -16 - s32e a5, a9, -12 - s32e a6, a9, -8 - s32e a7, a9, -4 + s32e a4, a8, -16 + s32e a5, a8, -12 + s32e a6, a8, -8 + s32e a7, a8, -4 rotw 3 _beqi a3, 1, .Lexit j .Lloop -.Linvalid_mask: +.Lexit: - /* We get here because of an unrecoverable error in the window - * registers. If we are in user space, we kill the application, - * however, this condition is unrecoverable in kernel space. - */ + /* Done. Do the final rotation and set WS */ - rsr a0, ps - _bbci.l a0, PS_UM_BIT, 1f + rotw 1 + rsr a3, windowbase + ssl a3 + movi a3, 1 + sll a3, a3 + wsr a3, windowstart +.Lnospill: - /* User space: Setup a dummy frame and kill application. + /* Advance PC, restore registers and SAR, and return from exception. */ + + l32i a3, a2, PT_SAR + l32i a0, a2, PT_AREG0 + wsr a3, sar + l32i a3, a2, PT_AREG3 + + /* Restore clobbered registers. */ + + l32i a4, a2, PT_AREG4 + l32i a7, a2, PT_AREG7 + l32i a8, a2, PT_AREG8 + l32i a11, a2, PT_AREG11 + l32i a12, a2, PT_AREG12 + l32i a15, a2, PT_AREG15 + + movi a2, 0 + rfe + +.Linvalid_mask: + + /* We get here because of an unrecoverable error in the window + * registers, so set up a dummy frame and kill the user application. * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer. */ @@ -1403,14 +1269,136 @@ ENTRY(_spill_registers) movi a4, do_exit callx4 a4 -1: /* Kernel space: PANIC! */ + /* shouldn't return, so panic */ wsr a0, excsave1 movi a0, unrecoverable_exception callx0 a0 # should not return 1: j 1b -ENDPROC(_spill_registers) + +ENDPROC(fast_syscall_spill_registers) + +/* Fixup handler. + * + * We get here if the spill routine causes an exception, e.g. tlb miss. + * We basically restore WINDOWBASE and WINDOWSTART to the condition when + * we entered the spill routine and jump to the user exception handler. + * + * Note that we only need to restore the bits in windowstart that have not + * been spilled yet by the _spill_register routine. Luckily, a3 contains a + * rotated windowstart with only those bits set for frames that haven't been + * spilled yet. Because a3 is rotated such that bit 0 represents the register + * frame for the current windowbase - 1, we need to rotate a3 left by the + * value of the current windowbase + 1 and move it to windowstart. + * + * a0: value of depc, original value in depc + * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE + * a3: exctable, original value in excsave1 + */ + +ENTRY(fast_syscall_spill_registers_fixup) + + rsr a2, windowbase # get current windowbase (a2 is saved) + xsr a0, depc # restore depc and a0 + ssl a2 # set shift (32 - WB) + + /* We need to make sure the current registers (a0-a3) are preserved. + * To do this, we simply set the bit for the current window frame + * in WS, so that the exception handlers save them to the task stack. + * + * Note: we use a3 to set the windowbase, so we take a special care + * of it, saving it in the original _spill_registers frame across + * the exception handler call. + */ + + xsr a3, excsave1 # get spill-mask + slli a3, a3, 1 # shift left by one + addi a3, a3, 1 # set the bit for the current window frame + + slli a2, a3, 32-WSBITS + src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy...... + wsr a2, windowstart # set corrected windowstart + + srli a3, a3, 1 + rsr a2, excsave1 + l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2 + xsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3 + l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task) + xsr a2, excsave1 + + /* Return to the original (user task) WINDOWBASE. + * We leave the following frame behind: + * a0, a1, a2 same + * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE) + * depc: depc (we have to return to that address) + * excsave_1: exctable + */ + + wsr a3, windowbase + rsync + + /* We are now in the original frame when we entered _spill_registers: + * a0: return address + * a1: used, stack pointer + * a2: kernel stack pointer + * a3: available + * depc: exception address + * excsave: exctable + * Note: This frame might be the same as above. + */ + + /* Setup stack pointer. */ + + addi a2, a2, -PT_USER_SIZE + s32i a0, a2, PT_AREG0 + + /* Make sure we return to this fixup handler. */ + + movi a3, fast_syscall_spill_registers_fixup_return + s32i a3, a2, PT_DEPC # setup depc + + /* Jump to the exception handler. */ + + rsr a3, excsave1 + rsr a0, exccause + addx4 a0, a0, a3 # find entry in table + l32i a0, a0, EXC_TABLE_FAST_USER # load handler + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + jx a0 + +ENDPROC(fast_syscall_spill_registers_fixup) + +ENTRY(fast_syscall_spill_registers_fixup_return) + + /* When we return here, all registers have been restored (a2: DEPC) */ + + wsr a2, depc # exception address + + /* Restore fixup handler. */ + + rsr a2, excsave1 + s32i a3, a2, EXC_TABLE_DOUBLE_SAVE + movi a3, fast_syscall_spill_registers_fixup + s32i a3, a2, EXC_TABLE_FIXUP + rsr a3, windowbase + s32i a3, a2, EXC_TABLE_PARAM + l32i a2, a2, EXC_TABLE_KSTK + + /* Load WB at the time the exception occurred. */ + + rsr a3, sar # WB is still in SAR + neg a3, a3 + wsr a3, windowbase + rsync + + rsr a3, excsave1 + l32i a3, a3, EXC_TABLE_DOUBLE_SAVE + + rfde + +ENDPROC(fast_syscall_spill_registers_fixup_return) #ifdef CONFIG_MMU /* @@ -1783,6 +1771,43 @@ ENTRY(system_call) ENDPROC(system_call) +/* + * Spill live registers on the kernel stack macro. + * + * Entry condition: ps.woe is set, ps.excm is cleared + * Exit condition: windowstart has single bit set + * May clobber: a12, a13 + */ + .macro spill_registers_kernel + +#if XCHAL_NUM_AREGS > 16 + call12 1f + _j 2f + retw + .align 4 +1: + _entry a1, 48 + addi a12, a0, 3 +#if XCHAL_NUM_AREGS > 32 + .rept (XCHAL_NUM_AREGS - 32) / 12 + _entry a1, 48 + mov a12, a0 + .endr +#endif + _entry a1, 48 +#if XCHAL_NUM_AREGS % 12 == 0 + mov a8, a8 +#elif XCHAL_NUM_AREGS % 12 == 4 + mov a12, a12 +#elif XCHAL_NUM_AREGS % 12 == 8 + mov a4, a4 +#endif + retw +2: +#else + mov a12, a12 +#endif + .endm /* * Task switch. @@ -1795,21 +1820,20 @@ ENTRY(_switch_to) entry a1, 16 - mov a12, a2 # preserve 'prev' (a2) - mov a13, a3 # and 'next' (a3) + mov a10, a2 # preserve 'prev' (a2) + mov a11, a3 # and 'next' (a3) l32i a4, a2, TASK_THREAD_INFO l32i a5, a3, TASK_THREAD_INFO - save_xtregs_user a4 a6 a8 a9 a10 a11 THREAD_XTREGS_USER + save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER - s32i a0, a12, THREAD_RA # save return address - s32i a1, a12, THREAD_SP # save stack pointer + s32i a0, a10, THREAD_RA # save return address + s32i a1, a10, THREAD_SP # save stack pointer /* Disable ints while we manipulate the stack pointer. */ - movi a14, (1 << PS_EXCM_BIT) | LOCKLEVEL - xsr a14, ps + rsil a14, LOCKLEVEL rsr a3, excsave1 rsync s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */ @@ -1824,7 +1848,7 @@ ENTRY(_switch_to) /* Flush register file. */ - call0 _spill_registers # destroys a3, a4, and SAR + spill_registers_kernel /* Set kernel stack (and leave critical section) * Note: It's save to set it here. The stack will not be overwritten @@ -1840,13 +1864,13 @@ ENTRY(_switch_to) /* restore context of the task 'next' */ - l32i a0, a13, THREAD_RA # restore return address - l32i a1, a13, THREAD_SP # restore stack pointer + l32i a0, a11, THREAD_RA # restore return address + l32i a1, a11, THREAD_SP # restore stack pointer - load_xtregs_user a5 a6 a8 a9 a10 a11 THREAD_XTREGS_USER + load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER wsr a14, ps - mov a2, a12 # return 'prev' + mov a2, a10 # return 'prev' rsync retw diff --git a/arch/xtensa/kernel/head.S b/arch/xtensa/kernel/head.S index 7d740ebbe19..aeeb3cc8a41 100644 --- a/arch/xtensa/kernel/head.S +++ b/arch/xtensa/kernel/head.S @@ -19,6 +19,7 @@ #include <asm/page.h> #include <asm/cacheasm.h> #include <asm/initialize_mmu.h> +#include <asm/mxregs.h> #include <linux/init.h> #include <linux/linkage.h> @@ -54,7 +55,7 @@ ENTRY(_start) /* Preserve the pointer to the boot parameter list in EXCSAVE_1 */ wsr a2, excsave1 - _j _SetupMMU + _j _SetupOCD .align 4 .literal_position @@ -62,6 +63,23 @@ ENTRY(_start) .word _startup .align 4 +_SetupOCD: + /* + * Initialize WB, WS, and clear PS.EXCM (to allow loop instructions). + * Set Interrupt Level just below XCHAL_DEBUGLEVEL to allow + * xt-gdb to single step via DEBUG exceptions received directly + * by ocd. + */ + movi a1, 1 + movi a0, 0 + wsr a1, windowstart + wsr a0, windowbase + rsync + + movi a1, LOCKLEVEL + wsr a1, ps + rsync + .global _SetupMMU _SetupMMU: Offset = _SetupMMU - _start @@ -85,24 +103,11 @@ _SetupMMU: ENDPROC(_start) - __INIT + __REF .literal_position ENTRY(_startup) - /* Disable interrupts and exceptions. */ - - movi a0, LOCKLEVEL - wsr a0, ps - - /* Start with a fresh windowbase and windowstart. */ - - movi a1, 1 - movi a0, 0 - wsr a1, windowstart - wsr a0, windowbase - rsync - /* Set a0 to 0 for the remaining initialization. */ movi a0, 0 @@ -154,17 +159,6 @@ ENTRY(_startup) wsr a0, cpenable #endif - /* Set PS.INTLEVEL=LOCKLEVEL, PS.WOE=0, kernel stack, PS.EXCM=0 - * - * Note: PS.EXCM must be cleared before using any loop - * instructions; otherwise, they are silently disabled, and - * at most one iteration of the loop is executed. - */ - - movi a1, LOCKLEVEL - wsr a1, ps - rsync - /* Initialize the caches. * a2, a3 are just working registers (clobbered). */ @@ -182,6 +176,37 @@ ENTRY(_startup) isync +#ifdef CONFIG_HAVE_SMP + movi a2, CCON # MX External Register to Configure Cache + movi a3, 1 + wer a3, a2 +#endif + + /* Setup stack and enable window exceptions (keep irqs disabled) */ + + movi a1, start_info + l32i a1, a1, 0 + + movi a2, (1 << PS_WOE_BIT) | LOCKLEVEL + # WOE=1, INTLEVEL=LOCKLEVEL, UM=0 + wsr a2, ps # (enable reg-windows; progmode stack) + rsync + + /* Set up EXCSAVE[DEBUGLEVEL] to point to the Debug Exception Handler.*/ + + movi a2, debug_exception + wsr a2, SREG_EXCSAVE + XCHAL_DEBUGLEVEL + +#ifdef CONFIG_SMP + /* + * Notice that we assume with SMP that cores have PRID + * supported by the cores. + */ + rsr a2, prid + bnez a2, .Lboot_secondary + +#endif /* CONFIG_SMP */ + /* Unpack data sections * * The linker script used to build the Linux kernel image @@ -234,24 +259,7 @@ ENTRY(_startup) ___invalidate_icache_all a2 a3 isync - /* Setup stack and enable window exceptions (keep irqs disabled) */ - - movi a1, init_thread_union - addi a1, a1, KERNEL_STACK_SIZE - - movi a2, (1 << PS_WOE_BIT) | LOCKLEVEL - # WOE=1, INTLEVEL=LOCKLEVEL, UM=0 - wsr a2, ps # (enable reg-windows; progmode stack) - rsync - - /* Set up EXCSAVE[DEBUGLEVEL] to point to the Debug Exception Handler.*/ - - movi a2, debug_exception - wsr a2, SREG_EXCSAVE + XCHAL_DEBUGLEVEL - - /* Set up EXCSAVE[1] to point to the exc_table. */ - - movi a6, exc_table + movi a6, 0 xsr a6, excsave1 /* init_arch kick-starts the linux kernel */ @@ -265,8 +273,93 @@ ENTRY(_startup) should_never_return: j should_never_return +#ifdef CONFIG_SMP +.Lboot_secondary: + + movi a2, cpu_start_ccount +1: + l32i a3, a2, 0 + beqi a3, 0, 1b + movi a3, 0 + s32i a3, a2, 0 + memw +1: + l32i a3, a2, 0 + beqi a3, 0, 1b + wsr a3, ccount + movi a3, 0 + s32i a3, a2, 0 + memw + + movi a6, 0 + wsr a6, excsave1 + + movi a4, secondary_start_kernel + callx4 a4 + j should_never_return + +#endif /* CONFIG_SMP */ + ENDPROC(_startup) +#ifdef CONFIG_HOTPLUG_CPU + +ENTRY(cpu_restart) + +#if XCHAL_DCACHE_IS_WRITEBACK + ___flush_invalidate_dcache_all a2 a3 +#else + ___invalidate_dcache_all a2 a3 +#endif + memw + movi a2, CCON # MX External Register to Configure Cache + movi a3, 0 + wer a3, a2 + extw + + rsr a0, prid + neg a2, a0 + movi a3, cpu_start_id + s32i a2, a3, 0 +#if XCHAL_DCACHE_IS_WRITEBACK + dhwbi a3, 0 +#endif +1: + l32i a2, a3, 0 + dhi a3, 0 + bne a2, a0, 1b + + /* + * Initialize WB, WS, and clear PS.EXCM (to allow loop instructions). + * Set Interrupt Level just below XCHAL_DEBUGLEVEL to allow + * xt-gdb to single step via DEBUG exceptions received directly + * by ocd. + */ + movi a1, 1 + movi a0, 0 + wsr a1, windowstart + wsr a0, windowbase + rsync + + movi a1, LOCKLEVEL + wsr a1, ps + rsync + + j _startup + +ENDPROC(cpu_restart) + +#endif /* CONFIG_HOTPLUG_CPU */ + +/* + * DATA section + */ + + .section ".data.init.refok" + .align 4 +ENTRY(start_info) + .long init_thread_union + KERNEL_STACK_SIZE + /* * BSS section */ diff --git a/arch/xtensa/kernel/irq.c b/arch/xtensa/kernel/irq.c index 6f4f9749cff..3eee94f621e 100644 --- a/arch/xtensa/kernel/irq.c +++ b/arch/xtensa/kernel/irq.c @@ -4,7 +4,7 @@ * Xtensa built-in interrupt controller and some generic functions copied * from i386. * - * Copyright (C) 2002 - 2006 Tensilica, Inc. + * Copyright (C) 2002 - 2013 Tensilica, Inc. * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar * * @@ -18,36 +18,27 @@ #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/kernel_stat.h> +#include <linux/irqchip.h> +#include <linux/irqchip/xtensa-mx.h> +#include <linux/irqchip/xtensa-pic.h> #include <linux/irqdomain.h> #include <linux/of.h> +#include <asm/mxregs.h> #include <asm/uaccess.h> #include <asm/platform.h> -static unsigned int cached_irq_mask; - atomic_t irq_err_count; -static struct irq_domain *root_domain; - -/* - * do_IRQ handles all normal device IRQ's (the special - * SMP cross-CPU interrupts have their own specific - * handlers). - */ - asmlinkage void do_IRQ(int hwirq, struct pt_regs *regs) { - struct pt_regs *old_regs = set_irq_regs(regs); - int irq = irq_find_mapping(root_domain, hwirq); + int irq = irq_find_mapping(NULL, hwirq); if (hwirq >= NR_IRQS) { printk(KERN_EMERG "%s: cannot handle IRQ %d\n", __func__, hwirq); } - irq_enter(); - #ifdef CONFIG_DEBUG_STACKOVERFLOW /* Debugging check for stack overflow: is there less than 1KB free? */ { @@ -62,95 +53,69 @@ asmlinkage void do_IRQ(int hwirq, struct pt_regs *regs) } #endif generic_handle_irq(irq); - - irq_exit(); - set_irq_regs(old_regs); } int arch_show_interrupts(struct seq_file *p, int prec) { +#ifdef CONFIG_SMP + show_ipi_list(p, prec); +#endif seq_printf(p, "%*s: ", prec, "ERR"); seq_printf(p, "%10u\n", atomic_read(&irq_err_count)); return 0; } -static void xtensa_irq_mask(struct irq_data *d) -{ - cached_irq_mask &= ~(1 << d->hwirq); - set_sr (cached_irq_mask, intenable); -} - -static void xtensa_irq_unmask(struct irq_data *d) -{ - cached_irq_mask |= 1 << d->hwirq; - set_sr (cached_irq_mask, intenable); -} - -static void xtensa_irq_enable(struct irq_data *d) -{ - variant_irq_enable(d->hwirq); - xtensa_irq_unmask(d); -} - -static void xtensa_irq_disable(struct irq_data *d) -{ - xtensa_irq_mask(d); - variant_irq_disable(d->hwirq); -} - -static void xtensa_irq_ack(struct irq_data *d) -{ - set_sr(1 << d->hwirq, intclear); -} - -static int xtensa_irq_retrigger(struct irq_data *d) +int xtensa_irq_domain_xlate(const u32 *intspec, unsigned int intsize, + unsigned long int_irq, unsigned long ext_irq, + unsigned long *out_hwirq, unsigned int *out_type) { - set_sr(1 << d->hwirq, intset); - return 1; + if (WARN_ON(intsize < 1 || intsize > 2)) + return -EINVAL; + if (intsize == 2 && intspec[1] == 1) { + int_irq = xtensa_map_ext_irq(ext_irq); + if (int_irq < XCHAL_NUM_INTERRUPTS) + *out_hwirq = int_irq; + else + return -EINVAL; + } else { + *out_hwirq = int_irq; + } + *out_type = IRQ_TYPE_NONE; + return 0; } -static struct irq_chip xtensa_irq_chip = { - .name = "xtensa", - .irq_enable = xtensa_irq_enable, - .irq_disable = xtensa_irq_disable, - .irq_mask = xtensa_irq_mask, - .irq_unmask = xtensa_irq_unmask, - .irq_ack = xtensa_irq_ack, - .irq_retrigger = xtensa_irq_retrigger, -}; - -static int xtensa_irq_map(struct irq_domain *d, unsigned int irq, +int xtensa_irq_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) { + struct irq_chip *irq_chip = d->host_data; u32 mask = 1 << hw; if (mask & XCHAL_INTTYPE_MASK_SOFTWARE) { - irq_set_chip_and_handler_name(irq, &xtensa_irq_chip, + irq_set_chip_and_handler_name(irq, irq_chip, handle_simple_irq, "level"); irq_set_status_flags(irq, IRQ_LEVEL); } else if (mask & XCHAL_INTTYPE_MASK_EXTERN_EDGE) { - irq_set_chip_and_handler_name(irq, &xtensa_irq_chip, + irq_set_chip_and_handler_name(irq, irq_chip, handle_edge_irq, "edge"); irq_clear_status_flags(irq, IRQ_LEVEL); } else if (mask & XCHAL_INTTYPE_MASK_EXTERN_LEVEL) { - irq_set_chip_and_handler_name(irq, &xtensa_irq_chip, + irq_set_chip_and_handler_name(irq, irq_chip, handle_level_irq, "level"); irq_set_status_flags(irq, IRQ_LEVEL); } else if (mask & XCHAL_INTTYPE_MASK_TIMER) { - irq_set_chip_and_handler_name(irq, &xtensa_irq_chip, - handle_edge_irq, "edge"); + irq_set_chip_and_handler_name(irq, irq_chip, + handle_percpu_irq, "timer"); irq_clear_status_flags(irq, IRQ_LEVEL); } else {/* XCHAL_INTTYPE_MASK_WRITE_ERROR */ /* XCHAL_INTTYPE_MASK_NMI */ - - irq_set_chip_and_handler_name(irq, &xtensa_irq_chip, + irq_set_chip_and_handler_name(irq, irq_chip, handle_level_irq, "level"); irq_set_status_flags(irq, IRQ_LEVEL); } return 0; } -static unsigned map_ext_irq(unsigned ext_irq) +unsigned xtensa_map_ext_irq(unsigned ext_irq) { unsigned mask = XCHAL_INTTYPE_MASK_EXTERN_EDGE | XCHAL_INTTYPE_MASK_EXTERN_LEVEL; @@ -163,55 +128,61 @@ static unsigned map_ext_irq(unsigned ext_irq) return XCHAL_NUM_INTERRUPTS; } -/* - * Device Tree IRQ specifier translation function which works with one or - * two cell bindings. First cell value maps directly to the hwirq number. - * Second cell if present specifies whether hwirq number is external (1) or - * internal (0). - */ -int xtensa_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, - const u32 *intspec, unsigned int intsize, - unsigned long *out_hwirq, unsigned int *out_type) +unsigned xtensa_get_ext_irq_no(unsigned irq) { - if (WARN_ON(intsize < 1 || intsize > 2)) - return -EINVAL; - if (intsize == 2 && intspec[1] == 1) { - unsigned int_irq = map_ext_irq(intspec[0]); - if (int_irq < XCHAL_NUM_INTERRUPTS) - *out_hwirq = int_irq; - else - return -EINVAL; - } else { - *out_hwirq = intspec[0]; - } - *out_type = IRQ_TYPE_NONE; - return 0; + unsigned mask = (XCHAL_INTTYPE_MASK_EXTERN_EDGE | + XCHAL_INTTYPE_MASK_EXTERN_LEVEL) & + ((1u << irq) - 1); + return hweight32(mask); } -static const struct irq_domain_ops xtensa_irq_domain_ops = { - .xlate = xtensa_irq_domain_xlate, - .map = xtensa_irq_map, -}; - void __init init_IRQ(void) { - struct device_node *intc = NULL; - - cached_irq_mask = 0; - set_sr(~0, intclear); - #ifdef CONFIG_OF - /* The interrupt controller device node is mandatory */ - intc = of_find_compatible_node(NULL, NULL, "xtensa,pic"); - BUG_ON(!intc); - - root_domain = irq_domain_add_linear(intc, NR_IRQS, - &xtensa_irq_domain_ops, NULL); + irqchip_init(); +#else +#ifdef CONFIG_HAVE_SMP + xtensa_mx_init_legacy(NULL); #else - root_domain = irq_domain_add_legacy(intc, NR_IRQS, 0, 0, - &xtensa_irq_domain_ops, NULL); + xtensa_pic_init_legacy(NULL); +#endif #endif - irq_set_default_host(root_domain); +#ifdef CONFIG_SMP + ipi_init(); +#endif variant_init_irq(); } + +#ifdef CONFIG_HOTPLUG_CPU +/* + * The CPU has been marked offline. Migrate IRQs off this CPU. If + * the affinity settings do not allow other CPUs, force them onto any + * available CPU. + */ +void migrate_irqs(void) +{ + unsigned int i, cpu = smp_processor_id(); + + for_each_active_irq(i) { + struct irq_data *data = irq_get_irq_data(i); + unsigned int newcpu; + + if (irqd_is_per_cpu(data)) + continue; + + if (!cpumask_test_cpu(cpu, data->affinity)) + continue; + + newcpu = cpumask_any_and(data->affinity, cpu_online_mask); + + if (newcpu >= nr_cpu_ids) { + pr_info_ratelimited("IRQ%u no longer affine to CPU%u\n", + i, cpu); + + cpumask_setall(data->affinity); + } + irq_set_affinity(i, data->affinity); + } +} +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/xtensa/kernel/mxhead.S b/arch/xtensa/kernel/mxhead.S new file mode 100644 index 00000000000..77a161a112c --- /dev/null +++ b/arch/xtensa/kernel/mxhead.S @@ -0,0 +1,85 @@ +/* + * Xtensa Secondary Processors startup code. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2013 Tensilica Inc. + * + * Joe Taylor <joe@tensilica.com> + * Chris Zankel <chris@zankel.net> + * Marc Gauthier <marc@tensilica.com, marc@alumni.uwaterloo.ca> + * Pete Delaney <piet@tensilica.com> + */ + +#include <linux/linkage.h> + +#include <asm/cacheasm.h> +#include <asm/initialize_mmu.h> +#include <asm/mxregs.h> +#include <asm/regs.h> + + + .section .SecondaryResetVector.text, "ax" + + +ENTRY(_SecondaryResetVector) + _j _SetupOCD + + .begin no-absolute-literals + .literal_position + +_SetupOCD: + /* + * Initialize WB, WS, and clear PS.EXCM (to allow loop instructions). + * Set Interrupt Level just below XCHAL_DEBUGLEVEL to allow + * xt-gdb to single step via DEBUG exceptions received directly + * by ocd. + */ + movi a1, 1 + movi a0, 0 + wsr a1, windowstart + wsr a0, windowbase + rsync + + movi a1, LOCKLEVEL + wsr a1, ps + rsync + +_SetupMMU: + Offset = _SetupMMU - _SecondaryResetVector + +#ifdef CONFIG_INITIALIZE_XTENSA_MMU_INSIDE_VMLINUX + initialize_mmu +#endif + + /* + * Start Secondary Processors with NULL pointer to boot params. + */ + movi a2, 0 # a2 == NULL + movi a3, _startup + jx a3 + + .end no-absolute-literals + + + .section .SecondaryResetVector.remapped_text, "ax" + .global _RemappedSecondaryResetVector + + .org 0 # Need to do org before literals + +_RemappedSecondaryResetVector: + .begin no-absolute-literals + .literal_position + + _j _RemappedSetupMMU + . = _RemappedSecondaryResetVector + Offset + +_RemappedSetupMMU: + +#ifdef CONFIG_INITIALIZE_XTENSA_MMU_INSIDE_VMLINUX + initialize_mmu +#endif + + .end no-absolute-literals diff --git a/arch/xtensa/kernel/setup.c b/arch/xtensa/kernel/setup.c index 946fb8d06c8..06370ccea9e 100644 --- a/arch/xtensa/kernel/setup.c +++ b/arch/xtensa/kernel/setup.c @@ -21,11 +21,11 @@ #include <linux/screen_info.h> #include <linux/bootmem.h> #include <linux/kernel.h> - -#ifdef CONFIG_OF +#include <linux/percpu.h> +#include <linux/clk-provider.h> +#include <linux/cpu.h> #include <linux/of_fdt.h> #include <linux/of_platform.h> -#endif #if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) # include <linux/console.h> @@ -40,6 +40,7 @@ #endif #include <asm/bootparam.h> +#include <asm/mmu_context.h> #include <asm/pgtable.h> #include <asm/processor.h> #include <asm/timex.h> @@ -48,6 +49,8 @@ #include <asm/setup.h> #include <asm/param.h> #include <asm/traps.h> +#include <asm/smp.h> +#include <asm/sysmem.h> #include <platform/hardware.h> @@ -64,14 +67,13 @@ extern struct rtc_ops no_rtc_ops; struct rtc_ops *rtc_ops; #ifdef CONFIG_BLK_DEV_INITRD -extern void *initrd_start; -extern void *initrd_end; +extern unsigned long initrd_start; +extern unsigned long initrd_end; int initrd_is_mapped = 0; extern int initrd_below_start_ok; #endif #ifdef CONFIG_OF -extern u32 __dtb_start[]; void *dtb_start = __dtb_start; #endif @@ -86,18 +88,6 @@ static char __initdata command_line[COMMAND_LINE_SIZE]; static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; #endif -sysmem_info_t __initdata sysmem; - -#ifdef CONFIG_MMU -extern void init_mmu(void); -#else -static inline void init_mmu(void) { } -#endif - -extern int mem_reserve(unsigned long, unsigned long, int); -extern void bootmem_init(void); -extern void zones_init(void); - /* * Boot parameter parsing. * @@ -117,31 +107,14 @@ typedef struct tagtable { /* parse current tag */ -static int __init add_sysmem_bank(unsigned long type, unsigned long start, - unsigned long end) -{ - if (sysmem.nr_banks >= SYSMEM_BANKS_MAX) { - printk(KERN_WARNING - "Ignoring memory bank 0x%08lx size %ldKB\n", - start, end - start); - return -EINVAL; - } - sysmem.bank[sysmem.nr_banks].type = type; - sysmem.bank[sysmem.nr_banks].start = PAGE_ALIGN(start); - sysmem.bank[sysmem.nr_banks].end = end & PAGE_MASK; - sysmem.nr_banks++; - - return 0; -} - static int __init parse_tag_mem(const bp_tag_t *tag) { - meminfo_t *mi = (meminfo_t *)(tag->data); + struct bp_meminfo *mi = (struct bp_meminfo *)(tag->data); if (mi->type != MEMORY_TYPE_CONVENTIONAL) return -1; - return add_sysmem_bank(mi->type, mi->start, mi->end); + return add_sysmem_bank(mi->start, mi->end); } __tagtable(BP_TAG_MEMORY, parse_tag_mem); @@ -150,10 +123,10 @@ __tagtable(BP_TAG_MEMORY, parse_tag_mem); static int __init parse_tag_initrd(const bp_tag_t* tag) { - meminfo_t* mi; - mi = (meminfo_t*)(tag->data); - initrd_start = __va(mi->start); - initrd_end = __va(mi->end); + struct bp_meminfo *mi = (struct bp_meminfo *)(tag->data); + + initrd_start = (unsigned long)__va(mi->start); + initrd_end = (unsigned long)__va(mi->end); return 0; } @@ -170,13 +143,6 @@ static int __init parse_tag_fdt(const bp_tag_t *tag) __tagtable(BP_TAG_FDT, parse_tag_fdt); -void __init early_init_dt_setup_initrd_arch(u64 start, u64 end) -{ - initrd_start = (void *)__va(start); - initrd_end = (void *)__va(end); - initrd_below_start_ok = 1; -} - #endif /* CONFIG_OF */ #endif /* CONFIG_BLK_DEV_INITRD */ @@ -222,11 +188,51 @@ static int __init parse_bootparam(const bp_tag_t* tag) } #ifdef CONFIG_OF +bool __initdata dt_memory_scan = false; + +#if XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY +unsigned long xtensa_kio_paddr = XCHAL_KIO_DEFAULT_PADDR; +EXPORT_SYMBOL(xtensa_kio_paddr); + +static int __init xtensa_dt_io_area(unsigned long node, const char *uname, + int depth, void *data) +{ + const __be32 *ranges; + int len; + + if (depth > 1) + return 0; + + if (!of_flat_dt_is_compatible(node, "simple-bus")) + return 0; + + ranges = of_get_flat_dt_prop(node, "ranges", &len); + if (!ranges) + return 1; + if (len == 0) + return 1; + + xtensa_kio_paddr = of_read_ulong(ranges+1, 1); + /* round down to nearest 256MB boundary */ + xtensa_kio_paddr &= 0xf0000000; + + return 1; +} +#else +static int __init xtensa_dt_io_area(unsigned long node, const char *uname, + int depth, void *data) +{ + return 1; +} +#endif void __init early_init_dt_add_memory_arch(u64 base, u64 size) { + if (!dt_memory_scan) + return; + size &= PAGE_MASK; - add_sysmem_bank(MEMORY_TYPE_CONVENTIONAL, base, base + size); + add_sysmem_bank(base, base + size); } void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) @@ -236,36 +242,20 @@ void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) void __init early_init_devtree(void *params) { - /* Setup flat device-tree pointer */ - initial_boot_params = params; - - /* Retrieve various informations from the /chosen node of the - * device-tree, including the platform type, initrd location and - * size, TCE reserve, and more ... - */ - if (!command_line[0]) - of_scan_flat_dt(early_init_dt_scan_chosen, command_line); - - /* Scan memory nodes and rebuild MEMBLOCKs */ - of_scan_flat_dt(early_init_dt_scan_root, NULL); if (sysmem.nr_banks == 0) - of_scan_flat_dt(early_init_dt_scan_memory, NULL); -} + dt_memory_scan = true; -static void __init copy_devtree(void) -{ - void *alloc = early_init_dt_alloc_memory_arch( - be32_to_cpu(initial_boot_params->totalsize), 8); - if (alloc) { - memcpy(alloc, initial_boot_params, - be32_to_cpu(initial_boot_params->totalsize)); - initial_boot_params = alloc; - } + early_init_dt_scan(params); + of_scan_flat_dt(xtensa_dt_io_area, NULL); + + if (!command_line[0]) + strlcpy(command_line, boot_command_line, COMMAND_LINE_SIZE); } static int __init xtensa_device_probe(void) { - of_platform_populate(NULL, NULL, NULL, NULL); + of_clk_init(NULL); + of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL); return 0; } @@ -279,8 +269,6 @@ device_initcall(xtensa_device_probe); void __init init_arch(bp_tag_t *bp_start) { - sysmem.nr_banks = 0; - /* Parse boot parameters */ if (bp_start) @@ -291,10 +279,9 @@ void __init init_arch(bp_tag_t *bp_start) #endif if (sysmem.nr_banks == 0) { - sysmem.nr_banks = 1; - sysmem.bank[0].start = PLATFORM_DEFAULT_MEM_START; - sysmem.bank[0].end = PLATFORM_DEFAULT_MEM_START - + PLATFORM_DEFAULT_MEM_SIZE; + add_sysmem_bank(PLATFORM_DEFAULT_MEM_START, + PLATFORM_DEFAULT_MEM_START + + PLATFORM_DEFAULT_MEM_SIZE); } #ifdef CONFIG_CMDLINE_BOOL @@ -378,7 +365,8 @@ static inline int probed_compare_swap(int *v, int cmp, int set) /* Handle probed exception */ -void __init do_probed_exception(struct pt_regs *regs, unsigned long exccause) +static void __init do_probed_exception(struct pt_regs *regs, + unsigned long exccause) { if (regs->pc == rcw_probe_pc) { /* exception on s32c1i ? */ regs->pc += 3; /* skip the s32c1i instruction */ @@ -390,7 +378,7 @@ void __init do_probed_exception(struct pt_regs *regs, unsigned long exccause) /* Simple test of S32C1I (soc bringup assist) */ -void __init check_s32c1i(void) +static int __init check_s32c1i(void) { int n, cause1, cause2; void *handbus, *handdata, *handaddr; /* temporarily saved handlers */ @@ -445,24 +433,21 @@ void __init check_s32c1i(void) trap_set_handler(EXCCAUSE_LOAD_STORE_ERROR, handbus); trap_set_handler(EXCCAUSE_LOAD_STORE_DATA_ERROR, handdata); trap_set_handler(EXCCAUSE_LOAD_STORE_ADDR_ERROR, handaddr); + return 0; } #else /* XCHAL_HAVE_S32C1I */ /* This condition should not occur with a commercially deployed processor. Display reminder for early engr test or demo chips / FPGA bitstreams */ -void __init check_s32c1i(void) +static int __init check_s32c1i(void) { pr_warn("Processor configuration lacks atomic compare-and-swap support!\n"); + return 0; } #endif /* XCHAL_HAVE_S32C1I */ -#else /* CONFIG_S32C1I_SELFTEST */ - -void __init check_s32c1i(void) -{ -} - +early_initcall(check_s32c1i); #endif /* CONFIG_S32C1I_SELFTEST */ @@ -471,14 +456,12 @@ void __init setup_arch(char **cmdline_p) strlcpy(boot_command_line, command_line, COMMAND_LINE_SIZE); *cmdline_p = command_line; - check_s32c1i(); - /* Reserve some memory regions */ #ifdef CONFIG_BLK_DEV_INITRD if (initrd_start < initrd_end) { initrd_is_mapped = mem_reserve(__pa(initrd_start), - __pa(initrd_end), 0); + __pa(initrd_end), 0) == 0; initrd_below_start_ok = 1; } else { initrd_start = 0; @@ -523,15 +506,17 @@ void __init setup_arch(char **cmdline_p) __pa(&_Level6InterruptVector_text_end), 0); #endif + parse_early_param(); bootmem_init(); -#ifdef CONFIG_OF - copy_devtree(); - unflatten_device_tree(); -#endif + unflatten_and_copy_device_tree(); platform_setup(cmdline_p); +#ifdef CONFIG_SMP + smp_init_cpus(); +#endif + paging_init(); zones_init(); @@ -548,6 +533,22 @@ void __init setup_arch(char **cmdline_p) #endif } +static DEFINE_PER_CPU(struct cpu, cpu_data); + +static int __init topology_init(void) +{ + int i; + + for_each_possible_cpu(i) { + struct cpu *cpu = &per_cpu(cpu_data, i); + cpu->hotpluggable = !!i; + register_cpu(cpu, i); + } + + return 0; +} +subsys_initcall(topology_init); + void machine_restart(char * cmd) { platform_restart(); @@ -573,21 +574,27 @@ void machine_power_off(void) static int c_show(struct seq_file *f, void *slot) { + char buf[NR_CPUS * 5]; + + cpulist_scnprintf(buf, sizeof(buf), cpu_online_mask); /* high-level stuff */ - seq_printf(f,"processor\t: 0\n" - "vendor_id\t: Tensilica\n" - "model\t\t: Xtensa " XCHAL_HW_VERSION_NAME "\n" - "core ID\t\t: " XCHAL_CORE_ID "\n" - "build ID\t: 0x%x\n" - "byte order\t: %s\n" - "cpu MHz\t\t: %lu.%02lu\n" - "bogomips\t: %lu.%02lu\n", - XCHAL_BUILD_UNIQUE_ID, - XCHAL_HAVE_BE ? "big" : "little", - ccount_freq/1000000, - (ccount_freq/10000) % 100, - loops_per_jiffy/(500000/HZ), - (loops_per_jiffy/(5000/HZ)) % 100); + seq_printf(f, "CPU count\t: %u\n" + "CPU list\t: %s\n" + "vendor_id\t: Tensilica\n" + "model\t\t: Xtensa " XCHAL_HW_VERSION_NAME "\n" + "core ID\t\t: " XCHAL_CORE_ID "\n" + "build ID\t: 0x%x\n" + "byte order\t: %s\n" + "cpu MHz\t\t: %lu.%02lu\n" + "bogomips\t: %lu.%02lu\n", + num_online_cpus(), + buf, + XCHAL_BUILD_UNIQUE_ID, + XCHAL_HAVE_BE ? "big" : "little", + ccount_freq/1000000, + (ccount_freq/10000) % 100, + loops_per_jiffy/(500000/HZ), + (loops_per_jiffy/(5000/HZ)) % 100); seq_printf(f,"flags\t\t: " #if XCHAL_HAVE_NMI @@ -699,7 +706,7 @@ c_show(struct seq_file *f, void *slot) static void * c_start(struct seq_file *f, loff_t *pos) { - return (void *) ((*pos == 0) ? (void *)1 : NULL); + return (*pos == 0) ? (void *)1 : NULL; } static void * @@ -715,10 +722,10 @@ c_stop(struct seq_file *f, void *v) const struct seq_operations cpuinfo_op = { - start: c_start, - next: c_next, - stop: c_stop, - show: c_show + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = c_show, }; #endif /* CONFIG_PROC_FS */ diff --git a/arch/xtensa/kernel/signal.c b/arch/xtensa/kernel/signal.c index 718eca1850b..98b67d5f151 100644 --- a/arch/xtensa/kernel/signal.c +++ b/arch/xtensa/kernel/signal.c @@ -341,7 +341,7 @@ static int setup_frame(int sig, struct k_sigaction *ka, siginfo_t *info, sp = regs->areg[1]; - if ((ka->sa.sa_flags & SA_ONSTACK) != 0 && ! on_sig_stack(sp)) { + if ((ka->sa.sa_flags & SA_ONSTACK) != 0 && sas_ss_flags(sp) == 0) { sp = current->sas_ss_sp + current->sas_ss_size; } diff --git a/arch/xtensa/kernel/smp.c b/arch/xtensa/kernel/smp.c new file mode 100644 index 00000000000..40b5a3771fb --- /dev/null +++ b/arch/xtensa/kernel/smp.c @@ -0,0 +1,607 @@ +/* + * Xtensa SMP support functions. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2008 - 2013 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor <joe@tensilica.com> + * Pete Delaney <piet@tensilica.com + */ + +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/irqdomain.h> +#include <linux/irq.h> +#include <linux/kdebug.h> +#include <linux/module.h> +#include <linux/reboot.h> +#include <linux/seq_file.h> +#include <linux/smp.h> +#include <linux/thread_info.h> + +#include <asm/cacheflush.h> +#include <asm/kdebug.h> +#include <asm/mmu_context.h> +#include <asm/mxregs.h> +#include <asm/platform.h> +#include <asm/tlbflush.h> +#include <asm/traps.h> + +#ifdef CONFIG_SMP +# if XCHAL_HAVE_S32C1I == 0 +# error "The S32C1I option is required for SMP." +# endif +#endif + +static void system_invalidate_dcache_range(unsigned long start, + unsigned long size); +static void system_flush_invalidate_dcache_range(unsigned long start, + unsigned long size); + +/* IPI (Inter Process Interrupt) */ + +#define IPI_IRQ 0 + +static irqreturn_t ipi_interrupt(int irq, void *dev_id); +static struct irqaction ipi_irqaction = { + .handler = ipi_interrupt, + .flags = IRQF_PERCPU, + .name = "ipi", +}; + +void ipi_init(void) +{ + unsigned irq = irq_create_mapping(NULL, IPI_IRQ); + setup_irq(irq, &ipi_irqaction); +} + +static inline unsigned int get_core_count(void) +{ + /* Bits 18..21 of SYSCFGID contain the core count minus 1. */ + unsigned int syscfgid = get_er(SYSCFGID); + return ((syscfgid >> 18) & 0xf) + 1; +} + +static inline int get_core_id(void) +{ + /* Bits 0...18 of SYSCFGID contain the core id */ + unsigned int core_id = get_er(SYSCFGID); + return core_id & 0x3fff; +} + +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + unsigned i; + + for (i = 0; i < max_cpus; ++i) + set_cpu_present(i, true); +} + +void __init smp_init_cpus(void) +{ + unsigned i; + unsigned int ncpus = get_core_count(); + unsigned int core_id = get_core_id(); + + pr_info("%s: Core Count = %d\n", __func__, ncpus); + pr_info("%s: Core Id = %d\n", __func__, core_id); + + for (i = 0; i < ncpus; ++i) + set_cpu_possible(i, true); +} + +void __init smp_prepare_boot_cpu(void) +{ + unsigned int cpu = smp_processor_id(); + BUG_ON(cpu != 0); + cpu_asid_cache(cpu) = ASID_USER_FIRST; +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ +} + +static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */ +static DECLARE_COMPLETION(cpu_running); + +void secondary_start_kernel(void) +{ + struct mm_struct *mm = &init_mm; + unsigned int cpu = smp_processor_id(); + + init_mmu(); + +#ifdef CONFIG_DEBUG_KERNEL + if (boot_secondary_processors == 0) { + pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n", + __func__, boot_secondary_processors, cpu); + for (;;) + __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL)); + } + + pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n", + __func__, boot_secondary_processors, cpu); +#endif + /* Init EXCSAVE1 */ + + secondary_trap_init(); + + /* All kernel threads share the same mm context. */ + + atomic_inc(&mm->mm_users); + atomic_inc(&mm->mm_count); + current->active_mm = mm; + cpumask_set_cpu(cpu, mm_cpumask(mm)); + enter_lazy_tlb(mm, current); + + preempt_disable(); + trace_hardirqs_off(); + + calibrate_delay(); + + notify_cpu_starting(cpu); + + secondary_init_irq(); + local_timer_setup(cpu); + + set_cpu_online(cpu, true); + + local_irq_enable(); + + complete(&cpu_running); + + cpu_startup_entry(CPUHP_ONLINE); +} + +static void mx_cpu_start(void *p) +{ + unsigned cpu = (unsigned)p; + unsigned long run_stall_mask = get_er(MPSCORE); + + set_er(run_stall_mask & ~(1u << cpu), MPSCORE); + pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", + __func__, cpu, run_stall_mask, get_er(MPSCORE)); +} + +static void mx_cpu_stop(void *p) +{ + unsigned cpu = (unsigned)p; + unsigned long run_stall_mask = get_er(MPSCORE); + + set_er(run_stall_mask | (1u << cpu), MPSCORE); + pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n", + __func__, cpu, run_stall_mask, get_er(MPSCORE)); +} + +#ifdef CONFIG_HOTPLUG_CPU +unsigned long cpu_start_id __cacheline_aligned; +#endif +unsigned long cpu_start_ccount; + +static int boot_secondary(unsigned int cpu, struct task_struct *ts) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(1000); + unsigned long ccount; + int i; + +#ifdef CONFIG_HOTPLUG_CPU + cpu_start_id = cpu; + system_flush_invalidate_dcache_range( + (unsigned long)&cpu_start_id, sizeof(cpu_start_id)); +#endif + smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1); + + for (i = 0; i < 2; ++i) { + do + ccount = get_ccount(); + while (!ccount); + + cpu_start_ccount = ccount; + + while (time_before(jiffies, timeout)) { + mb(); + if (!cpu_start_ccount) + break; + } + + if (cpu_start_ccount) { + smp_call_function_single(0, mx_cpu_stop, + (void *)cpu, 1); + cpu_start_ccount = 0; + return -EIO; + } + } + return 0; +} + +int __cpu_up(unsigned int cpu, struct task_struct *idle) +{ + int ret = 0; + + if (cpu_asid_cache(cpu) == 0) + cpu_asid_cache(cpu) = ASID_USER_FIRST; + + start_info.stack = (unsigned long)task_pt_regs(idle); + wmb(); + + pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n", + __func__, cpu, idle, start_info.stack); + + ret = boot_secondary(cpu, idle); + if (ret == 0) { + wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(1000)); + if (!cpu_online(cpu)) + ret = -EIO; + } + + if (ret) + pr_err("CPU %u failed to boot\n", cpu); + + return ret; +} + +#ifdef CONFIG_HOTPLUG_CPU + +/* + * __cpu_disable runs on the processor to be shutdown. + */ +int __cpu_disable(void) +{ + unsigned int cpu = smp_processor_id(); + + /* + * Take this CPU offline. Once we clear this, we can't return, + * and we must not schedule until we're ready to give up the cpu. + */ + set_cpu_online(cpu, false); + + /* + * OK - migrate IRQs away from this CPU + */ + migrate_irqs(); + + /* + * Flush user cache and TLB mappings, and then remove this CPU + * from the vm mask set of all processes. + */ + local_flush_cache_all(); + local_flush_tlb_all(); + invalidate_page_directory(); + + clear_tasks_mm_cpumask(cpu); + + return 0; +} + +static void platform_cpu_kill(unsigned int cpu) +{ + smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true); +} + +/* + * called on the thread which is asking for a CPU to be shutdown - + * waits until shutdown has completed, or it is timed out. + */ +void __cpu_die(unsigned int cpu) +{ + unsigned long timeout = jiffies + msecs_to_jiffies(1000); + while (time_before(jiffies, timeout)) { + system_invalidate_dcache_range((unsigned long)&cpu_start_id, + sizeof(cpu_start_id)); + if (cpu_start_id == -cpu) { + platform_cpu_kill(cpu); + return; + } + } + pr_err("CPU%u: unable to kill\n", cpu); +} + +void arch_cpu_idle_dead(void) +{ + cpu_die(); +} +/* + * Called from the idle thread for the CPU which has been shutdown. + * + * Note that we disable IRQs here, but do not re-enable them + * before returning to the caller. This is also the behaviour + * of the other hotplug-cpu capable cores, so presumably coming + * out of idle fixes this. + */ +void __ref cpu_die(void) +{ + idle_task_exit(); + local_irq_disable(); + __asm__ __volatile__( + " movi a2, cpu_restart\n" + " jx a2\n"); +} + +#endif /* CONFIG_HOTPLUG_CPU */ + +enum ipi_msg_type { + IPI_RESCHEDULE = 0, + IPI_CALL_FUNC, + IPI_CPU_STOP, + IPI_MAX +}; + +static const struct { + const char *short_text; + const char *long_text; +} ipi_text[] = { + { .short_text = "RES", .long_text = "Rescheduling interrupts" }, + { .short_text = "CAL", .long_text = "Function call interrupts" }, + { .short_text = "DIE", .long_text = "CPU shutdown interrupts" }, +}; + +struct ipi_data { + unsigned long ipi_count[IPI_MAX]; +}; + +static DEFINE_PER_CPU(struct ipi_data, ipi_data); + +static void send_ipi_message(const struct cpumask *callmask, + enum ipi_msg_type msg_id) +{ + int index; + unsigned long mask = 0; + + for_each_cpu(index, callmask) + if (index != smp_processor_id()) + mask |= 1 << index; + + set_er(mask, MIPISET(msg_id)); +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + send_ipi_message(mask, IPI_CALL_FUNC); +} + +void arch_send_call_function_single_ipi(int cpu) +{ + send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC); +} + +void smp_send_reschedule(int cpu) +{ + send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); +} + +void smp_send_stop(void) +{ + struct cpumask targets; + + cpumask_copy(&targets, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &targets); + send_ipi_message(&targets, IPI_CPU_STOP); +} + +static void ipi_cpu_stop(unsigned int cpu) +{ + set_cpu_online(cpu, false); + machine_halt(); +} + +irqreturn_t ipi_interrupt(int irq, void *dev_id) +{ + unsigned int cpu = smp_processor_id(); + struct ipi_data *ipi = &per_cpu(ipi_data, cpu); + unsigned int msg; + unsigned i; + + msg = get_er(MIPICAUSE(cpu)); + for (i = 0; i < IPI_MAX; i++) + if (msg & (1 << i)) { + set_er(1 << i, MIPICAUSE(cpu)); + ++ipi->ipi_count[i]; + } + + if (msg & (1 << IPI_RESCHEDULE)) + scheduler_ipi(); + if (msg & (1 << IPI_CALL_FUNC)) + generic_smp_call_function_interrupt(); + if (msg & (1 << IPI_CPU_STOP)) + ipi_cpu_stop(cpu); + + return IRQ_HANDLED; +} + +void show_ipi_list(struct seq_file *p, int prec) +{ + unsigned int cpu; + unsigned i; + + for (i = 0; i < IPI_MAX; ++i) { + seq_printf(p, "%*s:", prec, ipi_text[i].short_text); + for_each_online_cpu(cpu) + seq_printf(p, " %10lu", + per_cpu(ipi_data, cpu).ipi_count[i]); + seq_printf(p, " %s\n", ipi_text[i].long_text); + } +} + +int setup_profiling_timer(unsigned int multiplier) +{ + pr_debug("setup_profiling_timer %d\n", multiplier); + return 0; +} + +/* TLB flush functions */ + +struct flush_data { + struct vm_area_struct *vma; + unsigned long addr1; + unsigned long addr2; +}; + +static void ipi_flush_tlb_all(void *arg) +{ + local_flush_tlb_all(); +} + +void flush_tlb_all(void) +{ + on_each_cpu(ipi_flush_tlb_all, NULL, 1); +} + +static void ipi_flush_tlb_mm(void *arg) +{ + local_flush_tlb_mm(arg); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + on_each_cpu(ipi_flush_tlb_mm, mm, 1); +} + +static void ipi_flush_tlb_page(void *arg) +{ + struct flush_data *fd = arg; + local_flush_tlb_page(fd->vma, fd->addr1); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr) +{ + struct flush_data fd = { + .vma = vma, + .addr1 = addr, + }; + on_each_cpu(ipi_flush_tlb_page, &fd, 1); +} + +static void ipi_flush_tlb_range(void *arg) +{ + struct flush_data *fd = arg; + local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2); +} + +void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + struct flush_data fd = { + .vma = vma, + .addr1 = start, + .addr2 = end, + }; + on_each_cpu(ipi_flush_tlb_range, &fd, 1); +} + +static void ipi_flush_tlb_kernel_range(void *arg) +{ + struct flush_data *fd = arg; + local_flush_tlb_kernel_range(fd->addr1, fd->addr2); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + struct flush_data fd = { + .addr1 = start, + .addr2 = end, + }; + on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1); +} + +/* Cache flush functions */ + +static void ipi_flush_cache_all(void *arg) +{ + local_flush_cache_all(); +} + +void flush_cache_all(void) +{ + on_each_cpu(ipi_flush_cache_all, NULL, 1); +} + +static void ipi_flush_cache_page(void *arg) +{ + struct flush_data *fd = arg; + local_flush_cache_page(fd->vma, fd->addr1, fd->addr2); +} + +void flush_cache_page(struct vm_area_struct *vma, + unsigned long address, unsigned long pfn) +{ + struct flush_data fd = { + .vma = vma, + .addr1 = address, + .addr2 = pfn, + }; + on_each_cpu(ipi_flush_cache_page, &fd, 1); +} + +static void ipi_flush_cache_range(void *arg) +{ + struct flush_data *fd = arg; + local_flush_cache_range(fd->vma, fd->addr1, fd->addr2); +} + +void flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + struct flush_data fd = { + .vma = vma, + .addr1 = start, + .addr2 = end, + }; + on_each_cpu(ipi_flush_cache_range, &fd, 1); +} + +static void ipi_flush_icache_range(void *arg) +{ + struct flush_data *fd = arg; + local_flush_icache_range(fd->addr1, fd->addr2); +} + +void flush_icache_range(unsigned long start, unsigned long end) +{ + struct flush_data fd = { + .addr1 = start, + .addr2 = end, + }; + on_each_cpu(ipi_flush_icache_range, &fd, 1); +} + +/* ------------------------------------------------------------------------- */ + +static void ipi_invalidate_dcache_range(void *arg) +{ + struct flush_data *fd = arg; + __invalidate_dcache_range(fd->addr1, fd->addr2); +} + +static void system_invalidate_dcache_range(unsigned long start, + unsigned long size) +{ + struct flush_data fd = { + .addr1 = start, + .addr2 = size, + }; + on_each_cpu(ipi_invalidate_dcache_range, &fd, 1); +} + +static void ipi_flush_invalidate_dcache_range(void *arg) +{ + struct flush_data *fd = arg; + __flush_invalidate_dcache_range(fd->addr1, fd->addr2); +} + +static void system_flush_invalidate_dcache_range(unsigned long start, + unsigned long size) +{ + struct flush_data fd = { + .addr1 = start, + .addr2 = size, + }; + on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1); +} diff --git a/arch/xtensa/kernel/time.c b/arch/xtensa/kernel/time.c index 9af3dd88ad7..2a1823de69c 100644 --- a/arch/xtensa/kernel/time.c +++ b/arch/xtensa/kernel/time.c @@ -30,13 +30,14 @@ #include <asm/platform.h> unsigned long ccount_freq; /* ccount Hz */ +EXPORT_SYMBOL(ccount_freq); static cycle_t ccount_read(struct clocksource *cs) { return (cycle_t)get_ccount(); } -static u32 notrace ccount_sched_clock_read(void) +static u64 notrace ccount_sched_clock_read(void) { return get_ccount(); } @@ -46,24 +47,19 @@ static struct clocksource ccount_clocksource = { .rating = 200, .read = ccount_read, .mask = CLOCKSOURCE_MASK(32), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, }; static int ccount_timer_set_next_event(unsigned long delta, struct clock_event_device *dev); static void ccount_timer_set_mode(enum clock_event_mode mode, struct clock_event_device *evt); -static struct ccount_timer_t { +struct ccount_timer { struct clock_event_device evt; int irq_enabled; -} ccount_timer = { - .evt = { - .name = "ccount_clockevent", - .features = CLOCK_EVT_FEAT_ONESHOT, - .rating = 300, - .set_next_event = ccount_timer_set_next_event, - .set_mode = ccount_timer_set_mode, - }, + char name[24]; }; +static DEFINE_PER_CPU(struct ccount_timer, ccount_timer); static int ccount_timer_set_next_event(unsigned long delta, struct clock_event_device *dev) @@ -84,8 +80,8 @@ static int ccount_timer_set_next_event(unsigned long delta, static void ccount_timer_set_mode(enum clock_event_mode mode, struct clock_event_device *evt) { - struct ccount_timer_t *timer = - container_of(evt, struct ccount_timer_t, evt); + struct ccount_timer *timer = + container_of(evt, struct ccount_timer, evt); /* * There is no way to disable the timer interrupt at the device level, @@ -117,9 +113,28 @@ static struct irqaction timer_irqaction = { .handler = timer_interrupt, .flags = IRQF_TIMER, .name = "timer", - .dev_id = &ccount_timer, }; +void local_timer_setup(unsigned cpu) +{ + struct ccount_timer *timer = &per_cpu(ccount_timer, cpu); + struct clock_event_device *clockevent = &timer->evt; + + timer->irq_enabled = 1; + clockevent->name = timer->name; + snprintf(timer->name, sizeof(timer->name), "ccount_clockevent_%u", cpu); + clockevent->features = CLOCK_EVT_FEAT_ONESHOT; + clockevent->rating = 300; + clockevent->set_next_event = ccount_timer_set_next_event; + clockevent->set_mode = ccount_timer_set_mode; + clockevent->cpumask = cpumask_of(cpu); + clockevent->irq = irq_create_mapping(NULL, LINUX_TIMER_INT); + if (WARN(!clockevent->irq, "error: can't map timer irq")) + return; + clockevents_config_and_register(clockevent, ccount_freq, + 0xf, 0xffffffff); +} + void __init time_init(void) { #ifdef CONFIG_XTENSA_CALIBRATE_CCOUNT @@ -131,28 +146,21 @@ void __init time_init(void) ccount_freq = CONFIG_XTENSA_CPU_CLOCK*1000000UL; #endif clocksource_register_hz(&ccount_clocksource, ccount_freq); - - ccount_timer.evt.cpumask = cpumask_of(0); - ccount_timer.evt.irq = irq_create_mapping(NULL, LINUX_TIMER_INT); - if (WARN(!ccount_timer.evt.irq, "error: can't map timer irq")) - return; - clockevents_config_and_register(&ccount_timer.evt, ccount_freq, 0xf, - 0xffffffff); - setup_irq(ccount_timer.evt.irq, &timer_irqaction); - ccount_timer.irq_enabled = 1; - - setup_sched_clock(ccount_sched_clock_read, 32, ccount_freq); + local_timer_setup(0); + setup_irq(this_cpu_ptr(&ccount_timer)->evt.irq, &timer_irqaction); + sched_clock_register(ccount_sched_clock_read, 32, ccount_freq); + clocksource_of_init(); } /* * The timer interrupt is called HZ times per second. */ -irqreturn_t timer_interrupt (int irq, void *dev_id) +irqreturn_t timer_interrupt(int irq, void *dev_id) { - struct ccount_timer_t *timer = dev_id; - struct clock_event_device *evt = &timer->evt; + struct clock_event_device *evt = &this_cpu_ptr(&ccount_timer)->evt; + set_linux_timer(get_linux_timer()); evt->event_handler(evt); /* Allow platform to do something useful (Wdog). */ diff --git a/arch/xtensa/kernel/traps.c b/arch/xtensa/kernel/traps.c index 3e8a05c874c..eebbfd8c26f 100644 --- a/arch/xtensa/kernel/traps.c +++ b/arch/xtensa/kernel/traps.c @@ -157,7 +157,7 @@ COPROCESSOR(7), * 2. it is a temporary memory buffer for the exception handlers. */ -unsigned long exc_table[EXC_TABLE_SIZE/4]; +DEFINE_PER_CPU(unsigned long, exc_table[EXC_TABLE_SIZE/4]); void die(const char*, struct pt_regs*, long); @@ -212,6 +212,9 @@ void do_interrupt(struct pt_regs *regs) XCHAL_INTLEVEL6_MASK, XCHAL_INTLEVEL7_MASK, }; + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); for (;;) { unsigned intread = get_sr(interrupt); @@ -227,21 +230,13 @@ void do_interrupt(struct pt_regs *regs) } if (level == 0) - return; - - /* - * Clear the interrupt before processing, in case it's - * edge-triggered or software-generated - */ - while (int_at_level) { - unsigned i = __ffs(int_at_level); - unsigned mask = 1 << i; - - int_at_level ^= mask; - set_sr(mask, intclear); - do_IRQ(i, regs); - } + break; + + do_IRQ(__ffs(int_at_level), regs); } + + irq_exit(); + set_irq_regs(old_regs); } /* @@ -318,17 +313,31 @@ do_debug(struct pt_regs *regs) } +static void set_handler(int idx, void *handler) +{ + unsigned int cpu; + + for_each_possible_cpu(cpu) + per_cpu(exc_table, cpu)[idx] = (unsigned long)handler; +} + /* Set exception C handler - for temporary use when probing exceptions */ void * __init trap_set_handler(int cause, void *handler) { - unsigned long *entry = &exc_table[EXC_TABLE_DEFAULT / 4 + cause]; - void *previous = (void *)*entry; - *entry = (unsigned long)handler; + void *previous = (void *)per_cpu(exc_table, 0)[ + EXC_TABLE_DEFAULT / 4 + cause]; + set_handler(EXC_TABLE_DEFAULT / 4 + cause, handler); return previous; } +static void trap_init_excsave(void) +{ + unsigned long excsave1 = (unsigned long)this_cpu_ptr(exc_table); + __asm__ __volatile__("wsr %0, excsave1\n" : : "a" (excsave1)); +} + /* * Initialize dispatch tables. * @@ -342,8 +351,6 @@ void * __init trap_set_handler(int cause, void *handler) * See vectors.S for more details. */ -#define set_handler(idx,handler) (exc_table[idx] = (unsigned long) (handler)) - void __init trap_init(void) { int i; @@ -373,10 +380,15 @@ void __init trap_init(void) } /* Initialize EXCSAVE_1 to hold the address of the exception table. */ + trap_init_excsave(); +} - i = (unsigned long)exc_table; - __asm__ __volatile__("wsr %0, excsave1\n" : : "a" (i)); +#ifdef CONFIG_SMP +void secondary_trap_init(void) +{ + trap_init_excsave(); } +#endif /* * This function dumps the current valid window frame and other base registers. diff --git a/arch/xtensa/kernel/vectors.S b/arch/xtensa/kernel/vectors.S index cb8fd44caab..8453e6e3989 100644 --- a/arch/xtensa/kernel/vectors.S +++ b/arch/xtensa/kernel/vectors.S @@ -235,7 +235,7 @@ ENTRY(_DoubleExceptionVector) /* Check for overflow/underflow exception, jump if overflow. */ - _bbci.l a0, 6, _DoubleExceptionVector_WindowOverflow + bbci.l a0, 6, _DoubleExceptionVector_WindowOverflow /* * Restart window underflow exception. @@ -376,38 +376,42 @@ _DoubleExceptionVector_WindowOverflow: beqz a2, 1f # if at start of vector, don't restore addi a0, a0, -128 - bbsi a0, 8, 1f # don't restore except for overflow 8 and 12 - bbsi a0, 7, 2f + bbsi.l a0, 8, 1f # don't restore except for overflow 8 and 12 + + /* + * This fixup handler is for the extremely unlikely case where the + * overflow handler's reference thru a0 gets a hardware TLB refill + * that bumps out the (distinct, aliasing) TLB entry that mapped its + * prior references thru a9/a13, and where our reference now thru + * a9/a13 gets a 2nd-level miss exception (not hardware TLB refill). + */ + movi a2, window_overflow_restore_a0_fixup + s32i a2, a3, EXC_TABLE_FIXUP + l32i a2, a3, EXC_TABLE_DOUBLE_SAVE + xsr a3, excsave1 + + bbsi.l a0, 7, 2f /* * Restore a0 as saved by _WindowOverflow8(). - * - * FIXME: we really need a fixup handler for this L32E, - * for the extremely unlikely case where the overflow handler's - * reference thru a0 gets a hardware TLB refill that bumps out - * the (distinct, aliasing) TLB entry that mapped its prior - * references thru a9, and where our reference now thru a9 - * gets a 2nd-level miss exception (not hardware TLB refill). */ - l32e a2, a9, -16 - wsr a2, depc # replace the saved a0 - j 1f + l32e a0, a9, -16 + wsr a0, depc # replace the saved a0 + j 3f 2: /* * Restore a0 as saved by _WindowOverflow12(). - * - * FIXME: we really need a fixup handler for this L32E, - * for the extremely unlikely case where the overflow handler's - * reference thru a0 gets a hardware TLB refill that bumps out - * the (distinct, aliasing) TLB entry that mapped its prior - * references thru a13, and where our reference now thru a13 - * gets a 2nd-level miss exception (not hardware TLB refill). */ - l32e a2, a13, -16 - wsr a2, depc # replace the saved a0 + l32e a0, a13, -16 + wsr a0, depc # replace the saved a0 +3: + xsr a3, excsave1 + movi a0, 0 + s32i a0, a3, EXC_TABLE_FIXUP + s32i a2, a3, EXC_TABLE_DOUBLE_SAVE 1: /* * Restore WindowBase while leaving all address registers restored. @@ -449,6 +453,7 @@ _DoubleExceptionVector_WindowOverflow: s32i a0, a2, PT_DEPC +_DoubleExceptionVector_handle_exception: addx4 a0, a0, a3 l32i a0, a0, EXC_TABLE_FAST_USER xsr a3, excsave1 @@ -464,11 +469,120 @@ _DoubleExceptionVector_WindowOverflow: rotw -3 j 1b - .end literal_prefix ENDPROC(_DoubleExceptionVector) /* + * Fixup handler for TLB miss in double exception handler for window owerflow. + * We get here with windowbase set to the window that was being spilled and + * a0 trashed. a0 bit 7 determines if this is a call8 (bit clear) or call12 + * (bit set) window. + * + * We do the following here: + * - go to the original window retaining a0 value; + * - set up exception stack to return back to appropriate a0 restore code + * (we'll need to rotate window back and there's no place to save this + * information, use different return address for that); + * - handle the exception; + * - go to the window that was being spilled; + * - set up window_overflow_restore_a0_fixup as a fixup routine; + * - reload a0; + * - restore the original window; + * - reset the default fixup routine; + * - return to user. By the time we get to this fixup handler all information + * about the conditions of the original double exception that happened in + * the window overflow handler is lost, so we just return to userspace to + * retry overflow from start. + * + * a0: value of depc, original value in depc + * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE + * a3: exctable, original value in excsave1 + */ + +ENTRY(window_overflow_restore_a0_fixup) + + rsr a0, ps + extui a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH + rsr a2, windowbase + sub a0, a2, a0 + extui a0, a0, 0, 3 + l32i a2, a3, EXC_TABLE_DOUBLE_SAVE + xsr a3, excsave1 + + _beqi a0, 1, .Lhandle_1 + _beqi a0, 3, .Lhandle_3 + + .macro overflow_fixup_handle_exception_pane n + + rsr a0, depc + rotw -\n + + xsr a3, excsave1 + wsr a2, depc + l32i a2, a3, EXC_TABLE_KSTK + s32i a0, a2, PT_AREG0 + + movi a0, .Lrestore_\n + s32i a0, a2, PT_DEPC + rsr a0, exccause + j _DoubleExceptionVector_handle_exception + + .endm + + overflow_fixup_handle_exception_pane 2 +.Lhandle_1: + overflow_fixup_handle_exception_pane 1 +.Lhandle_3: + overflow_fixup_handle_exception_pane 3 + + .macro overflow_fixup_restore_a0_pane n + + rotw \n + /* Need to preserve a0 value here to be able to handle exception + * that may occur on a0 reload from stack. It may occur because + * TLB miss handler may not be atomic and pointer to page table + * may be lost before we get here. There are no free registers, + * so we need to use EXC_TABLE_DOUBLE_SAVE area. + */ + xsr a3, excsave1 + s32i a2, a3, EXC_TABLE_DOUBLE_SAVE + movi a2, window_overflow_restore_a0_fixup + s32i a2, a3, EXC_TABLE_FIXUP + l32i a2, a3, EXC_TABLE_DOUBLE_SAVE + xsr a3, excsave1 + bbsi.l a0, 7, 1f + l32e a0, a9, -16 + j 2f +1: + l32e a0, a13, -16 +2: + rotw -\n + + .endm + +.Lrestore_2: + overflow_fixup_restore_a0_pane 2 + +.Lset_default_fixup: + xsr a3, excsave1 + s32i a2, a3, EXC_TABLE_DOUBLE_SAVE + movi a2, 0 + s32i a2, a3, EXC_TABLE_FIXUP + l32i a2, a3, EXC_TABLE_DOUBLE_SAVE + xsr a3, excsave1 + rfe + +.Lrestore_1: + overflow_fixup_restore_a0_pane 1 + j .Lset_default_fixup +.Lrestore_3: + overflow_fixup_restore_a0_pane 3 + j .Lset_default_fixup + +ENDPROC(window_overflow_restore_a0_fixup) + + .end literal_prefix +/* * Debug interrupt vector * * There is not much space here, so simply jump to another handler. diff --git a/arch/xtensa/kernel/vmlinux.lds.S b/arch/xtensa/kernel/vmlinux.lds.S index 21acd11b5df..d16db6df86f 100644 --- a/arch/xtensa/kernel/vmlinux.lds.S +++ b/arch/xtensa/kernel/vmlinux.lds.S @@ -165,6 +165,13 @@ SECTIONS .DoubleExceptionVector.text); RELOCATE_ENTRY(_DebugInterruptVector_text, .DebugInterruptVector.text); +#if defined(CONFIG_SMP) + RELOCATE_ENTRY(_SecondaryResetVector_literal, + .SecondaryResetVector.literal); + RELOCATE_ENTRY(_SecondaryResetVector_text, + .SecondaryResetVector.text); +#endif + __boot_reloc_table_end = ABSOLUTE(.) ; @@ -262,16 +269,35 @@ SECTIONS .UserExceptionVector.literal) SECTION_VECTOR (_DoubleExceptionVector_literal, .DoubleExceptionVector.literal, - DOUBLEEXC_VECTOR_VADDR - 16, + DOUBLEEXC_VECTOR_VADDR - 40, SIZEOF(.UserExceptionVector.text), .UserExceptionVector.text) SECTION_VECTOR (_DoubleExceptionVector_text, .DoubleExceptionVector.text, DOUBLEEXC_VECTOR_VADDR, - 32, + 40, .DoubleExceptionVector.literal) . = (LOADADDR( .DoubleExceptionVector.text ) + SIZEOF( .DoubleExceptionVector.text ) + 3) & ~ 3; + +#if defined(CONFIG_SMP) + + SECTION_VECTOR (_SecondaryResetVector_literal, + .SecondaryResetVector.literal, + RESET_VECTOR1_VADDR - 4, + SIZEOF(.DoubleExceptionVector.text), + .DoubleExceptionVector.text) + + SECTION_VECTOR (_SecondaryResetVector_text, + .SecondaryResetVector.text, + RESET_VECTOR1_VADDR, + 4, + .SecondaryResetVector.literal) + + . = LOADADDR(.SecondaryResetVector.text)+SIZEOF(.SecondaryResetVector.text); + +#endif + . = ALIGN(PAGE_SIZE); __init_end = .; diff --git a/arch/xtensa/kernel/xtensa_ksyms.c b/arch/xtensa/kernel/xtensa_ksyms.c index 74a60c7e085..4d2872fd9bb 100644 --- a/arch/xtensa/kernel/xtensa_ksyms.c +++ b/arch/xtensa/kernel/xtensa_ksyms.c @@ -20,6 +20,7 @@ #include <linux/in6.h> #include <asm/uaccess.h> +#include <asm/cacheflush.h> #include <asm/checksum.h> #include <asm/dma.h> #include <asm/io.h> @@ -105,6 +106,7 @@ EXPORT_SYMBOL(csum_partial_copy_generic); * Architecture-specific symbols */ EXPORT_SYMBOL(__xtensa_copy_user); +EXPORT_SYMBOL(__invalidate_icache_range); /* * Kernel hacking ... @@ -122,10 +124,13 @@ EXPORT_SYMBOL(insw); EXPORT_SYMBOL(insl); extern long common_exception_return; -extern long _spill_registers; EXPORT_SYMBOL(common_exception_return); -EXPORT_SYMBOL(_spill_registers); #ifdef CONFIG_FUNCTION_TRACER EXPORT_SYMBOL(_mcount); #endif + +EXPORT_SYMBOL(__invalidate_dcache_range); +#if XCHAL_DCACHE_IS_WRITEBACK +EXPORT_SYMBOL(__flush_dcache_range); +#endif |