aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/kernel
diff options
context:
space:
mode:
authorBenjamin Herrenschmidt <benh@kernel.crashing.org>2014-03-24 10:26:10 +1100
committerBenjamin Herrenschmidt <benh@kernel.crashing.org>2014-03-24 10:26:10 +1100
commitcd427485357c0c4b99f69719251baacf25946e11 (patch)
treeca1ff7eb2277dec42f1e20adc0b98c3c215ecfbe /arch/powerpc/kernel
parentd410ae2126481a74bc5be4a6242c4a232c19a984 (diff)
parent48b16180d0d91324e5d2423c6d53d97bbe3dcc14 (diff)
Merge remote-tracking branch 'scott/next' into next
Freescale updates from Scott. Mostly support for critical and machine check exceptions on 64-bit BookE, some new PCI suspend/resume work and misc bits.
Diffstat (limited to 'arch/powerpc/kernel')
-rw-r--r--arch/powerpc/kernel/asm-offsets.c2
-rw-r--r--arch/powerpc/kernel/exceptions-64e.S435
-rw-r--r--arch/powerpc/kernel/setup_64.c20
-rw-r--r--arch/powerpc/kernel/vdso.c8
-rw-r--r--arch/powerpc/kernel/vdso32/getcpu.S2
-rw-r--r--arch/powerpc/kernel/vdso64/getcpu.S2
6 files changed, 358 insertions, 111 deletions
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index b5aacf72ae6..dba8140ebc2 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -253,7 +253,7 @@ int main(void)
DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time));
DEFINE(PACA_TRAP_SAVE, offsetof(struct paca_struct, trap_save));
DEFINE(PACA_NAPSTATELOST, offsetof(struct paca_struct, nap_state_lost));
- DEFINE(PACA_SPRG3, offsetof(struct paca_struct, sprg3));
+ DEFINE(PACA_SPRG_VDSO, offsetof(struct paca_struct, sprg_vdso));
#endif /* CONFIG_PPC64 */
/* RTAS */
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 063b65dd4f2..c1bee3ce9d1 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -34,7 +34,250 @@
* special interrupts from within a non-standard level will probably
* blow you up
*/
-#define SPECIAL_EXC_FRAME_SIZE INT_FRAME_SIZE
+#define SPECIAL_EXC_SRR0 0
+#define SPECIAL_EXC_SRR1 1
+#define SPECIAL_EXC_SPRG_GEN 2
+#define SPECIAL_EXC_SPRG_TLB 3
+#define SPECIAL_EXC_MAS0 4
+#define SPECIAL_EXC_MAS1 5
+#define SPECIAL_EXC_MAS2 6
+#define SPECIAL_EXC_MAS3 7
+#define SPECIAL_EXC_MAS6 8
+#define SPECIAL_EXC_MAS7 9
+#define SPECIAL_EXC_MAS5 10 /* E.HV only */
+#define SPECIAL_EXC_MAS8 11 /* E.HV only */
+#define SPECIAL_EXC_IRQHAPPENED 12
+#define SPECIAL_EXC_DEAR 13
+#define SPECIAL_EXC_ESR 14
+#define SPECIAL_EXC_SOFTE 15
+#define SPECIAL_EXC_CSRR0 16
+#define SPECIAL_EXC_CSRR1 17
+/* must be even to keep 16-byte stack alignment */
+#define SPECIAL_EXC_END 18
+
+#define SPECIAL_EXC_FRAME_SIZE (INT_FRAME_SIZE + SPECIAL_EXC_END * 8)
+#define SPECIAL_EXC_FRAME_OFFS (INT_FRAME_SIZE - 288)
+
+#define SPECIAL_EXC_STORE(reg, name) \
+ std reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+#define SPECIAL_EXC_LOAD(reg, name) \
+ ld reg, (SPECIAL_EXC_##name * 8 + SPECIAL_EXC_FRAME_OFFS)(r1)
+
+special_reg_save:
+ lbz r9,PACAIRQHAPPENED(r13)
+ RECONCILE_IRQ_STATE(r3,r4)
+
+ /*
+ * We only need (or have stack space) to save this stuff if
+ * we interrupted the kernel.
+ */
+ ld r3,_MSR(r1)
+ andi. r3,r3,MSR_PR
+ bnelr
+
+ /* Copy info into temporary exception thread info */
+ ld r11,PACAKSAVE(r13)
+ CURRENT_THREAD_INFO(r11, r11)
+ CURRENT_THREAD_INFO(r12, r1)
+ ld r10,TI_FLAGS(r11)
+ std r10,TI_FLAGS(r12)
+ ld r10,TI_PREEMPT(r11)
+ std r10,TI_PREEMPT(r12)
+ ld r10,TI_TASK(r11)
+ std r10,TI_TASK(r12)
+
+ /*
+ * Advance to the next TLB exception frame for handler
+ * types that don't do it automatically.
+ */
+ LOAD_REG_ADDR(r11,extlb_level_exc)
+ lwz r12,0(r11)
+ mfspr r10,SPRN_SPRG_TLB_EXFRAME
+ add r10,r10,r12
+ mtspr SPRN_SPRG_TLB_EXFRAME,r10
+
+ /*
+ * Save registers needed to allow nesting of certain exceptions
+ * (such as TLB misses) inside special exception levels
+ */
+ mfspr r10,SPRN_SRR0
+ SPECIAL_EXC_STORE(r10,SRR0)
+ mfspr r10,SPRN_SRR1
+ SPECIAL_EXC_STORE(r10,SRR1)
+ mfspr r10,SPRN_SPRG_GEN_SCRATCH
+ SPECIAL_EXC_STORE(r10,SPRG_GEN)
+ mfspr r10,SPRN_SPRG_TLB_SCRATCH
+ SPECIAL_EXC_STORE(r10,SPRG_TLB)
+ mfspr r10,SPRN_MAS0
+ SPECIAL_EXC_STORE(r10,MAS0)
+ mfspr r10,SPRN_MAS1
+ SPECIAL_EXC_STORE(r10,MAS1)
+ mfspr r10,SPRN_MAS2
+ SPECIAL_EXC_STORE(r10,MAS2)
+ mfspr r10,SPRN_MAS3
+ SPECIAL_EXC_STORE(r10,MAS3)
+ mfspr r10,SPRN_MAS6
+ SPECIAL_EXC_STORE(r10,MAS6)
+ mfspr r10,SPRN_MAS7
+ SPECIAL_EXC_STORE(r10,MAS7)
+BEGIN_FTR_SECTION
+ mfspr r10,SPRN_MAS5
+ SPECIAL_EXC_STORE(r10,MAS5)
+ mfspr r10,SPRN_MAS8
+ SPECIAL_EXC_STORE(r10,MAS8)
+
+ /* MAS5/8 could have inappropriate values if we interrupted KVM code */
+ li r10,0
+ mtspr SPRN_MAS5,r10
+ mtspr SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+ SPECIAL_EXC_STORE(r9,IRQHAPPENED)
+
+ mfspr r10,SPRN_DEAR
+ SPECIAL_EXC_STORE(r10,DEAR)
+ mfspr r10,SPRN_ESR
+ SPECIAL_EXC_STORE(r10,ESR)
+
+ lbz r10,PACASOFTIRQEN(r13)
+ SPECIAL_EXC_STORE(r10,SOFTE)
+ ld r10,_NIP(r1)
+ SPECIAL_EXC_STORE(r10,CSRR0)
+ ld r10,_MSR(r1)
+ SPECIAL_EXC_STORE(r10,CSRR1)
+
+ blr
+
+ret_from_level_except:
+ ld r3,_MSR(r1)
+ andi. r3,r3,MSR_PR
+ beq 1f
+ b ret_from_except
+1:
+
+ LOAD_REG_ADDR(r11,extlb_level_exc)
+ lwz r12,0(r11)
+ mfspr r10,SPRN_SPRG_TLB_EXFRAME
+ sub r10,r10,r12
+ mtspr SPRN_SPRG_TLB_EXFRAME,r10
+
+ /*
+ * It's possible that the special level exception interrupted a
+ * TLB miss handler, and inserted the same entry that the
+ * interrupted handler was about to insert. On CPUs without TLB
+ * write conditional, this can result in a duplicate TLB entry.
+ * Wipe all non-bolted entries to be safe.
+ *
+ * Note that this doesn't protect against any TLB misses
+ * we may take accessing the stack from here to the end of
+ * the special level exception. It's not clear how we can
+ * reasonably protect against that, but only CPUs with
+ * neither TLB write conditional nor bolted kernel memory
+ * are affected. Do any such CPUs even exist?
+ */
+ PPC_TLBILX_ALL(0,R0)
+
+ REST_NVGPRS(r1)
+
+ SPECIAL_EXC_LOAD(r10,SRR0)
+ mtspr SPRN_SRR0,r10
+ SPECIAL_EXC_LOAD(r10,SRR1)
+ mtspr SPRN_SRR1,r10
+ SPECIAL_EXC_LOAD(r10,SPRG_GEN)
+ mtspr SPRN_SPRG_GEN_SCRATCH,r10
+ SPECIAL_EXC_LOAD(r10,SPRG_TLB)
+ mtspr SPRN_SPRG_TLB_SCRATCH,r10
+ SPECIAL_EXC_LOAD(r10,MAS0)
+ mtspr SPRN_MAS0,r10
+ SPECIAL_EXC_LOAD(r10,MAS1)
+ mtspr SPRN_MAS1,r10
+ SPECIAL_EXC_LOAD(r10,MAS2)
+ mtspr SPRN_MAS2,r10
+ SPECIAL_EXC_LOAD(r10,MAS3)
+ mtspr SPRN_MAS3,r10
+ SPECIAL_EXC_LOAD(r10,MAS6)
+ mtspr SPRN_MAS6,r10
+ SPECIAL_EXC_LOAD(r10,MAS7)
+ mtspr SPRN_MAS7,r10
+BEGIN_FTR_SECTION
+ SPECIAL_EXC_LOAD(r10,MAS5)
+ mtspr SPRN_MAS5,r10
+ SPECIAL_EXC_LOAD(r10,MAS8)
+ mtspr SPRN_MAS8,r10
+END_FTR_SECTION_IFSET(CPU_FTR_EMB_HV)
+
+ lbz r6,PACASOFTIRQEN(r13)
+ ld r5,SOFTE(r1)
+
+ /* Interrupts had better not already be enabled... */
+ twnei r6,0
+
+ cmpwi cr0,r5,0
+ beq 1f
+
+ TRACE_ENABLE_INTS
+ stb r5,PACASOFTIRQEN(r13)
+1:
+ /*
+ * Restore PACAIRQHAPPENED rather than setting it based on
+ * the return MSR[EE], since we could have interrupted
+ * __check_irq_replay() or other inconsistent transitory
+ * states that must remain that way.
+ */
+ SPECIAL_EXC_LOAD(r10,IRQHAPPENED)
+ stb r10,PACAIRQHAPPENED(r13)
+
+ SPECIAL_EXC_LOAD(r10,DEAR)
+ mtspr SPRN_DEAR,r10
+ SPECIAL_EXC_LOAD(r10,ESR)
+ mtspr SPRN_ESR,r10
+
+ stdcx. r0,0,r1 /* to clear the reservation */
+
+ REST_4GPRS(2, r1)
+ REST_4GPRS(6, r1)
+
+ ld r10,_CTR(r1)
+ ld r11,_XER(r1)
+ mtctr r10
+ mtxer r11
+
+ blr
+
+.macro ret_from_level srr0 srr1 paca_ex scratch
+ bl ret_from_level_except
+
+ ld r10,_LINK(r1)
+ ld r11,_CCR(r1)
+ ld r0,GPR13(r1)
+ mtlr r10
+ mtcr r11
+
+ ld r10,GPR10(r1)
+ ld r11,GPR11(r1)
+ ld r12,GPR12(r1)
+ mtspr \scratch,r0
+
+ std r10,\paca_ex+EX_R10(r13);
+ std r11,\paca_ex+EX_R11(r13);
+ ld r10,_NIP(r1)
+ ld r11,_MSR(r1)
+ ld r0,GPR0(r1)
+ ld r1,GPR1(r1)
+ mtspr \srr0,r10
+ mtspr \srr1,r11
+ ld r10,\paca_ex+EX_R10(r13)
+ ld r11,\paca_ex+EX_R11(r13)
+ mfspr r13,\scratch
+.endm
+
+ret_from_crit_except:
+ ret_from_level SPRN_CSRR0 SPRN_CSRR1 PACA_EXCRIT SPRN_SPRG_CRIT_SCRATCH
+ rfci
+
+ret_from_mc_except:
+ ret_from_level SPRN_MCSRR0 SPRN_MCSRR1 PACA_EXMC SPRN_SPRG_MC_SCRATCH
+ rfmci
/* Exception prolog code for all exceptions */
#define EXCEPTION_PROLOG(n, intnum, type, addition) \
@@ -42,7 +285,6 @@
mfspr r13,SPRN_SPRG_PACA; /* get PACA */ \
std r10,PACA_EX##type+EX_R10(r13); \
std r11,PACA_EX##type+EX_R11(r13); \
- PROLOG_STORE_RESTORE_SCRATCH_##type; \
mfcr r10; /* save CR */ \
mfspr r11,SPRN_##type##_SRR1;/* what are we coming from */ \
DO_KVM intnum,SPRN_##type##_SRR1; /* KVM hook */ \
@@ -69,19 +311,19 @@
#define CRIT_SET_KSTACK \
ld r1,PACA_CRIT_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE;
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_CRIT_SRR0 SPRN_CSRR0
#define SPRN_CRIT_SRR1 SPRN_CSRR1
#define DBG_SET_KSTACK \
ld r1,PACA_DBG_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE;
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_DBG_SRR0 SPRN_DSRR0
#define SPRN_DBG_SRR1 SPRN_DSRR1
#define MC_SET_KSTACK \
ld r1,PACA_MC_STACK(r13); \
- subi r1,r1,SPECIAL_EXC_FRAME_SIZE;
+ subi r1,r1,SPECIAL_EXC_FRAME_SIZE
#define SPRN_MC_SRR0 SPRN_MCSRR0
#define SPRN_MC_SRR1 SPRN_MCSRR1
@@ -100,20 +342,6 @@
#define GDBELL_EXCEPTION_PROLOG(n, intnum, addition) \
EXCEPTION_PROLOG(n, intnum, GDBELL, addition##_GDBELL(n))
-/*
- * Store user-visible scratch in PACA exception slots and restore proper value
- */
-#define PROLOG_STORE_RESTORE_SCRATCH_GEN
-#define PROLOG_STORE_RESTORE_SCRATCH_GDBELL
-#define PROLOG_STORE_RESTORE_SCRATCH_DBG
-#define PROLOG_STORE_RESTORE_SCRATCH_MC
-
-#define PROLOG_STORE_RESTORE_SCRATCH_CRIT \
- mfspr r10,SPRN_SPRG_CRIT_SCRATCH; /* get r13 */ \
- std r10,PACA_EXCRIT+EX_R13(r13); \
- ld r11,PACA_SPRG3(r13); \
- mtspr SPRN_SPRG_CRIT_SCRATCH,r11;
-
/* Variants of the "addition" argument for the prolog
*/
#define PROLOG_ADDITION_NONE_GEN(n)
@@ -147,10 +375,8 @@
std r15,PACA_EXMC+EX_R15(r13)
-/* Core exception code for all exceptions except TLB misses.
- * XXX: Needs to make SPRN_SPRG_GEN depend on exception type
- */
-#define EXCEPTION_COMMON(n, excf, ints) \
+/* Core exception code for all exceptions except TLB misses. */
+#define EXCEPTION_COMMON_LVL(n, scratch, excf) \
exc_##n##_common: \
std r0,GPR0(r1); /* save r0 in stackframe */ \
std r2,GPR2(r1); /* save r2 in stackframe */ \
@@ -163,7 +389,7 @@ exc_##n##_common: \
ACCOUNT_CPU_USER_ENTRY(r10,r11);/* accounting (uses cr0+eq) */ \
2: ld r3,excf+EX_R10(r13); /* get back r10 */ \
ld r4,excf+EX_R11(r13); /* get back r11 */ \
- mfspr r5,SPRN_SPRG_GEN_SCRATCH;/* get back r13 */ \
+ mfspr r5,scratch; /* get back r13 */ \
std r12,GPR12(r1); /* save r12 in stackframe */ \
ld r2,PACATOC(r13); /* get kernel TOC into r2 */ \
mflr r6; /* save LR in stackframe */ \
@@ -187,24 +413,29 @@ exc_##n##_common: \
std r11,SOFTE(r1); /* and save it to stackframe */ \
std r12,STACK_FRAME_OVERHEAD-16(r1); /* mark the frame */ \
std r3,_TRAP(r1); /* set trap number */ \
- std r0,RESULT(r1); /* clear regs->result */ \
- ints;
+ std r0,RESULT(r1); /* clear regs->result */
-/* Variants for the "ints" argument. This one does nothing when we want
- * to keep interrupts in their original state
- */
-#define INTS_KEEP
+#define EXCEPTION_COMMON(n) \
+ EXCEPTION_COMMON_LVL(n, SPRN_SPRG_GEN_SCRATCH, PACA_EXGEN)
+#define EXCEPTION_COMMON_CRIT(n) \
+ EXCEPTION_COMMON_LVL(n, SPRN_SPRG_CRIT_SCRATCH, PACA_EXCRIT)
+#define EXCEPTION_COMMON_MC(n) \
+ EXCEPTION_COMMON_LVL(n, SPRN_SPRG_MC_SCRATCH, PACA_EXMC)
+#define EXCEPTION_COMMON_DBG(n) \
+ EXCEPTION_COMMON_LVL(n, SPRN_SPRG_DBG_SCRATCH, PACA_EXDBG)
-/* This second version is meant for exceptions that don't immediately
- * hard-enable. We set a bit in paca->irq_happened to ensure that
- * a subsequent call to arch_local_irq_restore() will properly
- * hard-enable and avoid the fast-path, and then reconcile irq state.
+/*
+ * This is meant for exceptions that don't immediately hard-enable. We
+ * set a bit in paca->irq_happened to ensure that a subsequent call to
+ * arch_local_irq_restore() will properly hard-enable and avoid the
+ * fast-path, and then reconcile irq state.
*/
#define INTS_DISABLE RECONCILE_IRQ_STATE(r3,r4)
-/* This is called by exceptions that used INTS_KEEP (that did not touch
- * irq indicators in the PACA). This will restore MSR:EE to it's previous
- * value
+/*
+ * This is called by exceptions that don't use INTS_DISABLE (that did not
+ * touch irq indicators in the PACA). This will restore MSR:EE to it's
+ * previous value
*
* XXX In the long run, we may want to open-code it in order to separate the
* load from the wrtee, thus limiting the latency caused by the dependency
@@ -262,7 +493,8 @@ exc_##n##_bad_stack: \
#define MASKABLE_EXCEPTION(trapnum, intnum, label, hdlr, ack) \
START_EXCEPTION(label); \
NORMAL_EXCEPTION_PROLOG(trapnum, intnum, PROLOG_ADDITION_MASKABLE)\
- EXCEPTION_COMMON(trapnum, PACA_EXGEN, INTS_DISABLE) \
+ EXCEPTION_COMMON(trapnum) \
+ INTS_DISABLE; \
ack(r8); \
CHECK_NAPPING(); \
addi r3,r1,STACK_FRAME_OVERHEAD; \
@@ -283,8 +515,8 @@ exception_marker:
.balign 0x1000
.globl interrupt_base_book3e
interrupt_base_book3e: /* fake trap */
- EXCEPTION_STUB(0x000, machine_check) /* 0x0200 */
- EXCEPTION_STUB(0x020, critical_input) /* 0x0580 */
+ EXCEPTION_STUB(0x000, machine_check)
+ EXCEPTION_STUB(0x020, critical_input) /* 0x0100 */
EXCEPTION_STUB(0x040, debug_crit) /* 0x0d00 */
EXCEPTION_STUB(0x060, data_storage) /* 0x0300 */
EXCEPTION_STUB(0x080, instruction_storage) /* 0x0400 */
@@ -299,8 +531,8 @@ interrupt_base_book3e: /* fake trap */
EXCEPTION_STUB(0x1a0, watchdog) /* 0x09f0 */
EXCEPTION_STUB(0x1c0, data_tlb_miss)
EXCEPTION_STUB(0x1e0, instruction_tlb_miss)
- EXCEPTION_STUB(0x200, altivec_unavailable) /* 0x0f20 */
- EXCEPTION_STUB(0x220, altivec_assist) /* 0x1700 */
+ EXCEPTION_STUB(0x200, altivec_unavailable)
+ EXCEPTION_STUB(0x220, altivec_assist)
EXCEPTION_STUB(0x260, perfmon)
EXCEPTION_STUB(0x280, doorbell)
EXCEPTION_STUB(0x2a0, doorbell_crit)
@@ -317,25 +549,25 @@ interrupt_end_book3e:
START_EXCEPTION(critical_input);
CRIT_EXCEPTION_PROLOG(0x100, BOOKE_INTERRUPT_CRITICAL,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON(0x100, PACA_EXCRIT, INTS_DISABLE)
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x100)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/* Machine Check Interrupt */
START_EXCEPTION(machine_check);
- MC_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_MACHINE_CHECK,
+ MC_EXCEPTION_PROLOG(0x000, BOOKE_INTERRUPT_MACHINE_CHECK,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON(0x200, PACA_EXMC, INTS_DISABLE)
-// bl special_reg_save_mc
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// CHECK_NAPPING();
-// bl .machine_check_exception
-// b ret_from_mc_except
- b .
+ EXCEPTION_COMMON_MC(0x000)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .machine_check_exception
+ b ret_from_mc_except
/* Data Storage Interrupt */
START_EXCEPTION(data_storage)
@@ -343,7 +575,8 @@ interrupt_end_book3e:
PROLOG_ADDITION_2REGS)
mfspr r14,SPRN_DEAR
mfspr r15,SPRN_ESR
- EXCEPTION_COMMON(0x300, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0x300)
+ INTS_DISABLE
b storage_fault_common
/* Instruction Storage Interrupt */
@@ -352,7 +585,8 @@ interrupt_end_book3e:
PROLOG_ADDITION_2REGS)
li r15,0
mr r14,r10
- EXCEPTION_COMMON(0x400, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0x400)
+ INTS_DISABLE
b storage_fault_common
/* External Input Interrupt */
@@ -365,7 +599,7 @@ interrupt_end_book3e:
PROLOG_ADDITION_2REGS)
mfspr r14,SPRN_DEAR
mfspr r15,SPRN_ESR
- EXCEPTION_COMMON(0x600, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x600)
b alignment_more /* no room, go out of line */
/* Program Interrupt */
@@ -373,7 +607,8 @@ interrupt_end_book3e:
NORMAL_EXCEPTION_PROLOG(0x700, BOOKE_INTERRUPT_PROGRAM,
PROLOG_ADDITION_1REG)
mfspr r14,SPRN_ESR
- EXCEPTION_COMMON(0x700, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0x700)
+ INTS_DISABLE
std r14,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
ld r14,PACA_EXGEN+EX_R14(r13)
@@ -386,7 +621,7 @@ interrupt_end_book3e:
NORMAL_EXCEPTION_PROLOG(0x800, BOOKE_INTERRUPT_FP_UNAVAIL,
PROLOG_ADDITION_NONE)
/* we can probably do a shorter exception entry for that one... */
- EXCEPTION_COMMON(0x800, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x800)
ld r12,_MSR(r1)
andi. r0,r12,MSR_PR;
beq- 1f
@@ -403,7 +638,7 @@ interrupt_end_book3e:
NORMAL_EXCEPTION_PROLOG(0x200, BOOKE_INTERRUPT_SPE_ALTIVEC_UNAVAIL,
PROLOG_ADDITION_NONE)
/* we can probably do a shorter exception entry for that one... */
- EXCEPTION_COMMON(0x200, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x200)
#ifdef CONFIG_ALTIVEC
BEGIN_FTR_SECTION
ld r12,_MSR(r1)
@@ -425,7 +660,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
NORMAL_EXCEPTION_PROLOG(0x220,
BOOKE_INTERRUPT_SPE_FP_DATA_ALTIVEC_ASSIST,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x220, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0x220)
+ INTS_DISABLE
bl .save_nvgprs
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_ALTIVEC
@@ -450,13 +686,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
START_EXCEPTION(watchdog);
CRIT_EXCEPTION_PROLOG(0x9f0, BOOKE_INTERRUPT_WATCHDOG,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON(0x9f0, PACA_EXCRIT, INTS_DISABLE)
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .unknown_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x9f0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_BOOKE_WDT
+ bl .WatchdogException
+#else
+ bl .unknown_exception
+#endif
+ b ret_from_crit_except
/* System Call Interrupt */
START_EXCEPTION(system_call)
@@ -470,7 +710,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
START_EXCEPTION(ap_unavailable);
NORMAL_EXCEPTION_PROLOG(0xf20, BOOKE_INTERRUPT_AP_UNAVAIL,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0xf20, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0xf20)
+ INTS_DISABLE
bl .save_nvgprs
addi r3,r1,STACK_FRAME_OVERHEAD
bl .unknown_exception
@@ -513,7 +754,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
mtcr r10
ld r10,PACA_EXCRIT+EX_R10(r13) /* restore registers */
ld r11,PACA_EXCRIT+EX_R11(r13)
- ld r13,PACA_EXCRIT+EX_R13(r13)
+ mfspr r13,SPRN_SPRG_CRIT_SCRATCH
rfci
/* Normal debug exception */
@@ -526,10 +767,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
/* Now we mash up things to make it look like we are coming on a
* normal exception
*/
- ld r15,PACA_EXCRIT+EX_R13(r13)
- mtspr SPRN_SPRG_GEN_SCRATCH,r15
mfspr r14,SPRN_DBSR
- EXCEPTION_COMMON(0xd00, PACA_EXCRIT, INTS_DISABLE)
+ EXCEPTION_COMMON_CRIT(0xd00)
std r14,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
mr r4,r14
@@ -592,10 +831,9 @@ kernel_dbg_exc:
/* Now we mash up things to make it look like we are coming on a
* normal exception
*/
- mfspr r15,SPRN_SPRG_DBG_SCRATCH
- mtspr SPRN_SPRG_GEN_SCRATCH,r15
mfspr r14,SPRN_DBSR
- EXCEPTION_COMMON(0xd08, PACA_EXDBG, INTS_DISABLE)
+ EXCEPTION_COMMON_DBG(0xd08)
+ INTS_DISABLE
std r14,_DSISR(r1)
addi r3,r1,STACK_FRAME_OVERHEAD
mr r4,r14
@@ -608,7 +846,8 @@ kernel_dbg_exc:
START_EXCEPTION(perfmon);
NORMAL_EXCEPTION_PROLOG(0x260, BOOKE_INTERRUPT_PERFORMANCE_MONITOR,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x260, PACA_EXGEN, INTS_DISABLE)
+ EXCEPTION_COMMON(0x260)
+ INTS_DISABLE
CHECK_NAPPING()
addi r3,r1,STACK_FRAME_OVERHEAD
bl .performance_monitor_exception
@@ -622,13 +861,13 @@ kernel_dbg_exc:
START_EXCEPTION(doorbell_crit);
CRIT_EXCEPTION_PROLOG(0x2a0, BOOKE_INTERRUPT_DOORBELL_CRITICAL,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON(0x2a0, PACA_EXCRIT, INTS_DISABLE)
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .doorbell_critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x2a0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/*
* Guest doorbell interrupt
@@ -637,7 +876,7 @@ kernel_dbg_exc:
START_EXCEPTION(guest_doorbell);
GDBELL_EXCEPTION_PROLOG(0x2c0, BOOKE_INTERRUPT_GUEST_DBELL,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x2c0, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x2c0)
addi r3,r1,STACK_FRAME_OVERHEAD
bl .save_nvgprs
INTS_RESTORE_HARD
@@ -648,19 +887,19 @@ kernel_dbg_exc:
START_EXCEPTION(guest_doorbell_crit);
CRIT_EXCEPTION_PROLOG(0x2e0, BOOKE_INTERRUPT_GUEST_DBELL_CRIT,
PROLOG_ADDITION_NONE)
-// EXCEPTION_COMMON(0x2e0, PACA_EXCRIT, INTS_DISABLE)
-// bl special_reg_save_crit
-// CHECK_NAPPING();
-// addi r3,r1,STACK_FRAME_OVERHEAD
-// bl .guest_doorbell_critical_exception
-// b ret_from_crit_except
- b .
+ EXCEPTION_COMMON_CRIT(0x2e0)
+ bl .save_nvgprs
+ bl special_reg_save
+ CHECK_NAPPING();
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl .unknown_exception
+ b ret_from_crit_except
/* Hypervisor call */
START_EXCEPTION(hypercall);
NORMAL_EXCEPTION_PROLOG(0x310, BOOKE_INTERRUPT_HV_SYSCALL,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x310, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x310)
addi r3,r1,STACK_FRAME_OVERHEAD
bl .save_nvgprs
INTS_RESTORE_HARD
@@ -671,7 +910,7 @@ kernel_dbg_exc:
START_EXCEPTION(ehpriv);
NORMAL_EXCEPTION_PROLOG(0x320, BOOKE_INTERRUPT_HV_PRIV,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x320, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x320)
addi r3,r1,STACK_FRAME_OVERHEAD
bl .save_nvgprs
INTS_RESTORE_HARD
@@ -682,7 +921,7 @@ kernel_dbg_exc:
START_EXCEPTION(lrat_error);
NORMAL_EXCEPTION_PROLOG(0x340, BOOKE_INTERRUPT_LRAT_ERROR,
PROLOG_ADDITION_NONE)
- EXCEPTION_COMMON(0x340, PACA_EXGEN, INTS_KEEP)
+ EXCEPTION_COMMON(0x340)
addi r3,r1,STACK_FRAME_OVERHEAD
bl .save_nvgprs
INTS_RESTORE_HARD
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index f5f11a7d30e..4933909cc5c 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -102,6 +102,8 @@ static void setup_tlb_core_data(void)
{
int cpu;
+ BUILD_BUG_ON(offsetof(struct tlb_core_data, lock) != 0);
+
for_each_possible_cpu(cpu) {
int first = cpu_first_thread_sibling(cpu);
@@ -552,14 +554,20 @@ static void __init irqstack_early_init(void)
static void __init exc_lvl_early_init(void)
{
unsigned int i;
+ unsigned long sp;
for_each_possible_cpu(i) {
- critirq_ctx[i] = (struct thread_info *)
- __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
- dbgirq_ctx[i] = (struct thread_info *)
- __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
- mcheckirq_ctx[i] = (struct thread_info *)
- __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
+ sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
+ critirq_ctx[i] = (struct thread_info *)__va(sp);
+ paca[i].crit_kstack = __va(sp + THREAD_SIZE);
+
+ sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
+ dbgirq_ctx[i] = (struct thread_info *)__va(sp);
+ paca[i].dbg_kstack = __va(sp + THREAD_SIZE);
+
+ sp = memblock_alloc(THREAD_SIZE, THREAD_SIZE);
+ mcheckirq_ctx[i] = (struct thread_info *)__va(sp);
+ paca[i].mc_kstack = __va(sp + THREAD_SIZE);
}
if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c
index 094e45c16a1..ce74c335a6a 100644
--- a/arch/powerpc/kernel/vdso.c
+++ b/arch/powerpc/kernel/vdso.c
@@ -715,8 +715,8 @@ int vdso_getcpu_init(void)
unsigned long cpu, node, val;
/*
- * SPRG3 contains the CPU in the bottom 16 bits and the NUMA node in
- * the next 16 bits. The VDSO uses this to implement getcpu().
+ * SPRG_VDSO contains the CPU in the bottom 16 bits and the NUMA node
+ * in the next 16 bits. The VDSO uses this to implement getcpu().
*/
cpu = get_cpu();
WARN_ON_ONCE(cpu > 0xffff);
@@ -725,8 +725,8 @@ int vdso_getcpu_init(void)
WARN_ON_ONCE(node > 0xffff);
val = (cpu & 0xfff) | ((node & 0xffff) << 16);
- mtspr(SPRN_SPRG3, val);
- get_paca()->sprg3 = val;
+ mtspr(SPRN_SPRG_VDSO_WRITE, val);
+ get_paca()->sprg_vdso = val;
put_cpu();
diff --git a/arch/powerpc/kernel/vdso32/getcpu.S b/arch/powerpc/kernel/vdso32/getcpu.S
index 47afd08c90f..23eb9a9441b 100644
--- a/arch/powerpc/kernel/vdso32/getcpu.S
+++ b/arch/powerpc/kernel/vdso32/getcpu.S
@@ -29,7 +29,7 @@
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
- mfspr r5,SPRN_USPRG3
+ mfspr r5,SPRN_SPRG_VDSO_READ
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16
diff --git a/arch/powerpc/kernel/vdso64/getcpu.S b/arch/powerpc/kernel/vdso64/getcpu.S
index 47afd08c90f..23eb9a9441b 100644
--- a/arch/powerpc/kernel/vdso64/getcpu.S
+++ b/arch/powerpc/kernel/vdso64/getcpu.S
@@ -29,7 +29,7 @@
*/
V_FUNCTION_BEGIN(__kernel_getcpu)
.cfi_startproc
- mfspr r5,SPRN_USPRG3
+ mfspr r5,SPRN_SPRG_VDSO_READ
cmpdi cr0,r3,0
cmpdi cr1,r4,0
clrlwi r6,r5,16