aboutsummaryrefslogtreecommitdiff
path: root/arch/powerpc/kernel/tm.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/powerpc/kernel/tm.S')
-rw-r--r--arch/powerpc/kernel/tm.S189
1 files changed, 123 insertions, 66 deletions
diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S
index 7b60b985146..2a324f4cb1b 100644
--- a/arch/powerpc/kernel/tm.S
+++ b/arch/powerpc/kernel/tm.S
@@ -10,18 +10,18 @@
#include <asm/ppc-opcode.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
+#include <asm/bug.h>
#ifdef CONFIG_VSX
-/* See fpu.S, this is very similar but to save/restore checkpointed FPRs/VSRs */
-#define __SAVE_32FPRS_VSRS_TRANSACT(n,c,base) \
+/* See fpu.S, this is borrowed from there */
+#define __SAVE_32FPRS_VSRS(n,c,base) \
BEGIN_FTR_SECTION \
b 2f; \
END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
- SAVE_32FPRS_TRANSACT(n,base); \
+ SAVE_32FPRS(n,base); \
b 3f; \
-2: SAVE_32VSRS_TRANSACT(n,c,base); \
+2: SAVE_32VSRS(n,c,base); \
3:
-/* ...and this is just plain borrowed from there. */
#define __REST_32FPRS_VSRS(n,c,base) \
BEGIN_FTR_SECTION \
b 2f; \
@@ -31,18 +31,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX); \
2: REST_32VSRS(n,c,base); \
3:
#else
-#define __SAVE_32FPRS_VSRS_TRANSACT(n,c,base) SAVE_32FPRS_TRANSACT(n, base)
-#define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base)
+#define __SAVE_32FPRS_VSRS(n,c,base) SAVE_32FPRS(n, base)
+#define __REST_32FPRS_VSRS(n,c,base) REST_32FPRS(n, base)
#endif
-#define SAVE_32FPRS_VSRS_TRANSACT(n,c,base) \
- __SAVE_32FPRS_VSRS_TRANSACT(n,__REG_##c,__REG_##base)
+#define SAVE_32FPRS_VSRS(n,c,base) \
+ __SAVE_32FPRS_VSRS(n,__REG_##c,__REG_##base)
#define REST_32FPRS_VSRS(n,c,base) \
__REST_32FPRS_VSRS(n,__REG_##c,__REG_##base)
/* Stack frame offsets for local variables. */
#define TM_FRAME_L0 TM_FRAME_SIZE-16
#define TM_FRAME_L1 TM_FRAME_SIZE-8
-#define STACK_PARAM(x) (48+((x)*8))
/* In order to access the TM SPRs, TM must be enabled. So, do so: */
@@ -79,7 +78,6 @@ _GLOBAL(tm_abort)
TABORT(R3)
blr
-
/* void tm_reclaim(struct thread_struct *thread,
* unsigned long orig_msr,
* uint8_t cause)
@@ -102,14 +100,14 @@ _GLOBAL(tm_abort)
_GLOBAL(tm_reclaim)
mfcr r6
mflr r0
- std r6, 8(r1)
+ stw r6, 8(r1)
std r0, 16(r1)
- std r2, 40(r1)
+ std r2, STK_GOT(r1)
stdu r1, -TM_FRAME_SIZE(r1)
/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD]. */
- std r3, STACK_PARAM(0)(r1)
+ std r3, STK_PARAM(R3)(r1)
SAVE_NVGPRS(r1)
/* We need to setup MSR for VSX register save instructions. Here we
@@ -123,6 +121,7 @@ _GLOBAL(tm_reclaim)
mr r15, r14
ori r15, r15, MSR_FP
li r16, MSR_RI
+ ori r16, r16, MSR_EE /* IRQs hard off */
andc r15, r15, r16
oris r15, r15, MSR_VEC@h
#ifdef CONFIG_VSX
@@ -151,10 +150,11 @@ _GLOBAL(tm_reclaim)
andis. r0, r4, MSR_VEC@h
beq dont_backup_vec
- SAVE_32VRS_TRANSACT(0, r6, r3) /* r6 scratch, r3 thread */
+ addi r7, r3, THREAD_TRANSACT_VRSTATE
+ SAVE_32VRS(0, r6, r7) /* r6 scratch, r7 transact vr state */
mfvscr vr0
- li r6, THREAD_TRANSACT_VSCR
- stvx vr0, r3, r6
+ li r6, VRSTATE_VSCR
+ stvx vr0, r7, r6
dont_backup_vec:
mfspr r0, SPRN_VRSAVE
std r0, THREAD_TRANSACT_VRSAVE(r3)
@@ -162,12 +162,20 @@ dont_backup_vec:
andi. r0, r4, MSR_FP
beq dont_backup_fp
- SAVE_32FPRS_VSRS_TRANSACT(0, R6, R3) /* r6 scratch, r3 thread */
+ addi r7, r3, THREAD_TRANSACT_FPSTATE
+ SAVE_32FPRS_VSRS(0, R6, R7) /* r6 scratch, r7 transact fp state */
mffs fr0
- stfd fr0,THREAD_TRANSACT_FPSCR(r3)
+ stfd fr0,FPSTATE_FPSCR(r7)
dont_backup_fp:
+ /* Do sanity check on MSR to make sure we are suspended */
+ li r7, (MSR_TS_S)@higher
+ srdi r6, r14, 32
+ and r6, r6, r7
+1: tdeqi r6, 0
+ EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
+
/* The moment we treclaim, ALL of our GPRs will switch
* to user register state. (FPRs, CCR etc. also!)
* Use an sprg and a tm_scratch in the PACA to shuffle.
@@ -187,10 +195,17 @@ dont_backup_fp:
std r1, PACATMSCRATCH(r13)
ld r1, PACAR1(r13)
+ /* Store the PPR in r11 and reset to decent value */
+ std r11, GPR11(r1) /* Temporary stash */
+ mfspr r11, SPRN_PPR
+ HMT_MEDIUM
+
/* Now get some more GPRS free */
std r7, GPR7(r1) /* Temporary stash */
std r12, GPR12(r1) /* '' '' '' */
- ld r12, STACK_PARAM(0)(r1) /* Param 0, thread_struct * */
+ ld r12, STK_PARAM(R3)(r1) /* Param 0, thread_struct * */
+
+ std r11, THREAD_TM_PPR(r12) /* Store PPR and free r11 */
addi r7, r12, PT_CKPT_REGS /* Thread's ckpt_regs */
@@ -203,15 +218,19 @@ dont_backup_fp:
SAVE_GPR(0, r7) /* user r0 */
SAVE_GPR(2, r7) /* user r2 */
SAVE_4GPRS(3, r7) /* user r3-r6 */
- SAVE_4GPRS(8, r7) /* user r8-r11 */
+ SAVE_GPR(8, r7) /* user r8 */
+ SAVE_GPR(9, r7) /* user r9 */
+ SAVE_GPR(10, r7) /* user r10 */
ld r3, PACATMSCRATCH(r13) /* user r1 */
ld r4, GPR7(r1) /* user r7 */
- ld r5, GPR12(r1) /* user r12 */
- GET_SCRATCH0(6) /* user r13 */
+ ld r5, GPR11(r1) /* user r11 */
+ ld r6, GPR12(r1) /* user r12 */
+ GET_SCRATCH0(8) /* user r13 */
std r3, GPR1(r7)
std r4, GPR7(r7)
- std r5, GPR12(r7)
- std r6, GPR13(r7)
+ std r5, GPR11(r7)
+ std r6, GPR12(r7)
+ std r8, GPR13(r7)
SAVE_NVGPRS(r7) /* user r14-r31 */
@@ -234,14 +253,12 @@ dont_backup_fp:
std r6, _XER(r7)
- /* ******************** TAR, PPR, DSCR ********** */
+ /* ******************** TAR, DSCR ********** */
mfspr r3, SPRN_TAR
- mfspr r4, SPRN_PPR
- mfspr r5, SPRN_DSCR
+ mfspr r4, SPRN_DSCR
std r3, THREAD_TM_TAR(r12)
- std r4, THREAD_TM_PPR(r12)
- std r5, THREAD_TM_DSCR(r12)
+ std r4, THREAD_TM_DSCR(r12)
/* MSR and flags: We don't change CRs, and we don't need to alter
* MSR.
@@ -258,7 +275,7 @@ dont_backup_fp:
std r3, THREAD_TM_TFHAR(r12)
std r4, THREAD_TM_TFIAR(r12)
- /* AMR and PPR are checkpointed too, but are unsupported by Linux. */
+ /* AMR is checkpointed too, but is unsupported by Linux. */
/* Restore original MSR/IRQ state & clear TM mode */
ld r14, TM_FRAME_L0(r1) /* Orig MSR */
@@ -269,11 +286,16 @@ dont_backup_fp:
REST_NVGPRS(r1)
addi r1, r1, TM_FRAME_SIZE
- ld r4, 8(r1)
+ lwz r4, 8(r1)
ld r0, 16(r1)
mtcr r4
mtlr r0
- ld r2, 40(r1)
+ ld r2, STK_GOT(r1)
+
+ /* Load CPU's default DSCR */
+ ld r0, PACA_DSCR(r13)
+ mtspr SPRN_DSCR, r0
+
blr
@@ -285,12 +307,12 @@ dont_backup_fp:
* Call with IRQs off, stacks get all out of sync for
* some periods in here!
*/
-_GLOBAL(tm_recheckpoint)
+_GLOBAL(__tm_recheckpoint)
mfcr r5
mflr r0
- std r5, 8(r1)
+ stw r5, 8(r1)
std r0, 16(r1)
- std r2, 40(r1)
+ std r2, STK_GOT(r1)
stdu r1, -TM_FRAME_SIZE(r1)
/* We've a struct pt_regs at [r1+STACK_FRAME_OVERHEAD].
@@ -298,8 +320,6 @@ _GLOBAL(tm_recheckpoint)
*/
SAVE_NVGPRS(r1)
- std r1, PACAR1(r13)
-
/* Load complete register state from ts_ckpt* registers */
addi r7, r3, PT_CKPT_REGS /* Thread's ckpt_regs */
@@ -337,10 +357,11 @@ _GLOBAL(tm_recheckpoint)
andis. r0, r4, MSR_VEC@h
beq dont_restore_vec
- li r5, THREAD_VSCR
- lvx vr0, r3, r5
+ addi r8, r3, THREAD_VRSTATE
+ li r5, VRSTATE_VSCR
+ lvx vr0, r8, r5
mtvscr vr0
- REST_32VRS(0, r5, r3) /* r5 scratch, r3 THREAD ptr */
+ REST_32VRS(0, r5, r8) /* r5 scratch, r8 ptr */
dont_restore_vec:
ld r5, THREAD_VRSAVE(r3)
mtspr SPRN_VRSAVE, r5
@@ -349,54 +370,85 @@ dont_restore_vec:
andi. r0, r4, MSR_FP
beq dont_restore_fp
- lfd fr0, THREAD_FPSCR(r3)
+ addi r8, r3, THREAD_FPSTATE
+ lfd fr0, FPSTATE_FPSCR(r8)
MTFSF_L(fr0)
- REST_32FPRS_VSRS(0, R4, R3)
+ REST_32FPRS_VSRS(0, R4, R8)
dont_restore_fp:
mtmsr r6 /* FP/Vec off again! */
restore_gprs:
- /* ******************** TAR, PPR, DSCR ********** */
- ld r4, THREAD_TM_TAR(r3)
- ld r5, THREAD_TM_PPR(r3)
- ld r6, THREAD_TM_DSCR(r3)
+ /* ******************** CR,LR,CCR,MSR ********** */
+ ld r4, _CTR(r7)
+ ld r5, _LINK(r7)
+ ld r8, _XER(r7)
- mtspr SPRN_TAR, r4
- mtspr SPRN_PPR, r5
- mtspr SPRN_DSCR, r6
+ mtctr r4
+ mtlr r5
+ mtxer r8
- /* ******************** CR,LR,CCR,MSR ********** */
- ld r3, _CTR(r7)
- ld r4, _LINK(r7)
- ld r5, _CCR(r7)
- ld r6, _XER(r7)
+ /* ******************** TAR ******************** */
+ ld r4, THREAD_TM_TAR(r3)
+ mtspr SPRN_TAR, r4
- mtctr r3
- mtlr r4
- mtcr r5
- mtxer r6
+ /* Load up the PPR and DSCR in GPRs only at this stage */
+ ld r5, THREAD_TM_DSCR(r3)
+ ld r6, THREAD_TM_PPR(r3)
/* Clear the MSR RI since we are about to change R1. EE is already off
*/
li r4, 0
mtmsrd r4, 1
- REST_4GPRS(0, r7) /* GPR0-3 */
- REST_GPR(4, r7) /* GPR4-6 */
- REST_GPR(5, r7)
- REST_GPR(6, r7)
+ REST_GPR(0, r7) /* GPR0 */
+ REST_2GPRS(2, r7) /* GPR2-3 */
+ REST_GPR(4, r7) /* GPR4 */
REST_4GPRS(8, r7) /* GPR8-11 */
REST_2GPRS(12, r7) /* GPR12-13 */
REST_NVGPRS(r7) /* GPR14-31 */
- ld r7, GPR7(r7) /* GPR7 */
+ /* Load up PPR and DSCR here so we don't run with user values for long
+ */
+ mtspr SPRN_DSCR, r5
+ mtspr SPRN_PPR, r6
+
+ /* Do final sanity check on TEXASR to make sure FS is set. Do this
+ * here before we load up the userspace r1 so any bugs we hit will get
+ * a call chain */
+ mfspr r5, SPRN_TEXASR
+ srdi r5, r5, 16
+ li r6, (TEXASR_FS)@h
+ and r6, r6, r5
+1: tdeqi r6, 0
+ EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
+
+ /* Do final sanity check on MSR to make sure we are not transactional
+ * or suspended
+ */
+ mfmsr r6
+ li r5, (MSR_TS_MASK)@higher
+ srdi r6, r6, 32
+ and r6, r6, r5
+1: tdnei r6, 0
+ EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,0
+
+ /* Restore CR */
+ ld r6, _CCR(r7)
+ mtcr r6
+
+ REST_GPR(1, r7) /* GPR1 */
+ REST_GPR(5, r7) /* GPR5-7 */
+ REST_GPR(6, r7)
+ ld r7, GPR7(r7)
/* Commit register state as checkpointed state: */
TRECHKPT
+ HMT_MEDIUM
+
/* Our transactional state has now changed.
*
* Now just get out of here. Transactional (current) state will be
@@ -414,11 +466,16 @@ restore_gprs:
REST_NVGPRS(r1)
addi r1, r1, TM_FRAME_SIZE
- ld r4, 8(r1)
+ lwz r4, 8(r1)
ld r0, 16(r1)
mtcr r4
mtlr r0
- ld r2, 40(r1)
+ ld r2, STK_GOT(r1)
+
+ /* Load CPU's default DSCR */
+ ld r0, PACA_DSCR(r13)
+ mtspr SPRN_DSCR, r0
+
blr
/* ****************************************************************** */
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-03 17:59:40 +0000