aboutsummaryrefslogtreecommitdiff
path: root/arch/arm/kernel/entry-common.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/kernel/entry-common.S')
-rw-r--r--arch/arm/kernel/entry-common.S409
1 files changed, 231 insertions, 178 deletions
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 8bfa98757cd..7139d4a7dea 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -10,9 +10,15 @@
#include <asm/unistd.h>
#include <asm/ftrace.h>
-#include <mach/entry-macro.S>
#include <asm/unwind.h>
+#ifdef CONFIG_NEED_RET_TO_USER
+#include <mach/entry-macro.S>
+#else
+ .macro arch_ret_to_user, tmp1, tmp2
+ .endm
+#endif
+
#include "entry-header.S"
@@ -29,9 +35,11 @@ ret_fast_syscall:
ldr r1, [tsk, #TI_FLAGS]
tst r1, #_TIF_WORK_MASK
bne fast_work_pending
+ asm_trace_hardirqs_on
/* perform architecture specific actions before user return */
arch_ret_to_user r1, lr
+ ct_user_enter
restore_user_regs fast = 1, offset = S_OFF
UNWIND(.fnend )
@@ -42,33 +50,34 @@ ret_fast_syscall:
fast_work_pending:
str r0, [sp, #S_R0+S_OFF]! @ returned r0
work_pending:
- tst r1, #_TIF_NEED_RESCHED
- bne work_resched
- tst r1, #_TIF_SIGPENDING|_TIF_NOTIFY_RESUME
- beq no_work_pending
mov r0, sp @ 'regs'
mov r2, why @ 'syscall'
- tst r1, #_TIF_SIGPENDING @ delivering a signal?
- movne why, #0 @ prevent further restarts
- bl do_notify_resume
- b ret_slow_syscall @ Check work again
+ bl do_work_pending
+ cmp r0, #0
+ beq no_work_pending
+ movlt scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
+ ldmia sp, {r0 - r6} @ have to reload r0 - r6
+ b local_restart @ ... and off we go
-work_resched:
- bl schedule
/*
* "slow" syscall return path. "why" tells us if this was a real syscall.
*/
ENTRY(ret_to_user)
ret_slow_syscall:
disable_irq @ disable interrupts
+ENTRY(ret_to_user_from_irq)
ldr r1, [tsk, #TI_FLAGS]
tst r1, #_TIF_WORK_MASK
bne work_pending
no_work_pending:
+ asm_trace_hardirqs_on
+
/* perform architecture specific actions before user return */
arch_ret_to_user r1, lr
+ ct_user_enter save = 0
restore_user_regs fast = 0, offset = 0
+ENDPROC(ret_to_user_from_irq)
ENDPROC(ret_to_user)
/*
@@ -76,20 +85,26 @@ ENDPROC(ret_to_user)
*/
ENTRY(ret_from_fork)
bl schedule_tail
- get_thread_info tsk
- ldr r1, [tsk, #TI_FLAGS] @ check for syscall tracing
- mov why, #1
- tst r1, #_TIF_SYSCALL_TRACE @ are we tracing syscalls?
- beq ret_slow_syscall
- mov r1, sp
- mov r0, #1 @ trace exit [IP = 1]
- bl syscall_trace
+ cmp r5, #0
+ movne r0, r4
+ adrne lr, BSYM(1f)
+ movne pc, r5
+1: get_thread_info tsk
b ret_slow_syscall
ENDPROC(ret_from_fork)
.equ NR_syscalls,0
#define CALL(x) .equ NR_syscalls,NR_syscalls+1
#include "calls.S"
+
+/*
+ * Ensure that the system call table is equal to __NR_syscalls,
+ * which is the value the rest of the system sees
+ */
+.ifne NR_syscalls - __NR_syscalls
+.error "__NR_syscalls is not equal to the size of the syscall table"
+.endif
+
#undef CALL
#define CALL(x) .long x
@@ -141,98 +156,185 @@ ENDPROC(ret_from_fork)
#endif
#endif
-#ifdef CONFIG_DYNAMIC_FTRACE
-ENTRY(__gnu_mcount_nc)
- mov ip, lr
- ldmia sp!, {lr}
- mov pc, ip
-ENDPROC(__gnu_mcount_nc)
+.macro mcount_adjust_addr rd, rn
+ bic \rd, \rn, #1 @ clear the Thumb bit if present
+ sub \rd, \rd, #MCOUNT_INSN_SIZE
+.endm
-ENTRY(ftrace_caller)
- stmdb sp!, {r0-r3, lr}
- mov r0, lr
- sub r0, r0, #MCOUNT_INSN_SIZE
- ldr r1, [sp, #20]
+.macro __mcount suffix
+ mcount_enter
+ ldr r0, =ftrace_trace_function
+ ldr r2, [r0]
+ adr r0, .Lftrace_stub
+ cmp r0, r2
+ bne 1f
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ ldr r1, =ftrace_graph_return
+ ldr r2, [r1]
+ cmp r0, r2
+ bne ftrace_graph_caller\suffix
+
+ ldr r1, =ftrace_graph_entry
+ ldr r2, [r1]
+ ldr r0, =ftrace_graph_entry_stub
+ cmp r0, r2
+ bne ftrace_graph_caller\suffix
+#endif
- .global ftrace_call
-ftrace_call:
+ mcount_exit
+
+1: mcount_get_lr r1 @ lr of instrumented func
+ mcount_adjust_addr r0, lr @ instrumented function
+ adr lr, BSYM(2f)
+ mov pc, r2
+2: mcount_exit
+.endm
+
+.macro __ftrace_caller suffix
+ mcount_enter
+
+ mcount_get_lr r1 @ lr of instrumented func
+ mcount_adjust_addr r0, lr @ instrumented function
+
+ .globl ftrace_call\suffix
+ftrace_call\suffix:
bl ftrace_stub
- ldmia sp!, {r0-r3, ip, lr}
- mov pc, ip
-ENDPROC(ftrace_caller)
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ .globl ftrace_graph_call\suffix
+ftrace_graph_call\suffix:
+ mov r0, r0
+#endif
+
+ mcount_exit
+.endm
+
+.macro __ftrace_graph_caller
+ sub r0, fp, #4 @ &lr of instrumented routine (&parent)
+#ifdef CONFIG_DYNAMIC_FTRACE
+ @ called from __ftrace_caller, saved in mcount_enter
+ ldr r1, [sp, #16] @ instrumented routine (func)
+ mcount_adjust_addr r1, r1
+#else
+ @ called from __mcount, untouched in lr
+ mcount_adjust_addr r1, lr @ instrumented routine (func)
+#endif
+ mov r2, fp @ frame pointer
+ bl prepare_ftrace_return
+ mcount_exit
+.endm
#ifdef CONFIG_OLD_MCOUNT
+/*
+ * mcount
+ */
+
+.macro mcount_enter
+ stmdb sp!, {r0-r3, lr}
+.endm
+
+.macro mcount_get_lr reg
+ ldr \reg, [fp, #-4]
+.endm
+
+.macro mcount_exit
+ ldr lr, [fp, #-4]
+ ldmia sp!, {r0-r3, pc}
+.endm
+
ENTRY(mcount)
+#ifdef CONFIG_DYNAMIC_FTRACE
stmdb sp!, {lr}
ldr lr, [fp, #-4]
ldmia sp!, {pc}
+#else
+ __mcount _old
+#endif
ENDPROC(mcount)
+#ifdef CONFIG_DYNAMIC_FTRACE
ENTRY(ftrace_caller_old)
- stmdb sp!, {r0-r3, lr}
- ldr r1, [fp, #-4]
- mov r0, lr
- sub r0, r0, #MCOUNT_INSN_SIZE
-
- .globl ftrace_call_old
-ftrace_call_old:
- bl ftrace_stub
- ldr lr, [fp, #-4] @ restore lr
- ldmia sp!, {r0-r3, pc}
+ __ftrace_caller _old
ENDPROC(ftrace_caller_old)
#endif
-#else
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller_old)
+ __ftrace_graph_caller
+ENDPROC(ftrace_graph_caller_old)
+#endif
-ENTRY(__gnu_mcount_nc)
+.purgem mcount_enter
+.purgem mcount_get_lr
+.purgem mcount_exit
+#endif
+
+/*
+ * __gnu_mcount_nc
+ */
+
+.macro mcount_enter
+/*
+ * This pad compensates for the push {lr} at the call site. Note that we are
+ * unable to unwind through a function which does not otherwise save its lr.
+ */
+ UNWIND(.pad #4)
stmdb sp!, {r0-r3, lr}
- ldr r0, =ftrace_trace_function
- ldr r2, [r0]
- adr r0, .Lftrace_stub
- cmp r0, r2
- bne gnu_trace
+ UNWIND(.save {r0-r3, lr})
+.endm
+
+.macro mcount_get_lr reg
+ ldr \reg, [sp, #20]
+.endm
+
+.macro mcount_exit
ldmia sp!, {r0-r3, ip, lr}
mov pc, ip
+.endm
-gnu_trace:
- ldr r1, [sp, #20] @ lr of instrumented routine
- mov r0, lr
- sub r0, r0, #MCOUNT_INSN_SIZE
- adr lr, BSYM(1f)
- mov pc, r2
-1:
- ldmia sp!, {r0-r3, ip, lr}
+ENTRY(__gnu_mcount_nc)
+UNWIND(.fnstart)
+#ifdef CONFIG_DYNAMIC_FTRACE
+ mov ip, lr
+ ldmia sp!, {lr}
mov pc, ip
+#else
+ __mcount
+#endif
+UNWIND(.fnend)
ENDPROC(__gnu_mcount_nc)
-#ifdef CONFIG_OLD_MCOUNT
-/*
- * This is under an ifdef in order to force link-time errors for people trying
- * to build with !FRAME_POINTER with a GCC which doesn't use the new-style
- * mcount.
- */
-ENTRY(mcount)
- stmdb sp!, {r0-r3, lr}
- ldr r0, =ftrace_trace_function
- ldr r2, [r0]
- adr r0, ftrace_stub
- cmp r0, r2
- bne trace
- ldr lr, [fp, #-4] @ restore lr
- ldmia sp!, {r0-r3, pc}
+#ifdef CONFIG_DYNAMIC_FTRACE
+ENTRY(ftrace_caller)
+UNWIND(.fnstart)
+ __ftrace_caller
+UNWIND(.fnend)
+ENDPROC(ftrace_caller)
+#endif
-trace:
- ldr r1, [fp, #-4] @ lr of instrumented routine
- mov r0, lr
- sub r0, r0, #MCOUNT_INSN_SIZE
- mov lr, pc
- mov pc, r2
- ldr lr, [fp, #-4] @ restore lr
- ldmia sp!, {r0-r3, pc}
-ENDPROC(mcount)
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller)
+UNWIND(.fnstart)
+ __ftrace_graph_caller
+UNWIND(.fnend)
+ENDPROC(ftrace_graph_caller)
#endif
-#endif /* CONFIG_DYNAMIC_FTRACE */
+.purgem mcount_enter
+.purgem mcount_get_lr
+.purgem mcount_exit
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ .globl return_to_handler
+return_to_handler:
+ stmdb sp!, {r0-r3}
+ mov r0, fp @ frame pointer
+ bl ftrace_return_to_handler
+ mov lr, r0 @ r0 has real ret addr
+ ldmia sp!, {r0-r3}
+ mov pc, lr
+#endif
ENTRY(ftrace_stub)
.Lftrace_stub:
@@ -246,22 +348,11 @@ ENDPROC(ftrace_stub)
*-----------------------------------------------------------------------------
*/
- /* If we're optimising for StrongARM the resulting code won't
- run on an ARM7 and we can save a couple of instructions.
- --pb */
-#ifdef CONFIG_CPU_ARM710
-#define A710(code...) code
-.Larm710bug:
- ldmia sp, {r0 - lr}^ @ Get calling r0 - lr
- mov r0, r0
- add sp, sp, #S_FRAME_SIZE
- subs pc, lr, #4
-#else
-#define A710(code...)
-#endif
-
.align 5
ENTRY(vector_swi)
+#ifdef CONFIG_CPU_V7M
+ v7m_exception_entry
+#else
sub sp, sp, #S_FRAME_SIZE
stmia sp, {r0 - r12} @ Calling r0 - r12
ARM( add r8, sp, #S_PC )
@@ -272,7 +363,12 @@ ENTRY(vector_swi)
str lr, [sp, #S_PC] @ Save calling PC
str r8, [sp, #S_PSR] @ Save CPSR
str r0, [sp, #S_OLD_R0] @ Save OLD_R0
+#endif
zero_fp
+ alignment_trap ip, __cr_alignment
+ enable_irq
+ ct_user_exit
+ get_thread_info tsk
/*
* Get the system call number.
@@ -287,52 +383,28 @@ ENTRY(vector_swi)
#ifdef CONFIG_ARM_THUMB
tst r8, #PSR_T_BIT
movne r10, #0 @ no thumb OABI emulation
- ldreq r10, [lr, #-4] @ get SWI instruction
+ USER( ldreq r10, [lr, #-4] ) @ get SWI instruction
#else
- ldr r10, [lr, #-4] @ get SWI instruction
- A710( and ip, r10, #0x0f000000 @ check for SWI )
- A710( teq ip, #0x0f000000 )
- A710( bne .Larm710bug )
-#endif
-#ifdef CONFIG_CPU_ENDIAN_BE8
- rev r10, r10 @ little endian instruction
+ USER( ldr r10, [lr, #-4] ) @ get SWI instruction
#endif
+ ARM_BE8(rev r10, r10) @ little endian instruction
#elif defined(CONFIG_AEABI)
/*
* Pure EABI user space always put syscall number into scno (r7).
*/
- A710( ldr ip, [lr, #-4] @ get SWI instruction )
- A710( and ip, ip, #0x0f000000 @ check for SWI )
- A710( teq ip, #0x0f000000 )
- A710( bne .Larm710bug )
-
#elif defined(CONFIG_ARM_THUMB)
-
/* Legacy ABI only, possibly thumb mode. */
tst r8, #PSR_T_BIT @ this is SPSR from save_user_regs
addne scno, r7, #__NR_SYSCALL_BASE @ put OS number in
- ldreq scno, [lr, #-4]
+ USER( ldreq scno, [lr, #-4] )
#else
-
/* Legacy ABI only. */
- ldr scno, [lr, #-4] @ get SWI instruction
- A710( and ip, scno, #0x0f000000 @ check for SWI )
- A710( teq ip, #0x0f000000 )
- A710( bne .Larm710bug )
-
+ USER( ldr scno, [lr, #-4] ) @ get SWI instruction
#endif
-#ifdef CONFIG_ALIGNMENT_TRAP
- ldr ip, __cr_alignment
- ldr ip, [ip]
- mcr p15, 0, ip, c1, c0 @ update control register
-#endif
- enable_irq
-
- get_thread_info tsk
adr tbl, sys_call_table @ load syscall table pointer
#if defined(CONFIG_OABI_COMPAT)
@@ -350,20 +422,11 @@ ENTRY(vector_swi)
eor scno, scno, #__NR_SYSCALL_BASE @ check OS number
#endif
+local_restart:
ldr r10, [tsk, #TI_FLAGS] @ check for syscall tracing
stmdb sp!, {r4, r5} @ push fifth and sixth args
-#ifdef CONFIG_SECCOMP
- tst r10, #_TIF_SECCOMP
- beq 1f
- mov r0, scno
- bl __secure_computing
- add r0, sp, #S_R0 + S_OFF @ pointer to regs
- ldmia r0, {r0 - r3} @ have to reload r0 - r3
-1:
-#endif
-
- tst r10, #_TIF_SYSCALL_TRACE @ are we tracing syscalls?
+ tst r10, #_TIF_SYSCALL_WORK @ are we tracing syscalls?
bne __sys_trace
cmp scno, #NR_syscalls @ check upper syscall limit
@@ -371,11 +434,26 @@ ENTRY(vector_swi)
ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine
add r1, sp, #S_OFF
-2: mov why, #0 @ no longer a real syscall
- cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
+2: cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
eor r0, scno, #__NR_SYSCALL_BASE @ put OS number back
- bcs arm_syscall
+ bcs arm_syscall
+ mov why, #0 @ no longer a real syscall
b sys_ni_syscall @ not private func
+
+#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
+ /*
+ * We failed to handle a fault trying to access the page
+ * containing the swi instruction, but we're not really in a
+ * position to return -EFAULT. Instead, return back to the
+ * instruction and re-enter the user fault handling path trying
+ * to page it in. This will likely result in sending SEGV to the
+ * current task.
+ */
+9001:
+ sub lr, lr, #4
+ str lr, [sp, #S_PC]
+ b ret_fast_syscall
+#endif
ENDPROC(vector_swi)
/*
@@ -383,25 +461,26 @@ ENDPROC(vector_swi)
* context switches, and waiting for our parent to respond.
*/
__sys_trace:
- mov r2, scno
- add r1, sp, #S_OFF
- mov r0, #0 @ trace entry [IP = 0]
- bl syscall_trace
+ mov r1, scno
+ add r0, sp, #S_OFF
+ bl syscall_trace_enter
adr lr, BSYM(__sys_trace_return) @ return address
mov scno, r0 @ syscall number (possibly new)
add r1, sp, #S_R0 + S_OFF @ pointer to regs
cmp scno, #NR_syscalls @ check upper syscall limit
- ldmccia r1, {r0 - r3} @ have to reload r0 - r3
+ ldmccia r1, {r0 - r6} @ have to reload r0 - r6
+ stmccia sp, {r4, r5} @ and update the stack args
ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine
- b 2b
+ cmp scno, #-1 @ skip the syscall?
+ bne 2b
+ add sp, sp, #S_OFF @ restore stack
+ b ret_slow_syscall
__sys_trace_return:
str r0, [sp, #S_R0 + S_OFF]! @ save returned r0
- mov r2, scno
- mov r1, sp
- mov r0, #1 @ trace exit [IP = 1]
- bl syscall_trace
+ mov r0, sp
+ bl syscall_trace_exit
b ret_slow_syscall
.align 5
@@ -447,27 +526,6 @@ sys_syscall:
b sys_ni_syscall
ENDPROC(sys_syscall)
-sys_fork_wrapper:
- add r0, sp, #S_OFF
- b sys_fork
-ENDPROC(sys_fork_wrapper)
-
-sys_vfork_wrapper:
- add r0, sp, #S_OFF
- b sys_vfork
-ENDPROC(sys_vfork_wrapper)
-
-sys_execve_wrapper:
- add r3, sp, #S_OFF
- b sys_execve
-ENDPROC(sys_execve_wrapper)
-
-sys_clone_wrapper:
- add ip, sp, #S_OFF
- str ip, [sp, #4]
- b sys_clone
-ENDPROC(sys_clone_wrapper)
-
sys_sigreturn_wrapper:
add r0, sp, #S_OFF
mov why, #0 @ prevent syscall restart handling
@@ -480,11 +538,6 @@ sys_rt_sigreturn_wrapper:
b sys_rt_sigreturn
ENDPROC(sys_rt_sigreturn_wrapper)
-sys_sigaltstack_wrapper:
- ldr r2, [sp, #S_OFF + S_SP]
- b do_sigaltstack
-ENDPROC(sys_sigaltstack_wrapper)
-
sys_statfs64_wrapper:
teq r1, #88
moveq r1, #84
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-05 05:57:13 +0000