From f9810a82536e0c730c57844753e6c08cc7f77881 Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Tue, 10 Aug 2010 19:39:20 +0100 Subject: ARM: 6317/1: ftrace: pass KBUILD_CFLAGS to record_mcount.pl On ARM, we have two ABIs, and the ABI used is controlled via a config option. Object files built with one ABI can't be merged with object files built with the other ABI. So, record_mcount.pl needs to use the same compiler flags as the kernel when generating the object file with the mcount locations. Ensure this by passing CFLAGS to the script. Acked-by: Steven Rostedt Signed-off-by: Rabin Vincent Signed-off-by: Russell King --- scripts/Makefile.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index a1a5cf95a68..108eeb99351 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -212,7 +212,8 @@ ifdef CONFIG_FTRACE_MCOUNT_RECORD cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ "$(if $(CONFIG_64BIT),64,32)" \ - "$(OBJDUMP)" "$(OBJCOPY)" "$(CC)" "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ + "$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \ + "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ "$(if $(part-of-module),1,0)" "$(@)"; endif -- cgit v1.2.3-70-g09d2 From 3b6c223b1b97ad60bbb0f4efda57d649414ac2a2 Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Tue, 10 Aug 2010 19:43:28 +0100 Subject: ARM: 6318/1: ftrace: fix and update dynamic ftrace This adds mcount recording and updates dynamic ftrace for ARM to work with the new ftrace dyamic tracing implementation. It also adds support for the mcount format used by newer ARM compilers. With dynamic tracing, mcount() is implemented as a nop. Callsites are patched on startup with nops, and dynamically patched to call to the ftrace_caller() routine as needed. Acked-by: Steven Rostedt [recordmcount.pl change] Signed-off-by: Rabin Vincent Signed-off-by: Russell King --- arch/arm/include/asm/ftrace.h | 19 ++++- arch/arm/kernel/entry-common.S | 37 +++++++--- arch/arm/kernel/ftrace.c | 155 ++++++++++++++++++++++++++++------------- scripts/recordmcount.pl | 2 + 4 files changed, 155 insertions(+), 58 deletions(-) (limited to 'scripts') diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index 103f7ee9731..4a56a2ee067 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -2,12 +2,29 @@ #define _ASM_ARM_FTRACE #ifdef CONFIG_FUNCTION_TRACER -#define MCOUNT_ADDR ((long)(mcount)) +#define MCOUNT_ADDR ((unsigned long)(__gnu_mcount_nc)) #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ #ifndef __ASSEMBLY__ extern void mcount(void); extern void __gnu_mcount_nc(void); + +#ifdef CONFIG_DYNAMIC_FTRACE +struct dyn_arch_ftrace { +#ifdef CONFIG_OLD_MCOUNT + bool old_mcount; +#endif +}; + +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + return addr; +} + +extern void ftrace_caller_old(void); +extern void ftrace_call_old(void); +#endif + #endif #endif diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index f5e75de0203..e02790f2887 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -127,6 +127,10 @@ ENDPROC(ret_from_fork) * clobber the ip register. This is OK because the ARM calling convention * allows it to be clobbered in subroutines and doesn't use it to hold * parameters.) + * + * When using dynamic ftrace, we patch out the mcount call by a "mov r0, r0" + * for the mcount case, and a "pop {lr}" for the __gnu_mcount_nc case (see + * arch/arm/kernel/ftrace.c). */ #ifndef CONFIG_OLD_MCOUNT @@ -136,30 +140,45 @@ ENDPROC(ret_from_fork) #endif #ifdef CONFIG_DYNAMIC_FTRACE -ENTRY(mcount) +ENTRY(__gnu_mcount_nc) + mov ip, lr + ldmia sp!, {lr} + mov pc, ip +ENDPROC(__gnu_mcount_nc) + +ENTRY(ftrace_caller) stmdb sp!, {r0-r3, lr} mov r0, lr sub r0, r0, #MCOUNT_INSN_SIZE + ldr r1, [sp, #20] - .globl mcount_call -mcount_call: + .global ftrace_call +ftrace_call: bl ftrace_stub - ldr lr, [fp, #-4] @ restore lr - ldmia sp!, {r0-r3, pc} + ldmia sp!, {r0-r3, ip, lr} + mov pc, ip +ENDPROC(ftrace_caller) + +#ifdef CONFIG_OLD_MCOUNT +ENTRY(mcount) + stmdb sp!, {lr} + ldr lr, [fp, #-4] + ldmia sp!, {pc} ENDPROC(mcount) -ENTRY(ftrace_caller) +ENTRY(ftrace_caller_old) stmdb sp!, {r0-r3, lr} ldr r1, [fp, #-4] mov r0, lr sub r0, r0, #MCOUNT_INSN_SIZE - .globl ftrace_call -ftrace_call: + .globl ftrace_call_old +ftrace_call_old: bl ftrace_stub ldr lr, [fp, #-4] @ restore lr ldmia sp!, {r0-r3, pc} -ENDPROC(ftrace_caller) +ENDPROC(ftrace_caller_old) +#endif #else diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index 0298286ad4a..f09014cfbf2 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -2,102 +2,161 @@ * Dynamic function tracing support. * * Copyright (C) 2008 Abhishek Sagar + * Copyright (C) 2010 Rabin Vincent * * For licencing details, see COPYING. * * Defines low-level handling of mcount calls when the kernel * is compiled with the -pg flag. When using dynamic ftrace, the - * mcount call-sites get patched lazily with NOP till they are - * enabled. All code mutation routines here take effect atomically. + * mcount call-sites get patched with NOP till they are enabled. + * All code mutation routines here are called under stop_machine(). */ #include +#include #include #include -#define PC_OFFSET 8 -#define BL_OPCODE 0xeb000000 -#define BL_OFFSET_MASK 0x00ffffff +#define NOP 0xe8bd4000 /* pop {lr} */ -static unsigned long bl_insn; -static const unsigned long NOP = 0xe1a00000; /* mov r0, r0 */ +#ifdef CONFIG_OLD_MCOUNT +#define OLD_MCOUNT_ADDR ((unsigned long) mcount) +#define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) -unsigned char *ftrace_nop_replace(void) +#define OLD_NOP 0xe1a00000 /* mov r0, r0 */ + +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) +{ + return rec->arch.old_mcount ? OLD_NOP : NOP; +} + +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +{ + if (!rec->arch.old_mcount) + return addr; + + if (addr == MCOUNT_ADDR) + addr = OLD_MCOUNT_ADDR; + else if (addr == FTRACE_ADDR) + addr = OLD_FTRACE_ADDR; + + return addr; +} +#else +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) +{ + return NOP; +} + +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) { - return (char *)&NOP; + return addr; } +#endif /* construct a branch (BL) instruction to addr */ -unsigned char *ftrace_call_replace(unsigned long pc, unsigned long addr) +static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) { long offset; - offset = (long)addr - (long)(pc + PC_OFFSET); + offset = (long)addr - (long)(pc + 8); if (unlikely(offset < -33554432 || offset > 33554428)) { /* Can't generate branches that far (from ARM ARM). Ftrace * doesn't generate branches outside of kernel text. */ WARN_ON_ONCE(1); - return NULL; + return 0; } - offset = (offset >> 2) & BL_OFFSET_MASK; - bl_insn = BL_OPCODE | offset; - return (unsigned char *)&bl_insn; -} -int ftrace_modify_code(unsigned long pc, unsigned char *old_code, - unsigned char *new_code) -{ - unsigned long err = 0, replaced = 0, old, new; + offset = (offset >> 2) & 0x00ffffff; - old = *(unsigned long *)old_code; - new = *(unsigned long *)new_code; + return 0xeb000000 | offset; +} - __asm__ __volatile__ ( - "1: ldr %1, [%2] \n" - " cmp %1, %4 \n" - "2: streq %3, [%2] \n" - " cmpne %1, %3 \n" - " movne %0, #2 \n" - "3:\n" +static int ftrace_modify_code(unsigned long pc, unsigned long old, + unsigned long new) +{ + unsigned long replaced; - ".pushsection .fixup, \"ax\"\n" - "4: mov %0, #1 \n" - " b 3b \n" - ".popsection\n" + if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) + return -EFAULT; - ".pushsection __ex_table, \"a\"\n" - " .long 1b, 4b \n" - " .long 2b, 4b \n" - ".popsection\n" + if (replaced != old) + return -EINVAL; - : "=r"(err), "=r"(replaced) - : "r"(pc), "r"(new), "r"(old), "0"(err), "1"(replaced) - : "memory"); + if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) + return -EPERM; - if (!err && (replaced == old)) - flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); + flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); - return err; + return 0; } int ftrace_update_ftrace_func(ftrace_func_t func) { - int ret; unsigned long pc, old; - unsigned char *new; + unsigned long new; + int ret; pc = (unsigned long)&ftrace_call; memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE); new = ftrace_call_replace(pc, (unsigned long)func); - ret = ftrace_modify_code(pc, (unsigned char *)&old, new); + + ret = ftrace_modify_code(pc, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (!ret) { + pc = (unsigned long)&ftrace_call_old; + memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE); + new = ftrace_call_replace(pc, (unsigned long)func); + + ret = ftrace_modify_code(pc, old, new); + } +#endif + + return ret; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long new, old; + unsigned long ip = rec->ip; + + old = ftrace_nop_replace(rec); + new = ftrace_call_replace(ip, adjust_address(rec, addr)); + + return ftrace_modify_code(rec->ip, old, new); +} + +int ftrace_make_nop(struct module *mod, + struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long ip = rec->ip; + unsigned long old; + unsigned long new; + int ret; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (ret == -EINVAL && addr == MCOUNT_ADDR) { + rec->arch.old_mcount = true; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + } +#endif + return ret; } -/* run from ftrace_init with irqs disabled */ int __init ftrace_dyn_arch_init(void *data) { - ftrace_mcount_set(data); + *(unsigned long *)data = 0; + return 0; } diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index e67f0548608..022d4679b1b 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -270,6 +270,8 @@ if ($arch eq "x86_64") { } elsif ($arch eq "arm") { $alignment = 2; $section_type = '%progbits'; + $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_ARM_(CALL|PC24)" . + "\\s+(__gnu_mcount_nc|mcount)\$"; } elsif ($arch eq "ia64") { $mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\s_mcount\$"; -- cgit v1.2.3-70-g09d2 From 72dc43a9eb123d2742bd413c80dbeab0c588f622 Mon Sep 17 00:00:00 2001 From: Rabin Vincent Date: Tue, 10 Aug 2010 19:52:35 +0100 Subject: ARM: 6319/1: ftrace: add Thumb-2 support to dynamic ftrace Handle the different nop and call instructions for Thumb-2. Also, we need to adjust the recorded mcount_loc addresses because they have the lsb set. Cc: Catalin Marinas Acked-by: Steven Rostedt [recordmcount.pl change] Signed-off-by: Rabin Vincent Signed-off-by: Russell King --- arch/arm/include/asm/ftrace.h | 3 ++- arch/arm/kernel/ftrace.c | 33 +++++++++++++++++++++++++++++++++ scripts/recordmcount.pl | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/arch/arm/include/asm/ftrace.h b/arch/arm/include/asm/ftrace.h index 4a56a2ee067..f89515adac6 100644 --- a/arch/arm/include/asm/ftrace.h +++ b/arch/arm/include/asm/ftrace.h @@ -18,7 +18,8 @@ struct dyn_arch_ftrace { static inline unsigned long ftrace_call_adjust(unsigned long addr) { - return addr; + /* With Thumb-2, the recorded addresses have the lsb set */ + return addr & ~1; } extern void ftrace_caller_old(void); diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c index f09014cfbf2..971ac8c36ea 100644 --- a/arch/arm/kernel/ftrace.c +++ b/arch/arm/kernel/ftrace.c @@ -18,7 +18,11 @@ #include #include +#ifdef CONFIG_THUMB2_KERNEL +#define NOP 0xeb04f85d /* pop.w {lr} */ +#else #define NOP 0xe8bd4000 /* pop {lr} */ +#endif #ifdef CONFIG_OLD_MCOUNT #define OLD_MCOUNT_ADDR ((unsigned long) mcount) @@ -56,6 +60,34 @@ static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) #endif /* construct a branch (BL) instruction to addr */ +#ifdef CONFIG_THUMB2_KERNEL +static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) +{ + unsigned long s, j1, j2, i1, i2, imm10, imm11; + unsigned long first, second; + long offset; + + offset = (long)addr - (long)(pc + 4); + if (offset < -16777216 || offset > 16777214) { + WARN_ON_ONCE(1); + return 0; + } + + s = (offset >> 24) & 0x1; + i1 = (offset >> 23) & 0x1; + i2 = (offset >> 22) & 0x1; + imm10 = (offset >> 12) & 0x3ff; + imm11 = (offset >> 1) & 0x7ff; + + j1 = (!i1) ^ s; + j2 = (!i2) ^ s; + + first = 0xf000 | (s << 10) | imm10; + second = 0xd000 | (j1 << 13) | (j2 << 11) | imm11; + + return (second << 16) | first; +} +#else static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) { long offset; @@ -73,6 +105,7 @@ static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) return 0xeb000000 | offset; } +#endif static int ftrace_modify_code(unsigned long pc, unsigned long old, unsigned long new) diff --git a/scripts/recordmcount.pl b/scripts/recordmcount.pl index 022d4679b1b..1d7963f4ee7 100755 --- a/scripts/recordmcount.pl +++ b/scripts/recordmcount.pl @@ -270,7 +270,7 @@ if ($arch eq "x86_64") { } elsif ($arch eq "arm") { $alignment = 2; $section_type = '%progbits'; - $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_ARM_(CALL|PC24)" . + $mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_ARM_(CALL|PC24|THM_CALL)" . "\\s+(__gnu_mcount_nc|mcount)\$"; } elsif ($arch eq "ia64") { -- cgit v1.2.3-70-g09d2 From bf5438fca2950b03c21ad868090cc1a8fcd49536 Mon Sep 17 00:00:00 2001 From: Jason Baron Date: Fri, 17 Sep 2010 11:09:00 -0400 Subject: jump label: Base patch for jump label base patch to implement 'jump labeling'. Based on a new 'asm goto' inline assembly gcc mechanism, we can now branch to labels from an 'asm goto' statment. This allows us to create a 'no-op' fastpath, which can subsequently be patched with a jump to the slowpath code. This is useful for code which might be rarely used, but which we'd like to be able to call, if needed. Tracepoints are the current usecase that these are being implemented for. Acked-by: David S. Miller Signed-off-by: Jason Baron LKML-Reference: [ cleaned up some formating ] Signed-off-by: Steven Rostedt --- Makefile | 5 + arch/Kconfig | 3 + arch/x86/include/asm/alternative.h | 3 +- arch/x86/kernel/alternative.c | 2 +- include/asm-generic/vmlinux.lds.h | 10 ++ include/linux/jump_label.h | 58 +++++++ include/linux/module.h | 5 +- kernel/Makefile | 2 +- kernel/jump_label.c | 346 +++++++++++++++++++++++++++++++++++++ kernel/kprobes.c | 1 + kernel/module.c | 6 + scripts/gcc-goto.sh | 5 + 12 files changed, 442 insertions(+), 4 deletions(-) create mode 100644 include/linux/jump_label.h create mode 100644 kernel/jump_label.c create mode 100644 scripts/gcc-goto.sh (limited to 'scripts') diff --git a/Makefile b/Makefile index 92ab33f16cf..a906378d505 100644 --- a/Makefile +++ b/Makefile @@ -591,6 +591,11 @@ KBUILD_CFLAGS += $(call cc-option,-fno-strict-overflow) # conserve stack if available KBUILD_CFLAGS += $(call cc-option,-fconserve-stack) +# check for 'asm goto' +ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y) + KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO +endif + # Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments # But warn user when we do so warn-assign = \ diff --git a/arch/Kconfig b/arch/Kconfig index 4877a8c8ee1..1462d8492d8 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -158,4 +158,7 @@ config HAVE_PERF_EVENTS_NMI subsystem. Also has support for calculating CPU cycle events to determine how many clock cycles in a given period. +config HAVE_ARCH_JUMP_LABEL + bool + source "kernel/gcov/Kconfig" diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 634bf782dca..76561d20ea2 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -4,6 +4,7 @@ #include #include #include +#include #include /* @@ -182,7 +183,7 @@ extern void *text_poke_early(void *addr, const void *opcode, size_t len); extern void *text_poke(void *addr, const void *opcode, size_t len); extern void *text_poke_smp(void *addr, const void *opcode, size_t len); -#if defined(CONFIG_DYNAMIC_FTRACE) +#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) #define IDEAL_NOP_SIZE_5 5 extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; extern void arch_init_ideal_nop5(void); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 083bd010d92..cb0e6d385f6 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -641,7 +641,7 @@ void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len) return addr; } -#if defined(CONFIG_DYNAMIC_FTRACE) +#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL) unsigned char ideal_nop5[IDEAL_NOP_SIZE_5]; diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 8a92a170fb7..ef2af9948ea 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -220,6 +220,8 @@ \ BUG_TABLE \ \ + JUMP_TABLE \ + \ /* PCI quirks */ \ .pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \ VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \ @@ -563,6 +565,14 @@ #define BUG_TABLE #endif +#define JUMP_TABLE \ + . = ALIGN(8); \ + __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \ + VMLINUX_SYMBOL(__start___jump_table) = .; \ + *(__jump_table) \ + VMLINUX_SYMBOL(__stop___jump_table) = .; \ + } + #ifdef CONFIG_PM_TRACE #define TRACEDATA \ . = ALIGN(4); \ diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h new file mode 100644 index 00000000000..de58656d28e --- /dev/null +++ b/include/linux/jump_label.h @@ -0,0 +1,58 @@ +#ifndef _LINUX_JUMP_LABEL_H +#define _LINUX_JUMP_LABEL_H + +#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_HAVE_ARCH_JUMP_LABEL) +# include +# define HAVE_JUMP_LABEL +#endif + +enum jump_label_type { + JUMP_LABEL_ENABLE, + JUMP_LABEL_DISABLE +}; + +struct module; + +#ifdef HAVE_JUMP_LABEL + +extern struct jump_entry __start___jump_table[]; +extern struct jump_entry __stop___jump_table[]; + +extern void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type); +extern void jump_label_update(unsigned long key, enum jump_label_type type); +extern void jump_label_apply_nops(struct module *mod); +extern void arch_jump_label_text_poke_early(jump_label_t addr); + +#define enable_jump_label(key) \ + jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE); + +#define disable_jump_label(key) \ + jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE); + +#else + +#define JUMP_LABEL(key, label) \ +do { \ + if (unlikely(*key)) \ + goto label; \ +} while (0) + +#define enable_jump_label(cond_var) \ +do { \ + *(cond_var) = 1; \ +} while (0) + +#define disable_jump_label(cond_var) \ +do { \ + *(cond_var) = 0; \ +} while (0) + +static inline int jump_label_apply_nops(struct module *mod) +{ + return 0; +} + +#endif + +#endif diff --git a/include/linux/module.h b/include/linux/module.h index 8a6b9fdc7ff..403ac26023c 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -350,7 +350,10 @@ struct module struct tracepoint *tracepoints; unsigned int num_tracepoints; #endif - +#ifdef HAVE_JUMP_LABEL + struct jump_entry *jump_entries; + unsigned int num_jump_entries; +#endif #ifdef CONFIG_TRACING const char **trace_bprintk_fmt_start; unsigned int num_trace_bprintk_fmt; diff --git a/kernel/Makefile b/kernel/Makefile index 0b72d1a74be..d52b473c99a 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -10,7 +10,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \ kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \ - async.o range.o + async.o range.o jump_label.o obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o obj-y += groups.o diff --git a/kernel/jump_label.c b/kernel/jump_label.c new file mode 100644 index 00000000000..460fd40112b --- /dev/null +++ b/kernel/jump_label.c @@ -0,0 +1,346 @@ +/* + * jump label support + * + * Copyright (C) 2009 Jason Baron + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_JUMP_LABEL + +#define JUMP_LABEL_HASH_BITS 6 +#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS) +static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE]; + +/* mutex to protect coming/going of the the jump_label table */ +static DEFINE_MUTEX(jump_label_mutex); + +struct jump_label_entry { + struct hlist_node hlist; + struct jump_entry *table; + int nr_entries; + /* hang modules off here */ + struct hlist_head modules; + unsigned long key; +}; + +struct jump_label_module_entry { + struct hlist_node hlist; + struct jump_entry *table; + int nr_entries; + struct module *mod; +}; + +static int jump_label_cmp(const void *a, const void *b) +{ + const struct jump_entry *jea = a; + const struct jump_entry *jeb = b; + + if (jea->key < jeb->key) + return -1; + + if (jea->key > jeb->key) + return 1; + + return 0; +} + +static void +sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop) +{ + unsigned long size; + + size = (((unsigned long)stop - (unsigned long)start) + / sizeof(struct jump_entry)); + sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL); +} + +static struct jump_label_entry *get_jump_label_entry(jump_label_t key) +{ + struct hlist_head *head; + struct hlist_node *node; + struct jump_label_entry *e; + u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0); + + head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; + hlist_for_each_entry(e, node, head, hlist) { + if (key == e->key) + return e; + } + return NULL; +} + +static struct jump_label_entry * +add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table) +{ + struct hlist_head *head; + struct jump_label_entry *e; + u32 hash; + + e = get_jump_label_entry(key); + if (e) + return ERR_PTR(-EEXIST); + + e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL); + if (!e) + return ERR_PTR(-ENOMEM); + + hash = jhash((void *)&key, sizeof(jump_label_t), 0); + head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)]; + e->key = key; + e->table = table; + e->nr_entries = nr_entries; + INIT_HLIST_HEAD(&(e->modules)); + hlist_add_head(&e->hlist, head); + return e; +} + +static int +build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop) +{ + struct jump_entry *iter, *iter_begin; + struct jump_label_entry *entry; + int count; + + sort_jump_label_entries(start, stop); + iter = start; + while (iter < stop) { + entry = get_jump_label_entry(iter->key); + if (!entry) { + iter_begin = iter; + count = 0; + while ((iter < stop) && + (iter->key == iter_begin->key)) { + iter++; + count++; + } + entry = add_jump_label_entry(iter_begin->key, + count, iter_begin); + if (IS_ERR(entry)) + return PTR_ERR(entry); + } else { + WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n"); + return -1; + } + } + return 0; +} + +/*** + * jump_label_update - update jump label text + * @key - key value associated with a a jump label + * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE + * + * Will enable/disable the jump for jump label @key, depending on the + * value of @type. + * + */ + +void jump_label_update(unsigned long key, enum jump_label_type type) +{ + struct jump_entry *iter; + struct jump_label_entry *entry; + struct hlist_node *module_node; + struct jump_label_module_entry *e_module; + int count; + + mutex_lock(&jump_label_mutex); + entry = get_jump_label_entry((jump_label_t)key); + if (entry) { + count = entry->nr_entries; + iter = entry->table; + while (count--) { + if (kernel_text_address(iter->code)) + arch_jump_label_transform(iter, type); + iter++; + } + /* eanble/disable jump labels in modules */ + hlist_for_each_entry(e_module, module_node, &(entry->modules), + hlist) { + count = e_module->nr_entries; + iter = e_module->table; + while (count--) { + if (kernel_text_address(iter->code)) + arch_jump_label_transform(iter, type); + iter++; + } + } + } + mutex_unlock(&jump_label_mutex); +} + +static __init int init_jump_label(void) +{ + int ret; + struct jump_entry *iter_start = __start___jump_table; + struct jump_entry *iter_stop = __stop___jump_table; + struct jump_entry *iter; + + mutex_lock(&jump_label_mutex); + ret = build_jump_label_hashtable(__start___jump_table, + __stop___jump_table); + iter = iter_start; + while (iter < iter_stop) { + arch_jump_label_text_poke_early(iter->code); + iter++; + } + mutex_unlock(&jump_label_mutex); + return ret; +} +early_initcall(init_jump_label); + +#ifdef CONFIG_MODULES + +static struct jump_label_module_entry * +add_jump_label_module_entry(struct jump_label_entry *entry, + struct jump_entry *iter_begin, + int count, struct module *mod) +{ + struct jump_label_module_entry *e; + + e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL); + if (!e) + return ERR_PTR(-ENOMEM); + e->mod = mod; + e->nr_entries = count; + e->table = iter_begin; + hlist_add_head(&e->hlist, &entry->modules); + return e; +} + +static int add_jump_label_module(struct module *mod) +{ + struct jump_entry *iter, *iter_begin; + struct jump_label_entry *entry; + struct jump_label_module_entry *module_entry; + int count; + + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return 0; + + sort_jump_label_entries(mod->jump_entries, + mod->jump_entries + mod->num_jump_entries); + iter = mod->jump_entries; + while (iter < mod->jump_entries + mod->num_jump_entries) { + entry = get_jump_label_entry(iter->key); + iter_begin = iter; + count = 0; + while ((iter < mod->jump_entries + mod->num_jump_entries) && + (iter->key == iter_begin->key)) { + iter++; + count++; + } + if (!entry) { + entry = add_jump_label_entry(iter_begin->key, 0, NULL); + if (IS_ERR(entry)) + return PTR_ERR(entry); + } + module_entry = add_jump_label_module_entry(entry, iter_begin, + count, mod); + if (IS_ERR(module_entry)) + return PTR_ERR(module_entry); + } + return 0; +} + +static void remove_jump_label_module(struct module *mod) +{ + struct hlist_head *head; + struct hlist_node *node, *node_next, *module_node, *module_node_next; + struct jump_label_entry *e; + struct jump_label_module_entry *e_module; + int i; + + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return; + + for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) { + head = &jump_label_table[i]; + hlist_for_each_entry_safe(e, node, node_next, head, hlist) { + hlist_for_each_entry_safe(e_module, module_node, + module_node_next, + &(e->modules), hlist) { + if (e_module->mod == mod) { + hlist_del(&e_module->hlist); + kfree(e_module); + } + } + if (hlist_empty(&e->modules) && (e->nr_entries == 0)) { + hlist_del(&e->hlist); + kfree(e); + } + } + } +} + +static int +jump_label_module_notify(struct notifier_block *self, unsigned long val, + void *data) +{ + struct module *mod = data; + int ret = 0; + + switch (val) { + case MODULE_STATE_COMING: + mutex_lock(&jump_label_mutex); + ret = add_jump_label_module(mod); + if (ret) + remove_jump_label_module(mod); + mutex_unlock(&jump_label_mutex); + break; + case MODULE_STATE_GOING: + mutex_lock(&jump_label_mutex); + remove_jump_label_module(mod); + mutex_unlock(&jump_label_mutex); + break; + } + return ret; +} + +/*** + * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop() + * @mod: module to patch + * + * Allow for run-time selection of the optimal nops. Before the module + * loads patch these with arch_get_jump_label_nop(), which is specified by + * the arch specific jump label code. + */ +void jump_label_apply_nops(struct module *mod) +{ + struct jump_entry *iter; + + /* if the module doesn't have jump label entries, just return */ + if (!mod->num_jump_entries) + return; + + iter = mod->jump_entries; + while (iter < mod->jump_entries + mod->num_jump_entries) { + arch_jump_label_text_poke_early(iter->code); + iter++; + } +} + +struct notifier_block jump_label_module_nb = { + .notifier_call = jump_label_module_notify, + .priority = 0, +}; + +static __init int init_jump_label_module(void) +{ + return register_module_notifier(&jump_label_module_nb); +} +early_initcall(init_jump_label_module); + +#endif /* CONFIG_MODULES */ + +#endif diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 6dd5359e1f0..18904e42a91 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include diff --git a/kernel/module.c b/kernel/module.c index d0b5f8db11b..eba134157ef 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -55,6 +55,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -2308,6 +2309,11 @@ static void find_module_sections(struct module *mod, struct load_info *info) sizeof(*mod->tracepoints), &mod->num_tracepoints); #endif +#ifdef HAVE_JUMP_LABEL + mod->jump_entries = section_objs(info, "__jump_table", + sizeof(*mod->jump_entries), + &mod->num_jump_entries); +#endif #ifdef CONFIG_EVENT_TRACING mod->trace_events = section_objs(info, "_ftrace_events", sizeof(*mod->trace_events), diff --git a/scripts/gcc-goto.sh b/scripts/gcc-goto.sh new file mode 100644 index 00000000000..8e82424be7a --- /dev/null +++ b/scripts/gcc-goto.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Test for gcc 'asm goto' suport +# Copyright (C) 2010, Jason Baron + +echo "int main(void) { entry: asm goto (\"\"::::entry); return 0; }" | $1 -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" -- cgit v1.2.3-70-g09d2 From 52159d98be6f26c48f5e02c7ab3c9848a85979b5 Mon Sep 17 00:00:00 2001 From: Jason Baron Date: Fri, 17 Sep 2010 11:09:17 -0400 Subject: jump label: Convert dynamic debug to use jump labels Convert the 'dynamic debug' infrastructure to use jump labels. Signed-off-by: Jason Baron LKML-Reference: Signed-off-by: Steven Rostedt --- include/linux/dynamic_debug.h | 39 ++++++++++++++------------ lib/dynamic_debug.c | 42 ++-------------------------- scripts/Makefile.lib | 11 +------- scripts/basic/Makefile | 2 +- scripts/basic/hash.c | 64 ------------------------------------------- 5 files changed, 26 insertions(+), 132 deletions(-) delete mode 100644 scripts/basic/hash.c (limited to 'scripts') diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index 52c0da4bdd1..bef3cda44c4 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -1,6 +1,8 @@ #ifndef _DYNAMIC_DEBUG_H #define _DYNAMIC_DEBUG_H +#include + /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They * use independent hash functions, to reduce the chance of false positives. @@ -22,8 +24,6 @@ struct _ddebug { const char *function; const char *filename; const char *format; - char primary_hash; - char secondary_hash; unsigned int lineno:24; /* * The flags field controls the behaviour at the callsite. @@ -33,6 +33,7 @@ struct _ddebug { #define _DPRINTK_FLAGS_PRINT (1<<0) /* printk() a message using the format */ #define _DPRINTK_FLAGS_DEFAULT 0 unsigned int flags:8; + char enabled; } __attribute__((aligned(8))); @@ -42,33 +43,35 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n, #if defined(CONFIG_DYNAMIC_DEBUG) extern int ddebug_remove_module(const char *mod_name); -#define __dynamic_dbg_enabled(dd) ({ \ - int __ret = 0; \ - if (unlikely((dynamic_debug_enabled & (1LL << DEBUG_HASH)) && \ - (dynamic_debug_enabled2 & (1LL << DEBUG_HASH2)))) \ - if (unlikely(dd.flags)) \ - __ret = 1; \ - __ret; }) - #define dynamic_pr_debug(fmt, ...) do { \ + __label__ do_printk; \ + __label__ out; \ static struct _ddebug descriptor \ __used \ __attribute__((section("__verbose"), aligned(8))) = \ - { KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH, \ - DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \ - if (__dynamic_dbg_enabled(descriptor)) \ - printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ + { KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__, \ + _DPRINTK_FLAGS_DEFAULT }; \ + JUMP_LABEL(&descriptor.enabled, do_printk); \ + goto out; \ +do_printk: \ + printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \ +out: ; \ } while (0) #define dynamic_dev_dbg(dev, fmt, ...) do { \ + __label__ do_printk; \ + __label__ out; \ static struct _ddebug descriptor \ __used \ __attribute__((section("__verbose"), aligned(8))) = \ - { KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH, \ - DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \ - if (__dynamic_dbg_enabled(descriptor)) \ - dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ + { KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__, \ + _DPRINTK_FLAGS_DEFAULT }; \ + JUMP_LABEL(&descriptor.enabled, do_printk); \ + goto out; \ +do_printk: \ + dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \ +out: ; \ } while (0) #else diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 02afc253372..e925c7b960f 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -26,19 +26,11 @@ #include #include #include +#include extern struct _ddebug __start___verbose[]; extern struct _ddebug __stop___verbose[]; -/* dynamic_debug_enabled, and dynamic_debug_enabled2 are bitmasks in which - * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They - * use independent hash functions, to reduce the chance of false positives. - */ -long long dynamic_debug_enabled; -EXPORT_SYMBOL_GPL(dynamic_debug_enabled); -long long dynamic_debug_enabled2; -EXPORT_SYMBOL_GPL(dynamic_debug_enabled2); - struct ddebug_table { struct list_head link; char *mod_name; @@ -87,26 +79,6 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf, return buf; } -/* - * must be called with ddebug_lock held - */ - -static int disabled_hash(char hash, bool first_table) -{ - struct ddebug_table *dt; - char table_hash_value; - - list_for_each_entry(dt, &ddebug_tables, link) { - if (first_table) - table_hash_value = dt->ddebugs->primary_hash; - else - table_hash_value = dt->ddebugs->secondary_hash; - if (dt->num_enabled && (hash == table_hash_value)) - return 0; - } - return 1; -} - /* * Search the tables for _ddebug's which match the given * `query' and apply the `flags' and `mask' to them. Tells @@ -170,17 +142,9 @@ static void ddebug_change(const struct ddebug_query *query, dt->num_enabled++; dp->flags = newflags; if (newflags) { - dynamic_debug_enabled |= - (1LL << dp->primary_hash); - dynamic_debug_enabled2 |= - (1LL << dp->secondary_hash); + enable_jump_label(&dp->enabled); } else { - if (disabled_hash(dp->primary_hash, true)) - dynamic_debug_enabled &= - ~(1LL << dp->primary_hash); - if (disabled_hash(dp->secondary_hash, false)) - dynamic_debug_enabled2 &= - ~(1LL << dp->secondary_hash); + disable_jump_label(&dp->enabled); } if (verbose) printk(KERN_INFO diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 54fd1b70013..7bfcf1a09ac 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -101,14 +101,6 @@ basename_flags = -D"KBUILD_BASENAME=KBUILD_STR($(call name-fix,$(basetarget)))" modname_flags = $(if $(filter 1,$(words $(modname))),\ -D"KBUILD_MODNAME=KBUILD_STR($(call name-fix,$(modname)))") -#hash values -ifdef CONFIG_DYNAMIC_DEBUG -debug_flags = -D"DEBUG_HASH=$(shell ./scripts/basic/hash djb2 $(@D)$(modname))"\ - -D"DEBUG_HASH2=$(shell ./scripts/basic/hash r5 $(@D)$(modname))" -else -debug_flags = -endif - orig_c_flags = $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(KBUILD_SUBDIR_CCFLAGS) \ $(ccflags-y) $(CFLAGS_$(basetarget).o) _c_flags = $(filter-out $(CFLAGS_REMOVE_$(basetarget).o), $(orig_c_flags)) @@ -152,8 +144,7 @@ endif c_flags = -Wp,-MD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(__c_flags) $(modkern_cflags) \ - -D"KBUILD_STR(s)=\#s" $(basename_flags) $(modname_flags) \ - $(debug_flags) + -D"KBUILD_STR(s)=\#s" $(basename_flags) $(modname_flags) a_flags = -Wp,-MD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(__a_flags) $(modkern_aflags) diff --git a/scripts/basic/Makefile b/scripts/basic/Makefile index 09559951df1..4c324a1f1e0 100644 --- a/scripts/basic/Makefile +++ b/scripts/basic/Makefile @@ -9,7 +9,7 @@ # fixdep: Used to generate dependency information during build process # docproc: Used in Documentation/DocBook -hostprogs-y := fixdep docproc hash +hostprogs-y := fixdep docproc always := $(hostprogs-y) # fixdep is needed to compile other host programs diff --git a/scripts/basic/hash.c b/scripts/basic/hash.c deleted file mode 100644 index 2ef5d3f666b..00000000000 --- a/scripts/basic/hash.c +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2008 Red Hat, Inc., Jason Baron - * - */ - -#include -#include -#include - -#define DYNAMIC_DEBUG_HASH_BITS 6 - -static const char *program; - -static void usage(void) -{ - printf("Usage: %s \n", program); - exit(1); -} - -/* djb2 hashing algorithm by Dan Bernstein. From: - * http://www.cse.yorku.ca/~oz/hash.html - */ - -static unsigned int djb2_hash(char *str) -{ - unsigned long hash = 5381; - int c; - - c = *str; - while (c) { - hash = ((hash << 5) + hash) + c; - c = *++str; - } - return (unsigned int)(hash & ((1 << DYNAMIC_DEBUG_HASH_BITS) - 1)); -} - -static unsigned int r5_hash(char *str) -{ - unsigned long hash = 0; - int c; - - c = *str; - while (c) { - hash = (hash + (c << 4) + (c >> 4)) * 11; - c = *++str; - } - return (unsigned int)(hash & ((1 << DYNAMIC_DEBUG_HASH_BITS) - 1)); -} - -int main(int argc, char *argv[]) -{ - program = argv[0]; - - if (argc != 3) - usage(); - if (!strcmp(argv[1], "djb2")) - printf("%d\n", djb2_hash(argv[2])); - else if (!strcmp(argv[1], "r5")) - printf("%d\n", r5_hash(argv[2])); - else - usage(); - exit(0); -} - -- cgit v1.2.3-70-g09d2 From d6dad199a10423ce37b8bfec1f055c380dc4a3d5 Mon Sep 17 00:00:00 2001 From: Ingo Molnar Date: Fri, 24 Sep 2010 09:12:25 +0200 Subject: jump label: Fix GCC feature check when distcc is used The following build bug occurs on distcc builds: CC arch/x86/kernel/asm-offsets.s In file included from include/linux/module.h:24, from include/linux/crypto.h:22, from arch/x86/kernel/asm-offsets_64.c:9, from arch/x86/kernel/asm-offsets.c:5: include/trace/events/module.h: In function 'trace_module_load': include/trace/events/module.h:18: error: expected '(' before 'goto' include/trace/events/module.h:18: error: expected identifier or '*' before '(' token It triggers because distcc is invoked by turning $CC into "distcc gcc", but gcc-goto.sh check script was using $1 not $@ to expand parameters. Cc: Jason Baron Cc: Steven Rostedt LKML-Reference: <20100923034910.867858597@goodmis.org> Signed-off-by: Ingo Molnar --- scripts/gcc-goto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/gcc-goto.sh b/scripts/gcc-goto.sh index 8e82424be7a..520d16b1ffa 100644 --- a/scripts/gcc-goto.sh +++ b/scripts/gcc-goto.sh @@ -2,4 +2,4 @@ # Test for gcc 'asm goto' suport # Copyright (C) 2010, Jason Baron -echo "int main(void) { entry: asm goto (\"\"::::entry); return 0; }" | $1 -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" +echo "int main(void) { entry: asm goto (\"\"::::entry); return 0; }" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y" -- cgit v1.2.3-70-g09d2 From 81d3858d3131a589cade0d8b57f95cc1fc699b89 Mon Sep 17 00:00:00 2001 From: John Reiser Date: Wed, 13 Oct 2010 15:12:54 -0400 Subject: ftrace: Add C version of recordmcount compile time code Currently, the mcount callers are found with a perl script that does an objdump on every file in the kernel. This is a C version of that same code which should increase the performance time of compiling the kernel with dynamic ftrace enabled. Signed-off-by: John Reiser [ Updated the code to include .text.unlikely section as well as changing the format to follow Linux coding style. ] Signed-off-by: Steven Rostedt --- scripts/recordmcount.c | 885 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 885 insertions(+) create mode 100644 scripts/recordmcount.c (limited to 'scripts') diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c new file mode 100644 index 00000000000..34f32be1709 --- /dev/null +++ b/scripts/recordmcount.c @@ -0,0 +1,885 @@ +/* + * recordmcount.c: construct a table of the locations of calls to 'mcount' + * so that ftrace can find them quickly. + * Copyright 2009 John F. Reiser . All rights reserved. + * Licensed under the GNU General Public License, version 2 (GPLv2). + * + * Restructured to fit Linux format, as well as other updates: + * Copyright 2010 Steven Rostedt , Red Hat Inc. + */ + +/* + * Strategy: alter the .o file in-place. + * + * Append a new STRTAB that has the new section names, followed by a new array + * ElfXX_Shdr[] that has the new section headers, followed by the section + * contents for __mcount_loc and its relocations. The old shstrtab strings, + * and the old ElfXX_Shdr[] array, remain as "garbage" (commonly, a couple + * kilobytes.) Subsequent processing by /bin/ld (or the kernel module loader) + * will ignore the garbage regions, because they are not designated by the + * new .e_shoff nor the new ElfXX_Shdr[]. [In order to remove the garbage, + * then use "ld -r" to create a new file that omits the garbage.] + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int fd_map; /* File descriptor for file being modified. */ +static int mmap_failed; /* Boolean flag. */ +static void *ehdr_curr; /* current ElfXX_Ehdr * for resource cleanup */ +static char gpfx; /* prefix for global symbol name (sometimes '_') */ +static struct stat sb; /* Remember .st_size, etc. */ +static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */ + +/* setjmp() return values */ +enum { + SJ_SETJMP = 0, /* hardwired first return */ + SJ_FAIL, + SJ_SUCCEED +}; + +/* Per-file resource cleanup when multiple files. */ +static void +cleanup(void) +{ + if (!mmap_failed) + munmap(ehdr_curr, sb.st_size); + else + free(ehdr_curr); + close(fd_map); +} + +static void __attribute__((noreturn)) +fail_file(void) +{ + cleanup(); + longjmp(jmpenv, SJ_FAIL); +} + +static void __attribute__((noreturn)) +succeed_file(void) +{ + cleanup(); + longjmp(jmpenv, SJ_SUCCEED); +} + +/* ulseek, uread, ...: Check return value for errors. */ + +static off_t +ulseek(int const fd, off_t const offset, int const whence) +{ + off_t const w = lseek(fd, offset, whence); + if ((off_t)-1 == w) { + perror("lseek"); + fail_file(); + } + return w; +} + +static size_t +uread(int const fd, void *const buf, size_t const count) +{ + size_t const n = read(fd, buf, count); + if (n != count) { + perror("read"); + fail_file(); + } + return n; +} + +static size_t +uwrite(int const fd, void const *const buf, size_t const count) +{ + size_t const n = write(fd, buf, count); + if (n != count) { + perror("write"); + fail_file(); + } + return n; +} + +static void * +umalloc(size_t size) +{ + void *const addr = malloc(size); + if (0 == addr) { + fprintf(stderr, "malloc failed: %zu bytes\n", size); + fail_file(); + } + return addr; +} + +/* + * Get the whole file as a programming convenience in order to avoid + * malloc+lseek+read+free of many pieces. If successful, then mmap + * avoids copying unused pieces; else just read the whole file. + * Open for both read and write; new info will be appended to the file. + * Use MAP_PRIVATE so that a few changes to the in-memory ElfXX_Ehdr + * do not propagate to the file until an explicit overwrite at the last. + * This preserves most aspects of consistency (all except .st_size) + * for simultaneous readers of the file while we are appending to it. + * However, multiple writers still are bad. We choose not to use + * locking because it is expensive and the use case of kernel build + * makes multiple writers unlikely. + */ +static void *mmap_file(char const *fname) +{ + void *addr; + + fd_map = open(fname, O_RDWR); + if (0 > fd_map || 0 > fstat(fd_map, &sb)) { + perror(fname); + fail_file(); + } + if (!S_ISREG(sb.st_mode)) { + fprintf(stderr, "not a regular file: %s\n", fname); + fail_file(); + } + addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE, + fd_map, 0); + mmap_failed = 0; + if (MAP_FAILED == addr) { + mmap_failed = 1; + addr = umalloc(sb.st_size); + uread(fd_map, addr, sb.st_size); + } + return addr; +} + +/* w8rev, w8nat, ...: Handle endianness. */ + +static uint64_t w8rev(uint64_t const x) +{ + return ((0xff & (x >> (0 * 8))) << (7 * 8)) + | ((0xff & (x >> (1 * 8))) << (6 * 8)) + | ((0xff & (x >> (2 * 8))) << (5 * 8)) + | ((0xff & (x >> (3 * 8))) << (4 * 8)) + | ((0xff & (x >> (4 * 8))) << (3 * 8)) + | ((0xff & (x >> (5 * 8))) << (2 * 8)) + | ((0xff & (x >> (6 * 8))) << (1 * 8)) + | ((0xff & (x >> (7 * 8))) << (0 * 8)); +} + +static uint32_t w4rev(uint32_t const x) +{ + return ((0xff & (x >> (0 * 8))) << (3 * 8)) + | ((0xff & (x >> (1 * 8))) << (2 * 8)) + | ((0xff & (x >> (2 * 8))) << (1 * 8)) + | ((0xff & (x >> (3 * 8))) << (0 * 8)); +} + +static uint32_t w2rev(uint16_t const x) +{ + return ((0xff & (x >> (0 * 8))) << (1 * 8)) + | ((0xff & (x >> (1 * 8))) << (0 * 8)); +} + +static uint64_t w8nat(uint64_t const x) +{ + return x; +} + +static uint32_t w4nat(uint32_t const x) +{ + return x; +} + +static uint32_t w2nat(uint16_t const x) +{ + return x; +} + +static uint64_t (*w8)(uint64_t); +static uint32_t (*w)(uint32_t); +static uint32_t (*w2)(uint16_t); + +/* Names of the sections that could contain calls to mcount. */ +static int +is_mcounted_section_name(char const *const txtname) +{ + return 0 == strcmp(".text", txtname) || + 0 == strcmp(".sched.text", txtname) || + 0 == strcmp(".spinlock.text", txtname) || + 0 == strcmp(".irqentry.text", txtname) || + 0 == strcmp(".text.unlikely", txtname); +} + +/* Append the new shstrtab, Elf32_Shdr[], __mcount_loc and its relocations. */ +static void append32(Elf32_Ehdr *const ehdr, + Elf32_Shdr *const shstr, + uint32_t const *const mloc0, + uint32_t const *const mlocp, + Elf32_Rel const *const mrel0, + Elf32_Rel const *const mrelp, + unsigned int const rel_entsize, + unsigned int const symsec_sh_link) +{ + /* Begin constructing output file */ + Elf32_Shdr mcsec; + char const *mc_name = (sizeof(Elf32_Rela) == rel_entsize) + ? ".rela__mcount_loc" + : ".rel__mcount_loc"; + unsigned const old_shnum = w2(ehdr->e_shnum); + uint32_t const old_shoff = w(ehdr->e_shoff); + uint32_t const old_shstr_sh_size = w(shstr->sh_size); + uint32_t const old_shstr_sh_offset = w(shstr->sh_offset); + uint32_t t = 1 + strlen(mc_name) + w(shstr->sh_size); + uint32_t new_e_shoff; + + shstr->sh_size = w(t); + shstr->sh_offset = w(sb.st_size); + t += sb.st_size; + t += (3u & -t); /* 4-byte align */ + new_e_shoff = t; + + /* body for new shstrtab */ + ulseek(fd_map, sb.st_size, SEEK_SET); + uwrite(fd_map, old_shstr_sh_offset + (void *)ehdr, old_shstr_sh_size); + uwrite(fd_map, mc_name, 1 + strlen(mc_name)); + + /* old(modified) Elf32_Shdr table, 4-byte aligned */ + ulseek(fd_map, t, SEEK_SET); + t += sizeof(Elf32_Shdr) * old_shnum; + uwrite(fd_map, old_shoff + (void *)ehdr, + sizeof(Elf32_Shdr) * old_shnum); + + /* new sections __mcount_loc and .rel__mcount_loc */ + t += 2*sizeof(mcsec); + mcsec.sh_name = w((sizeof(Elf32_Rela) == rel_entsize) + strlen(".rel") + + old_shstr_sh_size); + mcsec.sh_type = w(SHT_PROGBITS); + mcsec.sh_flags = w(SHF_ALLOC); + mcsec.sh_addr = 0; + mcsec.sh_offset = w(t); + mcsec.sh_size = w((void *)mlocp - (void *)mloc0); + mcsec.sh_link = 0; + mcsec.sh_info = 0; + mcsec.sh_addralign = w(4); + mcsec.sh_entsize = w(4); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + mcsec.sh_name = w(old_shstr_sh_size); + mcsec.sh_type = (sizeof(Elf32_Rela) == rel_entsize) + ? w(SHT_RELA) + : w(SHT_REL); + mcsec.sh_flags = 0; + mcsec.sh_addr = 0; + mcsec.sh_offset = w((void *)mlocp - (void *)mloc0 + t); + mcsec.sh_size = w((void *)mrelp - (void *)mrel0); + mcsec.sh_link = w(symsec_sh_link); + mcsec.sh_info = w(old_shnum); + mcsec.sh_addralign = w(4); + mcsec.sh_entsize = w(rel_entsize); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + uwrite(fd_map, mloc0, (void *)mlocp - (void *)mloc0); + uwrite(fd_map, mrel0, (void *)mrelp - (void *)mrel0); + + ehdr->e_shoff = w(new_e_shoff); + ehdr->e_shnum = w2(2 + w2(ehdr->e_shnum)); /* {.rel,}__mcount_loc */ + ulseek(fd_map, 0, SEEK_SET); + uwrite(fd_map, ehdr, sizeof(*ehdr)); +} + +/* + * append64 and append32 (and other analogous pairs) could be templated + * using C++, but the complexity is high. (For an example, look at p_elf.h + * in the source for UPX, http://upx.sourceforge.net) So: remember to make + * the corresponding change in the routine for the other size. + */ +static void append64(Elf64_Ehdr *const ehdr, + Elf64_Shdr *const shstr, + uint64_t const *const mloc0, + uint64_t const *const mlocp, + Elf64_Rel const *const mrel0, + Elf64_Rel const *const mrelp, + unsigned int const rel_entsize, + unsigned int const symsec_sh_link) +{ + /* Begin constructing output file */ + Elf64_Shdr mcsec; + char const *mc_name = (sizeof(Elf64_Rela) == rel_entsize) + ? ".rela__mcount_loc" + : ".rel__mcount_loc"; + unsigned const old_shnum = w2(ehdr->e_shnum); + uint64_t const old_shoff = w8(ehdr->e_shoff); + uint64_t const old_shstr_sh_size = w8(shstr->sh_size); + uint64_t const old_shstr_sh_offset = w8(shstr->sh_offset); + uint64_t t = 1 + strlen(mc_name) + w8(shstr->sh_size); + uint64_t new_e_shoff; + + shstr->sh_size = w8(t); + shstr->sh_offset = w8(sb.st_size); + t += sb.st_size; + t += (7u & -t); /* 8-byte align */ + new_e_shoff = t; + + /* body for new shstrtab */ + ulseek(fd_map, sb.st_size, SEEK_SET); + uwrite(fd_map, old_shstr_sh_offset + (void *)ehdr, old_shstr_sh_size); + uwrite(fd_map, mc_name, 1 + strlen(mc_name)); + + /* old(modified) Elf64_Shdr table, 8-byte aligned */ + ulseek(fd_map, t, SEEK_SET); + t += sizeof(Elf64_Shdr) * old_shnum; + uwrite(fd_map, old_shoff + (void *)ehdr, + sizeof(Elf64_Shdr) * old_shnum); + + /* new sections __mcount_loc and .rel__mcount_loc */ + t += 2*sizeof(mcsec); + mcsec.sh_name = w((sizeof(Elf64_Rela) == rel_entsize) + strlen(".rel") + + old_shstr_sh_size); + mcsec.sh_type = w(SHT_PROGBITS); + mcsec.sh_flags = w8(SHF_ALLOC); + mcsec.sh_addr = 0; + mcsec.sh_offset = w8(t); + mcsec.sh_size = w8((void *)mlocp - (void *)mloc0); + mcsec.sh_link = 0; + mcsec.sh_info = 0; + mcsec.sh_addralign = w8(8); + mcsec.sh_entsize = w8(8); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + mcsec.sh_name = w(old_shstr_sh_size); + mcsec.sh_type = (sizeof(Elf64_Rela) == rel_entsize) + ? w(SHT_RELA) + : w(SHT_REL); + mcsec.sh_flags = 0; + mcsec.sh_addr = 0; + mcsec.sh_offset = w8((void *)mlocp - (void *)mloc0 + t); + mcsec.sh_size = w8((void *)mrelp - (void *)mrel0); + mcsec.sh_link = w(symsec_sh_link); + mcsec.sh_info = w(old_shnum); + mcsec.sh_addralign = w8(8); + mcsec.sh_entsize = w8(rel_entsize); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + uwrite(fd_map, mloc0, (void *)mlocp - (void *)mloc0); + uwrite(fd_map, mrel0, (void *)mrelp - (void *)mrel0); + + ehdr->e_shoff = w8(new_e_shoff); + ehdr->e_shnum = w2(2 + w2(ehdr->e_shnum)); /* {.rel,}__mcount_loc */ + ulseek(fd_map, 0, SEEK_SET); + uwrite(fd_map, ehdr, sizeof(*ehdr)); +} + +/* + * Look at the relocations in order to find the calls to mcount. + * Accumulate the section offsets that are found, and their relocation info, + * onto the end of the existing arrays. + */ +static uint32_t *sift32_rel_mcount(uint32_t *mlocp, + unsigned const offbase, + Elf32_Rel **const mrelpp, + Elf32_Shdr const *const relhdr, + Elf32_Ehdr const *const ehdr, + unsigned const recsym, + uint32_t const recval, + unsigned const reltype) +{ + uint32_t *const mloc0 = mlocp; + Elf32_Rel *mrelp = *mrelpp; + Elf32_Shdr *const shdr0 = (Elf32_Shdr *)(w(ehdr->e_shoff) + + (void *)ehdr); + unsigned const symsec_sh_link = w(relhdr->sh_link); + Elf32_Shdr const *const symsec = &shdr0[symsec_sh_link]; + Elf32_Sym const *const sym0 = (Elf32_Sym const *)(w(symsec->sh_offset) + + (void *)ehdr); + + Elf32_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; + char const *const str0 = (char const *)(w(strsec->sh_offset) + + (void *)ehdr); + + Elf32_Rel const *const rel0 = (Elf32_Rel const *)(w(relhdr->sh_offset) + + (void *)ehdr); + unsigned rel_entsize = w(relhdr->sh_entsize); + unsigned const nrel = w(relhdr->sh_size) / rel_entsize; + Elf32_Rel const *relp = rel0; + + unsigned mcountsym = 0; + unsigned t; + + for (t = nrel; t; --t) { + if (!mcountsym) { + Elf32_Sym const *const symp = + &sym0[ELF32_R_SYM(w(relp->r_info))]; + + if (0 == strcmp((('_' == gpfx) ? "_mcount" : "mcount"), + &str0[w(symp->st_name)])) + mcountsym = ELF32_R_SYM(w(relp->r_info)); + } + if (mcountsym == ELF32_R_SYM(w(relp->r_info))) { + uint32_t const addend = w(w(relp->r_offset) - recval); + mrelp->r_offset = w(offbase + + ((void *)mlocp - (void *)mloc0)); + mrelp->r_info = w(ELF32_R_INFO(recsym, reltype)); + if (sizeof(Elf32_Rela) == rel_entsize) { + ((Elf32_Rela *)mrelp)->r_addend = addend; + *mlocp++ = 0; + } else + *mlocp++ = addend; + + mrelp = (Elf32_Rel *)(rel_entsize + (void *)mrelp); + } + relp = (Elf32_Rel const *)(rel_entsize + (void *)relp); + } + *mrelpp = mrelp; + return mlocp; +} + +static uint64_t *sift64_rel_mcount(uint64_t *mlocp, + unsigned const offbase, + Elf64_Rel **const mrelpp, + Elf64_Shdr const *const relhdr, + Elf64_Ehdr const *const ehdr, + unsigned const recsym, + uint64_t const recval, + unsigned const reltype) +{ + uint64_t *const mloc0 = mlocp; + Elf64_Rel *mrelp = *mrelpp; + Elf64_Shdr *const shdr0 = (Elf64_Shdr *)(w8(ehdr->e_shoff) + + (void *)ehdr); + unsigned const symsec_sh_link = w(relhdr->sh_link); + Elf64_Shdr const *const symsec = &shdr0[symsec_sh_link]; + Elf64_Sym const *const sym0 = (Elf64_Sym const *)(w8(symsec->sh_offset) + + (void *)ehdr); + + Elf64_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; + char const *const str0 = (char const *)(w8(strsec->sh_offset) + + (void *)ehdr); + + Elf64_Rel const *const rel0 = (Elf64_Rel const *)(w8(relhdr->sh_offset) + + (void *)ehdr); + unsigned rel_entsize = w8(relhdr->sh_entsize); + unsigned const nrel = w8(relhdr->sh_size) / rel_entsize; + Elf64_Rel const *relp = rel0; + + unsigned mcountsym = 0; + unsigned t; + + for (t = nrel; 0 != t; --t) { + if (!mcountsym) { + Elf64_Sym const *const symp = + &sym0[ELF64_R_SYM(w8(relp->r_info))]; + char const *symname = &str0[w(symp->st_name)]; + + if ('.' == symname[0]) + ++symname; /* ppc64 hack */ + if (0 == strcmp((('_' == gpfx) ? "_mcount" : "mcount"), + symname)) + mcountsym = ELF64_R_SYM(w8(relp->r_info)); + } + + if (mcountsym == ELF64_R_SYM(w8(relp->r_info))) { + uint64_t const addend = w8(w8(relp->r_offset) - recval); + + mrelp->r_offset = w8(offbase + + ((void *)mlocp - (void *)mloc0)); + mrelp->r_info = w8(ELF64_R_INFO(recsym, reltype)); + if (sizeof(Elf64_Rela) == rel_entsize) { + ((Elf64_Rela *)mrelp)->r_addend = addend; + *mlocp++ = 0; + } else + *mlocp++ = addend; + + mrelp = (Elf64_Rel *)(rel_entsize + (void *)mrelp); + } + relp = (Elf64_Rel const *)(rel_entsize + (void *)relp); + } + *mrelpp = mrelp; + + return mlocp; +} + +/* + * Find a symbol in the given section, to be used as the base for relocating + * the table of offsets of calls to mcount. A local or global symbol suffices, + * but avoid a Weak symbol because it may be overridden; the change in value + * would invalidate the relocations of the offsets of the calls to mcount. + * Often the found symbol will be the unnamed local symbol generated by + * GNU 'as' for the start of each section. For example: + * Num: Value Size Type Bind Vis Ndx Name + * 2: 00000000 0 SECTION LOCAL DEFAULT 1 + */ +static unsigned find32_secsym_ndx(unsigned const txtndx, + char const *const txtname, + uint32_t *const recvalp, + Elf32_Shdr const *const symhdr, + Elf32_Ehdr const *const ehdr) +{ + Elf32_Sym const *const sym0 = (Elf32_Sym const *)(w(symhdr->sh_offset) + + (void *)ehdr); + unsigned const nsym = w(symhdr->sh_size) / w(symhdr->sh_entsize); + Elf32_Sym const *symp; + unsigned t; + + for (symp = sym0, t = nsym; t; --t, ++symp) { + unsigned int const st_bind = ELF32_ST_BIND(symp->st_info); + + if (txtndx == w2(symp->st_shndx) + /* avoid STB_WEAK */ + && (STB_LOCAL == st_bind || STB_GLOBAL == st_bind)) { + *recvalp = w(symp->st_value); + return symp - sym0; + } + } + fprintf(stderr, "Cannot find symbol for section %d: %s.\n", + txtndx, txtname); + fail_file(); +} + +static unsigned find64_secsym_ndx(unsigned const txtndx, + char const *const txtname, + uint64_t *const recvalp, + Elf64_Shdr const *const symhdr, + Elf64_Ehdr const *const ehdr) +{ + Elf64_Sym const *const sym0 = (Elf64_Sym const *)(w8(symhdr->sh_offset) + + (void *)ehdr); + unsigned const nsym = w8(symhdr->sh_size) / w8(symhdr->sh_entsize); + Elf64_Sym const *symp; + unsigned t; + + for (symp = sym0, t = nsym; t; --t, ++symp) { + unsigned int const st_bind = ELF64_ST_BIND(symp->st_info); + + if (txtndx == w2(symp->st_shndx) + /* avoid STB_WEAK */ + && (STB_LOCAL == st_bind || STB_GLOBAL == st_bind)) { + *recvalp = w8(symp->st_value); + return symp - sym0; + } + } + fprintf(stderr, "Cannot find symbol for section %d: %s.\n", + txtndx, txtname); + fail_file(); +} + +/* + * Evade ISO C restriction: no declaration after statement in + * has32_rel_mcount. + */ +static char const * +__has32_rel_mcount(Elf32_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */ + Elf32_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + /* .sh_info depends on .sh_type == SHT_REL[,A] */ + Elf32_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; + char const *const txtname = &shstrtab[w(txthdr->sh_name)]; + + if (0 == strcmp("__mcount_loc", txtname)) { + fprintf(stderr, "warning: __mcount_loc already exists: %s\n", + fname); + succeed_file(); + } + if (SHT_PROGBITS != w(txthdr->sh_type) || + !is_mcounted_section_name(txtname)) + return NULL; + return txtname; +} + +static char const *has32_rel_mcount(Elf32_Shdr const *const relhdr, + Elf32_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) + return NULL; + return __has32_rel_mcount(relhdr, shdr0, shstrtab, fname); +} + +static char const *__has64_rel_mcount(Elf64_Shdr const *const relhdr, + Elf64_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + /* .sh_info depends on .sh_type == SHT_REL[,A] */ + Elf64_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; + char const *const txtname = &shstrtab[w(txthdr->sh_name)]; + + if (0 == strcmp("__mcount_loc", txtname)) { + fprintf(stderr, "warning: __mcount_loc already exists: %s\n", + fname); + succeed_file(); + } + if (SHT_PROGBITS != w(txthdr->sh_type) || + !is_mcounted_section_name(txtname)) + return NULL; + return txtname; +} + +static char const *has64_rel_mcount(Elf64_Shdr const *const relhdr, + Elf64_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) + return NULL; + return __has64_rel_mcount(relhdr, shdr0, shstrtab, fname); +} + +static unsigned tot32_relsize(Elf32_Shdr const *const shdr0, + unsigned nhdr, + const char *const shstrtab, + const char *const fname) +{ + unsigned totrelsz = 0; + Elf32_Shdr const *shdrp = shdr0; + for (; 0 != nhdr; --nhdr, ++shdrp) { + if (has32_rel_mcount(shdrp, shdr0, shstrtab, fname)) + totrelsz += w(shdrp->sh_size); + } + return totrelsz; +} + +static unsigned tot64_relsize(Elf64_Shdr const *const shdr0, + unsigned nhdr, + const char *const shstrtab, + const char *const fname) +{ + unsigned totrelsz = 0; + Elf64_Shdr const *shdrp = shdr0; + + for (; nhdr; --nhdr, ++shdrp) { + if (has64_rel_mcount(shdrp, shdr0, shstrtab, fname)) + totrelsz += w8(shdrp->sh_size); + } + return totrelsz; +} + +/* Overall supervision for Elf32 ET_REL file. */ +static void +do32(Elf32_Ehdr *const ehdr, char const *const fname, unsigned const reltype) +{ + Elf32_Shdr *const shdr0 = (Elf32_Shdr *)(w(ehdr->e_shoff) + + (void *)ehdr); + unsigned const nhdr = w2(ehdr->e_shnum); + Elf32_Shdr *const shstr = &shdr0[w2(ehdr->e_shstrndx)]; + char const *const shstrtab = (char const *)(w(shstr->sh_offset) + + (void *)ehdr); + + Elf32_Shdr const *relhdr; + unsigned k; + + /* Upper bound on space: assume all relevant relocs are for mcount. */ + unsigned const totrelsz = tot32_relsize(shdr0, nhdr, shstrtab, fname); + Elf32_Rel *const mrel0 = umalloc(totrelsz); + Elf32_Rel * mrelp = mrel0; + + /* 2*sizeof(address) <= sizeof(Elf32_Rel) */ + uint32_t *const mloc0 = umalloc(totrelsz>>1); + uint32_t * mlocp = mloc0; + + unsigned rel_entsize = 0; + unsigned symsec_sh_link = 0; + + for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { + char const *const txtname = has32_rel_mcount(relhdr, shdr0, + shstrtab, fname); + if (txtname) { + uint32_t recval = 0; + unsigned const recsym = find32_secsym_ndx( + w(relhdr->sh_info), txtname, &recval, + &shdr0[symsec_sh_link = w(relhdr->sh_link)], + ehdr); + + rel_entsize = w(relhdr->sh_entsize); + mlocp = sift32_rel_mcount(mlocp, + (void *)mlocp - (void *)mloc0, &mrelp, + relhdr, ehdr, recsym, recval, reltype); + } + } + if (mloc0 != mlocp) { + append32(ehdr, shstr, mloc0, mlocp, mrel0, mrelp, + rel_entsize, symsec_sh_link); + } + free(mrel0); + free(mloc0); +} + +static void +do64(Elf64_Ehdr *const ehdr, char const *const fname, unsigned const reltype) +{ + Elf64_Shdr *const shdr0 = (Elf64_Shdr *)(w8(ehdr->e_shoff) + + (void *)ehdr); + unsigned const nhdr = w2(ehdr->e_shnum); + Elf64_Shdr *const shstr = &shdr0[w2(ehdr->e_shstrndx)]; + char const *const shstrtab = (char const *)(w8(shstr->sh_offset) + + (void *)ehdr); + + Elf64_Shdr const *relhdr; + unsigned k; + + /* Upper bound on space: assume all relevant relocs are for mcount. */ + unsigned const totrelsz = tot64_relsize(shdr0, nhdr, shstrtab, fname); + Elf64_Rel *const mrel0 = umalloc(totrelsz); + Elf64_Rel * mrelp = mrel0; + + /* 2*sizeof(address) <= sizeof(Elf64_Rel) */ + uint64_t *const mloc0 = umalloc(totrelsz>>1); + uint64_t * mlocp = mloc0; + + unsigned rel_entsize = 0; + unsigned symsec_sh_link = 0; + + for ((relhdr = shdr0), k = nhdr; k; --k, ++relhdr) { + char const *const txtname = has64_rel_mcount(relhdr, shdr0, + shstrtab, fname); + if (txtname) { + uint64_t recval = 0; + unsigned const recsym = find64_secsym_ndx( + w(relhdr->sh_info), txtname, &recval, + &shdr0[symsec_sh_link = w(relhdr->sh_link)], + ehdr); + + rel_entsize = w8(relhdr->sh_entsize); + mlocp = sift64_rel_mcount(mlocp, + (void *)mlocp - (void *)mloc0, &mrelp, + relhdr, ehdr, recsym, recval, reltype); + } + } + if (mloc0 != mlocp) { + append64(ehdr, shstr, mloc0, mlocp, mrel0, mrelp, + rel_entsize, symsec_sh_link); + } + free(mrel0); + free(mloc0); +} + +static void +do_file(char const *const fname) +{ + Elf32_Ehdr *const ehdr = mmap_file(fname); + unsigned int reltype = 0; + + ehdr_curr = ehdr; + w = w4nat; + w2 = w2nat; + w8 = w8nat; + switch (ehdr->e_ident[EI_DATA]) { + static unsigned int const endian = 1; + default: { + fprintf(stderr, "unrecognized ELF data encoding %d: %s\n", + ehdr->e_ident[EI_DATA], fname); + fail_file(); + } break; + case ELFDATA2LSB: { + if (1 != *(unsigned char const *)&endian) { + /* main() is big endian, file.o is little endian. */ + w = w4rev; + w2 = w2rev; + w8 = w8rev; + } + } break; + case ELFDATA2MSB: { + if (0 != *(unsigned char const *)&endian) { + /* main() is little endian, file.o is big endian. */ + w = w4rev; + w2 = w2rev; + w8 = w8rev; + } + } break; + } /* end switch */ + if (0 != memcmp(ELFMAG, ehdr->e_ident, SELFMAG) + || ET_REL != w2(ehdr->e_type) + || EV_CURRENT != ehdr->e_ident[EI_VERSION]) { + fprintf(stderr, "unrecognized ET_REL file %s\n", fname); + fail_file(); + } + + gpfx = 0; + switch (w2(ehdr->e_machine)) { + default: { + fprintf(stderr, "unrecognized e_machine %d %s\n", + w2(ehdr->e_machine), fname); + fail_file(); + } break; + case EM_386: reltype = R_386_32; break; + case EM_ARM: reltype = R_ARM_ABS32; break; + case EM_IA_64: reltype = R_IA64_IMM64; gpfx = '_'; break; + case EM_PPC: reltype = R_PPC_ADDR32; gpfx = '_'; break; + case EM_PPC64: reltype = R_PPC64_ADDR64; gpfx = '_'; break; + case EM_S390: /* reltype: e_class */ gpfx = '_'; break; + case EM_SH: reltype = R_SH_DIR32; break; + case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break; + case EM_X86_64: reltype = R_X86_64_64; break; + } /* end switch */ + + switch (ehdr->e_ident[EI_CLASS]) { + default: { + fprintf(stderr, "unrecognized ELF class %d %s\n", + ehdr->e_ident[EI_CLASS], fname); + fail_file(); + } break; + case ELFCLASS32: { + if (sizeof(Elf32_Ehdr) != w2(ehdr->e_ehsize) + || sizeof(Elf32_Shdr) != w2(ehdr->e_shentsize)) { + fprintf(stderr, + "unrecognized ET_REL file: %s\n", fname); + fail_file(); + } + if (EM_S390 == w2(ehdr->e_machine)) + reltype = R_390_32; + do32(ehdr, fname, reltype); + } break; + case ELFCLASS64: { + Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr; + if (sizeof(Elf64_Ehdr) != w2(ghdr->e_ehsize) + || sizeof(Elf64_Shdr) != w2(ghdr->e_shentsize)) { + fprintf(stderr, + "unrecognized ET_REL file: %s\n", fname); + fail_file(); + } + if (EM_S390 == w2(ghdr->e_machine)) + reltype = R_390_64; + do64(ghdr, fname, reltype); + } break; + } /* end switch */ + + cleanup(); +} + +int +main(int argc, char const *argv[]) +{ + int n_error = 0; /* gcc-4.3.0 false positive complaint */ + if (argc <= 1) + fprintf(stderr, "usage: recordmcount file.o...\n"); + else /* Process each file in turn, allowing deep failure. */ + for (--argc, ++argv; 0 < argc; --argc, ++argv) { + int const sjval = setjmp(jmpenv); + switch (sjval) { + default: { + fprintf(stderr, "internal error: %s\n", argv[0]); + exit(1); + } break; + case SJ_SETJMP: { /* normal sequence */ + /* Avoid problems if early cleanup() */ + fd_map = -1; + ehdr_curr = NULL; + mmap_failed = 1; + do_file(argv[0]); + } break; + case SJ_FAIL: { /* error in do_file or below */ + ++n_error; + } break; + case SJ_SUCCEED: { /* premature success */ + /* do nothing */ + } break; + } /* end switch */ + } + return !!n_error; +} + + -- cgit v1.2.3-70-g09d2 From 72441cb1fd77d092f09ddfac748955703884c9a7 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 13 Oct 2010 17:12:30 -0400 Subject: ftrace/x86: Add support for C version of recordmcount This patch adds the support for the C version of recordmcount and compile times show ~ 12% improvement. After verifying this works, other archs can add: HAVE_C_MCOUNT_RECORD in its Kconfig and it will use the C version of recordmcount instead of the perl version. Cc: Cc: Michal Marek Cc: linux-kbuild@vger.kernel.org Cc: John Reiser Signed-off-by: Steven Rostedt --- Makefile | 6 ++++++ arch/x86/Kconfig | 1 + kernel/trace/Kconfig | 5 +++++ scripts/Makefile | 1 + scripts/Makefile.build | 4 ++++ 5 files changed, 17 insertions(+) (limited to 'scripts') diff --git a/Makefile b/Makefile index 534c09c255d..0dd3a8d9313 100644 --- a/Makefile +++ b/Makefile @@ -568,6 +568,12 @@ endif ifdef CONFIG_FUNCTION_TRACER KBUILD_CFLAGS += -pg +ifdef CONFIG_DYNAMIC_FTRACE + ifdef CONFIG_HAVE_C_MCOUNT_RECORD + BUILD_C_RECORDMCOUNT := y + export BUILD_C_RECORDMCOUNT + endif +endif endif # We trigger additional mismatches with less inlining diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index c14d8b4d2f7..788b50ef5fc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -33,6 +33,7 @@ config X86 select HAVE_KRETPROBES select HAVE_OPTPROBES select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_C_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 538501c6ea5..df00fbbaf60 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -49,6 +49,11 @@ config HAVE_SYSCALL_TRACEPOINTS help See Documentation/trace/ftrace-design.txt +config HAVE_C_MCOUNT_RECORD + bool + help + C version of recordmcount available? + config TRACER_MAX_TRACE bool diff --git a/scripts/Makefile b/scripts/Makefile index 842dbc2d5ae..2e088109fbd 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -11,6 +11,7 @@ hostprogs-$(CONFIG_KALLSYMS) += kallsyms hostprogs-$(CONFIG_LOGO) += pnmtologo hostprogs-$(CONFIG_VT) += conmakehash hostprogs-$(CONFIG_IKCONFIG) += bin2c +hostprogs-$(BUILD_C_RECORDMCOUNT) += recordmcount always := $(hostprogs-y) $(hostprogs-m) diff --git a/scripts/Makefile.build b/scripts/Makefile.build index a1a5cf95a68..4d03a7efc68 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -209,12 +209,16 @@ cmd_modversions = \ endif ifdef CONFIG_FTRACE_MCOUNT_RECORD +ifdef BUILD_C_RECORDMCOUNT +cmd_record_mcount = $(srctree)/scripts/recordmcount "$(@)"; +else cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ "$(if $(CONFIG_64BIT),64,32)" \ "$(OBJDUMP)" "$(OBJCOPY)" "$(CC)" "$(LD)" "$(NM)" "$(RM)" "$(MV)" \ "$(if $(part-of-module),1,0)" "$(@)"; endif +endif define rule_cc_o_c $(call echo-cmd,checksrc) $(cmd_checksrc) \ -- cgit v1.2.3-70-g09d2 From c28d5077f8d79bfce1e3f88db2e261cf2b6473dc Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Wed, 13 Oct 2010 19:06:14 -0400 Subject: ftrace: Remove duplicate code for 64 and 32 bit in recordmcount.c The elf reader for recordmcount.c had duplicate functions for both 32 bit and 64 bit elf handling. This was due to the need of using the 32 and 64 bit elf structures. This patch consolidates the two by using macros to define the 32 and 64 bit names in a recordmcount.h file, and then by just defining a RECORD_MCOUNT_64 macro and including recordmcount.h twice we create the funtions for both the 32 bit version as well as the 64 bit version using one code source. Cc: John Reiser Signed-off-by: Steven Rostedt --- scripts/recordmcount.c | 548 +------------------------------------------------ scripts/recordmcount.h | 366 +++++++++++++++++++++++++++++++++ 2 files changed, 370 insertions(+), 544 deletions(-) create mode 100644 scripts/recordmcount.h (limited to 'scripts') diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 34f32be1709..7f7f7180fe2 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -212,550 +212,10 @@ is_mcounted_section_name(char const *const txtname) 0 == strcmp(".text.unlikely", txtname); } -/* Append the new shstrtab, Elf32_Shdr[], __mcount_loc and its relocations. */ -static void append32(Elf32_Ehdr *const ehdr, - Elf32_Shdr *const shstr, - uint32_t const *const mloc0, - uint32_t const *const mlocp, - Elf32_Rel const *const mrel0, - Elf32_Rel const *const mrelp, - unsigned int const rel_entsize, - unsigned int const symsec_sh_link) -{ - /* Begin constructing output file */ - Elf32_Shdr mcsec; - char const *mc_name = (sizeof(Elf32_Rela) == rel_entsize) - ? ".rela__mcount_loc" - : ".rel__mcount_loc"; - unsigned const old_shnum = w2(ehdr->e_shnum); - uint32_t const old_shoff = w(ehdr->e_shoff); - uint32_t const old_shstr_sh_size = w(shstr->sh_size); - uint32_t const old_shstr_sh_offset = w(shstr->sh_offset); - uint32_t t = 1 + strlen(mc_name) + w(shstr->sh_size); - uint32_t new_e_shoff; - - shstr->sh_size = w(t); - shstr->sh_offset = w(sb.st_size); - t += sb.st_size; - t += (3u & -t); /* 4-byte align */ - new_e_shoff = t; - - /* body for new shstrtab */ - ulseek(fd_map, sb.st_size, SEEK_SET); - uwrite(fd_map, old_shstr_sh_offset + (void *)ehdr, old_shstr_sh_size); - uwrite(fd_map, mc_name, 1 + strlen(mc_name)); - - /* old(modified) Elf32_Shdr table, 4-byte aligned */ - ulseek(fd_map, t, SEEK_SET); - t += sizeof(Elf32_Shdr) * old_shnum; - uwrite(fd_map, old_shoff + (void *)ehdr, - sizeof(Elf32_Shdr) * old_shnum); - - /* new sections __mcount_loc and .rel__mcount_loc */ - t += 2*sizeof(mcsec); - mcsec.sh_name = w((sizeof(Elf32_Rela) == rel_entsize) + strlen(".rel") - + old_shstr_sh_size); - mcsec.sh_type = w(SHT_PROGBITS); - mcsec.sh_flags = w(SHF_ALLOC); - mcsec.sh_addr = 0; - mcsec.sh_offset = w(t); - mcsec.sh_size = w((void *)mlocp - (void *)mloc0); - mcsec.sh_link = 0; - mcsec.sh_info = 0; - mcsec.sh_addralign = w(4); - mcsec.sh_entsize = w(4); - uwrite(fd_map, &mcsec, sizeof(mcsec)); - - mcsec.sh_name = w(old_shstr_sh_size); - mcsec.sh_type = (sizeof(Elf32_Rela) == rel_entsize) - ? w(SHT_RELA) - : w(SHT_REL); - mcsec.sh_flags = 0; - mcsec.sh_addr = 0; - mcsec.sh_offset = w((void *)mlocp - (void *)mloc0 + t); - mcsec.sh_size = w((void *)mrelp - (void *)mrel0); - mcsec.sh_link = w(symsec_sh_link); - mcsec.sh_info = w(old_shnum); - mcsec.sh_addralign = w(4); - mcsec.sh_entsize = w(rel_entsize); - uwrite(fd_map, &mcsec, sizeof(mcsec)); - - uwrite(fd_map, mloc0, (void *)mlocp - (void *)mloc0); - uwrite(fd_map, mrel0, (void *)mrelp - (void *)mrel0); - - ehdr->e_shoff = w(new_e_shoff); - ehdr->e_shnum = w2(2 + w2(ehdr->e_shnum)); /* {.rel,}__mcount_loc */ - ulseek(fd_map, 0, SEEK_SET); - uwrite(fd_map, ehdr, sizeof(*ehdr)); -} - -/* - * append64 and append32 (and other analogous pairs) could be templated - * using C++, but the complexity is high. (For an example, look at p_elf.h - * in the source for UPX, http://upx.sourceforge.net) So: remember to make - * the corresponding change in the routine for the other size. - */ -static void append64(Elf64_Ehdr *const ehdr, - Elf64_Shdr *const shstr, - uint64_t const *const mloc0, - uint64_t const *const mlocp, - Elf64_Rel const *const mrel0, - Elf64_Rel const *const mrelp, - unsigned int const rel_entsize, - unsigned int const symsec_sh_link) -{ - /* Begin constructing output file */ - Elf64_Shdr mcsec; - char const *mc_name = (sizeof(Elf64_Rela) == rel_entsize) - ? ".rela__mcount_loc" - : ".rel__mcount_loc"; - unsigned const old_shnum = w2(ehdr->e_shnum); - uint64_t const old_shoff = w8(ehdr->e_shoff); - uint64_t const old_shstr_sh_size = w8(shstr->sh_size); - uint64_t const old_shstr_sh_offset = w8(shstr->sh_offset); - uint64_t t = 1 + strlen(mc_name) + w8(shstr->sh_size); - uint64_t new_e_shoff; - - shstr->sh_size = w8(t); - shstr->sh_offset = w8(sb.st_size); - t += sb.st_size; - t += (7u & -t); /* 8-byte align */ - new_e_shoff = t; - - /* body for new shstrtab */ - ulseek(fd_map, sb.st_size, SEEK_SET); - uwrite(fd_map, old_shstr_sh_offset + (void *)ehdr, old_shstr_sh_size); - uwrite(fd_map, mc_name, 1 + strlen(mc_name)); - - /* old(modified) Elf64_Shdr table, 8-byte aligned */ - ulseek(fd_map, t, SEEK_SET); - t += sizeof(Elf64_Shdr) * old_shnum; - uwrite(fd_map, old_shoff + (void *)ehdr, - sizeof(Elf64_Shdr) * old_shnum); - - /* new sections __mcount_loc and .rel__mcount_loc */ - t += 2*sizeof(mcsec); - mcsec.sh_name = w((sizeof(Elf64_Rela) == rel_entsize) + strlen(".rel") - + old_shstr_sh_size); - mcsec.sh_type = w(SHT_PROGBITS); - mcsec.sh_flags = w8(SHF_ALLOC); - mcsec.sh_addr = 0; - mcsec.sh_offset = w8(t); - mcsec.sh_size = w8((void *)mlocp - (void *)mloc0); - mcsec.sh_link = 0; - mcsec.sh_info = 0; - mcsec.sh_addralign = w8(8); - mcsec.sh_entsize = w8(8); - uwrite(fd_map, &mcsec, sizeof(mcsec)); - - mcsec.sh_name = w(old_shstr_sh_size); - mcsec.sh_type = (sizeof(Elf64_Rela) == rel_entsize) - ? w(SHT_RELA) - : w(SHT_REL); - mcsec.sh_flags = 0; - mcsec.sh_addr = 0; - mcsec.sh_offset = w8((void *)mlocp - (void *)mloc0 + t); - mcsec.sh_size = w8((void *)mrelp - (void *)mrel0); - mcsec.sh_link = w(symsec_sh_link); - mcsec.sh_info = w(old_shnum); - mcsec.sh_addralign = w8(8); - mcsec.sh_entsize = w8(rel_entsize); - uwrite(fd_map, &mcsec, sizeof(mcsec)); - - uwrite(fd_map, mloc0, (void *)mlocp - (void *)mloc0); - uwrite(fd_map, mrel0, (void *)mrelp - (void *)mrel0); - - ehdr->e_shoff = w8(new_e_shoff); - ehdr->e_shnum = w2(2 + w2(ehdr->e_shnum)); /* {.rel,}__mcount_loc */ - ulseek(fd_map, 0, SEEK_SET); - uwrite(fd_map, ehdr, sizeof(*ehdr)); -} - -/* - * Look at the relocations in order to find the calls to mcount. - * Accumulate the section offsets that are found, and their relocation info, - * onto the end of the existing arrays. - */ -static uint32_t *sift32_rel_mcount(uint32_t *mlocp, - unsigned const offbase, - Elf32_Rel **const mrelpp, - Elf32_Shdr const *const relhdr, - Elf32_Ehdr const *const ehdr, - unsigned const recsym, - uint32_t const recval, - unsigned const reltype) -{ - uint32_t *const mloc0 = mlocp; - Elf32_Rel *mrelp = *mrelpp; - Elf32_Shdr *const shdr0 = (Elf32_Shdr *)(w(ehdr->e_shoff) - + (void *)ehdr); - unsigned const symsec_sh_link = w(relhdr->sh_link); - Elf32_Shdr const *const symsec = &shdr0[symsec_sh_link]; - Elf32_Sym const *const sym0 = (Elf32_Sym const *)(w(symsec->sh_offset) - + (void *)ehdr); - - Elf32_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; - char const *const str0 = (char const *)(w(strsec->sh_offset) - + (void *)ehdr); - - Elf32_Rel const *const rel0 = (Elf32_Rel const *)(w(relhdr->sh_offset) - + (void *)ehdr); - unsigned rel_entsize = w(relhdr->sh_entsize); - unsigned const nrel = w(relhdr->sh_size) / rel_entsize; - Elf32_Rel const *relp = rel0; - - unsigned mcountsym = 0; - unsigned t; - - for (t = nrel; t; --t) { - if (!mcountsym) { - Elf32_Sym const *const symp = - &sym0[ELF32_R_SYM(w(relp->r_info))]; - - if (0 == strcmp((('_' == gpfx) ? "_mcount" : "mcount"), - &str0[w(symp->st_name)])) - mcountsym = ELF32_R_SYM(w(relp->r_info)); - } - if (mcountsym == ELF32_R_SYM(w(relp->r_info))) { - uint32_t const addend = w(w(relp->r_offset) - recval); - mrelp->r_offset = w(offbase - + ((void *)mlocp - (void *)mloc0)); - mrelp->r_info = w(ELF32_R_INFO(recsym, reltype)); - if (sizeof(Elf32_Rela) == rel_entsize) { - ((Elf32_Rela *)mrelp)->r_addend = addend; - *mlocp++ = 0; - } else - *mlocp++ = addend; - - mrelp = (Elf32_Rel *)(rel_entsize + (void *)mrelp); - } - relp = (Elf32_Rel const *)(rel_entsize + (void *)relp); - } - *mrelpp = mrelp; - return mlocp; -} - -static uint64_t *sift64_rel_mcount(uint64_t *mlocp, - unsigned const offbase, - Elf64_Rel **const mrelpp, - Elf64_Shdr const *const relhdr, - Elf64_Ehdr const *const ehdr, - unsigned const recsym, - uint64_t const recval, - unsigned const reltype) -{ - uint64_t *const mloc0 = mlocp; - Elf64_Rel *mrelp = *mrelpp; - Elf64_Shdr *const shdr0 = (Elf64_Shdr *)(w8(ehdr->e_shoff) - + (void *)ehdr); - unsigned const symsec_sh_link = w(relhdr->sh_link); - Elf64_Shdr const *const symsec = &shdr0[symsec_sh_link]; - Elf64_Sym const *const sym0 = (Elf64_Sym const *)(w8(symsec->sh_offset) - + (void *)ehdr); - - Elf64_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; - char const *const str0 = (char const *)(w8(strsec->sh_offset) - + (void *)ehdr); - - Elf64_Rel const *const rel0 = (Elf64_Rel const *)(w8(relhdr->sh_offset) - + (void *)ehdr); - unsigned rel_entsize = w8(relhdr->sh_entsize); - unsigned const nrel = w8(relhdr->sh_size) / rel_entsize; - Elf64_Rel const *relp = rel0; - - unsigned mcountsym = 0; - unsigned t; - - for (t = nrel; 0 != t; --t) { - if (!mcountsym) { - Elf64_Sym const *const symp = - &sym0[ELF64_R_SYM(w8(relp->r_info))]; - char const *symname = &str0[w(symp->st_name)]; - - if ('.' == symname[0]) - ++symname; /* ppc64 hack */ - if (0 == strcmp((('_' == gpfx) ? "_mcount" : "mcount"), - symname)) - mcountsym = ELF64_R_SYM(w8(relp->r_info)); - } - - if (mcountsym == ELF64_R_SYM(w8(relp->r_info))) { - uint64_t const addend = w8(w8(relp->r_offset) - recval); - - mrelp->r_offset = w8(offbase - + ((void *)mlocp - (void *)mloc0)); - mrelp->r_info = w8(ELF64_R_INFO(recsym, reltype)); - if (sizeof(Elf64_Rela) == rel_entsize) { - ((Elf64_Rela *)mrelp)->r_addend = addend; - *mlocp++ = 0; - } else - *mlocp++ = addend; - - mrelp = (Elf64_Rel *)(rel_entsize + (void *)mrelp); - } - relp = (Elf64_Rel const *)(rel_entsize + (void *)relp); - } - *mrelpp = mrelp; - - return mlocp; -} - -/* - * Find a symbol in the given section, to be used as the base for relocating - * the table of offsets of calls to mcount. A local or global symbol suffices, - * but avoid a Weak symbol because it may be overridden; the change in value - * would invalidate the relocations of the offsets of the calls to mcount. - * Often the found symbol will be the unnamed local symbol generated by - * GNU 'as' for the start of each section. For example: - * Num: Value Size Type Bind Vis Ndx Name - * 2: 00000000 0 SECTION LOCAL DEFAULT 1 - */ -static unsigned find32_secsym_ndx(unsigned const txtndx, - char const *const txtname, - uint32_t *const recvalp, - Elf32_Shdr const *const symhdr, - Elf32_Ehdr const *const ehdr) -{ - Elf32_Sym const *const sym0 = (Elf32_Sym const *)(w(symhdr->sh_offset) - + (void *)ehdr); - unsigned const nsym = w(symhdr->sh_size) / w(symhdr->sh_entsize); - Elf32_Sym const *symp; - unsigned t; - - for (symp = sym0, t = nsym; t; --t, ++symp) { - unsigned int const st_bind = ELF32_ST_BIND(symp->st_info); - - if (txtndx == w2(symp->st_shndx) - /* avoid STB_WEAK */ - && (STB_LOCAL == st_bind || STB_GLOBAL == st_bind)) { - *recvalp = w(symp->st_value); - return symp - sym0; - } - } - fprintf(stderr, "Cannot find symbol for section %d: %s.\n", - txtndx, txtname); - fail_file(); -} - -static unsigned find64_secsym_ndx(unsigned const txtndx, - char const *const txtname, - uint64_t *const recvalp, - Elf64_Shdr const *const symhdr, - Elf64_Ehdr const *const ehdr) -{ - Elf64_Sym const *const sym0 = (Elf64_Sym const *)(w8(symhdr->sh_offset) - + (void *)ehdr); - unsigned const nsym = w8(symhdr->sh_size) / w8(symhdr->sh_entsize); - Elf64_Sym const *symp; - unsigned t; - - for (symp = sym0, t = nsym; t; --t, ++symp) { - unsigned int const st_bind = ELF64_ST_BIND(symp->st_info); - - if (txtndx == w2(symp->st_shndx) - /* avoid STB_WEAK */ - && (STB_LOCAL == st_bind || STB_GLOBAL == st_bind)) { - *recvalp = w8(symp->st_value); - return symp - sym0; - } - } - fprintf(stderr, "Cannot find symbol for section %d: %s.\n", - txtndx, txtname); - fail_file(); -} - -/* - * Evade ISO C restriction: no declaration after statement in - * has32_rel_mcount. - */ -static char const * -__has32_rel_mcount(Elf32_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */ - Elf32_Shdr const *const shdr0, - char const *const shstrtab, - char const *const fname) -{ - /* .sh_info depends on .sh_type == SHT_REL[,A] */ - Elf32_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; - char const *const txtname = &shstrtab[w(txthdr->sh_name)]; - - if (0 == strcmp("__mcount_loc", txtname)) { - fprintf(stderr, "warning: __mcount_loc already exists: %s\n", - fname); - succeed_file(); - } - if (SHT_PROGBITS != w(txthdr->sh_type) || - !is_mcounted_section_name(txtname)) - return NULL; - return txtname; -} - -static char const *has32_rel_mcount(Elf32_Shdr const *const relhdr, - Elf32_Shdr const *const shdr0, - char const *const shstrtab, - char const *const fname) -{ - if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) - return NULL; - return __has32_rel_mcount(relhdr, shdr0, shstrtab, fname); -} - -static char const *__has64_rel_mcount(Elf64_Shdr const *const relhdr, - Elf64_Shdr const *const shdr0, - char const *const shstrtab, - char const *const fname) -{ - /* .sh_info depends on .sh_type == SHT_REL[,A] */ - Elf64_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; - char const *const txtname = &shstrtab[w(txthdr->sh_name)]; - - if (0 == strcmp("__mcount_loc", txtname)) { - fprintf(stderr, "warning: __mcount_loc already exists: %s\n", - fname); - succeed_file(); - } - if (SHT_PROGBITS != w(txthdr->sh_type) || - !is_mcounted_section_name(txtname)) - return NULL; - return txtname; -} - -static char const *has64_rel_mcount(Elf64_Shdr const *const relhdr, - Elf64_Shdr const *const shdr0, - char const *const shstrtab, - char const *const fname) -{ - if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) - return NULL; - return __has64_rel_mcount(relhdr, shdr0, shstrtab, fname); -} - -static unsigned tot32_relsize(Elf32_Shdr const *const shdr0, - unsigned nhdr, - const char *const shstrtab, - const char *const fname) -{ - unsigned totrelsz = 0; - Elf32_Shdr const *shdrp = shdr0; - for (; 0 != nhdr; --nhdr, ++shdrp) { - if (has32_rel_mcount(shdrp, shdr0, shstrtab, fname)) - totrelsz += w(shdrp->sh_size); - } - return totrelsz; -} - -static unsigned tot64_relsize(Elf64_Shdr const *const shdr0, - unsigned nhdr, - const char *const shstrtab, - const char *const fname) -{ - unsigned totrelsz = 0; - Elf64_Shdr const *shdrp = shdr0; - - for (; nhdr; --nhdr, ++shdrp) { - if (has64_rel_mcount(shdrp, shdr0, shstrtab, fname)) - totrelsz += w8(shdrp->sh_size); - } - return totrelsz; -} - -/* Overall supervision for Elf32 ET_REL file. */ -static void -do32(Elf32_Ehdr *const ehdr, char const *const fname, unsigned const reltype) -{ - Elf32_Shdr *const shdr0 = (Elf32_Shdr *)(w(ehdr->e_shoff) - + (void *)ehdr); - unsigned const nhdr = w2(ehdr->e_shnum); - Elf32_Shdr *const shstr = &shdr0[w2(ehdr->e_shstrndx)]; - char const *const shstrtab = (char const *)(w(shstr->sh_offset) - + (void *)ehdr); - - Elf32_Shdr const *relhdr; - unsigned k; - - /* Upper bound on space: assume all relevant relocs are for mcount. */ - unsigned const totrelsz = tot32_relsize(shdr0, nhdr, shstrtab, fname); - Elf32_Rel *const mrel0 = umalloc(totrelsz); - Elf32_Rel * mrelp = mrel0; - - /* 2*sizeof(address) <= sizeof(Elf32_Rel) */ - uint32_t *const mloc0 = umalloc(totrelsz>>1); - uint32_t * mlocp = mloc0; - - unsigned rel_entsize = 0; - unsigned symsec_sh_link = 0; - - for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { - char const *const txtname = has32_rel_mcount(relhdr, shdr0, - shstrtab, fname); - if (txtname) { - uint32_t recval = 0; - unsigned const recsym = find32_secsym_ndx( - w(relhdr->sh_info), txtname, &recval, - &shdr0[symsec_sh_link = w(relhdr->sh_link)], - ehdr); - - rel_entsize = w(relhdr->sh_entsize); - mlocp = sift32_rel_mcount(mlocp, - (void *)mlocp - (void *)mloc0, &mrelp, - relhdr, ehdr, recsym, recval, reltype); - } - } - if (mloc0 != mlocp) { - append32(ehdr, shstr, mloc0, mlocp, mrel0, mrelp, - rel_entsize, symsec_sh_link); - } - free(mrel0); - free(mloc0); -} - -static void -do64(Elf64_Ehdr *const ehdr, char const *const fname, unsigned const reltype) -{ - Elf64_Shdr *const shdr0 = (Elf64_Shdr *)(w8(ehdr->e_shoff) - + (void *)ehdr); - unsigned const nhdr = w2(ehdr->e_shnum); - Elf64_Shdr *const shstr = &shdr0[w2(ehdr->e_shstrndx)]; - char const *const shstrtab = (char const *)(w8(shstr->sh_offset) - + (void *)ehdr); - - Elf64_Shdr const *relhdr; - unsigned k; - - /* Upper bound on space: assume all relevant relocs are for mcount. */ - unsigned const totrelsz = tot64_relsize(shdr0, nhdr, shstrtab, fname); - Elf64_Rel *const mrel0 = umalloc(totrelsz); - Elf64_Rel * mrelp = mrel0; - - /* 2*sizeof(address) <= sizeof(Elf64_Rel) */ - uint64_t *const mloc0 = umalloc(totrelsz>>1); - uint64_t * mlocp = mloc0; - - unsigned rel_entsize = 0; - unsigned symsec_sh_link = 0; - - for ((relhdr = shdr0), k = nhdr; k; --k, ++relhdr) { - char const *const txtname = has64_rel_mcount(relhdr, shdr0, - shstrtab, fname); - if (txtname) { - uint64_t recval = 0; - unsigned const recsym = find64_secsym_ndx( - w(relhdr->sh_info), txtname, &recval, - &shdr0[symsec_sh_link = w(relhdr->sh_link)], - ehdr); - - rel_entsize = w8(relhdr->sh_entsize); - mlocp = sift64_rel_mcount(mlocp, - (void *)mlocp - (void *)mloc0, &mrelp, - relhdr, ehdr, recsym, recval, reltype); - } - } - if (mloc0 != mlocp) { - append64(ehdr, shstr, mloc0, mlocp, mrel0, mrelp, - rel_entsize, symsec_sh_link); - } - free(mrel0); - free(mloc0); -} +/* 32 bit and 64 bit are very similar */ +#include "recordmcount.h" +#define RECORD_MCOUNT_64 +#include "recordmcount.h" static void do_file(char const *const fname) diff --git a/scripts/recordmcount.h b/scripts/recordmcount.h new file mode 100644 index 00000000000..7f39d0943d2 --- /dev/null +++ b/scripts/recordmcount.h @@ -0,0 +1,366 @@ +/* + * recordmcount.h + * + * This code was taken out of recordmcount.c written by + * Copyright 2009 John F. Reiser . All rights reserved. + * + * The original code had the same algorithms for both 32bit + * and 64bit ELF files, but the code was duplicated to support + * the difference in structures that were used. This + * file creates a macro of everything that is different between + * the 64 and 32 bit code, such that by including this header + * twice we can create both sets of functions by including this + * header once with RECORD_MCOUNT_64 undefined, and again with + * it defined. + * + * This conversion to macros was done by: + * Copyright 2010 Steven Rostedt , Red Hat Inc. + * + * Licensed under the GNU General Public License, version 2 (GPLv2). + */ +#undef append_func +#undef sift_rel_mcount +#undef find_secsym_ndx +#undef __has_rel_mcount +#undef has_rel_mcount +#undef tot_relsize +#undef do_func +#undef Elf_Ehdr +#undef Elf_Shdr +#undef Elf_Rel +#undef Elf_Rela +#undef Elf_Sym +#undef ELF_R_SYM +#undef ELF_R_INFO +#undef ELF_ST_BIND +#undef uint_t +#undef _w +#undef _align +#undef _size + +#ifdef RECORD_MCOUNT_64 +# define append_func append64 +# define sift_rel_mcount sift64_rel_mcount +# define find_secsym_ndx find64_secsym_ndx +# define __has_rel_mcount __has64_rel_mcount +# define has_rel_mcount has64_rel_mcount +# define tot_relsize tot64_relsize +# define do_func do64 +# define Elf_Ehdr Elf64_Ehdr +# define Elf_Shdr Elf64_Shdr +# define Elf_Rel Elf64_Rel +# define Elf_Rela Elf64_Rela +# define Elf_Sym Elf64_Sym +# define ELF_R_SYM ELF64_R_SYM +# define ELF_R_INFO ELF64_R_INFO +# define ELF_ST_BIND ELF64_ST_BIND +# define uint_t uint64_t +# define _w w8 +# define _align 7u +# define _size 8 +#else +# define append_func append32 +# define sift_rel_mcount sift32_rel_mcount +# define find_secsym_ndx find32_secsym_ndx +# define __has_rel_mcount __has32_rel_mcount +# define has_rel_mcount has32_rel_mcount +# define tot_relsize tot32_relsize +# define do_func do32 +# define Elf_Ehdr Elf32_Ehdr +# define Elf_Shdr Elf32_Shdr +# define Elf_Rel Elf32_Rel +# define Elf_Rela Elf32_Rela +# define Elf_Sym Elf32_Sym +# define ELF_R_SYM ELF32_R_SYM +# define ELF_R_INFO ELF32_R_INFO +# define ELF_ST_BIND ELF32_ST_BIND +# define uint_t uint32_t +# define _w w +# define _align 3u +# define _size 4 +#endif + +/* Append the new shstrtab, Elf_Shdr[], __mcount_loc and its relocations. */ +static void append_func(Elf_Ehdr *const ehdr, + Elf_Shdr *const shstr, + uint_t const *const mloc0, + uint_t const *const mlocp, + Elf_Rel const *const mrel0, + Elf_Rel const *const mrelp, + unsigned int const rel_entsize, + unsigned int const symsec_sh_link) +{ + /* Begin constructing output file */ + Elf_Shdr mcsec; + char const *mc_name = (sizeof(Elf_Rela) == rel_entsize) + ? ".rela__mcount_loc" + : ".rel__mcount_loc"; + unsigned const old_shnum = w2(ehdr->e_shnum); + uint_t const old_shoff = _w(ehdr->e_shoff); + uint_t const old_shstr_sh_size = _w(shstr->sh_size); + uint_t const old_shstr_sh_offset = _w(shstr->sh_offset); + uint_t t = 1 + strlen(mc_name) + _w(shstr->sh_size); + uint_t new_e_shoff; + + shstr->sh_size = _w(t); + shstr->sh_offset = _w(sb.st_size); + t += sb.st_size; + t += (_align & -t); /* word-byte align */ + new_e_shoff = t; + + /* body for new shstrtab */ + ulseek(fd_map, sb.st_size, SEEK_SET); + uwrite(fd_map, old_shstr_sh_offset + (void *)ehdr, old_shstr_sh_size); + uwrite(fd_map, mc_name, 1 + strlen(mc_name)); + + /* old(modified) Elf_Shdr table, word-byte aligned */ + ulseek(fd_map, t, SEEK_SET); + t += sizeof(Elf_Shdr) * old_shnum; + uwrite(fd_map, old_shoff + (void *)ehdr, + sizeof(Elf_Shdr) * old_shnum); + + /* new sections __mcount_loc and .rel__mcount_loc */ + t += 2*sizeof(mcsec); + mcsec.sh_name = w((sizeof(Elf_Rela) == rel_entsize) + strlen(".rel") + + old_shstr_sh_size); + mcsec.sh_type = w(SHT_PROGBITS); + mcsec.sh_flags = _w(SHF_ALLOC); + mcsec.sh_addr = 0; + mcsec.sh_offset = _w(t); + mcsec.sh_size = _w((void *)mlocp - (void *)mloc0); + mcsec.sh_link = 0; + mcsec.sh_info = 0; + mcsec.sh_addralign = _w(_size); + mcsec.sh_entsize = _w(_size); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + mcsec.sh_name = w(old_shstr_sh_size); + mcsec.sh_type = (sizeof(Elf_Rela) == rel_entsize) + ? w(SHT_RELA) + : w(SHT_REL); + mcsec.sh_flags = 0; + mcsec.sh_addr = 0; + mcsec.sh_offset = _w((void *)mlocp - (void *)mloc0 + t); + mcsec.sh_size = _w((void *)mrelp - (void *)mrel0); + mcsec.sh_link = w(symsec_sh_link); + mcsec.sh_info = w(old_shnum); + mcsec.sh_addralign = _w(_size); + mcsec.sh_entsize = _w(rel_entsize); + uwrite(fd_map, &mcsec, sizeof(mcsec)); + + uwrite(fd_map, mloc0, (void *)mlocp - (void *)mloc0); + uwrite(fd_map, mrel0, (void *)mrelp - (void *)mrel0); + + ehdr->e_shoff = _w(new_e_shoff); + ehdr->e_shnum = w2(2 + w2(ehdr->e_shnum)); /* {.rel,}__mcount_loc */ + ulseek(fd_map, 0, SEEK_SET); + uwrite(fd_map, ehdr, sizeof(*ehdr)); +} + + +/* + * Look at the relocations in order to find the calls to mcount. + * Accumulate the section offsets that are found, and their relocation info, + * onto the end of the existing arrays. + */ +static uint_t *sift_rel_mcount(uint_t *mlocp, + unsigned const offbase, + Elf_Rel **const mrelpp, + Elf_Shdr const *const relhdr, + Elf_Ehdr const *const ehdr, + unsigned const recsym, + uint_t const recval, + unsigned const reltype) +{ + uint_t *const mloc0 = mlocp; + Elf_Rel *mrelp = *mrelpp; + Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) + + (void *)ehdr); + unsigned const symsec_sh_link = w(relhdr->sh_link); + Elf_Shdr const *const symsec = &shdr0[symsec_sh_link]; + Elf_Sym const *const sym0 = (Elf_Sym const *)(_w(symsec->sh_offset) + + (void *)ehdr); + + Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)]; + char const *const str0 = (char const *)(_w(strsec->sh_offset) + + (void *)ehdr); + + Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset) + + (void *)ehdr); + unsigned rel_entsize = _w(relhdr->sh_entsize); + unsigned const nrel = _w(relhdr->sh_size) / rel_entsize; + Elf_Rel const *relp = rel0; + + unsigned mcountsym = 0; + unsigned t; + + for (t = nrel; t; --t) { + if (!mcountsym) { + Elf_Sym const *const symp = + &sym0[ELF_R_SYM(_w(relp->r_info))]; + char const *symname = &str0[w(symp->st_name)]; + + if ('.' == symname[0]) + ++symname; /* ppc64 hack */ + if (0 == strcmp((('_' == gpfx) ? "_mcount" : "mcount"), + symname)) + mcountsym = ELF_R_SYM(_w(relp->r_info)); + } + + if (mcountsym == ELF_R_SYM(_w(relp->r_info))) { + uint_t const addend = _w(_w(relp->r_offset) - recval); + + mrelp->r_offset = _w(offbase + + ((void *)mlocp - (void *)mloc0)); + mrelp->r_info = _w(ELF_R_INFO(recsym, reltype)); + if (sizeof(Elf_Rela) == rel_entsize) { + ((Elf_Rela *)mrelp)->r_addend = addend; + *mlocp++ = 0; + } else + *mlocp++ = addend; + + mrelp = (Elf_Rel *)(rel_entsize + (void *)mrelp); + } + relp = (Elf_Rel const *)(rel_entsize + (void *)relp); + } + *mrelpp = mrelp; + return mlocp; +} + + +/* + * Find a symbol in the given section, to be used as the base for relocating + * the table of offsets of calls to mcount. A local or global symbol suffices, + * but avoid a Weak symbol because it may be overridden; the change in value + * would invalidate the relocations of the offsets of the calls to mcount. + * Often the found symbol will be the unnamed local symbol generated by + * GNU 'as' for the start of each section. For example: + * Num: Value Size Type Bind Vis Ndx Name + * 2: 00000000 0 SECTION LOCAL DEFAULT 1 + */ +static unsigned find_secsym_ndx(unsigned const txtndx, + char const *const txtname, + uint_t *const recvalp, + Elf_Shdr const *const symhdr, + Elf_Ehdr const *const ehdr) +{ + Elf_Sym const *const sym0 = (Elf_Sym const *)(_w(symhdr->sh_offset) + + (void *)ehdr); + unsigned const nsym = _w(symhdr->sh_size) / _w(symhdr->sh_entsize); + Elf_Sym const *symp; + unsigned t; + + for (symp = sym0, t = nsym; t; --t, ++symp) { + unsigned int const st_bind = ELF_ST_BIND(symp->st_info); + + if (txtndx == w2(symp->st_shndx) + /* avoid STB_WEAK */ + && (STB_LOCAL == st_bind || STB_GLOBAL == st_bind)) { + *recvalp = _w(symp->st_value); + return symp - sym0; + } + } + fprintf(stderr, "Cannot find symbol for section %d: %s.\n", + txtndx, txtname); + fail_file(); +} + + +/* Evade ISO C restriction: no declaration after statement in has_rel_mcount. */ +static char const * +__has_rel_mcount(Elf_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */ + Elf_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + /* .sh_info depends on .sh_type == SHT_REL[,A] */ + Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)]; + char const *const txtname = &shstrtab[w(txthdr->sh_name)]; + + if (0 == strcmp("__mcount_loc", txtname)) { + fprintf(stderr, "warning: __mcount_loc already exists: %s\n", + fname); + succeed_file(); + } + if (SHT_PROGBITS != w(txthdr->sh_type) || + !is_mcounted_section_name(txtname)) + return NULL; + return txtname; +} + +static char const *has_rel_mcount(Elf_Shdr const *const relhdr, + Elf_Shdr const *const shdr0, + char const *const shstrtab, + char const *const fname) +{ + if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type)) + return NULL; + return __has_rel_mcount(relhdr, shdr0, shstrtab, fname); +} + + +static unsigned tot_relsize(Elf_Shdr const *const shdr0, + unsigned nhdr, + const char *const shstrtab, + const char *const fname) +{ + unsigned totrelsz = 0; + Elf_Shdr const *shdrp = shdr0; + + for (; nhdr; --nhdr, ++shdrp) { + if (has_rel_mcount(shdrp, shdr0, shstrtab, fname)) + totrelsz += _w(shdrp->sh_size); + } + return totrelsz; +} + + +/* Overall supervision for Elf32 ET_REL file. */ +static void +do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype) +{ + Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff) + + (void *)ehdr); + unsigned const nhdr = w2(ehdr->e_shnum); + Elf_Shdr *const shstr = &shdr0[w2(ehdr->e_shstrndx)]; + char const *const shstrtab = (char const *)(_w(shstr->sh_offset) + + (void *)ehdr); + + Elf_Shdr const *relhdr; + unsigned k; + + /* Upper bound on space: assume all relevant relocs are for mcount. */ + unsigned const totrelsz = tot_relsize(shdr0, nhdr, shstrtab, fname); + Elf_Rel *const mrel0 = umalloc(totrelsz); + Elf_Rel * mrelp = mrel0; + + /* 2*sizeof(address) <= sizeof(Elf_Rel) */ + uint_t *const mloc0 = umalloc(totrelsz>>1); + uint_t * mlocp = mloc0; + + unsigned rel_entsize = 0; + unsigned symsec_sh_link = 0; + + for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) { + char const *const txtname = has_rel_mcount(relhdr, shdr0, + shstrtab, fname); + if (txtname) { + uint_t recval = 0; + unsigned const recsym = find_secsym_ndx( + w(relhdr->sh_info), txtname, &recval, + &shdr0[symsec_sh_link = w(relhdr->sh_link)], + ehdr); + + rel_entsize = _w(relhdr->sh_entsize); + mlocp = sift_rel_mcount(mlocp, + (void *)mlocp - (void *)mloc0, &mrelp, + relhdr, ehdr, recsym, recval, reltype); + } + } + if (mloc0 != mlocp) { + append_func(ehdr, shstr, mloc0, mlocp, mrel0, mrelp, + rel_entsize, symsec_sh_link); + } + free(mrel0); + free(mloc0); +} -- cgit v1.2.3-70-g09d2 From 444758636439810f110f86a3042c2dfd3626a9e2 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 15 Oct 2010 11:49:47 -0400 Subject: ftrace: Do not process kernel/trace/ftrace.o with C recordmcount program The file kernel/trace/ftrace.c references the mcount() call to convert the mcount() callers to nops. But because it references mcount(), the mcount() address is placed in the relocation table. The C version of recordmcount reads the relocation table of all object files, and it will add all references to mcount to the __mcount_loc table that is used to find the places that call mcount() and change the call to a nop. When recordmcount finds the mcount reference in kernel/trace/ftrace.o, it saves that location even though the code is not a call, but references mcount as data. On boot up, when all calls are converted to nops, the code has a safety check to determine what op code it is actually replacing before it replaces it. If that op code at the address does not match, then a warning is printed and the function tracer is disabled. The reference to mcount in ftrace.c, causes this warning to trigger, since the reference is not a call to mcount(). The ftrace.c file is not compiled with the -pg flag, so no calls to mcount() should be expected. This patch simply makes recordmcount.c skip the kernel/trace/ftrace.c file. This was the same solution used by the perl version of recordmcount. Reported-by: Ingo Molnar Cc: John Reiser Signed-off-by: Steven Rostedt --- scripts/recordmcount.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'scripts') diff --git a/scripts/recordmcount.c b/scripts/recordmcount.c index 7f7f7180fe2..26e1271259b 100644 --- a/scripts/recordmcount.c +++ b/scripts/recordmcount.c @@ -313,12 +313,30 @@ do_file(char const *const fname) int main(int argc, char const *argv[]) { + const char ftrace[] = "kernel/trace/ftrace.o"; + int ftrace_size = sizeof(ftrace) - 1; int n_error = 0; /* gcc-4.3.0 false positive complaint */ - if (argc <= 1) + + if (argc <= 1) { fprintf(stderr, "usage: recordmcount file.o...\n"); - else /* Process each file in turn, allowing deep failure. */ + return 0; + } + + /* Process each file in turn, allowing deep failure. */ for (--argc, ++argv; 0 < argc; --argc, ++argv) { int const sjval = setjmp(jmpenv); + int len; + + /* + * The file kernel/trace/ftrace.o references the mcount + * function but does not call it. Since ftrace.o should + * not be traced anyway, we just skip it. + */ + len = strlen(argv[0]); + if (len >= ftrace_size && + strcmp(argv[0] + (len - ftrace_size), ftrace) == 0) + continue; + switch (sjval) { default: { fprintf(stderr, "internal error: %s\n", argv[0]); -- cgit v1.2.3-70-g09d2 From 85caa993d7f218b7c2abcfcb28e212396b6a3313 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Fri, 15 Oct 2010 12:09:25 -0400 Subject: ftrace: Use objtree for C version of recordmcount The C version of recordmcount is compiled to a binary, which will end up located in the objtree. If the kernel is built with O=path, the srctree will not include the binary recordmcount caller. Cc: Michal Marek Cc: linux-kbuild@vger.kernel.org Signed-off-by: Steven Rostedt --- scripts/Makefile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 4d03a7efc68..4db60b2e2a7 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -210,7 +210,7 @@ endif ifdef CONFIG_FTRACE_MCOUNT_RECORD ifdef BUILD_C_RECORDMCOUNT -cmd_record_mcount = $(srctree)/scripts/recordmcount "$(@)"; +cmd_record_mcount = $(objtree)/scripts/recordmcount "$(@)"; else cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ -- cgit v1.2.3-70-g09d2 From d7b4d6de57d414a6384376880f2caf7125a45494 Mon Sep 17 00:00:00 2001 From: Steven Rostedt Date: Mon, 18 Oct 2010 14:42:00 -0400 Subject: ftrace: Remove recursion between recordmcount and scripts/mod/empty When DYNAMIC_FTRACE is enabled and we use the C version of recordmcount, all objects are run through the recordmcount program to create a separate section that stores all the callers of mcount. The build process has a special file: scripts/mod/empty.o. This is built from empty.c which is literally an empty file (except for a single comment). This file is used to find information about the target elf format, like endianness and word size. The problem comes up when we need to build recordmcount. The build process requires that empty.o is built first. The build rules for empty.o will try to execute recordmcount on the empty.o file. We get an error that recordmcount does not exist. To avoid this recursion, the build file will skip running recordmcount if the file that it is building is script/mod/empty.o. [ extra comment Suggested-by: Sam Ravnborg ] Reported-by: Ingo Molnar Tested-by: Ingo Molnar Cc: Michal Marek Cc: linux-kbuild@vger.kernel.org Signed-off-by: Steven Rostedt --- scripts/Makefile.build | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 4db60b2e2a7..843bd4f4ffc 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -210,7 +210,13 @@ endif ifdef CONFIG_FTRACE_MCOUNT_RECORD ifdef BUILD_C_RECORDMCOUNT -cmd_record_mcount = $(objtree)/scripts/recordmcount "$(@)"; +# Due to recursion, we must skip empty.o. +# The empty.o file is created in the make process in order to determine +# the target endianness and word size. It is made before all other C +# files, including recordmcount. +cmd_record_mcount = if [ $(@) != "scripts/mod/empty.o" ]; then \ + $(objtree)/scripts/recordmcount "$(@)"; \ + fi; else cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \ "$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \ -- cgit v1.2.3-70-g09d2