diff options
| -rw-r--r-- | Documentation/00-INDEX | 2 | ||||
| -rw-r--r-- | Documentation/kdump/gdbmacros.txt | 179 | ||||
| -rw-r--r-- | Documentation/kdump/kdump.txt | 135 |
3 files changed, 316 insertions, 0 deletions
diff --git a/Documentation/00-INDEX b/Documentation/00-INDEX index 8de8a01a247..f28a24e0279 100644 --- a/Documentation/00-INDEX +++ b/Documentation/00-INDEX @@ -138,6 +138,8 @@ java.txt - info on the in-kernel binary support for Java(tm). kbuild/ - directory with info about the kernel build process. +kdumpt.txt + - mini HowTo on getting the crash dump code to work. kernel-doc-nano-HOWTO.txt - mini HowTo on generation and location of kernel documentation files. kernel-docs.txt diff --git a/Documentation/kdump/gdbmacros.txt b/Documentation/kdump/gdbmacros.txt new file mode 100644 index 00000000000..bc1b9eb92ae --- /dev/null +++ b/Documentation/kdump/gdbmacros.txt @@ -0,0 +1,179 @@ +# +# This file contains a few gdb macros (user defined commands) to extract +# useful information from kernel crashdump (kdump) like stack traces of +# all the processes or a particular process and trapinfo. +# +# These macros can be used by copying this file in .gdbinit (put in home +# directory or current directory) or by invoking gdb command with +# --command=<command-file-name> option +# +# Credits: +# Alexander Nyberg <alexn@telia.com> +# V Srivatsa <vatsa@in.ibm.com> +# Maneesh Soni <maneesh@in.ibm.com> +# + +define bttnobp + set $tasks_off=((size_t)&((struct task_struct *)0)->tasks) + set $pid_off=((size_t)&((struct task_struct *)0)->pids[1].pid_list.next) + set $init_t=&init_task + set $next_t=(((char *)($init_t->tasks).next) - $tasks_off) + while ($next_t != $init_t) + set $next_t=(struct task_struct *)$next_t + printf "\npid %d; comm %s:\n", $next_t.pid, $next_t.comm + printf "===================\n" + set var $stackp = $next_t.thread.esp + set var $stack_top = ($stackp & ~4095) + 4096 + + while ($stackp < $stack_top) + if (*($stackp) > _stext && *($stackp) < _sinittext) + info symbol *($stackp) + end + set $stackp += 4 + end + set $next_th=(((char *)$next_t->pids[1].pid_list.next) - $pid_off) + while ($next_th != $next_t) + set $next_th=(struct task_struct *)$next_th + printf "\npid %d; comm %s:\n", $next_t.pid, $next_t.comm + printf "===================\n" + set var $stackp = $next_t.thread.esp + set var $stack_top = ($stackp & ~4095) + 4096 + + while ($stackp < $stack_top) + if (*($stackp) > _stext && *($stackp) < _sinittext) + info symbol *($stackp) + end + set $stackp += 4 + end + set $next_th=(((char *)$next_th->pids[1].pid_list.next) - $pid_off) + end + set $next_t=(char *)($next_t->tasks.next) - $tasks_off + end +end +document bttnobp + dump all thread stack traces on a kernel compiled with !CONFIG_FRAME_POINTER +end + +define btt + set $tasks_off=((size_t)&((struct task_struct *)0)->tasks) + set $pid_off=((size_t)&((struct task_struct *)0)->pids[1].pid_list.next) + set $init_t=&init_task + set $next_t=(((char *)($init_t->tasks).next) - $tasks_off) + while ($next_t != $init_t) + set $next_t=(struct task_struct *)$next_t + printf "\npid %d; comm %s:\n", $next_t.pid, $next_t.comm + printf "===================\n" + set var $stackp = $next_t.thread.esp + set var $stack_top = ($stackp & ~4095) + 4096 + set var $stack_bot = ($stackp & ~4095) + + set $stackp = *($stackp) + while (($stackp < $stack_top) && ($stackp > $stack_bot)) + set var $addr = *($stackp + 4) + info symbol $addr + set $stackp = *($stackp) + end + + set $next_th=(((char *)$next_t->pids[1].pid_list.next) - $pid_off) + while ($next_th != $next_t) + set $next_th=(struct task_struct *)$next_th + printf "\npid %d; comm %s:\n", $next_t.pid, $next_t.comm + printf "===================\n" + set var $stackp = $next_t.thread.esp + set var $stack_top = ($stackp & ~4095) + 4096 + set var $stack_bot = ($stackp & ~4095) + + set $stackp = *($stackp) + while (($stackp < $stack_top) && ($stackp > $stack_bot)) + set var $addr = *($stackp + 4) + info symbol $addr + set $stackp = *($stackp) + end + set $next_th=(((char *)$next_th->pids[1].pid_list.next) - $pid_off) + end + set $next_t=(char *)($next_t->tasks.next) - $tasks_off + end +end +document btt + dump all thread stack traces on a kernel compiled with CONFIG_FRAME_POINTER +end + +define btpid + set var $pid = $arg0 + set $tasks_off=((size_t)&((struct task_struct *)0)->tasks) + set $pid_off=((size_t)&((struct task_struct *)0)->pids[1].pid_list.next) + set $init_t=&init_task + set $next_t=(((char *)($init_t->tasks).next) - $tasks_off) + set var $pid_task = 0 + + while ($next_t != $init_t) + set $next_t=(struct task_struct *)$next_t + + if ($next_t.pid == $pid) + set $pid_task = $next_t + end + + set $next_th=(((char *)$next_t->pids[1].pid_list.next) - $pid_off) + while ($next_th != $next_t) + set $next_th=(struct task_struct *)$next_th + if ($next_th.pid == $pid) + set $pid_task = $next_th + end + set $next_th=(((char *)$next_th->pids[1].pid_list.next) - $pid_off) + end + set $next_t=(char *)($next_t->tasks.next) - $tasks_off + end + + printf "\npid %d; comm %s:\n", $pid_task.pid, $pid_task.comm + printf "===================\n" + set var $stackp = $pid_task.thread.esp + set var $stack_top = ($stackp & ~4095) + 4096 + set var $stack_bot = ($stackp & ~4095) + + set $stackp = *($stackp) + while (($stackp < $stack_top) && ($stackp > $stack_bot)) + set var $addr = *($stackp + 4) + info symbol $addr + set $stackp = *($stackp) + end +end +document btpid + backtrace of pid +end + + +define trapinfo + set var $pid = $arg0 + set $tasks_off=((size_t)&((struct task_struct *)0)->tasks) + set $pid_off=((size_t)&((struct task_struct *)0)->pids[1].pid_list.next) + set $init_t=&init_task + set $next_t=(((char *)($init_t->tasks).next) - $tasks_off) + set var $pid_task = 0 + + while ($next_t != $init_t) + set $next_t=(struct task_struct *)$next_t + + if ($next_t.pid == $pid) + set $pid_task = $next_t + end + + set $next_th=(((char *)$next_t->pids[1].pid_list.next) - $pid_off) + while ($next_th != $next_t) + set $next_th=(struct task_struct *)$next_th + if ($next_th.pid == $pid) + set $pid_task = $next_th + end + set $next_th=(((char *)$next_th->pids[1].pid_list.next) - $pid_off) + end + set $next_t=(char *)($next_t->tasks.next) - $tasks_off + end + + printf "Trapno %ld, cr2 0x%lx, error_code %ld\n", $pid_task.thread.trap_no, \ + $pid_task.thread.cr2, $pid_task.thread.error_code + +end +document trapinfo + Run info threads and lookup pid of thread #1 + 'trapinfo <pid>' will tell you by which trap & possibly + addresthe kernel paniced. +end diff --git a/Documentation/kdump/kdump.txt b/Documentation/kdump/kdump.txt new file mode 100644 index 00000000000..b0f412e4279 --- /dev/null +++ b/Documentation/kdump/kdump.txt @@ -0,0 +1,135 @@ +Documentation for kdump - the kexec-based crash dumping solution +================================================================ + +DESIGN +====== + +Kdump uses kexec to reboot to a second kernel whenever a dump needs to be taken. +This second kernel is booted with very little memory. The first kernel reserves +the section of memory that the second kernel uses. This ensures that on-going +DMA from the first kernel does not corrupt the second kernel. + +All the necessary information about Core image is encoded in ELF format and +stored in reserved area of memory before crash. Physical address of start of +ELF header is passed to new kernel through command line parameter elfcorehdr=. + +On i386, the first 640 KB of physical memory is needed to boot, irrespective +of where the kernel loads. Hence, this region is backed up by kexec just before +rebooting into the new kernel. + +In the second kernel, "old memory" can be accessed in two ways. + +- The first one is through a /dev/oldmem device interface. A capture utility + can read the device file and write out the memory in raw format. This is raw + dump of memory and analysis/capture tool should be intelligent enough to + determine where to look for the right information. ELF headers (elfcorehdr=) + can become handy here. + +- The second interface is through /proc/vmcore. This exports the dump as an ELF + format file which can be written out using any file copy command + (cp, scp, etc). Further, gdb can be used to perform limited debugging on + the dump file. This method ensures methods ensure that there is correct + ordering of the dump pages (corresponding to the first 640 KB that has been + relocated). + +SETUP +===== + +1) Download http://www.xmission.com/~ebiederm/files/kexec/kexec-tools-1.101.tar.gz + and apply http://lse.sourceforge.net/kdump/patches/kexec-tools-1.101-kdump.patch + and after that build the source. + +2) Download and build the appropriate (latest) kexec/kdump (-mm) kernel + patchset and apply it to the vanilla kernel tree. + + Two kernels need to be built in order to get this feature working. + + A) First kernel: + a) Enable "kexec system call" feature (in Processor type and features). + CONFIG_KEXEC=y + b) This kernel's physical load address should be the default value of + 0x100000 (0x100000, 1 MB) (in Processor type and features). + CONFIG_PHYSICAL_START=0x100000 + c) Enable "sysfs file system support" (in Pseudo filesystems). + CONFIG_SYSFS=y + d) Boot into first kernel with the command line parameter "crashkernel=Y@X". + Use appropriate values for X and Y. Y denotes how much memory to reserve + for the second kernel, and X denotes at what physical address the reserved + memory section starts. For example: "crashkernel=64M@16M". + + B) Second kernel: + a) Enable "kernel crash dumps" feature (in Processor type and features). + CONFIG_CRASH_DUMP=y + b) Specify a suitable value for "Physical address where the kernel is + loaded" (in Processor type and features). Typically this value + should be same as X (See option d) above, e.g., 16 MB or 0x1000000. + CONFIG_PHYSICAL_START=0x1000000 + c) Enable "/proc/vmcore support" (Optional, in Pseudo filesystems). + CONFIG_PROC_VMCORE=y + + Note: Options a) and b) depend upon "Configure standard kernel features + (for small systems)" (under General setup). + Option a) also depends on CONFIG_HIGHMEM (under Processor + type and features). + Both option a) and b) are under "Processor type and features". + +3) Boot into the first kernel. You are now ready to try out kexec-based crash + dumps. + +4) Load the second kernel to be booted using: + + kexec -p <second-kernel> --crash-dump --args-linux --append="root=<root-dev> + maxcpus=1 init 1" + + Note: i) <second-kernel> has to be a vmlinux image. bzImage will not work, + as of now. + ii) By default ELF headers are stored in ELF32 format (for i386). This + is sufficient to represent the physical memory up to 4GB. To store + headers in ELF64 format, specifiy "--elf64-core-headers" on the + kexec command line additionally. + iii) For now (or until it is fixed), it's best to build the + second-kernel without multi-processor support, i.e., make it + a uniprocessor kernel. + +5) System reboots into the second kernel when a panic occurs. A module can be + written to force the panic, for testing purposes. + +6) Write out the dump file using + + cp /proc/vmcore <dump-file> + + Dump memory can also be accessed as a /dev/oldmem device for a linear/raw + view. To create the device, type: + + mknod /dev/oldmem c 1 12 + + Use "dd" with suitable options for count, bs and skip to access specific + portions of the dump. + + Entire memory: dd if=/dev/oldmem of=oldmem.001 + +ANALYSIS +======== + +Limited analysis can be done using gdb on the dump file copied out of +/proc/vmcore. Use vmlinux built with -g and run + + gdb vmlinux <dump-file> + +Stack trace for the task on processor 0, register display, memory display +work fine. + +Note: gdb cannot analyse core files generated in ELF64 format for i386. + +TODO +==== + +1) Provide a kernel pages filtering mechanism so that core file size is not + insane on systems having huge memory banks. +2) Modify "crash" tool to make it recognize this dump. + +CONTACT +======= + +Hariprasad Nellitheertha - hari at in dot ibm dot com +Vivek Goyal (vgoyal@in.ibm.com) |