diff options
Diffstat (limited to 'arch/sparc')
498 files changed, 25139 insertions, 25789 deletions
diff --git a/arch/sparc/Kbuild b/arch/sparc/Kbuild new file mode 100644 index 00000000000..675afa285dd --- /dev/null +++ b/arch/sparc/Kbuild @@ -0,0 +1,9 @@ +# +# core part of the sparc kernel +# + +obj-y += kernel/ +obj-y += mm/ +obj-y += math-emu/ +obj-y += net/ +obj-y += crypto/ diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 95695e97703..407c87d9879 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -12,77 +12,83 @@ config 64BIT config SPARC bool default y + select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI + select ARCH_MIGHT_HAVE_PC_SERIO select OF select OF_PROMTREE select HAVE_IDE select HAVE_OPROFILE select HAVE_ARCH_KGDB if !SMP || SPARC64 select HAVE_ARCH_TRACEHOOK + select SYSCTL_EXCEPTION_TRACE select ARCH_WANT_OPTIONAL_GPIOLIB + select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select RTC_CLASS select RTC_DRV_M48T59 - select HAVE_IRQ_WORK select HAVE_DMA_ATTRS select HAVE_DMA_API_DEBUG - select HAVE_ARCH_JUMP_LABEL + select HAVE_ARCH_JUMP_LABEL if SPARC64 + select GENERIC_IRQ_SHOW + select ARCH_WANT_IPC_PARSE_VERSION + select GENERIC_PCI_IOMAP + select HAVE_NMI_WATCHDOG if SPARC64 + select HAVE_BPF_JIT + select HAVE_DEBUG_BUGVERBOSE + select GENERIC_SMP_IDLE_THREAD + select GENERIC_CMOS_UPDATE + select GENERIC_CLOCKEVENTS + select GENERIC_STRNCPY_FROM_USER + select GENERIC_STRNLEN_USER + select MODULES_USE_ELF_RELA + select ODD_RT_SIGACTION + select OLD_SIGSUSPEND config SPARC32 def_bool !64BIT + select GENERIC_ATOMIC64 + select CLZ_TAB + select HAVE_UID16 + select OLD_SIGACTION config SPARC64 def_bool 64BIT - select ARCH_SUPPORTS_MSI select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER select HAVE_FUNCTION_GRAPH_FP_TEST select HAVE_FUNCTION_TRACE_MCOUNT_TEST select HAVE_KRETPROBES select HAVE_KPROBES + select HAVE_RCU_TABLE_FREE if SMP select HAVE_MEMBLOCK - select HAVE_SYSCALL_WRAPPERS + select HAVE_MEMBLOCK_NODE_MAP + select HAVE_ARCH_TRANSPARENT_HUGEPAGE select HAVE_DYNAMIC_FTRACE select HAVE_FTRACE_MCOUNT_RECORD select HAVE_SYSCALL_TRACEPOINTS - select USE_GENERIC_SMP_HELPERS if SMP + select HAVE_CONTEXT_TRACKING + select HAVE_DEBUG_KMEMLEAK select RTC_DRV_CMOS select RTC_DRV_BQ4802 select RTC_DRV_SUN4V select RTC_DRV_STARFIRE select HAVE_PERF_EVENTS select PERF_USE_VMALLOC - select HAVE_GENERIC_HARDIRQS + select IRQ_PREFLOW_FASTEOI + select ARCH_HAVE_NMI_SAFE_CMPXCHG + select HAVE_C_RECORDMCOUNT + select NO_BOOTMEM + select HAVE_ARCH_AUDITSYSCALL + select ARCH_SUPPORTS_ATOMIC_RMW config ARCH_DEFCONFIG string default "arch/sparc/configs/sparc32_defconfig" if SPARC32 default "arch/sparc/configs/sparc64_defconfig" if SPARC64 -# CONFIG_BITS can be used at source level to get 32/64 bits -config BITS - int - default 32 if SPARC32 - default 64 if SPARC64 - -config ARCH_USES_GETTIMEOFFSET - bool - default y if SPARC32 - -config GENERIC_CMOS_UPDATE - bool - default y - -config GENERIC_CLOCKEVENTS - bool - default y if SPARC64 - config IOMMU_HELPER bool default y if SPARC64 -config QUICKLIST - bool - default y if SPARC64 - config STACKTRACE_SUPPORT bool default y if SPARC64 @@ -95,6 +101,9 @@ config HAVE_LATENCYTOP_SUPPORT bool default y if SPARC64 +config ARCH_HIBERNATION_POSSIBLE + def_bool y if SPARC64 + config AUDIT_ARCH bool default y @@ -130,14 +139,6 @@ config GENERIC_ISA_DMA bool default y if SPARC32 -config GENERIC_GPIO - bool - help - Generic GPIO API support - -config ARCH_NO_VIRT_TO_BUS - def_bool y - config ARCH_SUPPORTS_DEBUG_PAGEALLOC def_bool y if SPARC64 @@ -148,16 +149,16 @@ source "kernel/Kconfig.freezer" menu "Processor type and features" config SMP - bool "Symmetric multi-processing support (does not work on sun4/sun4c)" + bool "Symmetric multi-processing support" ---help--- This enables support for systems with more than one CPU. If you have a system with only one CPU, say N. If you have a system with more than one CPU, say Y. - If you say N here, the kernel will run on single and multiprocessor + If you say N here, the kernel will run on uni- and multiprocessor machines, but will use only one CPU of a multiprocessor machine. If you say Y here, the kernel will run on many, but not all, - singleprocessor machines. On a singleprocessor machine, the kernel + uniprocessor machines. On a uniprocessor machine, the kernel will run faster if you say N here. People using multiprocessor machines who say Y here should also say @@ -187,13 +188,9 @@ config RWSEM_XCHGADD_ALGORITHM bool default y if SPARC64 -config GENERIC_FIND_NEXT_BIT - bool - default y - config GENERIC_HWEIGHT bool - default y if !ULTRA_HAS_POPULATION_COUNT + default y config GENERIC_CALIBRATE_DELAY bool @@ -228,25 +225,6 @@ config EARLYFB help Say Y here to enable a faster early framebuffer boot console. -choice - prompt "Kernel page size" if SPARC64 - default SPARC64_PAGE_SIZE_8KB - -config SPARC64_PAGE_SIZE_8KB - bool "8KB" - help - This lets you select the page size of the kernel. - - 8KB and 64KB work quite well, since SPARC ELF sections - provide for up to 64KB alignment. - - If you don't know what to do, choose 8KB. - -config SPARC64_PAGE_SIZE_64KB - bool "64KB" - -endchoice - config SECCOMP bool "Enable seccomp to safely compute untrusted bytecode" depends on SPARC64 && PROC_FS @@ -267,39 +245,13 @@ config SECCOMP config HOTPLUG_CPU bool "Support for hot-pluggable CPUs" depends on SPARC64 && SMP - select HOTPLUG help Say Y here to experiment with turning CPUs off and on. CPUs can be controlled through /sys/devices/system/cpu/cpu#. Say N if you want to disable CPU hotplug. -source "kernel/time/Kconfig" - if SPARC64 source "drivers/cpufreq/Kconfig" - -config US3_FREQ - tristate "UltraSPARC-III CPU Frequency driver" - depends on CPU_FREQ - select CPU_FREQ_TABLE - help - This adds the CPUFreq driver for UltraSPARC-III processors. - - For details, take a look at <file:Documentation/cpu-freq>. - - If in doubt, say N. - -config US2E_FREQ - tristate "UltraSPARC-IIe CPU Frequency driver" - depends on CPU_FREQ - select CPU_FREQ_TABLE - help - This adds the CPUFreq driver for UltraSPARC-IIe processors. - - For details, take a look at <file:Documentation/cpu-freq>. - - If in doubt, say N. - endif config US3_MC @@ -320,23 +272,6 @@ config GENERIC_LOCKBREAK default y depends on SPARC64 && SMP && PREEMPT -choice - prompt "SPARC64 Huge TLB Page Size" - depends on SPARC64 && HUGETLB_PAGE - default HUGETLB_PAGE_SIZE_4MB - -config HUGETLB_PAGE_SIZE_4MB - bool "4MB" - -config HUGETLB_PAGE_SIZE_512K - bool "512K" - -config HUGETLB_PAGE_SIZE_64K - depends on !SPARC64_PAGE_SIZE_64KB - bool "64K" - -endchoice - config NUMA bool "NUMA support" depends on SPARC64 && SMP @@ -355,9 +290,6 @@ config NODES_SPAN_OTHER_NODES def_bool y depends on NEED_MULTIPLE_NODES -config ARCH_POPULATES_NODE_MAP - def_bool y if SPARC64 - config ARCH_SELECT_MEMORY_MODEL def_bool y if SPARC64 @@ -370,6 +302,10 @@ config ARCH_SPARSEMEM_DEFAULT source "mm/Kconfig" +if SPARC64 +source "kernel/power/Kconfig" +endif + config SCHED_SMT bool "SMT (Hyperthreading) scheduler support" depends on SPARC64 && SMP @@ -388,9 +324,7 @@ config SCHED_MC making when dealing with multi-core CPU chips at a cost of slightly increased overhead in some places. If unsure say N here. -if SPARC64 source "kernel/Kconfig.preempt" -endif config CMDLINE_BOOL bool "Default bootloader kernel arguments" @@ -452,6 +386,8 @@ config SERIAL_CONSOLE config SPARC_LEON bool "Sparc Leon processor family" depends on SPARC32 + select USB_EHCI_BIG_ENDIAN_MMIO + select USB_EHCI_BIG_ENDIAN_DESC ---help--- If you say Y here if you are running on a SPARC-LEON processor. The LEON processor is a synthesizable VHDL model of the @@ -460,6 +396,39 @@ config SPARC_LEON from www.gaisler.com. You can download a sparc-linux cross-compilation toolchain at www.gaisler.com. +if SPARC_LEON +menu "U-Boot options" + +config UBOOT_LOAD_ADDR + hex "uImage Load Address" + default 0x40004000 + ---help--- + U-Boot kernel load address, the address in physical address space + where u-boot will place the Linux kernel before booting it. + This address is normally the base address of main memory + 0x4000. + +config UBOOT_FLASH_ADDR + hex "uImage.o Load Address" + default 0x00080000 + ---help--- + Optional setting only affecting the uImage.o ELF-image used to + download the uImage file to the target using a ELF-loader other than + U-Boot. It may for example be used to download an uImage to FLASH with + the GRMON utility before even starting u-boot. + +config UBOOT_ENTRY_ADDR + hex "uImage Entry Address" + default 0xf0004000 + ---help--- + Do not change this unless you know what you're doing. This is + hardcoded by the SPARC32 and LEON port. + + This is the virtual address u-boot jumps to when booting the Linux + Kernel. + +endmenu +endif + endmenu menu "Bus options (PCI etc.)" @@ -496,6 +465,34 @@ config PCI_DOMAINS config PCI_SYSCALL def_bool PCI +config PCIC_PCI + bool + depends on PCI && SPARC32 && !SPARC_LEON + default y + +config LEON_PCI + bool + depends on PCI && SPARC_LEON + default y + +config SPARC_GRPCI1 + bool "GRPCI Host Bridge Support" + depends on LEON_PCI + default y + help + Say Y here to include the GRPCI Host Bridge Driver. The GRPCI + PCI host controller is typically found in GRLIB SPARC32/LEON + systems. The driver has one property (all_pci_errors) controlled + from the bootloader that makes the GRPCI to generate interrupts + on detected PCI Parity and System errors. + +config SPARC_GRPCI2 + bool "GRPCI2 Host Bridge Support" + depends on LEON_PCI + default y + help + Say Y here to include the GRPCI2 Host Bridge Driver. + source "drivers/pci/Kconfig" source "drivers/pcmcia/Kconfig" @@ -514,15 +511,15 @@ config SUN_OPENPROMFS OpenPROM settings on the running system. # Makefile helpers -config SPARC32_PCI +config SPARC64_PCI bool default y - depends on SPARC32 && PCI + depends on SPARC64 && PCI -config SPARC64_PCI +config SPARC64_PCI_MSI bool default y - depends on SPARC64 && PCI + depends on SPARC64_PCI && PCI_MSI endmenu @@ -535,12 +532,18 @@ config COMPAT depends on SPARC64 default y select COMPAT_BINFMT_ELF + select HAVE_UID16 + select ARCH_WANT_OLD_COMPAT_IPC + select COMPAT_OLD_SIGACTION config SYSVIPC_COMPAT bool depends on COMPAT && SYSVIPC default y +config KEYS_COMPAT + def_bool y if COMPAT && KEYS + endmenu source "net/Kconfig" diff --git a/arch/sparc/Kconfig.debug b/arch/sparc/Kconfig.debug index d9a795efbc0..6db35fba79f 100644 --- a/arch/sparc/Kconfig.debug +++ b/arch/sparc/Kconfig.debug @@ -6,15 +6,6 @@ config TRACE_IRQFLAGS_SUPPORT source "lib/Kconfig.debug" -config DEBUG_STACK_USAGE - bool "Enable stack utilization instrumentation" - depends on DEBUG_KERNEL - help - Enables the display of the minimum amount of free stack which each - task has ever had available in the sysrq-T and sysrq-P debug output. - - This option will slow down process creation somewhat. - config DEBUG_DCFLUSH bool "D-cache flush debugging" depends on SPARC64 && DEBUG_KERNEL diff --git a/arch/sparc/Makefile b/arch/sparc/Makefile index 113225b241e..9ff423678cb 100644 --- a/arch/sparc/Makefile +++ b/arch/sparc/Makefile @@ -19,39 +19,27 @@ ifeq ($(CONFIG_SPARC32),y) # sparc32 # -# -# Uncomment the first KBUILD_CFLAGS if you are doing kgdb source level -# debugging of the kernel to get the proper debugging information. - -AS := $(AS) -32 -LDFLAGS := -m elf32_sparc CHECKFLAGS += -D__sparc__ +LDFLAGS := -m elf32_sparc export BITS := 32 UTS_MACHINE := sparc -#KBUILD_CFLAGS += -g -pipe -fcall-used-g5 -fcall-used-g7 -KBUILD_CFLAGS += -m32 -pipe -mno-fpu -fcall-used-g5 -fcall-used-g7 -KBUILD_AFLAGS += -m32 - -#LDFLAGS_vmlinux = -N -Ttext 0xf0004000 -# Since 2.5.40, the first stage is left not btfix-ed. -# Actual linking is done with "make image". -LDFLAGS_vmlinux = -r +KBUILD_CFLAGS += -m32 -mcpu=v8 -pipe -mno-fpu -fcall-used-g5 -fcall-used-g7 +KBUILD_AFLAGS += -m32 -Wa,-Av8 else ##### # sparc64 # -CHECKFLAGS += -D__sparc__ -D__sparc_v9__ -D__arch64__ -m64 +CHECKFLAGS += -D__sparc__ -D__sparc_v9__ -D__arch64__ -m64 +LDFLAGS := -m elf64_sparc +export BITS := 64 +UTS_MACHINE := sparc64 -LDFLAGS := -m elf64_sparc -export BITS := 64 -UTS_MACHINE := sparc64 - -KBUILD_CFLAGS += -m64 -pipe -mno-fpu -mcpu=ultrasparc -mcmodel=medlow \ - -ffixed-g4 -ffixed-g5 -fcall-used-g7 -Wno-sign-compare \ - -Wa,--undeclared-regs +KBUILD_CFLAGS += -m64 -pipe -mno-fpu -mcpu=ultrasparc -mcmodel=medlow +KBUILD_CFLAGS += -ffixed-g4 -ffixed-g5 -fcall-used-g7 -Wno-sign-compare +KBUILD_CFLAGS += -Wa,--undeclared-regs KBUILD_CFLAGS += $(call cc-option,-mtune=ultrasparc3) KBUILD_AFLAGS += -m64 -mcpu=ultrasparc -Wa,--undeclared-regs @@ -62,33 +50,22 @@ endif endif head-y := arch/sparc/kernel/head_$(BITS).o -head-y += arch/sparc/kernel/init_task.o -core-y += arch/sparc/kernel/ -core-y += arch/sparc/mm/ arch/sparc/math-emu/ +# See arch/sparc/Kbuild for the core part of the kernel +core-y += arch/sparc/ libs-y += arch/sparc/prom/ libs-y += arch/sparc/lib/ +drivers-$(CONFIG_PM) += arch/sparc/power/ drivers-$(CONFIG_OPROFILE) += arch/sparc/oprofile/ -# Export what is needed by arch/sparc/boot/Makefile -export VMLINUX_INIT VMLINUX_MAIN -VMLINUX_INIT := $(head-y) $(init-y) -VMLINUX_MAIN := $(core-y) kernel/ mm/ fs/ ipc/ security/ crypto/ block/ -VMLINUX_MAIN += $(patsubst %/, %/lib.a, $(libs-y)) $(libs-y) -VMLINUX_MAIN += $(drivers-y) $(net-y) - -ifdef CONFIG_KALLSYMS -export kallsyms.o := .tmp_kallsyms2.o -endif - boot := arch/sparc/boot # Default target all: zImage -image zImage tftpboot.img vmlinux.aout: vmlinux +image zImage uImage tftpboot.img vmlinux.aout: vmlinux $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ archclean: @@ -102,6 +79,7 @@ ifeq ($(ARCH),sparc) define archhelp echo '* image - kernel image ($(boot)/image)' echo '* zImage - stripped kernel image ($(boot)/zImage)' + echo ' uImage - U-Boot SPARC32 Image (only for LEON)' echo ' tftpboot.img - image prepared for tftp' endef else diff --git a/arch/sparc/boot/Makefile b/arch/sparc/boot/Makefile index a2c5898c1ab..6e63afb128d 100644 --- a/arch/sparc/boot/Makefile +++ b/arch/sparc/boot/Makefile @@ -6,8 +6,8 @@ ROOT_IMG := /usr/src/root.img ELFTOAOUT := elftoaout -hostprogs-y := piggyback btfixupprep -targets := tftpboot.img btfix.o btfix.S image zImage vmlinux.aout +hostprogs-y := piggyback +targets := tftpboot.img image zImage vmlinux.aout clean-files := System.map quiet_cmd_elftoaout = ELFTOAOUT $@ @@ -17,58 +17,9 @@ quiet_cmd_piggy = PIGGY $@ quiet_cmd_strip = STRIP $@ cmd_strip = $(STRIP) -R .comment -R .note -K sun4u_init -K _end -K _start $< -o $@ -ifeq ($(CONFIG_SPARC32),y) -quiet_cmd_btfix = BTFIX $@ - cmd_btfix = $(OBJDUMP) -x vmlinux | $(obj)/btfixupprep > $@ -quiet_cmd_sysmap = SYSMAP $(obj)/System.map - cmd_sysmap = $(CONFIG_SHELL) $(srctree)/scripts/mksysmap -quiet_cmd_image = LD $@ - cmd_image = $(LD) $(LDFLAGS) $(EXTRA_LDFLAGS) $(LDFLAGS_$(@F)) -o $@ - -define rule_image - $(if $($(quiet)cmd_image), \ - echo ' $($(quiet)cmd_image)' &&) \ - $(cmd_image); \ - $(if $($(quiet)cmd_sysmap), \ - echo ' $($(quiet)cmd_sysmap)' &&) \ - $(cmd_sysmap) $@ $(obj)/System.map; \ - if [ $$? -ne 0 ]; then \ - rm -f $@; \ - /bin/false; \ - fi; \ - echo 'cmd_$@ := $(cmd_image)' > $(@D)/.$(@F).cmd -endef - -BTOBJS := $(patsubst %/, %/built-in.o, $(VMLINUX_INIT)) -BTLIBS := $(patsubst %/, %/built-in.o, $(VMLINUX_MAIN)) -LDFLAGS_image := -T arch/sparc/kernel/vmlinux.lds $(BTOBJS) \ - --start-group $(BTLIBS) --end-group \ - $(kallsyms.o) $(obj)/btfix.o - -# Link the final image including btfixup'ed symbols. -# This is a replacement for the link done in the top-level Makefile. -# Note: No dependency on the prerequisite files since that would require -# make to try check if they are updated - and due to changes -# in gcc options (path for example) this would result in -# these files being recompiled for each build. -$(obj)/image: $(obj)/btfix.o FORCE - $(call if_changed_rule,image) - -$(obj)/zImage: $(obj)/image - $(call if_changed,strip) - @echo ' kernel: $@ is ready' - -$(obj)/btfix.S: $(obj)/btfixupprep vmlinux FORCE - $(call if_changed,btfix) - -endif - ifeq ($(CONFIG_SPARC64),y) # Actual linking -$(obj)/image: vmlinux FORCE - $(call if_changed,strip) - @echo ' kernel: $@ is ready' $(obj)/zImage: $(obj)/image $(call if_changed,gzip) @@ -77,8 +28,44 @@ $(obj)/zImage: $(obj)/image $(obj)/vmlinux.aout: vmlinux FORCE $(call if_changed,elftoaout) @echo ' kernel: $@ is ready' +else + +$(obj)/zImage: $(obj)/image + $(call if_changed,strip) + @echo ' kernel: $@ is ready' + +# The following lines make a readable image for U-Boot. +# uImage - Binary file read by U-boot +# uImage.o - object file of uImage for loading with a +# flash programmer understanding ELF. + +OBJCOPYFLAGS_image.bin := -S -O binary -R .note -R .comment +$(obj)/image.bin: $(obj)/image FORCE + $(call if_changed,objcopy) + +$(obj)/image.gz: $(obj)/image.bin + $(call if_changed,gzip) + +UIMAGE_LOADADDR = $(CONFIG_UBOOT_LOAD_ADDR) +UIMAGE_ENTRYADDR = $(CONFIG_UBOOT_ENTRY_ADDR) +UIMAGE_COMPRESSION = gzip + +quiet_cmd_uimage.o = UIMAGE.O $@ + cmd_uimage.o = $(LD) -Tdata $(CONFIG_UBOOT_FLASH_ADDR) \ + -r -b binary $@ -o $@.o + +targets += uImage +$(obj)/uImage: $(obj)/image.gz + $(call if_changed,uimage) + $(call if_changed,uimage.o) + @echo ' Image $@ is ready' + endif +$(obj)/image: vmlinux FORCE + $(call if_changed,strip) + @echo ' kernel: $@ is ready' + $(obj)/tftpboot.img: $(obj)/image $(obj)/piggyback System.map $(ROOT_IMG) FORCE $(call if_changed,elftoaout) $(call if_changed,piggy) diff --git a/arch/sparc/boot/btfixupprep.c b/arch/sparc/boot/btfixupprep.c deleted file mode 100644 index da031159e2b..00000000000 --- a/arch/sparc/boot/btfixupprep.c +++ /dev/null @@ -1,386 +0,0 @@ -/* - Simple utility to prepare vmlinux image for sparc. - Resolves all BTFIXUP uses and settings and creates - a special .s object to link to the image. - - Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ - -#include <stdio.h> -#include <string.h> -#include <ctype.h> -#include <errno.h> -#include <unistd.h> -#include <stdlib.h> -#include <malloc.h> - -#define MAXSYMS 1024 - -static char *symtab = "SYMBOL TABLE:"; -static char *relrec = "RELOCATION RECORDS FOR ["; -static int rellen; -static int symlen; -int mode; - -struct _btfixup; - -typedef struct _btfixuprel { - char *sect; - unsigned long offset; - struct _btfixup *f; - int frel; - struct _btfixuprel *next; -} btfixuprel; - -typedef struct _btfixup { - int type; - int setinitval; - unsigned int initval; - char *initvalstr; - char *name; - btfixuprel *rel; -} btfixup; - -btfixup array[MAXSYMS]; -int last = 0; -char buffer[1024]; -unsigned long lastfoffset = -1; -unsigned long lastfrelno; -btfixup *lastf; - -static void fatal(void) __attribute__((noreturn)); -static void fatal(void) -{ - fprintf(stderr, "Malformed output from objdump\n%s\n", buffer); - exit(1); -} - -static btfixup *find(int type, char *name) -{ - int i; - for (i = 0; i < last; i++) { - if (array[i].type == type && !strcmp(array[i].name, name)) - return array + i; - } - array[last].type = type; - array[last].name = strdup(name); - array[last].setinitval = 0; - if (!array[last].name) fatal(); - array[last].rel = NULL; - last++; - if (last >= MAXSYMS) { - fprintf(stderr, "Ugh. Something strange. More than %d different BTFIXUP symbols\n", MAXSYMS); - exit(1); - } - return array + last - 1; -} - -static void set_mode (char *buffer) -{ - for (mode = 0;; mode++) - if (buffer[mode] < '0' || buffer[mode] > '9') - break; - if (mode != 8 && mode != 16) - fatal(); -} - - -int main(int argc,char **argv) -{ - char *p, *q; - char *sect; - int i, j, k; - unsigned int initval; - int shift; - btfixup *f; - btfixuprel *r, **rr; - unsigned long offset; - char *initvalstr; - - symlen = strlen(symtab); - while (fgets (buffer, 1024, stdin) != NULL) - if (!strncmp (buffer, symtab, symlen)) - goto main0; - fatal(); -main0: - rellen = strlen(relrec); - while (fgets (buffer, 1024, stdin) != NULL) - if (!strncmp (buffer, relrec, rellen)) - goto main1; - fatal(); -main1: - sect = malloc(strlen (buffer + rellen) + 1); - if (!sect) fatal(); - strcpy (sect, buffer + rellen); - p = strchr (sect, ']'); - if (!p) fatal(); - *p = 0; - if (fgets (buffer, 1024, stdin) == NULL) - fatal(); - while (fgets (buffer, 1024, stdin) != NULL) { - int nbase; - if (!strncmp (buffer, relrec, rellen)) - goto main1; - if (mode == 0) - set_mode (buffer); - p = strchr (buffer, '\n'); - if (p) *p = 0; - if (strlen (buffer) < 22+mode) - continue; - if (strncmp (buffer + mode, " R_SPARC_", 9)) - continue; - nbase = 27 - 8 + mode; - if (buffer[nbase] != '_' || buffer[nbase+1] != '_' || buffer[nbase+2] != '_') - continue; - switch (buffer[nbase+3]) { - case 'f': /* CALL */ - case 'b': /* BLACKBOX */ - case 's': /* SIMM13 */ - case 'a': /* HALF */ - case 'h': /* SETHI */ - case 'i': /* INT */ - break; - default: - continue; - } - p = strchr (buffer + nbase+5, '+'); - if (p) *p = 0; - shift = nbase + 5; - if (buffer[nbase+4] == 's' && buffer[nbase+5] == '_') { - shift = nbase + 6; - if (strcmp (sect, ".init.text")) { - fprintf(stderr, - "Wrong use of '%s' BTFIXUPSET in '%s' section.\n" - "BTFIXUPSET_CALL can be used only in" - " __init sections\n", - buffer + shift, sect); - exit(1); - } - } else if (buffer[nbase+4] != '_') - continue; - if (!strcmp (sect, ".text.exit")) - continue; - if (strcmp (sect, ".text") && - strcmp (sect, ".init.text") && - strcmp (sect, ".fixup") && - (strcmp (sect, "__ksymtab") || buffer[nbase+3] != 'f')) { - if (buffer[nbase+3] == 'f') - fprintf(stderr, - "Wrong use of '%s' in '%s' section.\n" - " It can be used only in .text, .init.text," - " .fixup and __ksymtab\n", - buffer + shift, sect); - else - fprintf(stderr, - "Wrong use of '%s' in '%s' section.\n" - " It can be only used in .text, .init.text," - " and .fixup\n", buffer + shift, sect); - exit(1); - } - p = strstr (buffer + shift, "__btset_"); - if (p && buffer[nbase+4] == 's') { - fprintf(stderr, "__btset_ in BTFIXUP name can only be used when defining the variable, not for setting\n%s\n", buffer); - exit(1); - } - initval = 0; - initvalstr = NULL; - if (p) { - if (p[8] != '0' || p[9] != 'x') { - fprintf(stderr, "Pre-initialized values can be only initialized with hexadecimal constants starting 0x\n%s\n", buffer); - exit(1); - } - initval = strtoul(p + 10, &q, 16); - if (*q || !initval) { - fprintf(stderr, "Pre-initialized values can be only in the form name__btset_0xXXXXXXXX where X are hex digits.\nThey cannot be name__btset_0x00000000 though. Use BTFIXUPDEF_XX instead of BTFIXUPDEF_XX_INIT then.\n%s\n", buffer); - exit(1); - } - initvalstr = p + 10; - *p = 0; - } - f = find(buffer[nbase+3], buffer + shift); - if (buffer[nbase+4] == 's') - continue; - switch (buffer[nbase+3]) { - case 'f': - if (initval) { - fprintf(stderr, "Cannot use pre-initialized fixups for calls\n%s\n", buffer); - exit(1); - } - if (!strcmp (sect, "__ksymtab")) { - if (strncmp (buffer + mode+9, "32 ", 10)) { - fprintf(stderr, "BTFIXUP_CALL in EXPORT_SYMBOL results in relocation other than R_SPARC_32\n\%s\n", buffer); - exit(1); - } - } else if (strncmp (buffer + mode+9, "WDISP30 ", 10) && - strncmp (buffer + mode+9, "HI22 ", 10) && - strncmp (buffer + mode+9, "LO10 ", 10)) { - fprintf(stderr, "BTFIXUP_CALL results in relocation other than R_SPARC_WDISP30, R_SPARC_HI22 or R_SPARC_LO10\n%s\n", buffer); - exit(1); - } - break; - case 'b': - if (initval) { - fprintf(stderr, "Cannot use pre-initialized fixups for blackboxes\n%s\n", buffer); - exit(1); - } - if (strncmp (buffer + mode+9, "HI22 ", 10)) { - fprintf(stderr, "BTFIXUP_BLACKBOX results in relocation other than R_SPARC_HI22\n%s\n", buffer); - exit(1); - } - break; - case 's': - if (initval + 0x1000 >= 0x2000) { - fprintf(stderr, "Wrong initializer for SIMM13. Has to be from $fffff000 to $00000fff\n%s\n", buffer); - exit(1); - } - if (strncmp (buffer + mode+9, "13 ", 10)) { - fprintf(stderr, "BTFIXUP_SIMM13 results in relocation other than R_SPARC_13\n%s\n", buffer); - exit(1); - } - break; - case 'a': - if (initval + 0x1000 >= 0x2000 && (initval & 0x3ff)) { - fprintf(stderr, "Wrong initializer for HALF.\n%s\n", buffer); - exit(1); - } - if (strncmp (buffer + mode+9, "13 ", 10)) { - fprintf(stderr, "BTFIXUP_HALF results in relocation other than R_SPARC_13\n%s\n", buffer); - exit(1); - } - break; - case 'h': - if (initval & 0x3ff) { - fprintf(stderr, "Wrong initializer for SETHI. Cannot have set low 10 bits\n%s\n", buffer); - exit(1); - } - if (strncmp (buffer + mode+9, "HI22 ", 10)) { - fprintf(stderr, "BTFIXUP_SETHI results in relocation other than R_SPARC_HI22\n%s\n", buffer); - exit(1); - } - break; - case 'i': - if (initval) { - fprintf(stderr, "Cannot use pre-initialized fixups for INT\n%s\n", buffer); - exit(1); - } - if (strncmp (buffer + mode+9, "HI22 ", 10) && strncmp (buffer + mode+9, "LO10 ", 10)) { - fprintf(stderr, "BTFIXUP_INT results in relocation other than R_SPARC_HI22 and R_SPARC_LO10\n%s\n", buffer); - exit(1); - } - break; - } - if (!f->setinitval) { - f->initval = initval; - if (initvalstr) { - f->initvalstr = strdup(initvalstr); - if (!f->initvalstr) fatal(); - } - f->setinitval = 1; - } else if (f->initval != initval) { - fprintf(stderr, "Btfixup %s previously used with initializer %s which doesn't match with current initializer\n%s\n", - f->name, f->initvalstr ? : "0x00000000", buffer); - exit(1); - } else if (initval && strcmp(f->initvalstr, initvalstr)) { - fprintf(stderr, "Btfixup %s previously used with initializer %s which doesn't match with current initializer.\n" - "Initializers have to match literally as well.\n%s\n", - f->name, f->initvalstr, buffer); - exit(1); - } - offset = strtoul(buffer, &q, 16); - if (q != buffer + mode || (!offset && (mode == 8 ? strncmp (buffer, "00000000 ", 9) : strncmp (buffer, "0000000000000000 ", 17)))) { - fprintf(stderr, "Malformed relocation address in\n%s\n", buffer); - exit(1); - } - for (k = 0, r = f->rel, rr = &f->rel; r; rr = &r->next, r = r->next, k++) - if (r->offset == offset && !strcmp(r->sect, sect)) { - fprintf(stderr, "Ugh. One address has two relocation records\n"); - exit(1); - } - *rr = malloc(sizeof(btfixuprel)); - if (!*rr) fatal(); - (*rr)->offset = offset; - (*rr)->f = NULL; - if (buffer[nbase+3] == 'f') { - lastf = f; - lastfoffset = offset; - lastfrelno = k; - } else if (lastfoffset + 4 == offset) { - (*rr)->f = lastf; - (*rr)->frel = lastfrelno; - } - (*rr)->sect = sect; - (*rr)->next = NULL; - } - printf("! Generated by btfixupprep. Do not edit.\n\n"); - printf("\t.section\t\".data..init\",#alloc,#write\n\t.align\t4\n\n"); - printf("\t.global\t___btfixup_start\n___btfixup_start:\n\n"); - for (i = 0; i < last; i++) { - f = array + i; - printf("\t.global\t___%cs_%s\n", f->type, f->name); - if (f->type == 'f') - printf("___%cs_%s:\n\t.word 0x%08x,0,0,", f->type, f->name, f->type << 24); - else - printf("___%cs_%s:\n\t.word 0x%08x,0,", f->type, f->name, f->type << 24); - for (j = 0, r = f->rel; r != NULL; j++, r = r->next); - if (j) - printf("%d\n\t.word\t", j * 2); - else - printf("0\n"); - for (r = f->rel, j--; r != NULL; j--, r = r->next) { - if (!strcmp (r->sect, ".text")) - printf ("_stext+0x%08lx", r->offset); - else if (!strcmp (r->sect, ".init.text")) - printf ("__init_begin+0x%08lx", r->offset); - else if (!strcmp (r->sect, "__ksymtab")) - printf ("__start___ksymtab+0x%08lx", r->offset); - else if (!strcmp (r->sect, ".fixup")) - printf ("__start___fixup+0x%08lx", r->offset); - else - fatal(); - if (f->type == 'f' || !r->f) - printf (",0"); - else - printf (",___fs_%s+0x%08x", r->f->name, (4 + r->frel*2)*4 + 4); - if (j) printf (","); - else printf ("\n"); - } - printf("\n"); - } - printf("\n\t.global\t___btfixup_end\n___btfixup_end:\n"); - printf("\n\n! Define undefined references\n\n"); - for (i = 0; i < last; i++) { - f = array + i; - if (f->type == 'f') { - printf("\t.global\t___f_%s\n", f->name); - printf("___f_%s:\n", f->name); - } - } - printf("\tretl\n\t nop\n\n"); - for (i = 0; i < last; i++) { - f = array + i; - if (f->type != 'f') { - if (!f->initval) { - printf("\t.global\t___%c_%s\n", f->type, f->name); - printf("___%c_%s = 0\n", f->type, f->name); - } else { - printf("\t.global\t___%c_%s__btset_0x%s\n", f->type, f->name, f->initvalstr); - printf("___%c_%s__btset_0x%s = 0x%08x\n", f->type, f->name, f->initvalstr, f->initval); - } - } - } - printf("\n\n"); - exit(0); -} diff --git a/arch/sparc/boot/piggyback.c b/arch/sparc/boot/piggyback.c index c0a798fcf03..bb7c95161d7 100644 --- a/arch/sparc/boot/piggyback.c +++ b/arch/sparc/boot/piggyback.c @@ -81,18 +81,18 @@ static void usage(void) static int start_line(const char *line) { - if (strcmp(line + 8, " T _start\n") == 0) + if (strcmp(line + 10, " _start\n") == 0) return 1; - else if (strcmp(line + 16, " T _start\n") == 0) + else if (strcmp(line + 18, " _start\n") == 0) return 1; return 0; } static int end_line(const char *line) { - if (strcmp(line + 8, " A _end\n") == 0) + if (strcmp(line + 10, " _end\n") == 0) return 1; - else if (strcmp (line + 16, " A _end\n") == 0) + else if (strcmp (line + 18, " _end\n") == 0) return 1; return 0; } @@ -100,8 +100,8 @@ static int end_line(const char *line) /* * Find address for start and end in System.map. * The file looks like this: - * f0004000 T _start - * f0379f79 A _end + * f0004000 ... _start + * f0379f79 ... _end * 1234567890123456 * ^coloumn 1 * There is support for 64 bit addresses too. diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index 3c1e8580740..9d8521b8c85 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -5,7 +5,7 @@ CONFIG_SYSVIPC=y CONFIG_POSIX_MQUEUE=y CONFIG_LOG_BUF_SHIFT=18 CONFIG_BLK_DEV_INITRD=y -CONFIG_PERF_COUNTERS=y +CONFIG_PERF_EVENTS=y # CONFIG_COMPAT_BRK is not set CONFIG_SLAB=y CONFIG_PROFILING=y diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile new file mode 100644 index 00000000000..5d469d81761 --- /dev/null +++ b/arch/sparc/crypto/Makefile @@ -0,0 +1,25 @@ +# +# Arch-specific CryptoAPI modules. +# + +obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o +obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o +obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o +obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o + +obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o +obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o +obj-$(CONFIG_CRYPTO_DES_SPARC64) += camellia-sparc64.o + +obj-$(CONFIG_CRYPTO_CRC32C_SPARC64) += crc32c-sparc64.o + +sha1-sparc64-y := sha1_asm.o sha1_glue.o +sha256-sparc64-y := sha256_asm.o sha256_glue.o +sha512-sparc64-y := sha512_asm.o sha512_glue.o +md5-sparc64-y := md5_asm.o md5_glue.o + +aes-sparc64-y := aes_asm.o aes_glue.o +des-sparc64-y := des_asm.o des_glue.o +camellia-sparc64-y := camellia_asm.o camellia_glue.o + +crc32c-sparc64-y := crc32c_asm.o crc32c_glue.o diff --git a/arch/sparc/crypto/aes_asm.S b/arch/sparc/crypto/aes_asm.S new file mode 100644 index 00000000000..1cda8aa7cb8 --- /dev/null +++ b/arch/sparc/crypto/aes_asm.S @@ -0,0 +1,1543 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +#define ENCRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \ + AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ + AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ + AES_EROUND01(KEY_BASE + 4, T0, T1, I0) \ + AES_EROUND23(KEY_BASE + 6, T0, T1, I1) + +#define ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ + AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ + AES_EROUND01(KEY_BASE + 0, I2, I3, T2) \ + AES_EROUND23(KEY_BASE + 2, I2, I3, T3) \ + AES_EROUND01(KEY_BASE + 4, T0, T1, I0) \ + AES_EROUND23(KEY_BASE + 6, T0, T1, I1) \ + AES_EROUND01(KEY_BASE + 4, T2, T3, I2) \ + AES_EROUND23(KEY_BASE + 6, T2, T3, I3) + +#define ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \ + AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ + AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ + AES_EROUND01_L(KEY_BASE + 4, T0, T1, I0) \ + AES_EROUND23_L(KEY_BASE + 6, T0, T1, I1) + +#define ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + AES_EROUND01(KEY_BASE + 0, I0, I1, T0) \ + AES_EROUND23(KEY_BASE + 2, I0, I1, T1) \ + AES_EROUND01(KEY_BASE + 0, I2, I3, T2) \ + AES_EROUND23(KEY_BASE + 2, I2, I3, T3) \ + AES_EROUND01_L(KEY_BASE + 4, T0, T1, I0) \ + AES_EROUND23_L(KEY_BASE + 6, T0, T1, I1) \ + AES_EROUND01_L(KEY_BASE + 4, T2, T3, I2) \ + AES_EROUND23_L(KEY_BASE + 6, T2, T3, I3) + + /* 10 rounds */ +#define ENCRYPT_128(KEY_BASE, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1) + +#define ENCRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) + + /* 12 rounds */ +#define ENCRYPT_192(KEY_BASE, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1) + +#define ENCRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \ + ENCRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3) + + /* 14 rounds */ +#define ENCRYPT_256(KEY_BASE, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \ + ENCRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1) + +#define ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \ + ENCRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \ + TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6) + +#define ENCRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, KEY_BASE + 48) \ + ldd [%o0 + 0xd0], %f56; \ + ldd [%o0 + 0xd8], %f58; \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, KEY_BASE + 0) \ + ldd [%o0 + 0xe0], %f60; \ + ldd [%o0 + 0xe8], %f62; \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE + 0) \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE + 0) \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE + 0) \ + ENCRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE + 0) \ + AES_EROUND01(KEY_BASE + 48, I0, I1, KEY_BASE + 0) \ + AES_EROUND23(KEY_BASE + 50, I0, I1, KEY_BASE + 2) \ + AES_EROUND01(KEY_BASE + 48, I2, I3, KEY_BASE + 4) \ + AES_EROUND23(KEY_BASE + 50, I2, I3, KEY_BASE + 6) \ + AES_EROUND01_L(KEY_BASE + 52, KEY_BASE + 0, KEY_BASE + 2, I0) \ + AES_EROUND23_L(KEY_BASE + 54, KEY_BASE + 0, KEY_BASE + 2, I1) \ + ldd [%o0 + 0x10], %f8; \ + ldd [%o0 + 0x18], %f10; \ + AES_EROUND01_L(KEY_BASE + 52, KEY_BASE + 4, KEY_BASE + 6, I2) \ + AES_EROUND23_L(KEY_BASE + 54, KEY_BASE + 4, KEY_BASE + 6, I3) \ + ldd [%o0 + 0x20], %f12; \ + ldd [%o0 + 0x28], %f14; + +#define DECRYPT_TWO_ROUNDS(KEY_BASE, I0, I1, T0, T1) \ + AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \ + AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \ + AES_DROUND23(KEY_BASE + 4, T0, T1, I1) \ + AES_DROUND01(KEY_BASE + 6, T0, T1, I0) + +#define DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \ + AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \ + AES_DROUND23(KEY_BASE + 0, I2, I3, T3) \ + AES_DROUND01(KEY_BASE + 2, I2, I3, T2) \ + AES_DROUND23(KEY_BASE + 4, T0, T1, I1) \ + AES_DROUND01(KEY_BASE + 6, T0, T1, I0) \ + AES_DROUND23(KEY_BASE + 4, T2, T3, I3) \ + AES_DROUND01(KEY_BASE + 6, T2, T3, I2) + +#define DECRYPT_TWO_ROUNDS_LAST(KEY_BASE, I0, I1, T0, T1) \ + AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \ + AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \ + AES_DROUND23_L(KEY_BASE + 4, T0, T1, I1) \ + AES_DROUND01_L(KEY_BASE + 6, T0, T1, I0) + +#define DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + AES_DROUND23(KEY_BASE + 0, I0, I1, T1) \ + AES_DROUND01(KEY_BASE + 2, I0, I1, T0) \ + AES_DROUND23(KEY_BASE + 0, I2, I3, T3) \ + AES_DROUND01(KEY_BASE + 2, I2, I3, T2) \ + AES_DROUND23_L(KEY_BASE + 4, T0, T1, I1) \ + AES_DROUND01_L(KEY_BASE + 6, T0, T1, I0) \ + AES_DROUND23_L(KEY_BASE + 4, T2, T3, I3) \ + AES_DROUND01_L(KEY_BASE + 6, T2, T3, I2) + + /* 10 rounds */ +#define DECRYPT_128(KEY_BASE, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 32, I0, I1, T0, T1) + +#define DECRYPT_128_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) + + /* 12 rounds */ +#define DECRYPT_192(KEY_BASE, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 40, I0, I1, T0, T1) + +#define DECRYPT_192_2(KEY_BASE, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, T0, T1, T2, T3) \ + DECRYPT_TWO_ROUNDS_LAST_2(KEY_BASE + 40, I0, I1, I2, I3, T0, T1, T2, T3) + + /* 14 rounds */ +#define DECRYPT_256(KEY_BASE, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 0, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 8, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 16, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 24, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 32, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS(KEY_BASE + 40, I0, I1, T0, T1) \ + DECRYPT_TWO_ROUNDS_LAST(KEY_BASE + 48, I0, I1, T0, T1) + +#define DECRYPT_256_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, TMP_BASE) \ + DECRYPT_TWO_ROUNDS_2(KEY_BASE, I0, I1, I2, I3, \ + TMP_BASE + 0, TMP_BASE + 2, TMP_BASE + 4, TMP_BASE + 6) + +#define DECRYPT_256_2(KEY_BASE, I0, I1, I2, I3) \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 0, I0, I1, I2, I3, KEY_BASE + 48) \ + ldd [%o0 + 0x18], %f56; \ + ldd [%o0 + 0x10], %f58; \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 8, I0, I1, I2, I3, KEY_BASE + 0) \ + ldd [%o0 + 0x08], %f60; \ + ldd [%o0 + 0x00], %f62; \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 16, I0, I1, I2, I3, KEY_BASE + 0) \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 24, I0, I1, I2, I3, KEY_BASE + 0) \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 32, I0, I1, I2, I3, KEY_BASE + 0) \ + DECRYPT_256_TWO_ROUNDS_2(KEY_BASE + 40, I0, I1, I2, I3, KEY_BASE + 0) \ + AES_DROUND23(KEY_BASE + 48, I0, I1, KEY_BASE + 2) \ + AES_DROUND01(KEY_BASE + 50, I0, I1, KEY_BASE + 0) \ + AES_DROUND23(KEY_BASE + 48, I2, I3, KEY_BASE + 6) \ + AES_DROUND01(KEY_BASE + 50, I2, I3, KEY_BASE + 4) \ + AES_DROUND23_L(KEY_BASE + 52, KEY_BASE + 0, KEY_BASE + 2, I1) \ + AES_DROUND01_L(KEY_BASE + 54, KEY_BASE + 0, KEY_BASE + 2, I0) \ + ldd [%o0 + 0xd8], %f8; \ + ldd [%o0 + 0xd0], %f10; \ + AES_DROUND23_L(KEY_BASE + 52, KEY_BASE + 4, KEY_BASE + 6, I3) \ + AES_DROUND01_L(KEY_BASE + 54, KEY_BASE + 4, KEY_BASE + 6, I2) \ + ldd [%o0 + 0xc8], %f12; \ + ldd [%o0 + 0xc0], %f14; + + .align 32 +ENTRY(aes_sparc64_key_expand) + /* %o0=input_key, %o1=output_key, %o2=key_len */ + VISEntry + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + ld [%o0 + 0x08], %f2 + ld [%o0 + 0x0c], %f3 + + std %f0, [%o1 + 0x00] + std %f2, [%o1 + 0x08] + add %o1, 0x10, %o1 + + cmp %o2, 24 + bl 2f + nop + + be 1f + nop + + /* 256-bit key expansion */ + ld [%o0 + 0x10], %f4 + ld [%o0 + 0x14], %f5 + ld [%o0 + 0x18], %f6 + ld [%o0 + 0x1c], %f7 + + std %f4, [%o1 + 0x00] + std %f6, [%o1 + 0x08] + add %o1, 0x10, %o1 + + AES_KEXPAND1(0, 6, 0x0, 8) + AES_KEXPAND2(2, 8, 10) + AES_KEXPAND0(4, 10, 12) + AES_KEXPAND2(6, 12, 14) + AES_KEXPAND1(8, 14, 0x1, 16) + AES_KEXPAND2(10, 16, 18) + AES_KEXPAND0(12, 18, 20) + AES_KEXPAND2(14, 20, 22) + AES_KEXPAND1(16, 22, 0x2, 24) + AES_KEXPAND2(18, 24, 26) + AES_KEXPAND0(20, 26, 28) + AES_KEXPAND2(22, 28, 30) + AES_KEXPAND1(24, 30, 0x3, 32) + AES_KEXPAND2(26, 32, 34) + AES_KEXPAND0(28, 34, 36) + AES_KEXPAND2(30, 36, 38) + AES_KEXPAND1(32, 38, 0x4, 40) + AES_KEXPAND2(34, 40, 42) + AES_KEXPAND0(36, 42, 44) + AES_KEXPAND2(38, 44, 46) + AES_KEXPAND1(40, 46, 0x5, 48) + AES_KEXPAND2(42, 48, 50) + AES_KEXPAND0(44, 50, 52) + AES_KEXPAND2(46, 52, 54) + AES_KEXPAND1(48, 54, 0x6, 56) + AES_KEXPAND2(50, 56, 58) + + std %f8, [%o1 + 0x00] + std %f10, [%o1 + 0x08] + std %f12, [%o1 + 0x10] + std %f14, [%o1 + 0x18] + std %f16, [%o1 + 0x20] + std %f18, [%o1 + 0x28] + std %f20, [%o1 + 0x30] + std %f22, [%o1 + 0x38] + std %f24, [%o1 + 0x40] + std %f26, [%o1 + 0x48] + std %f28, [%o1 + 0x50] + std %f30, [%o1 + 0x58] + std %f32, [%o1 + 0x60] + std %f34, [%o1 + 0x68] + std %f36, [%o1 + 0x70] + std %f38, [%o1 + 0x78] + std %f40, [%o1 + 0x80] + std %f42, [%o1 + 0x88] + std %f44, [%o1 + 0x90] + std %f46, [%o1 + 0x98] + std %f48, [%o1 + 0xa0] + std %f50, [%o1 + 0xa8] + std %f52, [%o1 + 0xb0] + std %f54, [%o1 + 0xb8] + std %f56, [%o1 + 0xc0] + ba,pt %xcc, 80f + std %f58, [%o1 + 0xc8] + +1: + /* 192-bit key expansion */ + ld [%o0 + 0x10], %f4 + ld [%o0 + 0x14], %f5 + + std %f4, [%o1 + 0x00] + add %o1, 0x08, %o1 + + AES_KEXPAND1(0, 4, 0x0, 6) + AES_KEXPAND2(2, 6, 8) + AES_KEXPAND2(4, 8, 10) + AES_KEXPAND1(6, 10, 0x1, 12) + AES_KEXPAND2(8, 12, 14) + AES_KEXPAND2(10, 14, 16) + AES_KEXPAND1(12, 16, 0x2, 18) + AES_KEXPAND2(14, 18, 20) + AES_KEXPAND2(16, 20, 22) + AES_KEXPAND1(18, 22, 0x3, 24) + AES_KEXPAND2(20, 24, 26) + AES_KEXPAND2(22, 26, 28) + AES_KEXPAND1(24, 28, 0x4, 30) + AES_KEXPAND2(26, 30, 32) + AES_KEXPAND2(28, 32, 34) + AES_KEXPAND1(30, 34, 0x5, 36) + AES_KEXPAND2(32, 36, 38) + AES_KEXPAND2(34, 38, 40) + AES_KEXPAND1(36, 40, 0x6, 42) + AES_KEXPAND2(38, 42, 44) + AES_KEXPAND2(40, 44, 46) + AES_KEXPAND1(42, 46, 0x7, 48) + AES_KEXPAND2(44, 48, 50) + + std %f6, [%o1 + 0x00] + std %f8, [%o1 + 0x08] + std %f10, [%o1 + 0x10] + std %f12, [%o1 + 0x18] + std %f14, [%o1 + 0x20] + std %f16, [%o1 + 0x28] + std %f18, [%o1 + 0x30] + std %f20, [%o1 + 0x38] + std %f22, [%o1 + 0x40] + std %f24, [%o1 + 0x48] + std %f26, [%o1 + 0x50] + std %f28, [%o1 + 0x58] + std %f30, [%o1 + 0x60] + std %f32, [%o1 + 0x68] + std %f34, [%o1 + 0x70] + std %f36, [%o1 + 0x78] + std %f38, [%o1 + 0x80] + std %f40, [%o1 + 0x88] + std %f42, [%o1 + 0x90] + std %f44, [%o1 + 0x98] + std %f46, [%o1 + 0xa0] + std %f48, [%o1 + 0xa8] + ba,pt %xcc, 80f + std %f50, [%o1 + 0xb0] + +2: + /* 128-bit key expansion */ + AES_KEXPAND1(0, 2, 0x0, 4) + AES_KEXPAND2(2, 4, 6) + AES_KEXPAND1(4, 6, 0x1, 8) + AES_KEXPAND2(6, 8, 10) + AES_KEXPAND1(8, 10, 0x2, 12) + AES_KEXPAND2(10, 12, 14) + AES_KEXPAND1(12, 14, 0x3, 16) + AES_KEXPAND2(14, 16, 18) + AES_KEXPAND1(16, 18, 0x4, 20) + AES_KEXPAND2(18, 20, 22) + AES_KEXPAND1(20, 22, 0x5, 24) + AES_KEXPAND2(22, 24, 26) + AES_KEXPAND1(24, 26, 0x6, 28) + AES_KEXPAND2(26, 28, 30) + AES_KEXPAND1(28, 30, 0x7, 32) + AES_KEXPAND2(30, 32, 34) + AES_KEXPAND1(32, 34, 0x8, 36) + AES_KEXPAND2(34, 36, 38) + AES_KEXPAND1(36, 38, 0x9, 40) + AES_KEXPAND2(38, 40, 42) + + std %f4, [%o1 + 0x00] + std %f6, [%o1 + 0x08] + std %f8, [%o1 + 0x10] + std %f10, [%o1 + 0x18] + std %f12, [%o1 + 0x20] + std %f14, [%o1 + 0x28] + std %f16, [%o1 + 0x30] + std %f18, [%o1 + 0x38] + std %f20, [%o1 + 0x40] + std %f22, [%o1 + 0x48] + std %f24, [%o1 + 0x50] + std %f26, [%o1 + 0x58] + std %f28, [%o1 + 0x60] + std %f30, [%o1 + 0x68] + std %f32, [%o1 + 0x70] + std %f34, [%o1 + 0x78] + std %f36, [%o1 + 0x80] + std %f38, [%o1 + 0x88] + std %f40, [%o1 + 0x90] + std %f42, [%o1 + 0x98] +80: + retl + VISExit +ENDPROC(aes_sparc64_key_expand) + + .align 32 +ENTRY(aes_sparc64_encrypt_128) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + ldd [%o0 + 0x00], %f8 + ldd [%o0 + 0x08], %f10 + ldd [%o0 + 0x10], %f12 + ldd [%o0 + 0x18], %f14 + ldd [%o0 + 0x20], %f16 + ldd [%o0 + 0x28], %f18 + ldd [%o0 + 0x30], %f20 + ldd [%o0 + 0x38], %f22 + ldd [%o0 + 0x40], %f24 + ldd [%o0 + 0x48], %f26 + ldd [%o0 + 0x50], %f28 + ldd [%o0 + 0x58], %f30 + ldd [%o0 + 0x60], %f32 + ldd [%o0 + 0x68], %f34 + ldd [%o0 + 0x70], %f36 + ldd [%o0 + 0x78], %f38 + ldd [%o0 + 0x80], %f40 + ldd [%o0 + 0x88], %f42 + ldd [%o0 + 0x90], %f44 + ldd [%o0 + 0x98], %f46 + ldd [%o0 + 0xa0], %f48 + ldd [%o0 + 0xa8], %f50 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + ENCRYPT_128(12, 4, 6, 0, 2) + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + retl + VISExit +ENDPROC(aes_sparc64_encrypt_128) + + .align 32 +ENTRY(aes_sparc64_encrypt_192) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + + ldd [%o0 + 0x00], %f8 + ldd [%o0 + 0x08], %f10 + + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + add %o0, 0x20, %o0 + + ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2) + + ldd [%o0 + 0x10], %f12 + ldd [%o0 + 0x18], %f14 + ldd [%o0 + 0x20], %f16 + ldd [%o0 + 0x28], %f18 + ldd [%o0 + 0x30], %f20 + ldd [%o0 + 0x38], %f22 + ldd [%o0 + 0x40], %f24 + ldd [%o0 + 0x48], %f26 + ldd [%o0 + 0x50], %f28 + ldd [%o0 + 0x58], %f30 + ldd [%o0 + 0x60], %f32 + ldd [%o0 + 0x68], %f34 + ldd [%o0 + 0x70], %f36 + ldd [%o0 + 0x78], %f38 + ldd [%o0 + 0x80], %f40 + ldd [%o0 + 0x88], %f42 + ldd [%o0 + 0x90], %f44 + ldd [%o0 + 0x98], %f46 + ldd [%o0 + 0xa0], %f48 + ldd [%o0 + 0xa8], %f50 + + + ENCRYPT_128(12, 4, 6, 0, 2) + + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + + retl + VISExit +ENDPROC(aes_sparc64_encrypt_192) + + .align 32 +ENTRY(aes_sparc64_encrypt_256) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + + ldd [%o0 + 0x00], %f8 + ldd [%o0 + 0x08], %f10 + + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + + ldd [%o0 + 0x10], %f8 + + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + add %o0, 0x20, %o0 + + ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2) + + ldd [%o0 + 0x10], %f8 + + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + add %o0, 0x20, %o0 + + ENCRYPT_TWO_ROUNDS(8, 4, 6, 0, 2) + + ldd [%o0 + 0x10], %f12 + ldd [%o0 + 0x18], %f14 + ldd [%o0 + 0x20], %f16 + ldd [%o0 + 0x28], %f18 + ldd [%o0 + 0x30], %f20 + ldd [%o0 + 0x38], %f22 + ldd [%o0 + 0x40], %f24 + ldd [%o0 + 0x48], %f26 + ldd [%o0 + 0x50], %f28 + ldd [%o0 + 0x58], %f30 + ldd [%o0 + 0x60], %f32 + ldd [%o0 + 0x68], %f34 + ldd [%o0 + 0x70], %f36 + ldd [%o0 + 0x78], %f38 + ldd [%o0 + 0x80], %f40 + ldd [%o0 + 0x88], %f42 + ldd [%o0 + 0x90], %f44 + ldd [%o0 + 0x98], %f46 + ldd [%o0 + 0xa0], %f48 + ldd [%o0 + 0xa8], %f50 + + ENCRYPT_128(12, 4, 6, 0, 2) + + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + + retl + VISExit +ENDPROC(aes_sparc64_encrypt_256) + + .align 32 +ENTRY(aes_sparc64_decrypt_128) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + ldd [%o0 + 0xa0], %f8 + ldd [%o0 + 0xa8], %f10 + ldd [%o0 + 0x98], %f12 + ldd [%o0 + 0x90], %f14 + ldd [%o0 + 0x88], %f16 + ldd [%o0 + 0x80], %f18 + ldd [%o0 + 0x78], %f20 + ldd [%o0 + 0x70], %f22 + ldd [%o0 + 0x68], %f24 + ldd [%o0 + 0x60], %f26 + ldd [%o0 + 0x58], %f28 + ldd [%o0 + 0x50], %f30 + ldd [%o0 + 0x48], %f32 + ldd [%o0 + 0x40], %f34 + ldd [%o0 + 0x38], %f36 + ldd [%o0 + 0x30], %f38 + ldd [%o0 + 0x28], %f40 + ldd [%o0 + 0x20], %f42 + ldd [%o0 + 0x18], %f44 + ldd [%o0 + 0x10], %f46 + ldd [%o0 + 0x08], %f48 + ldd [%o0 + 0x00], %f50 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + DECRYPT_128(12, 4, 6, 0, 2) + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + retl + VISExit +ENDPROC(aes_sparc64_decrypt_128) + + .align 32 +ENTRY(aes_sparc64_decrypt_192) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + ldd [%o0 + 0xc0], %f8 + ldd [%o0 + 0xc8], %f10 + ldd [%o0 + 0xb8], %f12 + ldd [%o0 + 0xb0], %f14 + ldd [%o0 + 0xa8], %f16 + ldd [%o0 + 0xa0], %f18 + fxor %f8, %f4, %f4 + fxor %f10, %f6, %f6 + ldd [%o0 + 0x98], %f20 + ldd [%o0 + 0x90], %f22 + ldd [%o0 + 0x88], %f24 + ldd [%o0 + 0x80], %f26 + DECRYPT_TWO_ROUNDS(12, 4, 6, 0, 2) + ldd [%o0 + 0x78], %f28 + ldd [%o0 + 0x70], %f30 + ldd [%o0 + 0x68], %f32 + ldd [%o0 + 0x60], %f34 + ldd [%o0 + 0x58], %f36 + ldd [%o0 + 0x50], %f38 + ldd [%o0 + 0x48], %f40 + ldd [%o0 + 0x40], %f42 + ldd [%o0 + 0x38], %f44 + ldd [%o0 + 0x30], %f46 + ldd [%o0 + 0x28], %f48 + ldd [%o0 + 0x20], %f50 + ldd [%o0 + 0x18], %f52 + ldd [%o0 + 0x10], %f54 + ldd [%o0 + 0x08], %f56 + ldd [%o0 + 0x00], %f58 + DECRYPT_128(20, 4, 6, 0, 2) + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + retl + VISExit +ENDPROC(aes_sparc64_decrypt_192) + + .align 32 +ENTRY(aes_sparc64_decrypt_256) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ld [%o1 + 0x00], %f4 + ld [%o1 + 0x04], %f5 + ld [%o1 + 0x08], %f6 + ld [%o1 + 0x0c], %f7 + ldd [%o0 + 0xe0], %f8 + ldd [%o0 + 0xe8], %f10 + ldd [%o0 + 0xd8], %f12 + ldd [%o0 + 0xd0], %f14 + ldd [%o0 + 0xc8], %f16 + fxor %f8, %f4, %f4 + ldd [%o0 + 0xc0], %f18 + fxor %f10, %f6, %f6 + ldd [%o0 + 0xb8], %f20 + AES_DROUND23(12, 4, 6, 2) + ldd [%o0 + 0xb0], %f22 + AES_DROUND01(14, 4, 6, 0) + ldd [%o0 + 0xa8], %f24 + AES_DROUND23(16, 0, 2, 6) + ldd [%o0 + 0xa0], %f26 + AES_DROUND01(18, 0, 2, 4) + ldd [%o0 + 0x98], %f12 + AES_DROUND23(20, 4, 6, 2) + ldd [%o0 + 0x90], %f14 + AES_DROUND01(22, 4, 6, 0) + ldd [%o0 + 0x88], %f16 + AES_DROUND23(24, 0, 2, 6) + ldd [%o0 + 0x80], %f18 + AES_DROUND01(26, 0, 2, 4) + ldd [%o0 + 0x78], %f20 + AES_DROUND23(12, 4, 6, 2) + ldd [%o0 + 0x70], %f22 + AES_DROUND01(14, 4, 6, 0) + ldd [%o0 + 0x68], %f24 + AES_DROUND23(16, 0, 2, 6) + ldd [%o0 + 0x60], %f26 + AES_DROUND01(18, 0, 2, 4) + ldd [%o0 + 0x58], %f28 + AES_DROUND23(20, 4, 6, 2) + ldd [%o0 + 0x50], %f30 + AES_DROUND01(22, 4, 6, 0) + ldd [%o0 + 0x48], %f32 + AES_DROUND23(24, 0, 2, 6) + ldd [%o0 + 0x40], %f34 + AES_DROUND01(26, 0, 2, 4) + ldd [%o0 + 0x38], %f36 + AES_DROUND23(28, 4, 6, 2) + ldd [%o0 + 0x30], %f38 + AES_DROUND01(30, 4, 6, 0) + ldd [%o0 + 0x28], %f40 + AES_DROUND23(32, 0, 2, 6) + ldd [%o0 + 0x20], %f42 + AES_DROUND01(34, 0, 2, 4) + ldd [%o0 + 0x18], %f44 + AES_DROUND23(36, 4, 6, 2) + ldd [%o0 + 0x10], %f46 + AES_DROUND01(38, 4, 6, 0) + ldd [%o0 + 0x08], %f48 + AES_DROUND23(40, 0, 2, 6) + ldd [%o0 + 0x00], %f50 + AES_DROUND01(42, 0, 2, 4) + AES_DROUND23(44, 4, 6, 2) + AES_DROUND01(46, 4, 6, 0) + AES_DROUND23_L(48, 0, 2, 6) + AES_DROUND01_L(50, 0, 2, 4) + st %f4, [%o2 + 0x00] + st %f5, [%o2 + 0x04] + st %f6, [%o2 + 0x08] + st %f7, [%o2 + 0x0c] + retl + VISExit +ENDPROC(aes_sparc64_decrypt_256) + + .align 32 +ENTRY(aes_sparc64_load_encrypt_keys_128) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + retl + ldd [%o0 + 0xa8], %f46 +ENDPROC(aes_sparc64_load_encrypt_keys_128) + + .align 32 +ENTRY(aes_sparc64_load_encrypt_keys_192) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + retl + ldd [%o0 + 0xc8], %f54 +ENDPROC(aes_sparc64_load_encrypt_keys_192) + + .align 32 +ENTRY(aes_sparc64_load_encrypt_keys_256) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + ldd [%o0 + 0xc8], %f54 + ldd [%o0 + 0xd0], %f56 + ldd [%o0 + 0xd8], %f58 + ldd [%o0 + 0xe0], %f60 + retl + ldd [%o0 + 0xe8], %f62 +ENDPROC(aes_sparc64_load_encrypt_keys_256) + + .align 32 +ENTRY(aes_sparc64_load_decrypt_keys_128) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x98], %f8 + ldd [%o0 + 0x90], %f10 + ldd [%o0 + 0x88], %f12 + ldd [%o0 + 0x80], %f14 + ldd [%o0 + 0x78], %f16 + ldd [%o0 + 0x70], %f18 + ldd [%o0 + 0x68], %f20 + ldd [%o0 + 0x60], %f22 + ldd [%o0 + 0x58], %f24 + ldd [%o0 + 0x50], %f26 + ldd [%o0 + 0x48], %f28 + ldd [%o0 + 0x40], %f30 + ldd [%o0 + 0x38], %f32 + ldd [%o0 + 0x30], %f34 + ldd [%o0 + 0x28], %f36 + ldd [%o0 + 0x20], %f38 + ldd [%o0 + 0x18], %f40 + ldd [%o0 + 0x10], %f42 + ldd [%o0 + 0x08], %f44 + retl + ldd [%o0 + 0x00], %f46 +ENDPROC(aes_sparc64_load_decrypt_keys_128) + + .align 32 +ENTRY(aes_sparc64_load_decrypt_keys_192) + /* %o0=key */ + VISEntry + ldd [%o0 + 0xb8], %f8 + ldd [%o0 + 0xb0], %f10 + ldd [%o0 + 0xa8], %f12 + ldd [%o0 + 0xa0], %f14 + ldd [%o0 + 0x98], %f16 + ldd [%o0 + 0x90], %f18 + ldd [%o0 + 0x88], %f20 + ldd [%o0 + 0x80], %f22 + ldd [%o0 + 0x78], %f24 + ldd [%o0 + 0x70], %f26 + ldd [%o0 + 0x68], %f28 + ldd [%o0 + 0x60], %f30 + ldd [%o0 + 0x58], %f32 + ldd [%o0 + 0x50], %f34 + ldd [%o0 + 0x48], %f36 + ldd [%o0 + 0x40], %f38 + ldd [%o0 + 0x38], %f40 + ldd [%o0 + 0x30], %f42 + ldd [%o0 + 0x28], %f44 + ldd [%o0 + 0x20], %f46 + ldd [%o0 + 0x18], %f48 + ldd [%o0 + 0x10], %f50 + ldd [%o0 + 0x08], %f52 + retl + ldd [%o0 + 0x00], %f54 +ENDPROC(aes_sparc64_load_decrypt_keys_192) + + .align 32 +ENTRY(aes_sparc64_load_decrypt_keys_256) + /* %o0=key */ + VISEntry + ldd [%o0 + 0xd8], %f8 + ldd [%o0 + 0xd0], %f10 + ldd [%o0 + 0xc8], %f12 + ldd [%o0 + 0xc0], %f14 + ldd [%o0 + 0xb8], %f16 + ldd [%o0 + 0xb0], %f18 + ldd [%o0 + 0xa8], %f20 + ldd [%o0 + 0xa0], %f22 + ldd [%o0 + 0x98], %f24 + ldd [%o0 + 0x90], %f26 + ldd [%o0 + 0x88], %f28 + ldd [%o0 + 0x80], %f30 + ldd [%o0 + 0x78], %f32 + ldd [%o0 + 0x70], %f34 + ldd [%o0 + 0x68], %f36 + ldd [%o0 + 0x60], %f38 + ldd [%o0 + 0x58], %f40 + ldd [%o0 + 0x50], %f42 + ldd [%o0 + 0x48], %f44 + ldd [%o0 + 0x40], %f46 + ldd [%o0 + 0x38], %f48 + ldd [%o0 + 0x30], %f50 + ldd [%o0 + 0x28], %f52 + ldd [%o0 + 0x20], %f54 + ldd [%o0 + 0x18], %f56 + ldd [%o0 + 0x10], %f58 + ldd [%o0 + 0x08], %f60 + retl + ldd [%o0 + 0x00], %f62 +ENDPROC(aes_sparc64_load_decrypt_keys_256) + + .align 32 +ENTRY(aes_sparc64_ecb_encrypt_128) + /* %o0=key, %o1=input, %o2=output, %o3=len */ + ldx [%o0 + 0x00], %g1 + subcc %o3, 0x10, %o3 + be 10f + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F60 + MOVXTOD_G7_F62 + ENCRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + ENCRYPT_128(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_encrypt_128) + + .align 32 +ENTRY(aes_sparc64_ecb_encrypt_192) + /* %o0=key, %o1=input, %o2=output, %o3=len */ + ldx [%o0 + 0x00], %g1 + subcc %o3, 0x10, %o3 + be 10f + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F60 + MOVXTOD_G7_F62 + ENCRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + ENCRYPT_192(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_encrypt_192) + + .align 32 +ENTRY(aes_sparc64_ecb_encrypt_256) + /* %o0=key, %o1=input, %o2=output, %o3=len */ + ldx [%o0 + 0x00], %g1 + subcc %o3, 0x10, %o3 + be 10f + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F0 + MOVXTOD_G7_F2 + ENCRYPT_256_2(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f0, [%o2 + 0x10] + std %f2, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldd [%o0 + 0xd0], %f56 + ldd [%o0 + 0xd8], %f58 + ldd [%o0 + 0xe0], %f60 + ldd [%o0 + 0xe8], %f62 + ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + ENCRYPT_256(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_encrypt_256) + + .align 32 +ENTRY(aes_sparc64_ecb_decrypt_128) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */ + ldx [%o0 - 0x10], %g1 + subcc %o3, 0x10, %o3 + be 10f + ldx [%o0 - 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F60 + MOVXTOD_G7_F62 + DECRYPT_128_2(8, 4, 6, 60, 62, 0, 2, 56, 58) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz,pt %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_128(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_decrypt_128) + + .align 32 +ENTRY(aes_sparc64_ecb_decrypt_192) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */ + ldx [%o0 - 0x10], %g1 + subcc %o3, 0x10, %o3 + be 10f + ldx [%o0 - 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F60 + MOVXTOD_G7_F62 + DECRYPT_192_2(8, 4, 6, 60, 62, 0, 2, 56, 58) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz,pt %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_192(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_decrypt_192) + + .align 32 +ENTRY(aes_sparc64_ecb_decrypt_256) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */ + ldx [%o0 - 0x10], %g1 + subcc %o3, 0x10, %o3 + ldx [%o0 - 0x08], %g2 + be 10f + sub %o0, 0xf0, %o0 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + xor %g1, %o4, %g3 + xor %g2, %o5, %g7 + MOVXTOD_G3_F0 + MOVXTOD_G7_F2 + DECRYPT_256_2(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + std %f0, [%o2 + 0x10] + std %f2, [%o2 + 0x18] + sub %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz,pt %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldd [%o0 + 0x18], %f56 + ldd [%o0 + 0x10], %f58 + ldd [%o0 + 0x08], %f60 + ldd [%o0 + 0x00], %f62 + ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_256(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: retl + nop +ENDPROC(aes_sparc64_ecb_decrypt_256) + + .align 32 +ENTRY(aes_sparc64_cbc_encrypt_128) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldd [%o4 + 0x00], %f4 + ldd [%o4 + 0x08], %f6 + ldx [%o0 + 0x00], %g1 + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F0 + MOVXTOD_G7_F2 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + ENCRYPT_128(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + std %f4, [%o4 + 0x00] + std %f6, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_encrypt_128) + + .align 32 +ENTRY(aes_sparc64_cbc_encrypt_192) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldd [%o4 + 0x00], %f4 + ldd [%o4 + 0x08], %f6 + ldx [%o0 + 0x00], %g1 + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F0 + MOVXTOD_G7_F2 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + ENCRYPT_192(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + std %f4, [%o4 + 0x00] + std %f6, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_encrypt_192) + + .align 32 +ENTRY(aes_sparc64_cbc_encrypt_256) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldd [%o4 + 0x00], %f4 + ldd [%o4 + 0x08], %f6 + ldx [%o0 + 0x00], %g1 + ldx [%o0 + 0x08], %g2 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F0 + MOVXTOD_G7_F2 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + ENCRYPT_256(8, 4, 6, 0, 2) + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + std %f4, [%o4 + 0x00] + std %f6, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_encrypt_256) + + .align 32 +ENTRY(aes_sparc64_cbc_decrypt_128) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */ + ldx [%o0 - 0x10], %g1 + ldx [%o0 - 0x08], %g2 + ldx [%o4 + 0x00], %o0 + ldx [%o4 + 0x08], %o5 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_128(8, 4, 6, 0, 2) + MOVXTOD_O0_F0 + MOVXTOD_O5_F2 + xor %g1, %g3, %o0 + xor %g2, %g7, %o5 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + stx %o0, [%o4 + 0x00] + stx %o5, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_decrypt_128) + + .align 32 +ENTRY(aes_sparc64_cbc_decrypt_192) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */ + ldx [%o0 - 0x10], %g1 + ldx [%o0 - 0x08], %g2 + ldx [%o4 + 0x00], %o0 + ldx [%o4 + 0x08], %o5 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_192(8, 4, 6, 0, 2) + MOVXTOD_O0_F0 + MOVXTOD_O5_F2 + xor %g1, %g3, %o0 + xor %g2, %g7, %o5 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + stx %o0, [%o4 + 0x00] + stx %o5, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_decrypt_192) + + .align 32 +ENTRY(aes_sparc64_cbc_decrypt_256) + /* %o0=&key[key_len], %o1=input, %o2=output, %o3=len, %o4=iv */ + ldx [%o0 - 0x10], %g1 + ldx [%o0 - 0x08], %g2 + ldx [%o4 + 0x00], %o0 + ldx [%o4 + 0x08], %o5 +1: ldx [%o1 + 0x00], %g3 + ldx [%o1 + 0x08], %g7 + add %o1, 0x10, %o1 + xor %g1, %g3, %g3 + xor %g2, %g7, %g7 + MOVXTOD_G3_F4 + MOVXTOD_G7_F6 + DECRYPT_256(8, 4, 6, 0, 2) + MOVXTOD_O0_F0 + MOVXTOD_O5_F2 + xor %g1, %g3, %o0 + xor %g2, %g7, %o5 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] + subcc %o3, 0x10, %o3 + bne,pt %xcc, 1b + add %o2, 0x10, %o2 + stx %o0, [%o4 + 0x00] + stx %o5, [%o4 + 0x08] + retl + nop +ENDPROC(aes_sparc64_cbc_decrypt_256) + + .align 32 +ENTRY(aes_sparc64_ctr_crypt_128) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldx [%o4 + 0x00], %g3 + ldx [%o4 + 0x08], %g7 + subcc %o3, 0x10, %o3 + ldx [%o0 + 0x00], %g1 + be 10f + ldx [%o0 + 0x08], %g2 +1: xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + xor %g1, %g3, %o5 + MOVXTOD_O5_F4 + xor %g2, %g7, %o5 + MOVXTOD_O5_F6 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_128_2(8, 0, 2, 4, 6, 56, 58, 60, 62) + ldd [%o1 + 0x00], %f56 + ldd [%o1 + 0x08], %f58 + ldd [%o1 + 0x10], %f60 + ldd [%o1 + 0x18], %f62 + fxor %f56, %f0, %f56 + fxor %f58, %f2, %f58 + fxor %f60, %f4, %f60 + fxor %f62, %f6, %f62 + std %f56, [%o2 + 0x00] + std %f58, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + subcc %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_128(8, 0, 2, 4, 6) + ldd [%o1 + 0x00], %f4 + ldd [%o1 + 0x08], %f6 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: stx %g3, [%o4 + 0x00] + retl + stx %g7, [%o4 + 0x08] +ENDPROC(aes_sparc64_ctr_crypt_128) + + .align 32 +ENTRY(aes_sparc64_ctr_crypt_192) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldx [%o4 + 0x00], %g3 + ldx [%o4 + 0x08], %g7 + subcc %o3, 0x10, %o3 + ldx [%o0 + 0x00], %g1 + be 10f + ldx [%o0 + 0x08], %g2 +1: xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + xor %g1, %g3, %o5 + MOVXTOD_O5_F4 + xor %g2, %g7, %o5 + MOVXTOD_O5_F6 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_192_2(8, 0, 2, 4, 6, 56, 58, 60, 62) + ldd [%o1 + 0x00], %f56 + ldd [%o1 + 0x08], %f58 + ldd [%o1 + 0x10], %f60 + ldd [%o1 + 0x18], %f62 + fxor %f56, %f0, %f56 + fxor %f58, %f2, %f58 + fxor %f60, %f4, %f60 + fxor %f62, %f6, %f62 + std %f56, [%o2 + 0x00] + std %f58, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + subcc %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_192(8, 0, 2, 4, 6) + ldd [%o1 + 0x00], %f4 + ldd [%o1 + 0x08], %f6 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: stx %g3, [%o4 + 0x00] + retl + stx %g7, [%o4 + 0x08] +ENDPROC(aes_sparc64_ctr_crypt_192) + + .align 32 +ENTRY(aes_sparc64_ctr_crypt_256) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldx [%o4 + 0x00], %g3 + ldx [%o4 + 0x08], %g7 + subcc %o3, 0x10, %o3 + ldx [%o0 + 0x00], %g1 + be 10f + ldx [%o0 + 0x08], %g2 +1: xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + xor %g1, %g3, %o5 + MOVXTOD_O5_F4 + xor %g2, %g7, %o5 + MOVXTOD_O5_F6 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_256_2(8, 0, 2, 4, 6) + ldd [%o1 + 0x00], %f56 + ldd [%o1 + 0x08], %f58 + ldd [%o1 + 0x10], %f60 + ldd [%o1 + 0x18], %f62 + fxor %f56, %f0, %f56 + fxor %f58, %f2, %f58 + fxor %f60, %f4, %f60 + fxor %f62, %f6, %f62 + std %f56, [%o2 + 0x00] + std %f58, [%o2 + 0x08] + std %f60, [%o2 + 0x10] + std %f62, [%o2 + 0x18] + subcc %o3, 0x20, %o3 + add %o1, 0x20, %o1 + brgz %o3, 1b + add %o2, 0x20, %o2 + brlz,pt %o3, 11f + nop +10: ldd [%o0 + 0xd0], %f56 + ldd [%o0 + 0xd8], %f58 + ldd [%o0 + 0xe0], %f60 + ldd [%o0 + 0xe8], %f62 + xor %g1, %g3, %o5 + MOVXTOD_O5_F0 + xor %g2, %g7, %o5 + MOVXTOD_O5_F2 + add %g7, 1, %g7 + add %g3, 1, %o5 + movrz %g7, %o5, %g3 + ENCRYPT_256(8, 0, 2, 4, 6) + ldd [%o1 + 0x00], %f4 + ldd [%o1 + 0x08], %f6 + fxor %f4, %f0, %f4 + fxor %f6, %f2, %f6 + std %f4, [%o2 + 0x00] + std %f6, [%o2 + 0x08] +11: stx %g3, [%o4 + 0x00] + retl + stx %g7, [%o4 + 0x08] +ENDPROC(aes_sparc64_ctr_crypt_256) diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c new file mode 100644 index 00000000000..df922f52d76 --- /dev/null +++ b/arch/sparc/crypto/aes_glue.c @@ -0,0 +1,504 @@ +/* Glue code for AES encryption optimized for sparc64 crypto opcodes. + * + * This is based largely upon arch/x86/crypto/aesni-intel_glue.c + * + * Copyright (C) 2008, Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Adrian Hoban <adrian.hoban@intel.com> + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Tadeusz Struk (tadeusz.struk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Copyright (c) 2010, Intel Corporation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> + +#include <asm/fpumacro.h> +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +struct aes_ops { + void (*encrypt)(const u64 *key, const u32 *input, u32 *output); + void (*decrypt)(const u64 *key, const u32 *input, u32 *output); + void (*load_encrypt_keys)(const u64 *key); + void (*load_decrypt_keys)(const u64 *key); + void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len); + void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len); + void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); + void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); + void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +}; + +struct crypto_sparc64_aes_ctx { + struct aes_ops *ops; + u64 key[AES_MAX_KEYLENGTH / sizeof(u64)]; + u32 key_length; + u32 expanded_key_length; +}; + +extern void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, + u32 *output); +extern void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, + u32 *output); +extern void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, + u32 *output); + +extern void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, + u32 *output); +extern void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, + u32 *output); +extern void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, + u32 *output); + +extern void aes_sparc64_load_encrypt_keys_128(const u64 *key); +extern void aes_sparc64_load_encrypt_keys_192(const u64 *key); +extern void aes_sparc64_load_encrypt_keys_256(const u64 *key); + +extern void aes_sparc64_load_decrypt_keys_128(const u64 *key); +extern void aes_sparc64_load_decrypt_keys_192(const u64 *key); +extern void aes_sparc64_load_decrypt_keys_256(const u64 *key); + +extern void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, + u64 *output, unsigned int len); +extern void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, + u64 *output, unsigned int len); +extern void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, + u64 *output, unsigned int len); + +extern void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, + u64 *output, unsigned int len); +extern void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, + u64 *output, unsigned int len); +extern void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, + u64 *output, unsigned int len); + +extern void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +extern void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); +extern void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); +extern void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +static struct aes_ops aes128_ops = { + .encrypt = aes_sparc64_encrypt_128, + .decrypt = aes_sparc64_decrypt_128, + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128, + .ecb_encrypt = aes_sparc64_ecb_encrypt_128, + .ecb_decrypt = aes_sparc64_ecb_decrypt_128, + .cbc_encrypt = aes_sparc64_cbc_encrypt_128, + .cbc_decrypt = aes_sparc64_cbc_decrypt_128, + .ctr_crypt = aes_sparc64_ctr_crypt_128, +}; + +static struct aes_ops aes192_ops = { + .encrypt = aes_sparc64_encrypt_192, + .decrypt = aes_sparc64_decrypt_192, + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192, + .ecb_encrypt = aes_sparc64_ecb_encrypt_192, + .ecb_decrypt = aes_sparc64_ecb_decrypt_192, + .cbc_encrypt = aes_sparc64_cbc_encrypt_192, + .cbc_decrypt = aes_sparc64_cbc_decrypt_192, + .ctr_crypt = aes_sparc64_ctr_crypt_192, +}; + +static struct aes_ops aes256_ops = { + .encrypt = aes_sparc64_encrypt_256, + .decrypt = aes_sparc64_decrypt_256, + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256, + .ecb_encrypt = aes_sparc64_ecb_encrypt_256, + .ecb_decrypt = aes_sparc64_ecb_decrypt_256, + .cbc_encrypt = aes_sparc64_cbc_encrypt_256, + .cbc_decrypt = aes_sparc64_cbc_decrypt_256, + .ctr_crypt = aes_sparc64_ctr_crypt_256, +}; + +extern void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key, + unsigned int key_len); + +static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *flags = &tfm->crt_flags; + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->expanded_key_length = 0xb0; + ctx->ops = &aes128_ops; + break; + + case AES_KEYSIZE_192: + ctx->expanded_key_length = 0xd0; + ctx->ops = &aes192_ops; + break; + + case AES_KEYSIZE_256: + ctx->expanded_key_length = 0xf0; + ctx->ops = &aes256_ops; + break; + + default: + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len); + ctx->key_length = key_len; + + return 0; +} + +static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->ops->encrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); +} + +static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->ops->decrypt(&ctx->key[0], (const u32 *) src, (u32 *) dst); +} + +#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) + +static int ecb_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & AES_BLOCK_MASK; + + if (likely(block_len)) { + ctx->ops->ecb_encrypt(&ctx->key[0], + (const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len); + } + nbytes &= AES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int ecb_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + u64 *key_end; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + ctx->ops->load_decrypt_keys(&ctx->key[0]); + key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & AES_BLOCK_MASK; + + if (likely(block_len)) { + ctx->ops->ecb_decrypt(key_end, + (const u64 *) walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, block_len); + } + nbytes &= AES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + + return err; +} + +static int cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & AES_BLOCK_MASK; + + if (likely(block_len)) { + ctx->ops->cbc_encrypt(&ctx->key[0], + (const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len, (u64 *) walk.iv); + } + nbytes &= AES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + u64 *key_end; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + ctx->ops->load_decrypt_keys(&ctx->key[0]); + key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & AES_BLOCK_MASK; + + if (likely(block_len)) { + ctx->ops->cbc_decrypt(key_end, + (const u64 *) walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len, (u64 *) walk.iv); + } + nbytes &= AES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + + return err; +} + +static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx, + struct blkcipher_walk *walk) +{ + u8 *ctrblk = walk->iv; + u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; + u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + + ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk, + keystream, AES_BLOCK_SIZE); + crypto_xor((u8 *) keystream, src, nbytes); + memcpy(dst, keystream, nbytes); + crypto_inc(ctrblk, AES_BLOCK_SIZE); +} + +static int ctr_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + unsigned int block_len = nbytes & AES_BLOCK_MASK; + + if (likely(block_len)) { + ctx->ops->ctr_crypt(&ctx->key[0], + (const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len, (u64 *) walk.iv); + } + nbytes &= AES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + if (walk.nbytes) { + ctr_crypt_final(ctx, &walk); + err = blkcipher_walk_done(desc, &walk, 0); + } + fprs_write(0); + return err; +} + +static struct crypto_alg algs[] = { { + .cra_name = "aes", + .cra_driver_name = "aes-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = AES_MIN_KEY_SIZE, + .cia_max_keysize = AES_MAX_KEY_SIZE, + .cia_setkey = aes_set_key, + .cia_encrypt = aes_encrypt, + .cia_decrypt = aes_decrypt + } + } +}, { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +} }; + +static bool __init sparc64_has_aes_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_AES)) + return false; + + return true; +} + +static int __init aes_sparc64_mod_init(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(algs); i++) + INIT_LIST_HEAD(&algs[i].cra_list); + + if (sparc64_has_aes_opcode()) { + pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); + return crypto_register_algs(algs, ARRAY_SIZE(algs)); + } + pr_info("sparc64 aes opcodes not available.\n"); + return -ENODEV; +} + +static void __exit aes_sparc64_mod_fini(void) +{ + crypto_unregister_algs(algs, ARRAY_SIZE(algs)); +} + +module_init(aes_sparc64_mod_init); +module_exit(aes_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); + +MODULE_ALIAS("aes"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S new file mode 100644 index 00000000000..cc39553a4e4 --- /dev/null +++ b/arch/sparc/crypto/camellia_asm.S @@ -0,0 +1,563 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +#define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ + CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \ + CAMELLIA_F(KEY_BASE + 4, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 6, I0, I1, I0) \ + CAMELLIA_F(KEY_BASE + 8, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 10, I0, I1, I0) + +#define CAMELLIA_6ROUNDS_FL_FLI(KEY_BASE, I0, I1) \ + CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ + CAMELLIA_FL(KEY_BASE + 12, I0, I0) \ + CAMELLIA_FLI(KEY_BASE + 14, I1, I1) + + .data + + .align 8 +SIGMA: .xword 0xA09E667F3BCC908B + .xword 0xB67AE8584CAA73B2 + .xword 0xC6EF372FE94F82BE + .xword 0x54FF53A5F1D36F1C + .xword 0x10E527FADE682D1D + .xword 0xB05688C2B3E6C1FD + + .text + + .align 32 +ENTRY(camellia_sparc64_key_expand) + /* %o0=in_key, %o1=encrypt_key, %o2=key_len, %o3=decrypt_key */ + VISEntry + ld [%o0 + 0x00], %f0 ! i0, k[0] + ld [%o0 + 0x04], %f1 ! i1, k[1] + ld [%o0 + 0x08], %f2 ! i2, k[2] + ld [%o0 + 0x0c], %f3 ! i3, k[3] + std %f0, [%o1 + 0x00] ! k[0, 1] + fsrc2 %f0, %f28 + std %f2, [%o1 + 0x08] ! k[2, 3] + cmp %o2, 16 + be 10f + fsrc2 %f2, %f30 + + ld [%o0 + 0x10], %f0 + ld [%o0 + 0x14], %f1 + std %f0, [%o1 + 0x20] ! k[8, 9] + cmp %o2, 24 + fone %f10 + be,a 1f + fxor %f10, %f0, %f2 + ld [%o0 + 0x18], %f2 + ld [%o0 + 0x1c], %f3 +1: + std %f2, [%o1 + 0x28] ! k[10, 11] + fxor %f28, %f0, %f0 + fxor %f30, %f2, %f2 + +10: + sethi %hi(SIGMA), %g3 + or %g3, %lo(SIGMA), %g3 + ldd [%g3 + 0x00], %f16 + ldd [%g3 + 0x08], %f18 + ldd [%g3 + 0x10], %f20 + ldd [%g3 + 0x18], %f22 + ldd [%g3 + 0x20], %f24 + ldd [%g3 + 0x28], %f26 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + fxor %f28, %f0, %f0 + fxor %f30, %f2, %f2 + CAMELLIA_F(20, 2, 0, 2) + CAMELLIA_F(22, 0, 2, 0) + +#define ROTL128(S01, S23, TMP1, TMP2, N) \ + srlx S01, (64 - N), TMP1; \ + sllx S01, N, S01; \ + srlx S23, (64 - N), TMP2; \ + sllx S23, N, S23; \ + or S01, TMP2, S01; \ + or S23, TMP1, S23 + + cmp %o2, 16 + bne 1f + nop + /* 128-bit key */ + std %f0, [%o1 + 0x10] ! k[ 4, 5] + std %f2, [%o1 + 0x18] ! k[ 6, 7] + MOVDTOX_F0_O4 + MOVDTOX_F2_O5 + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x30] ! k[12, 13] + stx %o5, [%o1 + 0x38] ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x40] ! k[16, 17] + stx %o5, [%o1 + 0x48] ! k[18, 19] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x60] ! k[24, 25] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x70] ! k[28, 29] + stx %o5, [%o1 + 0x78] ! k[30, 31] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xa0] ! k[40, 41] + stx %o5, [%o1 + 0xa8] ! k[42, 43] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xc0] ! k[48, 49] + stx %o5, [%o1 + 0xc8] ! k[50, 51] + + ldx [%o1 + 0x00], %o4 ! k[ 0, 1] + ldx [%o1 + 0x08], %o5 ! k[ 2, 3] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x20] ! k[ 8, 9] + stx %o5, [%o1 + 0x28] ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x50] ! k[20, 21] + stx %o5, [%o1 + 0x58] ! k[22, 23] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o5, [%o1 + 0x68] ! k[26, 27] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0x80] ! k[32, 33] + stx %o5, [%o1 + 0x88] ! k[34, 35] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0x90] ! k[36, 37] + stx %o5, [%o1 + 0x98] ! k[38, 39] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xb0] ! k[44, 45] + stx %o5, [%o1 + 0xb8] ! k[46, 47] + + ba,pt %xcc, 2f + mov (3 * 16 * 4), %o0 + +1: + /* 192-bit or 256-bit key */ + std %f0, [%o1 + 0x30] ! k[12, 13] + std %f2, [%o1 + 0x38] ! k[14, 15] + ldd [%o1 + 0x20], %f4 ! k[ 8, 9] + ldd [%o1 + 0x28], %f6 ! k[10, 11] + fxor %f0, %f4, %f0 + fxor %f2, %f6, %f2 + CAMELLIA_F(24, 2, 0, 2) + CAMELLIA_F(26, 0, 2, 0) + std %f0, [%o1 + 0x10] ! k[ 4, 5] + std %f2, [%o1 + 0x18] ! k[ 6, 7] + MOVDTOX_F0_O4 + MOVDTOX_F2_O5 + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x50] ! k[20, 21] + stx %o5, [%o1 + 0x58] ! k[22, 23] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0xa0] ! k[40, 41] + stx %o5, [%o1 + 0xa8] ! k[42, 43] + ROTL128(%o4, %o5, %g2, %g3, 51) + stx %o4, [%o1 + 0x100] ! k[64, 65] + stx %o5, [%o1 + 0x108] ! k[66, 67] + ldx [%o1 + 0x20], %o4 ! k[ 8, 9] + ldx [%o1 + 0x28], %o5 ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x20] ! k[ 8, 9] + stx %o5, [%o1 + 0x28] ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x40] ! k[16, 17] + stx %o5, [%o1 + 0x48] ! k[18, 19] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x90] ! k[36, 37] + stx %o5, [%o1 + 0x98] ! k[38, 39] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xd0] ! k[52, 53] + stx %o5, [%o1 + 0xd8] ! k[54, 55] + ldx [%o1 + 0x30], %o4 ! k[12, 13] + ldx [%o1 + 0x38], %o5 ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x30] ! k[12, 13] + stx %o5, [%o1 + 0x38] ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x70] ! k[28, 29] + stx %o5, [%o1 + 0x78] ! k[30, 31] + srlx %o4, 32, %g2 + srlx %o5, 32, %g3 + stw %o4, [%o1 + 0xc0] ! k[48] + stw %g3, [%o1 + 0xc4] ! k[49] + stw %o5, [%o1 + 0xc8] ! k[50] + stw %g2, [%o1 + 0xcc] ! k[51] + ROTL128(%o4, %o5, %g2, %g3, 49) + stx %o4, [%o1 + 0xe0] ! k[56, 57] + stx %o5, [%o1 + 0xe8] ! k[58, 59] + ldx [%o1 + 0x00], %o4 ! k[ 0, 1] + ldx [%o1 + 0x08], %o5 ! k[ 2, 3] + ROTL128(%o4, %o5, %g2, %g3, 45) + stx %o4, [%o1 + 0x60] ! k[24, 25] + stx %o5, [%o1 + 0x68] ! k[26, 27] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x80] ! k[32, 33] + stx %o5, [%o1 + 0x88] ! k[34, 35] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xb0] ! k[44, 45] + stx %o5, [%o1 + 0xb8] ! k[46, 47] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xf0] ! k[60, 61] + stx %o5, [%o1 + 0xf8] ! k[62, 63] + mov (4 * 16 * 4), %o0 +2: + add %o1, %o0, %o1 + ldd [%o1 + 0x00], %f0 + ldd [%o1 + 0x08], %f2 + std %f0, [%o3 + 0x00] + std %f2, [%o3 + 0x08] + add %o3, 0x10, %o3 +1: + sub %o1, (16 * 4), %o1 + ldd [%o1 + 0x38], %f0 + ldd [%o1 + 0x30], %f2 + ldd [%o1 + 0x28], %f4 + ldd [%o1 + 0x20], %f6 + ldd [%o1 + 0x18], %f8 + ldd [%o1 + 0x10], %f10 + std %f0, [%o3 + 0x00] + std %f2, [%o3 + 0x08] + std %f4, [%o3 + 0x10] + std %f6, [%o3 + 0x18] + std %f8, [%o3 + 0x20] + std %f10, [%o3 + 0x28] + + ldd [%o1 + 0x08], %f0 + ldd [%o1 + 0x00], %f2 + std %f0, [%o3 + 0x30] + std %f2, [%o3 + 0x38] + subcc %o0, (16 * 4), %o0 + bne,pt %icc, 1b + add %o3, (16 * 4), %o3 + + std %f2, [%o3 - 0x10] + std %f0, [%o3 - 0x08] + + retl + VISExit +ENDPROC(camellia_sparc64_key_expand) + + .align 32 +ENTRY(camellia_sparc64_crypt) + /* %o0=key, %o1=input, %o2=output, %o3=key_len */ + VISEntry + + ld [%o1 + 0x00], %f0 + ld [%o1 + 0x04], %f1 + ld [%o1 + 0x08], %f2 + ld [%o1 + 0x0c], %f3 + + ldd [%o0 + 0x00], %f4 + ldd [%o0 + 0x08], %f6 + + cmp %o3, 16 + fxor %f4, %f0, %f0 + be 1f + fxor %f6, %f2, %f2 + + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + add %o0, 0x40, %o0 + + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + +1: + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + ldd [%o0 + 0xc8], %f54 + + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + + st %f2, [%o2 + 0x00] + st %f3, [%o2 + 0x04] + st %f0, [%o2 + 0x08] + st %f1, [%o2 + 0x0c] + + retl + VISExit +ENDPROC(camellia_sparc64_crypt) + + .align 32 +ENTRY(camellia_sparc64_load_keys) + /* %o0=key, %o1=key_len */ + VISEntry + ldd [%o0 + 0x00], %f4 + ldd [%o0 + 0x08], %f6 + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + retl + ldd [%o0 + 0xc8], %f54 +ENDPROC(camellia_sparc64_load_keys) + + .align 32 +ENTRY(camellia_sparc64_ecb_crypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key */ +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + retl + nop +ENDPROC(camellia_sparc64_ecb_crypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_ecb_crypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key */ +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f2 + fxor %f22, %f0, %f0 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + retl + nop +ENDPROC(camellia_sparc64_ecb_crypt_4_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_encrypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f60, %f0, %f0 + fxor %f62, %f2, %f2 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f60 + fxor %f54, %f0, %f62 + std %f60, [%o1 + 0x00] + std %f62, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_encrypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_encrypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f60, %f0, %f0 + fxor %f62, %f2, %f2 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f60 + fxor %f22, %f0, %f62 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + std %f60, [%o1 + 0x00] + std %f62, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_encrypt_4_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_decrypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f56 + ldd [%o0 + 0x08], %f58 + add %o0, 0x10, %o0 + fxor %f4, %f56, %f0 + fxor %f6, %f58, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + fxor %f60, %f2, %f2 + fxor %f62, %f0, %f0 + fsrc2 %f56, %f60 + fsrc2 %f58, %f62 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_decrypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_decrypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f56 + ldd [%o0 + 0x08], %f58 + add %o0, 0x10, %o0 + fxor %f4, %f56, %f0 + fxor %f6, %f58, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f2 + fxor %f22, %f0, %f0 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + fxor %f60, %f2, %f2 + fxor %f62, %f0, %f0 + fsrc2 %f56, %f60 + fsrc2 %f58, %f62 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_decrypt_4_grand_rounds) diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c new file mode 100644 index 00000000000..888f6260b4e --- /dev/null +++ b/arch/sparc/crypto/camellia_glue.c @@ -0,0 +1,327 @@ +/* Glue code for CAMELLIA encryption optimized for sparc64 crypto opcodes. + * + * Copyright (C) 2012 David S. Miller <davem@davemloft.net> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <crypto/algapi.h> + +#include <asm/fpumacro.h> +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +#define CAMELLIA_MIN_KEY_SIZE 16 +#define CAMELLIA_MAX_KEY_SIZE 32 +#define CAMELLIA_BLOCK_SIZE 16 +#define CAMELLIA_TABLE_BYTE_LEN 272 + +struct camellia_sparc64_ctx { + u64 encrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)]; + u64 decrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)]; + int key_len; +}; + +extern void camellia_sparc64_key_expand(const u32 *in_key, u64 *encrypt_key, + unsigned int key_len, u64 *decrypt_key); + +static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key, + unsigned int key_len) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u32 *in_key = (const u32 *) _in_key; + u32 *flags = &tfm->crt_flags; + + if (key_len != 16 && key_len != 24 && key_len != 32) { + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + ctx->key_len = key_len; + + camellia_sparc64_key_expand(in_key, &ctx->encrypt_key[0], + key_len, &ctx->decrypt_key[0]); + return 0; +} + +extern void camellia_sparc64_crypt(const u64 *key, const u32 *input, + u32 *output, unsigned int key_len); + +static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + + camellia_sparc64_crypt(&ctx->encrypt_key[0], + (const u32 *) src, + (u32 *) dst, ctx->key_len); +} + +static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + + camellia_sparc64_crypt(&ctx->decrypt_key[0], + (const u32 *) src, + (u32 *) dst, ctx->key_len); +} + +extern void camellia_sparc64_load_keys(const u64 *key, unsigned int key_len); + +typedef void ecb_crypt_op(const u64 *input, u64 *output, unsigned int len, + const u64 *key); + +extern ecb_crypt_op camellia_sparc64_ecb_crypt_3_grand_rounds; +extern ecb_crypt_op camellia_sparc64_ecb_crypt_4_grand_rounds; + +#define CAMELLIA_BLOCK_MASK (~(CAMELLIA_BLOCK_SIZE - 1)) + +static int __ecb_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, bool encrypt) +{ + struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + ecb_crypt_op *op; + const u64 *key; + int err; + + op = camellia_sparc64_ecb_crypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_ecb_crypt_4_grand_rounds; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + if (encrypt) + key = &ctx->encrypt_key[0]; + else + key = &ctx->decrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64; + u64 *dst64; + + src64 = (const u64 *)walk.src.virt.addr; + dst64 = (u64 *) walk.dst.virt.addr; + op(src64, dst64, block_len, key); + } + nbytes &= CAMELLIA_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int ecb_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb_crypt(desc, dst, src, nbytes, true); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb_crypt(desc, dst, src, nbytes, false); +} + +typedef void cbc_crypt_op(const u64 *input, u64 *output, unsigned int len, + const u64 *key, u64 *iv); + +extern cbc_crypt_op camellia_sparc64_cbc_encrypt_3_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_encrypt_4_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_decrypt_3_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_decrypt_4_grand_rounds; + +static int cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + cbc_crypt_op *op; + const u64 *key; + int err; + + op = camellia_sparc64_cbc_encrypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_cbc_encrypt_4_grand_rounds; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + key = &ctx->encrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64; + u64 *dst64; + + src64 = (const u64 *)walk.src.virt.addr; + dst64 = (u64 *) walk.dst.virt.addr; + op(src64, dst64, block_len, key, + (u64 *) walk.iv); + } + nbytes &= CAMELLIA_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct camellia_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + cbc_crypt_op *op; + const u64 *key; + int err; + + op = camellia_sparc64_cbc_decrypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_cbc_decrypt_4_grand_rounds; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + key = &ctx->decrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & CAMELLIA_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64; + u64 *dst64; + + src64 = (const u64 *)walk.src.virt.addr; + dst64 = (u64 *) walk.dst.virt.addr; + op(src64, dst64, block_len, key, + (u64 *) walk.iv); + } + nbytes &= CAMELLIA_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static struct crypto_alg algs[] = { { + .cra_name = "camellia", + .cra_driver_name = "camellia-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, + .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE, + .cia_setkey = camellia_set_key, + .cia_encrypt = camellia_encrypt, + .cia_decrypt = camellia_decrypt + } + } +}, { + .cra_name = "ecb(camellia)", + .cra_driver_name = "ecb-camellia-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_set_key, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(camellia)", + .cra_driver_name = "cbc-camellia-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_set_key, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +} +}; + +static bool __init sparc64_has_camellia_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_CAMELLIA)) + return false; + + return true; +} + +static int __init camellia_sparc64_mod_init(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(algs); i++) + INIT_LIST_HEAD(&algs[i].cra_list); + + if (sparc64_has_camellia_opcode()) { + pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n"); + return crypto_register_algs(algs, ARRAY_SIZE(algs)); + } + pr_info("sparc64 camellia opcodes not available.\n"); + return -ENODEV; +} + +static void __exit camellia_sparc64_mod_fini(void) +{ + crypto_unregister_algs(algs, ARRAY_SIZE(algs)); +} + +module_init(camellia_sparc64_mod_init); +module_exit(camellia_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated"); + +MODULE_ALIAS("aes"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/crc32c_asm.S b/arch/sparc/crypto/crc32c_asm.S new file mode 100644 index 00000000000..2b1976e765b --- /dev/null +++ b/arch/sparc/crypto/crc32c_asm.S @@ -0,0 +1,20 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> +#include <asm/asi.h> + +#include "opcodes.h" + +ENTRY(crc32c_sparc64) + /* %o0=crc32p, %o1=data_ptr, %o2=len */ + VISEntryHalf + lda [%o0] ASI_PL, %f1 +1: ldd [%o1], %f2 + CRC32C(0,2,0) + subcc %o2, 8, %o2 + bne,pt %icc, 1b + add %o1, 0x8, %o1 + sta %f1, [%o0] ASI_PL + VISExitHalf +2: retl + nop +ENDPROC(crc32c_sparc64) diff --git a/arch/sparc/crypto/crc32c_glue.c b/arch/sparc/crypto/crc32c_glue.c new file mode 100644 index 00000000000..5162fad912c --- /dev/null +++ b/arch/sparc/crypto/crc32c_glue.c @@ -0,0 +1,181 @@ +/* Glue code for CRC32C optimized for sparc64 crypto opcodes. + * + * This is based largely upon arch/x86/crypto/crc32c-intel.c + * + * Copyright (C) 2008 Intel Corporation + * Authors: Austin Zhang <austin_zhang@linux.intel.com> + * Kent Liu <kent.liu@intel.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/string.h> +#include <linux/kernel.h> +#include <linux/crc32.h> + +#include <crypto/internal/hash.h> + +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +/* + * Setting the seed allows arbitrary accumulators and flexible XOR policy + * If your algorithm starts with ~0, then XOR with ~0 before you set + * the seed. + */ +static int crc32c_sparc64_setkey(struct crypto_shash *hash, const u8 *key, + unsigned int keylen) +{ + u32 *mctx = crypto_shash_ctx(hash); + + if (keylen != sizeof(u32)) { + crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + *(__le32 *)mctx = le32_to_cpup((__le32 *)key); + return 0; +} + +static int crc32c_sparc64_init(struct shash_desc *desc) +{ + u32 *mctx = crypto_shash_ctx(desc->tfm); + u32 *crcp = shash_desc_ctx(desc); + + *crcp = *mctx; + + return 0; +} + +extern void crc32c_sparc64(u32 *crcp, const u64 *data, unsigned int len); + +static void crc32c_compute(u32 *crcp, const u64 *data, unsigned int len) +{ + unsigned int asm_len; + + asm_len = len & ~7U; + if (asm_len) { + crc32c_sparc64(crcp, data, asm_len); + data += asm_len / 8; + len -= asm_len; + } + if (len) + *crcp = __crc32c_le(*crcp, (const unsigned char *) data, len); +} + +static int crc32c_sparc64_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + u32 *crcp = shash_desc_ctx(desc); + + crc32c_compute(crcp, (const u64 *) data, len); + + return 0; +} + +static int __crc32c_sparc64_finup(u32 *crcp, const u8 *data, unsigned int len, + u8 *out) +{ + u32 tmp = *crcp; + + crc32c_compute(&tmp, (const u64 *) data, len); + + *(__le32 *) out = ~cpu_to_le32(tmp); + return 0; +} + +static int crc32c_sparc64_finup(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32c_sparc64_finup(shash_desc_ctx(desc), data, len, out); +} + +static int crc32c_sparc64_final(struct shash_desc *desc, u8 *out) +{ + u32 *crcp = shash_desc_ctx(desc); + + *(__le32 *) out = ~cpu_to_le32p(crcp); + return 0; +} + +static int crc32c_sparc64_digest(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32c_sparc64_finup(crypto_shash_ctx(desc->tfm), data, len, + out); +} + +static int crc32c_sparc64_cra_init(struct crypto_tfm *tfm) +{ + u32 *key = crypto_tfm_ctx(tfm); + + *key = ~0; + + return 0; +} + +#define CHKSUM_BLOCK_SIZE 1 +#define CHKSUM_DIGEST_SIZE 4 + +static struct shash_alg alg = { + .setkey = crc32c_sparc64_setkey, + .init = crc32c_sparc64_init, + .update = crc32c_sparc64_update, + .final = crc32c_sparc64_final, + .finup = crc32c_sparc64_finup, + .digest = crc32c_sparc64_digest, + .descsize = sizeof(u32), + .digestsize = CHKSUM_DIGEST_SIZE, + .base = { + .cra_name = "crc32c", + .cra_driver_name = "crc32c-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_blocksize = CHKSUM_BLOCK_SIZE, + .cra_ctxsize = sizeof(u32), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + .cra_init = crc32c_sparc64_cra_init, + } +}; + +static bool __init sparc64_has_crc32c_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_CRC32C)) + return false; + + return true; +} + +static int __init crc32c_sparc64_mod_init(void) +{ + if (sparc64_has_crc32c_opcode()) { + pr_info("Using sparc64 crc32c opcode optimized CRC32C implementation\n"); + return crypto_register_shash(&alg); + } + pr_info("sparc64 crc32c opcode not available.\n"); + return -ENODEV; +} + +static void __exit crc32c_sparc64_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(crc32c_sparc64_mod_init); +module_exit(crc32c_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("CRC32c (Castagnoli), sparc64 crc32c opcode accelerated"); + +MODULE_ALIAS("crc32c"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/crop_devid.c b/arch/sparc/crypto/crop_devid.c new file mode 100644 index 00000000000..5f5724a0ae2 --- /dev/null +++ b/arch/sparc/crypto/crop_devid.c @@ -0,0 +1,14 @@ +#include <linux/module.h> +#include <linux/of_device.h> + +/* This is a dummy device table linked into all of the crypto + * opcode drivers. It serves to trigger the module autoloading + * mechanisms in userspace which scan the OF device tree and + * load any modules which have device table entries that + * match OF device nodes. + */ +static const struct of_device_id crypto_opcode_match[] = { + { .name = "cpu", .compatible = "sun4v", }, + {}, +}; +MODULE_DEVICE_TABLE(of, crypto_opcode_match); diff --git a/arch/sparc/crypto/des_asm.S b/arch/sparc/crypto/des_asm.S new file mode 100644 index 00000000000..b5c8fc269b5 --- /dev/null +++ b/arch/sparc/crypto/des_asm.S @@ -0,0 +1,419 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + + .align 32 +ENTRY(des_sparc64_key_expand) + /* %o0=input_key, %o1=output_key */ + VISEntryHalf + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + DES_KEXPAND(0, 0, 0) + DES_KEXPAND(0, 1, 2) + DES_KEXPAND(2, 3, 6) + DES_KEXPAND(2, 2, 4) + DES_KEXPAND(6, 3, 10) + DES_KEXPAND(6, 2, 8) + DES_KEXPAND(10, 3, 14) + DES_KEXPAND(10, 2, 12) + DES_KEXPAND(14, 1, 16) + DES_KEXPAND(16, 3, 20) + DES_KEXPAND(16, 2, 18) + DES_KEXPAND(20, 3, 24) + DES_KEXPAND(20, 2, 22) + DES_KEXPAND(24, 3, 28) + DES_KEXPAND(24, 2, 26) + DES_KEXPAND(28, 1, 30) + std %f0, [%o1 + 0x00] + std %f2, [%o1 + 0x08] + std %f4, [%o1 + 0x10] + std %f6, [%o1 + 0x18] + std %f8, [%o1 + 0x20] + std %f10, [%o1 + 0x28] + std %f12, [%o1 + 0x30] + std %f14, [%o1 + 0x38] + std %f16, [%o1 + 0x40] + std %f18, [%o1 + 0x48] + std %f20, [%o1 + 0x50] + std %f22, [%o1 + 0x58] + std %f24, [%o1 + 0x60] + std %f26, [%o1 + 0x68] + std %f28, [%o1 + 0x70] + std %f30, [%o1 + 0x78] + retl + VISExitHalf +ENDPROC(des_sparc64_key_expand) + + .align 32 +ENTRY(des_sparc64_crypt) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ldd [%o1 + 0x00], %f32 + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + ldd [%o0 + 0x30], %f12 + ldd [%o0 + 0x38], %f14 + ldd [%o0 + 0x40], %f16 + ldd [%o0 + 0x48], %f18 + ldd [%o0 + 0x50], %f20 + ldd [%o0 + 0x58], %f22 + ldd [%o0 + 0x60], %f24 + ldd [%o0 + 0x68], %f26 + ldd [%o0 + 0x70], %f28 + ldd [%o0 + 0x78], %f30 + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + DES_ROUND(4, 6, 32, 32) + DES_ROUND(8, 10, 32, 32) + DES_ROUND(12, 14, 32, 32) + DES_ROUND(16, 18, 32, 32) + DES_ROUND(20, 22, 32, 32) + DES_ROUND(24, 26, 32, 32) + DES_ROUND(28, 30, 32, 32) + DES_IIP(32, 32) + std %f32, [%o2 + 0x00] + retl + VISExit +ENDPROC(des_sparc64_crypt) + + .align 32 +ENTRY(des_sparc64_load_keys) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + ldd [%o0 + 0x30], %f12 + ldd [%o0 + 0x38], %f14 + ldd [%o0 + 0x40], %f16 + ldd [%o0 + 0x48], %f18 + ldd [%o0 + 0x50], %f20 + ldd [%o0 + 0x58], %f22 + ldd [%o0 + 0x60], %f24 + ldd [%o0 + 0x68], %f26 + ldd [%o0 + 0x70], %f28 + retl + ldd [%o0 + 0x78], %f30 +ENDPROC(des_sparc64_load_keys) + + .align 32 +ENTRY(des_sparc64_ecb_crypt) + /* %o0=input, %o1=output, %o2=len */ +1: ldd [%o0 + 0x00], %f32 + add %o0, 0x08, %o0 + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + DES_ROUND(4, 6, 32, 32) + DES_ROUND(8, 10, 32, 32) + DES_ROUND(12, 14, 32, 32) + DES_ROUND(16, 18, 32, 32) + DES_ROUND(20, 22, 32, 32) + DES_ROUND(24, 26, 32, 32) + DES_ROUND(28, 30, 32, 32) + DES_IIP(32, 32) + std %f32, [%o1 + 0x00] + subcc %o2, 0x08, %o2 + bne,pt %icc, 1b + add %o1, 0x08, %o1 + retl + nop +ENDPROC(des_sparc64_ecb_crypt) + + .align 32 +ENTRY(des_sparc64_cbc_encrypt) + /* %o0=input, %o1=output, %o2=len, %o3=IV */ + ldd [%o3 + 0x00], %f32 +1: ldd [%o0 + 0x00], %f34 + fxor %f32, %f34, %f32 + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + DES_ROUND(4, 6, 32, 32) + DES_ROUND(8, 10, 32, 32) + DES_ROUND(12, 14, 32, 32) + DES_ROUND(16, 18, 32, 32) + DES_ROUND(20, 22, 32, 32) + DES_ROUND(24, 26, 32, 32) + DES_ROUND(28, 30, 32, 32) + DES_IIP(32, 32) + std %f32, [%o1 + 0x00] + add %o0, 0x08, %o0 + subcc %o2, 0x08, %o2 + bne,pt %icc, 1b + add %o1, 0x08, %o1 + retl + std %f32, [%o3 + 0x00] +ENDPROC(des_sparc64_cbc_encrypt) + + .align 32 +ENTRY(des_sparc64_cbc_decrypt) + /* %o0=input, %o1=output, %o2=len, %o3=IV */ + ldd [%o3 + 0x00], %f34 +1: ldd [%o0 + 0x00], %f36 + DES_IP(36, 32) + DES_ROUND(0, 2, 32, 32) + DES_ROUND(4, 6, 32, 32) + DES_ROUND(8, 10, 32, 32) + DES_ROUND(12, 14, 32, 32) + DES_ROUND(16, 18, 32, 32) + DES_ROUND(20, 22, 32, 32) + DES_ROUND(24, 26, 32, 32) + DES_ROUND(28, 30, 32, 32) + DES_IIP(32, 32) + fxor %f32, %f34, %f32 + fsrc2 %f36, %f34 + std %f32, [%o1 + 0x00] + add %o0, 0x08, %o0 + subcc %o2, 0x08, %o2 + bne,pt %icc, 1b + add %o1, 0x08, %o1 + retl + std %f36, [%o3 + 0x00] +ENDPROC(des_sparc64_cbc_decrypt) + + .align 32 +ENTRY(des3_ede_sparc64_crypt) + /* %o0=key, %o1=input, %o2=output */ + VISEntry + ldd [%o1 + 0x00], %f32 + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + ldd [%o0 + 0x30], %f12 + ldd [%o0 + 0x38], %f14 + ldd [%o0 + 0x40], %f16 + ldd [%o0 + 0x48], %f18 + ldd [%o0 + 0x50], %f20 + ldd [%o0 + 0x58], %f22 + ldd [%o0 + 0x60], %f24 + ldd [%o0 + 0x68], %f26 + ldd [%o0 + 0x70], %f28 + ldd [%o0 + 0x78], %f30 + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + ldd [%o0 + 0x80], %f0 + ldd [%o0 + 0x88], %f2 + DES_ROUND(4, 6, 32, 32) + ldd [%o0 + 0x90], %f4 + ldd [%o0 + 0x98], %f6 + DES_ROUND(8, 10, 32, 32) + ldd [%o0 + 0xa0], %f8 + ldd [%o0 + 0xa8], %f10 + DES_ROUND(12, 14, 32, 32) + ldd [%o0 + 0xb0], %f12 + ldd [%o0 + 0xb8], %f14 + DES_ROUND(16, 18, 32, 32) + ldd [%o0 + 0xc0], %f16 + ldd [%o0 + 0xc8], %f18 + DES_ROUND(20, 22, 32, 32) + ldd [%o0 + 0xd0], %f20 + ldd [%o0 + 0xd8], %f22 + DES_ROUND(24, 26, 32, 32) + ldd [%o0 + 0xe0], %f24 + ldd [%o0 + 0xe8], %f26 + DES_ROUND(28, 30, 32, 32) + ldd [%o0 + 0xf0], %f28 + ldd [%o0 + 0xf8], %f30 + DES_IIP(32, 32) + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + ldd [%o0 + 0x100], %f0 + ldd [%o0 + 0x108], %f2 + DES_ROUND(4, 6, 32, 32) + ldd [%o0 + 0x110], %f4 + ldd [%o0 + 0x118], %f6 + DES_ROUND(8, 10, 32, 32) + ldd [%o0 + 0x120], %f8 + ldd [%o0 + 0x128], %f10 + DES_ROUND(12, 14, 32, 32) + ldd [%o0 + 0x130], %f12 + ldd [%o0 + 0x138], %f14 + DES_ROUND(16, 18, 32, 32) + ldd [%o0 + 0x140], %f16 + ldd [%o0 + 0x148], %f18 + DES_ROUND(20, 22, 32, 32) + ldd [%o0 + 0x150], %f20 + ldd [%o0 + 0x158], %f22 + DES_ROUND(24, 26, 32, 32) + ldd [%o0 + 0x160], %f24 + ldd [%o0 + 0x168], %f26 + DES_ROUND(28, 30, 32, 32) + ldd [%o0 + 0x170], %f28 + ldd [%o0 + 0x178], %f30 + DES_IIP(32, 32) + DES_IP(32, 32) + DES_ROUND(0, 2, 32, 32) + DES_ROUND(4, 6, 32, 32) + DES_ROUND(8, 10, 32, 32) + DES_ROUND(12, 14, 32, 32) + DES_ROUND(16, 18, 32, 32) + DES_ROUND(20, 22, 32, 32) + DES_ROUND(24, 26, 32, 32) + DES_ROUND(28, 30, 32, 32) + DES_IIP(32, 32) + + std %f32, [%o2 + 0x00] + retl + VISExit +ENDPROC(des3_ede_sparc64_crypt) + + .align 32 +ENTRY(des3_ede_sparc64_load_keys) + /* %o0=key */ + VISEntry + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + ldd [%o0 + 0x30], %f12 + ldd [%o0 + 0x38], %f14 + ldd [%o0 + 0x40], %f16 + ldd [%o0 + 0x48], %f18 + ldd [%o0 + 0x50], %f20 + ldd [%o0 + 0x58], %f22 + ldd [%o0 + 0x60], %f24 + ldd [%o0 + 0x68], %f26 + ldd [%o0 + 0x70], %f28 + ldd [%o0 + 0x78], %f30 + ldd [%o0 + 0x80], %f32 + ldd [%o0 + 0x88], %f34 + ldd [%o0 + 0x90], %f36 + ldd [%o0 + 0x98], %f38 + ldd [%o0 + 0xa0], %f40 + ldd [%o0 + 0xa8], %f42 + ldd [%o0 + 0xb0], %f44 + ldd [%o0 + 0xb8], %f46 + ldd [%o0 + 0xc0], %f48 + ldd [%o0 + 0xc8], %f50 + ldd [%o0 + 0xd0], %f52 + ldd [%o0 + 0xd8], %f54 + ldd [%o0 + 0xe0], %f56 + retl + ldd [%o0 + 0xe8], %f58 +ENDPROC(des3_ede_sparc64_load_keys) + +#define DES3_LOOP_BODY(X) \ + DES_IP(X, X) \ + DES_ROUND(0, 2, X, X) \ + DES_ROUND(4, 6, X, X) \ + DES_ROUND(8, 10, X, X) \ + DES_ROUND(12, 14, X, X) \ + DES_ROUND(16, 18, X, X) \ + ldd [%o0 + 0xf0], %f16; \ + ldd [%o0 + 0xf8], %f18; \ + DES_ROUND(20, 22, X, X) \ + ldd [%o0 + 0x100], %f20; \ + ldd [%o0 + 0x108], %f22; \ + DES_ROUND(24, 26, X, X) \ + ldd [%o0 + 0x110], %f24; \ + ldd [%o0 + 0x118], %f26; \ + DES_ROUND(28, 30, X, X) \ + ldd [%o0 + 0x120], %f28; \ + ldd [%o0 + 0x128], %f30; \ + DES_IIP(X, X) \ + DES_IP(X, X) \ + DES_ROUND(32, 34, X, X) \ + ldd [%o0 + 0x130], %f0; \ + ldd [%o0 + 0x138], %f2; \ + DES_ROUND(36, 38, X, X) \ + ldd [%o0 + 0x140], %f4; \ + ldd [%o0 + 0x148], %f6; \ + DES_ROUND(40, 42, X, X) \ + ldd [%o0 + 0x150], %f8; \ + ldd [%o0 + 0x158], %f10; \ + DES_ROUND(44, 46, X, X) \ + ldd [%o0 + 0x160], %f12; \ + ldd [%o0 + 0x168], %f14; \ + DES_ROUND(48, 50, X, X) \ + DES_ROUND(52, 54, X, X) \ + DES_ROUND(56, 58, X, X) \ + DES_ROUND(16, 18, X, X) \ + ldd [%o0 + 0x170], %f16; \ + ldd [%o0 + 0x178], %f18; \ + DES_IIP(X, X) \ + DES_IP(X, X) \ + DES_ROUND(20, 22, X, X) \ + ldd [%o0 + 0x50], %f20; \ + ldd [%o0 + 0x58], %f22; \ + DES_ROUND(24, 26, X, X) \ + ldd [%o0 + 0x60], %f24; \ + ldd [%o0 + 0x68], %f26; \ + DES_ROUND(28, 30, X, X) \ + ldd [%o0 + 0x70], %f28; \ + ldd [%o0 + 0x78], %f30; \ + DES_ROUND(0, 2, X, X) \ + ldd [%o0 + 0x00], %f0; \ + ldd [%o0 + 0x08], %f2; \ + DES_ROUND(4, 6, X, X) \ + ldd [%o0 + 0x10], %f4; \ + ldd [%o0 + 0x18], %f6; \ + DES_ROUND(8, 10, X, X) \ + ldd [%o0 + 0x20], %f8; \ + ldd [%o0 + 0x28], %f10; \ + DES_ROUND(12, 14, X, X) \ + ldd [%o0 + 0x30], %f12; \ + ldd [%o0 + 0x38], %f14; \ + DES_ROUND(16, 18, X, X) \ + ldd [%o0 + 0x40], %f16; \ + ldd [%o0 + 0x48], %f18; \ + DES_IIP(X, X) + + .align 32 +ENTRY(des3_ede_sparc64_ecb_crypt) + /* %o0=key, %o1=input, %o2=output, %o3=len */ +1: ldd [%o1 + 0x00], %f60 + DES3_LOOP_BODY(60) + std %f60, [%o2 + 0x00] + add %o1, 0x08, %o1 + subcc %o3, 0x08, %o3 + bne,pt %icc, 1b + add %o2, 0x08, %o2 + retl + nop +ENDPROC(des3_ede_sparc64_ecb_crypt) + + .align 32 +ENTRY(des3_ede_sparc64_cbc_encrypt) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldd [%o4 + 0x00], %f60 +1: ldd [%o1 + 0x00], %f62 + fxor %f60, %f62, %f60 + DES3_LOOP_BODY(60) + std %f60, [%o2 + 0x00] + add %o1, 0x08, %o1 + subcc %o3, 0x08, %o3 + bne,pt %icc, 1b + add %o2, 0x08, %o2 + retl + std %f60, [%o4 + 0x00] +ENDPROC(des3_ede_sparc64_cbc_encrypt) + + .align 32 +ENTRY(des3_ede_sparc64_cbc_decrypt) + /* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */ + ldd [%o4 + 0x00], %f62 +1: ldx [%o1 + 0x00], %g1 + MOVXTOD_G1_F60 + DES3_LOOP_BODY(60) + fxor %f62, %f60, %f60 + MOVXTOD_G1_F62 + std %f60, [%o2 + 0x00] + add %o1, 0x08, %o1 + subcc %o3, 0x08, %o3 + bne,pt %icc, 1b + add %o2, 0x08, %o2 + retl + stx %g1, [%o4 + 0x00] +ENDPROC(des3_ede_sparc64_cbc_decrypt) diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c new file mode 100644 index 00000000000..3065bc61f9d --- /dev/null +++ b/arch/sparc/crypto/des_glue.c @@ -0,0 +1,537 @@ +/* Glue code for DES encryption optimized for sparc64 crypto opcodes. + * + * Copyright (C) 2012 David S. Miller <davem@davemloft.net> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/types.h> +#include <crypto/algapi.h> +#include <crypto/des.h> + +#include <asm/fpumacro.h> +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +struct des_sparc64_ctx { + u64 encrypt_expkey[DES_EXPKEY_WORDS / 2]; + u64 decrypt_expkey[DES_EXPKEY_WORDS / 2]; +}; + +struct des3_ede_sparc64_ctx { + u64 encrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; + u64 decrypt_expkey[DES3_EDE_EXPKEY_WORDS / 2]; +}; + +static void encrypt_to_decrypt(u64 *d, const u64 *e) +{ + const u64 *s = e + (DES_EXPKEY_WORDS / 2) - 1; + int i; + + for (i = 0; i < DES_EXPKEY_WORDS / 2; i++) + *d++ = *s--; +} + +extern void des_sparc64_key_expand(const u32 *input_key, u64 *key); + +static int des_set_key(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct des_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); + u32 *flags = &tfm->crt_flags; + u32 tmp[DES_EXPKEY_WORDS]; + int ret; + + /* Even though we have special instructions for key expansion, + * we call des_ekey() so that we don't have to write our own + * weak key detection code. + */ + ret = des_ekey(tmp, key); + if (unlikely(ret == 0) && (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + des_sparc64_key_expand((const u32 *) key, &dctx->encrypt_expkey[0]); + encrypt_to_decrypt(&dctx->decrypt_expkey[0], &dctx->encrypt_expkey[0]); + + return 0; +} + +extern void des_sparc64_crypt(const u64 *key, const u64 *input, + u64 *output); + +static void des_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u64 *K = ctx->encrypt_expkey; + + des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); +} + +static void des_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct des_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u64 *K = ctx->decrypt_expkey; + + des_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); +} + +extern void des_sparc64_load_keys(const u64 *key); + +extern void des_sparc64_ecb_crypt(const u64 *input, u64 *output, + unsigned int len); + +#define DES_BLOCK_MASK (~(DES_BLOCK_SIZE - 1)) + +static int __ecb_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, bool encrypt) +{ + struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + if (encrypt) + des_sparc64_load_keys(&ctx->encrypt_expkey[0]); + else + des_sparc64_load_keys(&ctx->decrypt_expkey[0]); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + des_sparc64_ecb_crypt((const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int ecb_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb_crypt(desc, dst, src, nbytes, true); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb_crypt(desc, dst, src, nbytes, false); +} + +extern void des_sparc64_cbc_encrypt(const u64 *input, u64 *output, + unsigned int len, u64 *iv); + +static int cbc_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + des_sparc64_load_keys(&ctx->encrypt_expkey[0]); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + des_sparc64_cbc_encrypt((const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len, (u64 *) walk.iv); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +extern void des_sparc64_cbc_decrypt(const u64 *input, u64 *output, + unsigned int len, u64 *iv); + +static int cbc_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct des_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + des_sparc64_load_keys(&ctx->decrypt_expkey[0]); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + des_sparc64_cbc_decrypt((const u64 *)walk.src.virt.addr, + (u64 *) walk.dst.virt.addr, + block_len, (u64 *) walk.iv); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int des3_ede_set_key(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct des3_ede_sparc64_ctx *dctx = crypto_tfm_ctx(tfm); + const u32 *K = (const u32 *)key; + u32 *flags = &tfm->crt_flags; + u64 k1[DES_EXPKEY_WORDS / 2]; + u64 k2[DES_EXPKEY_WORDS / 2]; + u64 k3[DES_EXPKEY_WORDS / 2]; + + if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || + !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && + (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + + des_sparc64_key_expand((const u32 *)key, k1); + key += DES_KEY_SIZE; + des_sparc64_key_expand((const u32 *)key, k2); + key += DES_KEY_SIZE; + des_sparc64_key_expand((const u32 *)key, k3); + + memcpy(&dctx->encrypt_expkey[0], &k1[0], sizeof(k1)); + encrypt_to_decrypt(&dctx->encrypt_expkey[DES_EXPKEY_WORDS / 2], &k2[0]); + memcpy(&dctx->encrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], + &k3[0], sizeof(k3)); + + encrypt_to_decrypt(&dctx->decrypt_expkey[0], &k3[0]); + memcpy(&dctx->decrypt_expkey[DES_EXPKEY_WORDS / 2], + &k2[0], sizeof(k2)); + encrypt_to_decrypt(&dctx->decrypt_expkey[(DES_EXPKEY_WORDS / 2) * 2], + &k1[0]); + + return 0; +} + +extern void des3_ede_sparc64_crypt(const u64 *key, const u64 *input, + u64 *output); + +static void des3_ede_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u64 *K = ctx->encrypt_expkey; + + des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); +} + +static void des3_ede_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct des3_ede_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u64 *K = ctx->decrypt_expkey; + + des3_ede_sparc64_crypt(K, (const u64 *) src, (u64 *) dst); +} + +extern void des3_ede_sparc64_load_keys(const u64 *key); + +extern void des3_ede_sparc64_ecb_crypt(const u64 *expkey, const u64 *input, + u64 *output, unsigned int len); + +static int __ecb3_crypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes, bool encrypt) +{ + struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + const u64 *K; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + if (encrypt) + K = &ctx->encrypt_expkey[0]; + else + K = &ctx->decrypt_expkey[0]; + des3_ede_sparc64_load_keys(K); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64 = (const u64 *)walk.src.virt.addr; + des3_ede_sparc64_ecb_crypt(K, src64, + (u64 *) walk.dst.virt.addr, + block_len); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static int ecb3_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb3_crypt(desc, dst, src, nbytes, true); +} + +static int ecb3_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + return __ecb3_crypt(desc, dst, src, nbytes, false); +} + +extern void des3_ede_sparc64_cbc_encrypt(const u64 *expkey, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +static int cbc3_encrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + const u64 *K; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + K = &ctx->encrypt_expkey[0]; + des3_ede_sparc64_load_keys(K); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64 = (const u64 *)walk.src.virt.addr; + des3_ede_sparc64_cbc_encrypt(K, src64, + (u64 *) walk.dst.virt.addr, + block_len, + (u64 *) walk.iv); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +extern void des3_ede_sparc64_cbc_decrypt(const u64 *expkey, const u64 *input, + u64 *output, unsigned int len, + u64 *iv); + +static int cbc3_decrypt(struct blkcipher_desc *desc, + struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct des3_ede_sparc64_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + struct blkcipher_walk walk; + const u64 *K; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + K = &ctx->decrypt_expkey[0]; + des3_ede_sparc64_load_keys(K); + while ((nbytes = walk.nbytes)) { + unsigned int block_len = nbytes & DES_BLOCK_MASK; + + if (likely(block_len)) { + const u64 *src64 = (const u64 *)walk.src.virt.addr; + des3_ede_sparc64_cbc_decrypt(K, src64, + (u64 *) walk.dst.virt.addr, + block_len, + (u64 *) walk.iv); + } + nbytes &= DES_BLOCK_SIZE - 1; + err = blkcipher_walk_done(desc, &walk, nbytes); + } + fprs_write(0); + return err; +} + +static struct crypto_alg algs[] = { { + .cra_name = "des", + .cra_driver_name = "des-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des_sparc64_ctx), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = DES_KEY_SIZE, + .cia_max_keysize = DES_KEY_SIZE, + .cia_setkey = des_set_key, + .cia_encrypt = des_encrypt, + .cia_decrypt = des_decrypt + } + } +}, { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = des_set_key, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = des_set_key, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "des3_ede", + .cra_driver_name = "des3_ede-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .cra_alignmask = 7, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = DES3_EDE_KEY_SIZE, + .cia_max_keysize = DES3_EDE_KEY_SIZE, + .cia_setkey = des3_ede_set_key, + .cia_encrypt = des3_ede_encrypt, + .cia_decrypt = des3_ede_decrypt + } + } +}, { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3_ede-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = des3_ede_set_key, + .encrypt = ecb3_encrypt, + .decrypt = ecb3_decrypt, + }, + }, +}, { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3_ede-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct des3_ede_sparc64_ctx), + .cra_alignmask = 7, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .blkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .setkey = des3_ede_set_key, + .encrypt = cbc3_encrypt, + .decrypt = cbc3_decrypt, + }, + }, +} }; + +static bool __init sparc64_has_des_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_DES)) + return false; + + return true; +} + +static int __init des_sparc64_mod_init(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(algs); i++) + INIT_LIST_HEAD(&algs[i].cra_list); + + if (sparc64_has_des_opcode()) { + pr_info("Using sparc64 des opcodes optimized DES implementation\n"); + return crypto_register_algs(algs, ARRAY_SIZE(algs)); + } + pr_info("sparc64 des opcodes not available.\n"); + return -ENODEV; +} + +static void __exit des_sparc64_mod_fini(void) +{ + crypto_unregister_algs(algs, ARRAY_SIZE(algs)); +} + +module_init(des_sparc64_mod_init); +module_exit(des_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); + +MODULE_ALIAS("des"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/md5_asm.S b/arch/sparc/crypto/md5_asm.S new file mode 100644 index 00000000000..3150404e602 --- /dev/null +++ b/arch/sparc/crypto/md5_asm.S @@ -0,0 +1,70 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +ENTRY(md5_sparc64_transform) + /* %o0 = digest, %o1 = data, %o2 = rounds */ + VISEntryHalf + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + andcc %o1, 0x7, %g0 + ld [%o0 + 0x08], %f2 + bne,pn %xcc, 10f + ld [%o0 + 0x0c], %f3 + +1: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + ldd [%o1 + 0x38], %f22 + + MD5 + + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + +5: + st %f0, [%o0 + 0x00] + st %f1, [%o0 + 0x04] + st %f2, [%o0 + 0x08] + st %f3, [%o0 + 0x0c] + retl + VISExitHalf +10: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +1: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + ldd [%o1 + 0x40], %f26 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + MD5 + + subcc %o2, 1, %o2 + fsrc2 %f26, %f10 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + + ba,a,pt %xcc, 5b +ENDPROC(md5_sparc64_transform) diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c new file mode 100644 index 00000000000..09a9ea1dfb6 --- /dev/null +++ b/arch/sparc/crypto/md5_glue.c @@ -0,0 +1,190 @@ +/* Glue code for MD5 hashing optimized for sparc64 crypto opcodes. + * + * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c + * and crypto/md5.c which are: + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> + * Copyright (c) Mathias Krause <minipli@googlemail.com> + * Copyright (c) Cryptoapi developers. + * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/md5.h> + +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +asmlinkage void md5_sparc64_transform(u32 *digest, const char *data, + unsigned int rounds); + +static int md5_sparc64_init(struct shash_desc *desc) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + + mctx->hash[0] = cpu_to_le32(0x67452301); + mctx->hash[1] = cpu_to_le32(0xefcdab89); + mctx->hash[2] = cpu_to_le32(0x98badcfe); + mctx->hash[3] = cpu_to_le32(0x10325476); + mctx->byte_count = 0; + + return 0; +} + +static void __md5_sparc64_update(struct md5_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + sctx->byte_count += len; + if (partial) { + done = MD5_HMAC_BLOCK_SIZE - partial; + memcpy((u8 *)sctx->block + partial, data, done); + md5_sparc64_transform(sctx->hash, (u8 *)sctx->block, 1); + } + if (len - done >= MD5_HMAC_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / MD5_HMAC_BLOCK_SIZE; + + md5_sparc64_transform(sctx->hash, data + done, rounds); + done += rounds * MD5_HMAC_BLOCK_SIZE; + } + + memcpy(sctx->block, data + done, len - done); +} + +static int md5_sparc64_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; + + /* Handle the fast case right here */ + if (partial + len < MD5_HMAC_BLOCK_SIZE) { + sctx->byte_count += len; + memcpy((u8 *)sctx->block + partial, data, len); + } else + __md5_sparc64_update(sctx, data, len, partial); + + return 0; +} + +/* Add padding and return the message digest. */ +static int md5_sparc64_final(struct shash_desc *desc, u8 *out) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + u32 *dst = (u32 *)out; + __le64 bits; + static const u8 padding[MD5_HMAC_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_le64(sctx->byte_count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->byte_count % MD5_HMAC_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((MD5_HMAC_BLOCK_SIZE+56) - index); + + /* We need to fill a whole block for __md5_sparc64_update() */ + if (padlen <= 56) { + sctx->byte_count += padlen; + memcpy((u8 *)sctx->block + index, padding, padlen); + } else { + __md5_sparc64_update(sctx, padding, padlen, index); + } + __md5_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + + /* Store state in digest */ + for (i = 0; i < MD5_HASH_WORDS; i++) + dst[i] = sctx->hash[i]; + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int md5_sparc64_export(struct shash_desc *desc, void *out) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int md5_sparc64_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = MD5_DIGEST_SIZE, + .init = md5_sparc64_init, + .update = md5_sparc64_update, + .final = md5_sparc64_final, + .export = md5_sparc64_export, + .import = md5_sparc64_import, + .descsize = sizeof(struct md5_state), + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name= "md5-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static bool __init sparc64_has_md5_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_MD5)) + return false; + + return true; +} + +static int __init md5_sparc64_mod_init(void) +{ + if (sparc64_has_md5_opcode()) { + pr_info("Using sparc64 md5 opcode optimized MD5 implementation\n"); + return crypto_register_shash(&alg); + } + pr_info("sparc64 md5 opcode not available.\n"); + return -ENODEV; +} + +static void __exit md5_sparc64_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(md5_sparc64_mod_init); +module_exit(md5_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated"); + +MODULE_ALIAS("md5"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/opcodes.h b/arch/sparc/crypto/opcodes.h new file mode 100644 index 00000000000..19cbaea6976 --- /dev/null +++ b/arch/sparc/crypto/opcodes.h @@ -0,0 +1,99 @@ +#ifndef _OPCODES_H +#define _OPCODES_H + +#define SPARC_CR_OPCODE_PRIORITY 300 + +#define F3F(x,y,z) (((x)<<30)|((y)<<19)|((z)<<5)) + +#define FPD_ENCODE(x) (((x) >> 5) | ((x) & ~(0x20))) + +#define RS1(x) (FPD_ENCODE(x) << 14) +#define RS2(x) (FPD_ENCODE(x) << 0) +#define RS3(x) (FPD_ENCODE(x) << 9) +#define RD(x) (FPD_ENCODE(x) << 25) +#define IMM5_0(x) ((x) << 0) +#define IMM5_9(x) ((x) << 9) + +#define CRC32C(a,b,c) \ + .word (F3F(2,0x36,0x147)|RS1(a)|RS2(b)|RD(c)); + +#define MD5 \ + .word 0x81b02800; +#define SHA1 \ + .word 0x81b02820; +#define SHA256 \ + .word 0x81b02840; +#define SHA512 \ + .word 0x81b02860; + +#define AES_EROUND01(a,b,c,d) \ + .word (F3F(2, 0x19, 0)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND23(a,b,c,d) \ + .word (F3F(2, 0x19, 1)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND01(a,b,c,d) \ + .word (F3F(2, 0x19, 2)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND23(a,b,c,d) \ + .word (F3F(2, 0x19, 3)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND01_L(a,b,c,d) \ + .word (F3F(2, 0x19, 4)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND23_L(a,b,c,d) \ + .word (F3F(2, 0x19, 5)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND01_L(a,b,c,d) \ + .word (F3F(2, 0x19, 6)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND23_L(a,b,c,d) \ + .word (F3F(2, 0x19, 7)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_KEXPAND1(a,b,c,d) \ + .word (F3F(2, 0x19, 8)|RS1(a)|RS2(b)|IMM5_9(c)|RD(d)); +#define AES_KEXPAND0(a,b,c) \ + .word (F3F(2, 0x36, 0x130)|RS1(a)|RS2(b)|RD(c)); +#define AES_KEXPAND2(a,b,c) \ + .word (F3F(2, 0x36, 0x131)|RS1(a)|RS2(b)|RD(c)); + +#define DES_IP(a,b) \ + .word (F3F(2, 0x36, 0x134)|RS1(a)|RD(b)); +#define DES_IIP(a,b) \ + .word (F3F(2, 0x36, 0x135)|RS1(a)|RD(b)); +#define DES_KEXPAND(a,b,c) \ + .word (F3F(2, 0x36, 0x136)|RS1(a)|IMM5_0(b)|RD(c)); +#define DES_ROUND(a,b,c,d) \ + .word (F3F(2, 0x19, 0x009)|RS1(a)|RS2(b)|RS3(c)|RD(d)); + +#define CAMELLIA_F(a,b,c,d) \ + .word (F3F(2, 0x19, 0x00c)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define CAMELLIA_FL(a,b,c) \ + .word (F3F(2, 0x36, 0x13c)|RS1(a)|RS2(b)|RD(c)); +#define CAMELLIA_FLI(a,b,c) \ + .word (F3F(2, 0x36, 0x13d)|RS1(a)|RS2(b)|RD(c)); + +#define MOVDTOX_F0_O4 \ + .word 0x99b02200 +#define MOVDTOX_F2_O5 \ + .word 0x9bb02202 +#define MOVXTOD_G1_F60 \ + .word 0xbbb02301 +#define MOVXTOD_G1_F62 \ + .word 0xbfb02301 +#define MOVXTOD_G3_F4 \ + .word 0x89b02303; +#define MOVXTOD_G7_F6 \ + .word 0x8db02307; +#define MOVXTOD_G3_F0 \ + .word 0x81b02303; +#define MOVXTOD_G7_F2 \ + .word 0x85b02307; +#define MOVXTOD_O0_F0 \ + .word 0x81b02308; +#define MOVXTOD_O5_F0 \ + .word 0x81b0230d; +#define MOVXTOD_O5_F2 \ + .word 0x85b0230d; +#define MOVXTOD_O5_F4 \ + .word 0x89b0230d; +#define MOVXTOD_O5_F6 \ + .word 0x8db0230d; +#define MOVXTOD_G3_F60 \ + .word 0xbbb02303; +#define MOVXTOD_G7_F62 \ + .word 0xbfb02307; + +#endif /* _OPCODES_H */ diff --git a/arch/sparc/crypto/sha1_asm.S b/arch/sparc/crypto/sha1_asm.S new file mode 100644 index 00000000000..219d10c5ae0 --- /dev/null +++ b/arch/sparc/crypto/sha1_asm.S @@ -0,0 +1,72 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +ENTRY(sha1_sparc64_transform) + /* %o0 = digest, %o1 = data, %o2 = rounds */ + VISEntryHalf + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + ld [%o0 + 0x08], %f2 + andcc %o1, 0x7, %g0 + ld [%o0 + 0x0c], %f3 + bne,pn %xcc, 10f + ld [%o0 + 0x10], %f4 + +1: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + ldd [%o1 + 0x38], %f22 + + SHA1 + + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + +5: + st %f0, [%o0 + 0x00] + st %f1, [%o0 + 0x04] + st %f2, [%o0 + 0x08] + st %f3, [%o0 + 0x0c] + st %f4, [%o0 + 0x10] + retl + VISExitHalf +10: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +1: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + ldd [%o1 + 0x40], %f26 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + SHA1 + + subcc %o2, 1, %o2 + fsrc2 %f26, %f10 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + + ba,a,pt %xcc, 5b +ENDPROC(sha1_sparc64_transform) diff --git a/arch/sparc/crypto/sha1_glue.c b/arch/sparc/crypto/sha1_glue.c new file mode 100644 index 00000000000..6cd5f29e1e0 --- /dev/null +++ b/arch/sparc/crypto/sha1_glue.c @@ -0,0 +1,185 @@ +/* Glue code for SHA1 hashing optimized for sparc64 crypto opcodes. + * + * This is based largely upon arch/x86/crypto/sha1_ssse3_glue.c + * + * Copyright (c) Alan Smithee. + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) Jean-Francois Dive <jef@linuxbe.org> + * Copyright (c) Mathias Krause <minipli@googlemail.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/sha.h> + +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data, + unsigned int rounds); + +static int sha1_sparc64_init(struct shash_desc *desc) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, + }; + + return 0; +} + +static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + sctx->count += len; + if (partial) { + done = SHA1_BLOCK_SIZE - partial; + memcpy(sctx->buffer + partial, data, done); + sha1_sparc64_transform(sctx->state, sctx->buffer, 1); + } + if (len - done >= SHA1_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA1_BLOCK_SIZE; + + sha1_sparc64_transform(sctx->state, data + done, rounds); + done += rounds * SHA1_BLOCK_SIZE; + } + + memcpy(sctx->buffer, data + done, len - done); +} + +static int sha1_sparc64_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA1_BLOCK_SIZE; + + /* Handle the fast case right here */ + if (partial + len < SHA1_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buffer + partial, data, len); + } else + __sha1_sparc64_update(sctx, data, len, partial); + + return 0; +} + +/* Add padding and return the message digest. */ +static int sha1_sparc64_final(struct shash_desc *desc, u8 *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA1_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA1_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA1_BLOCK_SIZE+56) - index); + + /* We need to fill a whole block for __sha1_sparc64_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buffer + index, padding, padlen); + } else { + __sha1_sparc64_update(sctx, padding, padlen, index); + } + __sha1_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + + /* Store state in digest */ + for (i = 0; i < 5; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha1_sparc64_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + + return 0; +} + +static int sha1_sparc64_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = sha1_sparc64_init, + .update = sha1_sparc64_update, + .final = sha1_sparc64_final, + .export = sha1_sparc64_export, + .import = sha1_sparc64_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), + .base = { + .cra_name = "sha1", + .cra_driver_name= "sha1-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static bool __init sparc64_has_sha1_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA1)) + return false; + + return true; +} + +static int __init sha1_sparc64_mod_init(void) +{ + if (sparc64_has_sha1_opcode()) { + pr_info("Using sparc64 sha1 opcode optimized SHA-1 implementation\n"); + return crypto_register_shash(&alg); + } + pr_info("sparc64 sha1 opcode not available.\n"); + return -ENODEV; +} + +static void __exit sha1_sparc64_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(sha1_sparc64_mod_init); +module_exit(sha1_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, sparc64 sha1 opcode accelerated"); + +MODULE_ALIAS("sha1"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/crypto/sha256_asm.S new file mode 100644 index 00000000000..b5f3d5826eb --- /dev/null +++ b/arch/sparc/crypto/sha256_asm.S @@ -0,0 +1,78 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +ENTRY(sha256_sparc64_transform) + /* %o0 = digest, %o1 = data, %o2 = rounds */ + VISEntryHalf + ld [%o0 + 0x00], %f0 + ld [%o0 + 0x04], %f1 + ld [%o0 + 0x08], %f2 + ld [%o0 + 0x0c], %f3 + ld [%o0 + 0x10], %f4 + ld [%o0 + 0x14], %f5 + andcc %o1, 0x7, %g0 + ld [%o0 + 0x18], %f6 + bne,pn %xcc, 10f + ld [%o0 + 0x1c], %f7 + +1: + ldd [%o1 + 0x00], %f8 + ldd [%o1 + 0x08], %f10 + ldd [%o1 + 0x10], %f12 + ldd [%o1 + 0x18], %f14 + ldd [%o1 + 0x20], %f16 + ldd [%o1 + 0x28], %f18 + ldd [%o1 + 0x30], %f20 + ldd [%o1 + 0x38], %f22 + + SHA256 + + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + +5: + st %f0, [%o0 + 0x00] + st %f1, [%o0 + 0x04] + st %f2, [%o0 + 0x08] + st %f3, [%o0 + 0x0c] + st %f4, [%o0 + 0x10] + st %f5, [%o0 + 0x14] + st %f6, [%o0 + 0x18] + st %f7, [%o0 + 0x1c] + retl + VISExitHalf +10: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f10 +1: + ldd [%o1 + 0x08], %f12 + ldd [%o1 + 0x10], %f14 + ldd [%o1 + 0x18], %f16 + ldd [%o1 + 0x20], %f18 + ldd [%o1 + 0x28], %f20 + ldd [%o1 + 0x30], %f22 + ldd [%o1 + 0x38], %f24 + ldd [%o1 + 0x40], %f26 + + faligndata %f10, %f12, %f8 + faligndata %f12, %f14, %f10 + faligndata %f14, %f16, %f12 + faligndata %f16, %f18, %f14 + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + + SHA256 + + subcc %o2, 1, %o2 + fsrc2 %f26, %f10 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + + ba,a,pt %xcc, 5b +ENDPROC(sha256_sparc64_transform) diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c new file mode 100644 index 00000000000..04f555ab268 --- /dev/null +++ b/arch/sparc/crypto/sha256_glue.c @@ -0,0 +1,243 @@ +/* Glue code for SHA256 hashing optimized for sparc64 crypto opcodes. + * + * This is based largely upon crypto/sha256_generic.c + * + * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> + * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/sha.h> + +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, + unsigned int rounds); + +static int sha224_sparc64_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + + return 0; +} + +static int sha256_sparc64_init(struct shash_desc *desc) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; + + return 0; +} + +static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + sctx->count += len; + if (partial) { + done = SHA256_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha256_sparc64_transform(sctx->state, sctx->buf, 1); + } + if (len - done >= SHA256_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA256_BLOCK_SIZE; + + sha256_sparc64_transform(sctx->state, data + done, rounds); + done += rounds * SHA256_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); +} + +static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; + + /* Handle the fast case right here */ + if (partial + len < SHA256_BLOCK_SIZE) { + sctx->count += len; + memcpy(sctx->buf + partial, data, len); + } else + __sha256_sparc64_update(sctx, data, len, partial); + + return 0; +} + +static int sha256_sparc64_final(struct shash_desc *desc, u8 *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be32 *dst = (__be32 *)out; + __be64 bits; + static const u8 padding[SHA256_BLOCK_SIZE] = { 0x80, }; + + bits = cpu_to_be64(sctx->count << 3); + + /* Pad out to 56 mod 64 and append length */ + index = sctx->count % SHA256_BLOCK_SIZE; + padlen = (index < 56) ? (56 - index) : ((SHA256_BLOCK_SIZE+56) - index); + + /* We need to fill a whole block for __sha256_sparc64_update() */ + if (padlen <= 56) { + sctx->count += padlen; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha256_sparc64_update(sctx, padding, padlen, index); + } + __sha256_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 56); + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be32(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha224_sparc64_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[SHA256_DIGEST_SIZE]; + + sha256_sparc64_final(desc, D); + + memcpy(hash, D, SHA224_DIGEST_SIZE); + memset(D, 0, SHA256_DIGEST_SIZE); + + return 0; +} + +static int sha256_sparc64_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int sha256_sparc64_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + +static struct shash_alg sha256 = { + .digestsize = SHA256_DIGEST_SIZE, + .init = sha256_sparc64_init, + .update = sha256_sparc64_update, + .final = sha256_sparc64_final, + .export = sha256_sparc64_export, + .import = sha256_sparc64_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha256", + .cra_driver_name= "sha256-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha224 = { + .digestsize = SHA224_DIGEST_SIZE, + .init = sha224_sparc64_init, + .update = sha256_sparc64_update, + .final = sha224_sparc64_final, + .descsize = sizeof(struct sha256_state), + .base = { + .cra_name = "sha224", + .cra_driver_name= "sha224-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA224_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static bool __init sparc64_has_sha256_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA256)) + return false; + + return true; +} + +static int __init sha256_sparc64_mod_init(void) +{ + if (sparc64_has_sha256_opcode()) { + int ret = crypto_register_shash(&sha224); + if (ret < 0) + return ret; + + ret = crypto_register_shash(&sha256); + if (ret < 0) { + crypto_unregister_shash(&sha224); + return ret; + } + + pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); + return 0; + } + pr_info("sparc64 sha256 opcode not available.\n"); + return -ENODEV; +} + +static void __exit sha256_sparc64_mod_fini(void) +{ + crypto_unregister_shash(&sha224); + crypto_unregister_shash(&sha256); +} + +module_init(sha256_sparc64_mod_init); +module_exit(sha256_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, sparc64 sha256 opcode accelerated"); + +MODULE_ALIAS("sha224"); +MODULE_ALIAS("sha256"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/sha512_asm.S b/arch/sparc/crypto/sha512_asm.S new file mode 100644 index 00000000000..54bfba713c0 --- /dev/null +++ b/arch/sparc/crypto/sha512_asm.S @@ -0,0 +1,102 @@ +#include <linux/linkage.h> +#include <asm/visasm.h> + +#include "opcodes.h" + +ENTRY(sha512_sparc64_transform) + /* %o0 = digest, %o1 = data, %o2 = rounds */ + VISEntry + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + andcc %o1, 0x7, %g0 + ldd [%o0 + 0x30], %f12 + bne,pn %xcc, 10f + ldd [%o0 + 0x38], %f14 + +1: + ldd [%o1 + 0x00], %f16 + ldd [%o1 + 0x08], %f18 + ldd [%o1 + 0x10], %f20 + ldd [%o1 + 0x18], %f22 + ldd [%o1 + 0x20], %f24 + ldd [%o1 + 0x28], %f26 + ldd [%o1 + 0x30], %f28 + ldd [%o1 + 0x38], %f30 + ldd [%o1 + 0x40], %f32 + ldd [%o1 + 0x48], %f34 + ldd [%o1 + 0x50], %f36 + ldd [%o1 + 0x58], %f38 + ldd [%o1 + 0x60], %f40 + ldd [%o1 + 0x68], %f42 + ldd [%o1 + 0x70], %f44 + ldd [%o1 + 0x78], %f46 + + SHA512 + + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + add %o1, 0x80, %o1 + +5: + std %f0, [%o0 + 0x00] + std %f2, [%o0 + 0x08] + std %f4, [%o0 + 0x10] + std %f6, [%o0 + 0x18] + std %f8, [%o0 + 0x20] + std %f10, [%o0 + 0x28] + std %f12, [%o0 + 0x30] + std %f14, [%o0 + 0x38] + retl + VISExit +10: + alignaddr %o1, %g0, %o1 + + ldd [%o1 + 0x00], %f18 +1: + ldd [%o1 + 0x08], %f20 + ldd [%o1 + 0x10], %f22 + ldd [%o1 + 0x18], %f24 + ldd [%o1 + 0x20], %f26 + ldd [%o1 + 0x28], %f28 + ldd [%o1 + 0x30], %f30 + ldd [%o1 + 0x38], %f32 + ldd [%o1 + 0x40], %f34 + ldd [%o1 + 0x48], %f36 + ldd [%o1 + 0x50], %f38 + ldd [%o1 + 0x58], %f40 + ldd [%o1 + 0x60], %f42 + ldd [%o1 + 0x68], %f44 + ldd [%o1 + 0x70], %f46 + ldd [%o1 + 0x78], %f48 + ldd [%o1 + 0x80], %f50 + + faligndata %f18, %f20, %f16 + faligndata %f20, %f22, %f18 + faligndata %f22, %f24, %f20 + faligndata %f24, %f26, %f22 + faligndata %f26, %f28, %f24 + faligndata %f28, %f30, %f26 + faligndata %f30, %f32, %f28 + faligndata %f32, %f34, %f30 + faligndata %f34, %f36, %f32 + faligndata %f36, %f38, %f34 + faligndata %f38, %f40, %f36 + faligndata %f40, %f42, %f38 + faligndata %f42, %f44, %f40 + faligndata %f44, %f46, %f42 + faligndata %f46, %f48, %f44 + faligndata %f48, %f50, %f46 + + SHA512 + + subcc %o2, 1, %o2 + fsrc2 %f50, %f18 + bne,pt %xcc, 1b + add %o1, 0x80, %o1 + + ba,a,pt %xcc, 5b +ENDPROC(sha512_sparc64_transform) diff --git a/arch/sparc/crypto/sha512_glue.c b/arch/sparc/crypto/sha512_glue.c new file mode 100644 index 00000000000..f04d1994d19 --- /dev/null +++ b/arch/sparc/crypto/sha512_glue.c @@ -0,0 +1,228 @@ +/* Glue code for SHA512 hashing optimized for sparc64 crypto opcodes. + * + * This is based largely upon crypto/sha512_generic.c + * + * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com> + * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk> + * Copyright (c) 2003 Kyle McMartin <kyle@debian.org> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/mm.h> +#include <linux/cryptohash.h> +#include <linux/types.h> +#include <crypto/sha.h> + +#include <asm/pstate.h> +#include <asm/elf.h> + +#include "opcodes.h" + +asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data, + unsigned int rounds); + +static int sha512_sparc64_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + sctx->state[0] = SHA512_H0; + sctx->state[1] = SHA512_H1; + sctx->state[2] = SHA512_H2; + sctx->state[3] = SHA512_H3; + sctx->state[4] = SHA512_H4; + sctx->state[5] = SHA512_H5; + sctx->state[6] = SHA512_H6; + sctx->state[7] = SHA512_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static int sha384_sparc64_init(struct shash_desc *desc) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + sctx->state[0] = SHA384_H0; + sctx->state[1] = SHA384_H1; + sctx->state[2] = SHA384_H2; + sctx->state[3] = SHA384_H3; + sctx->state[4] = SHA384_H4; + sctx->state[5] = SHA384_H5; + sctx->state[6] = SHA384_H6; + sctx->state[7] = SHA384_H7; + sctx->count[0] = sctx->count[1] = 0; + + return 0; +} + +static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data, + unsigned int len, unsigned int partial) +{ + unsigned int done = 0; + + if ((sctx->count[0] += len) < len) + sctx->count[1]++; + if (partial) { + done = SHA512_BLOCK_SIZE - partial; + memcpy(sctx->buf + partial, data, done); + sha512_sparc64_transform(sctx->state, sctx->buf, 1); + } + if (len - done >= SHA512_BLOCK_SIZE) { + const unsigned int rounds = (len - done) / SHA512_BLOCK_SIZE; + + sha512_sparc64_transform(sctx->state, data + done, rounds); + done += rounds * SHA512_BLOCK_SIZE; + } + + memcpy(sctx->buf, data + done, len - done); +} + +static int sha512_sparc64_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE; + + /* Handle the fast case right here */ + if (partial + len < SHA512_BLOCK_SIZE) { + if ((sctx->count[0] += len) < len) + sctx->count[1]++; + memcpy(sctx->buf + partial, data, len); + } else + __sha512_sparc64_update(sctx, data, len, partial); + + return 0; +} + +static int sha512_sparc64_final(struct shash_desc *desc, u8 *out) +{ + struct sha512_state *sctx = shash_desc_ctx(desc); + unsigned int i, index, padlen; + __be64 *dst = (__be64 *)out; + __be64 bits[2]; + static const u8 padding[SHA512_BLOCK_SIZE] = { 0x80, }; + + /* Save number of bits */ + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); + + /* Pad out to 112 mod 128 and append length */ + index = sctx->count[0] % SHA512_BLOCK_SIZE; + padlen = (index < 112) ? (112 - index) : ((SHA512_BLOCK_SIZE+112) - index); + + /* We need to fill a whole block for __sha512_sparc64_update() */ + if (padlen <= 112) { + if ((sctx->count[0] += padlen) < padlen) + sctx->count[1]++; + memcpy(sctx->buf + index, padding, padlen); + } else { + __sha512_sparc64_update(sctx, padding, padlen, index); + } + __sha512_sparc64_update(sctx, (const u8 *)&bits, sizeof(bits), 112); + + /* Store state in digest */ + for (i = 0; i < 8; i++) + dst[i] = cpu_to_be64(sctx->state[i]); + + /* Wipe context */ + memset(sctx, 0, sizeof(*sctx)); + + return 0; +} + +static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash) +{ + u8 D[64]; + + sha512_sparc64_final(desc, D); + + memcpy(hash, D, 48); + memset(D, 0, 64); + + return 0; +} + +static struct shash_alg sha512 = { + .digestsize = SHA512_DIGEST_SIZE, + .init = sha512_sparc64_init, + .update = sha512_sparc64_update, + .final = sha512_sparc64_final, + .descsize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha512", + .cra_driver_name= "sha512-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA512_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static struct shash_alg sha384 = { + .digestsize = SHA384_DIGEST_SIZE, + .init = sha384_sparc64_init, + .update = sha512_sparc64_update, + .final = sha384_sparc64_final, + .descsize = sizeof(struct sha512_state), + .base = { + .cra_name = "sha384", + .cra_driver_name= "sha384-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = SHA384_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static bool __init sparc64_has_sha512_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA512)) + return false; + + return true; +} + +static int __init sha512_sparc64_mod_init(void) +{ + if (sparc64_has_sha512_opcode()) { + int ret = crypto_register_shash(&sha384); + if (ret < 0) + return ret; + + ret = crypto_register_shash(&sha512); + if (ret < 0) { + crypto_unregister_shash(&sha384); + return ret; + } + + pr_info("Using sparc64 sha512 opcode optimized SHA-512/SHA-384 implementation\n"); + return 0; + } + pr_info("sparc64 sha512 opcode not available.\n"); + return -ENODEV; +} + +static void __exit sha512_sparc64_mod_fini(void) +{ + crypto_unregister_shash(&sha384); + crypto_unregister_shash(&sha512); +} + +module_init(sha512_sparc64_mod_init); +module_exit(sha512_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-384 and SHA-512 Secure Hash Algorithm, sparc64 sha512 opcode accelerated"); + +MODULE_ALIAS("sha384"); +MODULE_ALIAS("sha512"); + +#include "crop_devid.c" diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index 3c93f08ce18..a4582181800 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild @@ -1,18 +1,21 @@ # User exported sparc header files -include include/asm-generic/Kbuild.asm -header-y += apc.h -header-y += asi.h -header-y += display7seg.h -header-y += envctrl.h -header-y += fbio.h -header-y += jsflash.h -header-y += openpromio.h -header-y += perfctr.h -header-y += psrcompat.h -header-y += psr.h -header-y += pstate.h -header-y += traps.h -header-y += uctx.h -header-y += utrap.h -header-y += watchdog.h + +generic-y += clkdev.h +generic-y += cputime.h +generic-y += div64.h +generic-y += emergency-restart.h +generic-y += exec.h +generic-y += hash.h +generic-y += irq_regs.h +generic-y += linkage.h +generic-y += local.h +generic-y += local64.h +generic-y += mcs_spinlock.h +generic-y += module.h +generic-y += mutex.h +generic-y += preempt.h +generic-y += serial.h +generic-y += trace_clock.h +generic-y += types.h +generic-y += word-at-a-time.h diff --git a/arch/sparc/include/asm/asmmacro.h b/arch/sparc/include/asm/asmmacro.h index a995bf8aba3..a0e28ef0255 100644 --- a/arch/sparc/include/asm/asmmacro.h +++ b/arch/sparc/include/asm/asmmacro.h @@ -6,17 +6,6 @@ #ifndef _SPARC_ASMMACRO_H #define _SPARC_ASMMACRO_H -#include <asm/btfixup.h> -#include <asm/asi.h> - -#define GET_PROCESSOR4M_ID(reg) \ - rd %tbr, %reg; \ - srl %reg, 12, %reg; \ - and %reg, 3, %reg; - -#define GET_PROCESSOR4D_ID(reg) \ - lda [%g0] ASI_M_VIKING_TMP1, %reg; - /* All trap entry points _must_ begin with this macro or else you * lose. It makes sure the kernel has a proper window so that * c-code can be called. @@ -31,10 +20,26 @@ /* All traps low-level code here must end with this macro. */ #define RESTORE_ALL b ret_trap_entry; clr %l6; -/* sun4 probably wants half word accesses to ASI_SEGMAP, while sun4c+ - likes byte accesses. These are to avoid ifdef mania. */ - -#define lduXa lduba -#define stXa stba +/* Support for run-time patching of single instructions. + * This is used to handle the differences in the ASI for + * MMUREGS for LEON and SUN. + * + * Sample: + * LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %o0 + * SUN_PI_(lda [%g0] ASI_M_MMUREGS, %o0 + * PI == Patch Instruction + * + * For LEON we will use the first variant, + * and for all other we will use the SUN variant. + * The order is important. + */ +#define LEON_PI(...) \ +662: __VA_ARGS__ + +#define SUN_PI_(...) \ + .section .leon_1insn_patch, "ax"; \ + .word 662b; \ + __VA_ARGS__; \ + .previous #endif /* !(_SPARC_ASMMACRO_H) */ diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 7ae128b19d3..7aed2be45b4 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -13,17 +13,18 @@ #include <linux/types.h> -#ifdef __KERNEL__ +#include <asm/cmpxchg.h> +#include <asm/barrier.h> +#include <asm-generic/atomic64.h> -#include <asm/system.h> #define ATOMIC_INIT(i) { (i) } -extern int __atomic_add_return(int, atomic_t *); -extern int atomic_cmpxchg(atomic_t *, int, int); +int __atomic_add_return(int, atomic_t *); +int atomic_cmpxchg(atomic_t *, int, int); #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -extern int atomic_add_unless(atomic_t *, int, int); -extern void atomic_set(atomic_t *, int); +int __atomic_add_unless(atomic_t *, int, int); +void atomic_set(atomic_t *, int); #define atomic_read(v) (*(volatile int *)&(v)->counter) @@ -52,114 +53,4 @@ extern void atomic_set(atomic_t *, int); #define atomic_dec_and_test(v) (atomic_dec_return(v) == 0) #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0) -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - -/* This is the old 24-bit implementation. It's still used internally - * by some sparc-specific code, notably the semaphore implementation. - */ -typedef struct { volatile int counter; } atomic24_t; - -#ifndef CONFIG_SMP - -#define ATOMIC24_INIT(i) { (i) } -#define atomic24_read(v) ((v)->counter) -#define atomic24_set(v, i) (((v)->counter) = i) - -#else -/* We do the bulk of the actual work out of line in two common - * routines in assembler, see arch/sparc/lib/atomic.S for the - * "fun" details. - * - * For SMP the trick is you embed the spin lock byte within - * the word, use the low byte so signedness is easily retained - * via a quick arithmetic shift. It looks like this: - * - * ---------------------------------------- - * | signed 24-bit counter value | lock | atomic_t - * ---------------------------------------- - * 31 8 7 0 - */ - -#define ATOMIC24_INIT(i) { ((i) << 8) } - -static inline int atomic24_read(const atomic24_t *v) -{ - int ret = v->counter; - - while(ret & 0xff) - ret = v->counter; - - return ret >> 8; -} - -#define atomic24_set(v, i) (((v)->counter) = ((i) << 8)) -#endif - -static inline int __atomic24_add(int i, atomic24_t *v) -{ - register volatile int *ptr asm("g1"); - register int increment asm("g2"); - register int tmp1 asm("g3"); - register int tmp2 asm("g4"); - register int tmp3 asm("g7"); - - ptr = &v->counter; - increment = i; - - __asm__ __volatile__( - "mov %%o7, %%g4\n\t" - "call ___atomic24_add\n\t" - " add %%o7, 8, %%o7\n" - : "=&r" (increment), "=r" (tmp1), "=r" (tmp2), "=r" (tmp3) - : "0" (increment), "r" (ptr) - : "memory", "cc"); - - return increment; -} - -static inline int __atomic24_sub(int i, atomic24_t *v) -{ - register volatile int *ptr asm("g1"); - register int increment asm("g2"); - register int tmp1 asm("g3"); - register int tmp2 asm("g4"); - register int tmp3 asm("g7"); - - ptr = &v->counter; - increment = i; - - __asm__ __volatile__( - "mov %%o7, %%g4\n\t" - "call ___atomic24_sub\n\t" - " add %%o7, 8, %%o7\n" - : "=&r" (increment), "=r" (tmp1), "=r" (tmp2), "=r" (tmp3) - : "0" (increment), "r" (ptr) - : "memory", "cc"); - - return increment; -} - -#define atomic24_add(i, v) ((void)__atomic24_add((i), (v))) -#define atomic24_sub(i, v) ((void)__atomic24_sub((i), (v))) - -#define atomic24_dec_return(v) __atomic24_sub(1, (v)) -#define atomic24_inc_return(v) __atomic24_add(1, (v)) - -#define atomic24_sub_and_test(i, v) (__atomic24_sub((i), (v)) == 0) -#define atomic24_dec_and_test(v) (__atomic24_sub(1, (v)) == 0) - -#define atomic24_inc(v) ((void)__atomic24_add(1, (v))) -#define atomic24_dec(v) ((void)__atomic24_sub(1, (v))) - -#define atomic24_add_negative(i, v) (__atomic24_add((i), (v)) < 0) - -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() - -#endif /* !(__KERNEL__) */ - -#include <asm-generic/atomic-long.h> #endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index bdb2ff880bd..bb894c8bec5 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -1,14 +1,15 @@ /* atomic.h: Thankfully the V9 is at least reasonable for this * stuff. * - * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com) + * Copyright (C) 1996, 1997, 2000, 2012 David S. Miller (davem@redhat.com) */ #ifndef __ARCH_SPARC64_ATOMIC__ #define __ARCH_SPARC64_ATOMIC__ #include <linux/types.h> -#include <asm/system.h> +#include <asm/cmpxchg.h> +#include <asm/barrier.h> #define ATOMIC_INIT(i) { (i) } #define ATOMIC64_INIT(i) { (i) } @@ -19,15 +20,15 @@ #define atomic_set(v, i) (((v)->counter) = i) #define atomic64_set(v, i) (((v)->counter) = i) -extern void atomic_add(int, atomic_t *); -extern void atomic64_add(long, atomic64_t *); -extern void atomic_sub(int, atomic_t *); -extern void atomic64_sub(long, atomic64_t *); +void atomic_add(int, atomic_t *); +void atomic64_add(long, atomic64_t *); +void atomic_sub(int, atomic_t *); +void atomic64_sub(long, atomic64_t *); -extern int atomic_add_ret(int, atomic_t *); -extern long atomic64_add_ret(long, atomic64_t *); -extern int atomic_sub_ret(int, atomic_t *); -extern long atomic64_sub_ret(long, atomic64_t *); +int atomic_add_ret(int, atomic_t *); +long atomic64_add_ret(long, atomic64_t *); +int atomic_sub_ret(int, atomic_t *); +long atomic64_sub_ret(long, atomic64_t *); #define atomic_dec_return(v) atomic_sub_ret(1, v) #define atomic64_dec_return(v) atomic64_sub_ret(1, v) @@ -70,7 +71,7 @@ extern long atomic64_sub_ret(long, atomic64_t *); #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_xchg(v, new) (xchg(&((v)->counter), new)) -static inline int atomic_add_unless(atomic_t *v, int a, int u) +static inline int __atomic_add_unless(atomic_t *v, int a, int u) { int c, old; c = atomic_read(v); @@ -82,11 +83,9 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u) break; c = old; } - return c != (u); + return c; } -#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) - #define atomic64_cmpxchg(v, o, n) \ ((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n))) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) @@ -108,11 +107,6 @@ static inline long atomic64_add_unless(atomic64_t *v, long a, long u) #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) -/* Atomic operations are already serializing */ -#define smp_mb__before_atomic_dec() barrier() -#define smp_mb__after_atomic_dec() barrier() -#define smp_mb__before_atomic_inc() barrier() -#define smp_mb__after_atomic_inc() barrier() +long atomic64_dec_if_positive(atomic64_t *v); -#include <asm-generic/atomic-long.h> #endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/auxio.h b/arch/sparc/include/asm/auxio.h index 13dc67f0301..3e09a07b77e 100644 --- a/arch/sparc/include/asm/auxio.h +++ b/arch/sparc/include/asm/auxio.h @@ -1,5 +1,12 @@ #ifndef ___ASM_SPARC_AUXIO_H #define ___ASM_SPARC_AUXIO_H + +#ifndef __ASSEMBLY__ + +extern void __iomem *auxio_register; + +#endif /* ifndef __ASSEMBLY__ */ + #if defined(__sparc__) && defined(__arch64__) #include <asm/auxio_64.h> #else diff --git a/arch/sparc/include/asm/auxio_32.h b/arch/sparc/include/asm/auxio_32.h index e03e088be95..5d685df427b 100644 --- a/arch/sparc/include/asm/auxio_32.h +++ b/arch/sparc/include/asm/auxio_32.h @@ -6,7 +6,6 @@ #ifndef _SPARC_AUXIO_H #define _SPARC_AUXIO_H -#include <asm/system.h> #include <asm/vaddrs.h> /* This register is an unsigned char in IO space. It does two things. @@ -35,8 +34,8 @@ * NOTE: these routines are implementation dependent-- * understand the hardware you are querying! */ -extern void set_auxio(unsigned char bits_on, unsigned char bits_off); -extern unsigned char get_auxio(void); /* .../asm/floppy.h */ +void set_auxio(unsigned char bits_on, unsigned char bits_off); +unsigned char get_auxio(void); /* .../asm/floppy.h */ /* * The following routines are provided for driver-compatibility @@ -79,7 +78,7 @@ do { \ /* AUXIO2 (Power Off Control) */ -extern __volatile__ unsigned char * auxio_power_register; +extern volatile u8 __iomem *auxio_power_register; #define AUXIO_POWER_DETECT_FAILURE 32 #define AUXIO_POWER_CLEAR_FAILURE 2 diff --git a/arch/sparc/include/asm/auxio_64.h b/arch/sparc/include/asm/auxio_64.h index f61cd1e3e39..6079e59a7ad 100644 --- a/arch/sparc/include/asm/auxio_64.h +++ b/arch/sparc/include/asm/auxio_64.h @@ -75,8 +75,6 @@ #ifndef __ASSEMBLY__ -extern void __iomem *auxio_register; - #define AUXIO_LTE_ON 1 #define AUXIO_LTE_OFF 0 @@ -84,7 +82,7 @@ extern void __iomem *auxio_register; * * on - AUXIO_LTE_ON or AUXIO_LTE_OFF */ -extern void auxio_set_lte(int on); +void auxio_set_lte(int on); #define AUXIO_LED_ON 1 #define AUXIO_LED_OFF 0 @@ -93,7 +91,7 @@ extern void auxio_set_lte(int on); * * on - AUXIO_LED_ON or AUXIO_LED_OFF */ -extern void auxio_set_led(int on); +void auxio_set_led(int on); #endif /* ifndef __ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h index db3af0d30fb..4e02086b839 100644 --- a/arch/sparc/include/asm/backoff.h +++ b/arch/sparc/include/asm/backoff.h @@ -1,6 +1,46 @@ #ifndef _SPARC64_BACKOFF_H #define _SPARC64_BACKOFF_H +/* The macros in this file implement an exponential backoff facility + * for atomic operations. + * + * When multiple threads compete on an atomic operation, it is + * possible for one thread to be continually denied a successful + * completion of the compare-and-swap instruction. Heavily + * threaded cpu implementations like Niagara can compound this + * problem even further. + * + * When an atomic operation fails and needs to be retried, we spin a + * certain number of times. At each subsequent failure of the same + * operation we double the spin count, realizing an exponential + * backoff. + * + * When we spin, we try to use an operation that will cause the + * current cpu strand to block, and therefore make the core fully + * available to any other other runnable strands. There are two + * options, based upon cpu capabilities. + * + * On all cpus prior to SPARC-T4 we do three dummy reads of the + * condition code register. Each read blocks the strand for something + * between 40 and 50 cpu cycles. + * + * For SPARC-T4 and later we have a special "pause" instruction + * available. This is implemented using writes to register %asr27. + * The cpu will block the number of cycles written into the register, + * unless a disrupting trap happens first. SPARC-T4 specifically + * implements pause with a granularity of 8 cycles. Each strand has + * an internal pause counter which decrements every 8 cycles. So the + * chip shifts the %asr27 value down by 3 bits, and writes the result + * into the pause counter. If a value smaller than 8 is written, the + * chip blocks for 1 cycle. + * + * To achieve the same amount of backoff as the three %ccr reads give + * on earlier chips, we shift the backoff value up by 7 bits. (Three + * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the + * whole amount we want to block into the pause register, rather than + * loop writing 128 each time. + */ + #define BACKOFF_LIMIT (4 * 1024) #ifdef CONFIG_SMP @@ -11,16 +51,25 @@ #define BACKOFF_LABEL(spin_label, continue_label) \ spin_label -#define BACKOFF_SPIN(reg, tmp, label) \ - mov reg, tmp; \ -88: brnz,pt tmp, 88b; \ - sub tmp, 1, tmp; \ - set BACKOFF_LIMIT, tmp; \ - cmp reg, tmp; \ - bg,pn %xcc, label; \ - nop; \ - ba,pt %xcc, label; \ - sllx reg, 1, reg; +#define BACKOFF_SPIN(reg, tmp, label) \ + mov reg, tmp; \ +88: rd %ccr, %g0; \ + rd %ccr, %g0; \ + rd %ccr, %g0; \ + .section .pause_3insn_patch,"ax";\ + .word 88b; \ + sllx tmp, 7, tmp; \ + wr tmp, 0, %asr27; \ + clr tmp; \ + .previous; \ + brnz,pt tmp, 88b; \ + sub tmp, 1, tmp; \ + set BACKOFF_LIMIT, tmp; \ + cmp reg, tmp; \ + bg,pn %xcc, label; \ + nop; \ + ba,pt %xcc, label; \ + sllx reg, 1, reg; #else diff --git a/arch/sparc/include/asm/barrier.h b/arch/sparc/include/asm/barrier.h new file mode 100644 index 00000000000..b25f02a029e --- /dev/null +++ b/arch/sparc/include/asm/barrier.h @@ -0,0 +1,8 @@ +#ifndef ___ASM_SPARC_BARRIER_H +#define ___ASM_SPARC_BARRIER_H +#if defined(__sparc__) && defined(__arch64__) +#include <asm/barrier_64.h> +#else +#include <asm/barrier_32.h> +#endif +#endif diff --git a/arch/sparc/include/asm/barrier_32.h b/arch/sparc/include/asm/barrier_32.h new file mode 100644 index 00000000000..ae69eda288f --- /dev/null +++ b/arch/sparc/include/asm/barrier_32.h @@ -0,0 +1,7 @@ +#ifndef __SPARC_BARRIER_H +#define __SPARC_BARRIER_H + +#include <asm/processor.h> /* for nop() */ +#include <asm-generic/barrier.h> + +#endif /* !(__SPARC_BARRIER_H) */ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h new file mode 100644 index 00000000000..305dcc3dc72 --- /dev/null +++ b/arch/sparc/include/asm/barrier_64.h @@ -0,0 +1,74 @@ +#ifndef __SPARC64_BARRIER_H +#define __SPARC64_BARRIER_H + +/* These are here in an effort to more fully work around Spitfire Errata + * #51. Essentially, if a memory barrier occurs soon after a mispredicted + * branch, the chip can stop executing instructions until a trap occurs. + * Therefore, if interrupts are disabled, the chip can hang forever. + * + * It used to be believed that the memory barrier had to be right in the + * delay slot, but a case has been traced recently wherein the memory barrier + * was one instruction after the branch delay slot and the chip still hung. + * The offending sequence was the following in sym_wakeup_done() of the + * sym53c8xx_2 driver: + * + * call sym_ccb_from_dsa, 0 + * movge %icc, 0, %l0 + * brz,pn %o0, .LL1303 + * mov %o0, %l2 + * membar #LoadLoad + * + * The branch has to be mispredicted for the bug to occur. Therefore, we put + * the memory barrier explicitly into a "branch always, predicted taken" + * delay slot to avoid the problem case. + */ +#define membar_safe(type) \ +do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ + " membar " type "\n" \ + "1:\n" \ + : : : "memory"); \ +} while (0) + +/* The kernel always executes in TSO memory model these days, + * and furthermore most sparc64 chips implement more stringent + * memory ordering than required by the specifications. + */ +#define mb() membar_safe("#StoreLoad") +#define rmb() __asm__ __volatile__("":::"memory") +#define wmb() __asm__ __volatile__("":::"memory") + +#define read_barrier_depends() do { } while(0) +#define set_mb(__var, __value) \ + do { __var = __value; membar_safe("#StoreLoad"); } while(0) + +#ifdef CONFIG_SMP +#define smp_mb() mb() +#define smp_rmb() rmb() +#define smp_wmb() wmb() +#else +#define smp_mb() __asm__ __volatile__("":::"memory") +#define smp_rmb() __asm__ __volatile__("":::"memory") +#define smp_wmb() __asm__ __volatile__("":::"memory") +#endif + +#define smp_read_barrier_depends() do { } while(0) + +#define smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + ACCESS_ONCE(*p) = (v); \ +} while (0) + +#define smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = ACCESS_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + ___p1; \ +}) + +#define smp_mb__before_atomic() barrier() +#define smp_mb__after_atomic() barrier() + +#endif /* !(__SPARC64_BARRIER_H) */ diff --git a/arch/sparc/include/asm/bitext.h b/arch/sparc/include/asm/bitext.h index 297b2f2fcb4..9c988bf3adb 100644 --- a/arch/sparc/include/asm/bitext.h +++ b/arch/sparc/include/asm/bitext.h @@ -20,8 +20,8 @@ struct bit_map { int num_colors; }; -extern int bit_map_string_get(struct bit_map *t, int len, int align); -extern void bit_map_clear(struct bit_map *t, int offset, int len); -extern void bit_map_init(struct bit_map *t, unsigned long *map, int size); +int bit_map_string_get(struct bit_map *t, int len, int align); +void bit_map_clear(struct bit_map *t, int offset, int len); +void bit_map_init(struct bit_map *t, unsigned long *map, int size); #endif /* defined(_SPARC_BITEXT_H) */ diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h index 9cf4ae0cd7b..600ed1d9c8c 100644 --- a/arch/sparc/include/asm/bitops_32.h +++ b/arch/sparc/include/asm/bitops_32.h @@ -18,9 +18,9 @@ #error only <linux/bitops.h> can be included directly #endif -extern unsigned long ___set_bit(unsigned long *addr, unsigned long mask); -extern unsigned long ___clear_bit(unsigned long *addr, unsigned long mask); -extern unsigned long ___change_bit(unsigned long *addr, unsigned long mask); +unsigned long ___set_bit(unsigned long *addr, unsigned long mask); +unsigned long ___clear_bit(unsigned long *addr, unsigned long mask); +unsigned long ___change_bit(unsigned long *addr, unsigned long mask); /* * Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0' @@ -90,9 +90,6 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include <asm-generic/bitops/non-atomic.h> -#define smp_mb__before_clear_bit() do { } while(0) -#define smp_mb__after_clear_bit() do { } while(0) - #include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/sched.h> @@ -103,9 +100,8 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *addr) #include <asm-generic/bitops/hweight.h> #include <asm-generic/bitops/lock.h> #include <asm-generic/bitops/find.h> -#include <asm-generic/bitops/ext2-non-atomic.h> +#include <asm-generic/bitops/le.h> #include <asm-generic/bitops/ext2-atomic.h> -#include <asm-generic/bitops/minix.h> #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h index 766121a67a2..2d522402a93 100644 --- a/arch/sparc/include/asm/bitops_64.h +++ b/arch/sparc/include/asm/bitops_64.h @@ -13,74 +13,39 @@ #include <linux/compiler.h> #include <asm/byteorder.h> +#include <asm/barrier.h> -extern int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); -extern int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); -extern int test_and_change_bit(unsigned long nr, volatile unsigned long *addr); -extern void set_bit(unsigned long nr, volatile unsigned long *addr); -extern void clear_bit(unsigned long nr, volatile unsigned long *addr); -extern void change_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_change_bit(unsigned long nr, volatile unsigned long *addr); +void set_bit(unsigned long nr, volatile unsigned long *addr); +void clear_bit(unsigned long nr, volatile unsigned long *addr); +void change_bit(unsigned long nr, volatile unsigned long *addr); #include <asm-generic/bitops/non-atomic.h> -#define smp_mb__before_clear_bit() barrier() -#define smp_mb__after_clear_bit() barrier() - -#include <asm-generic/bitops/ffz.h> -#include <asm-generic/bitops/__ffs.h> #include <asm-generic/bitops/fls.h> #include <asm-generic/bitops/__fls.h> #include <asm-generic/bitops/fls64.h> #ifdef __KERNEL__ +int ffs(int x); +unsigned long __ffs(unsigned long); + +#include <asm-generic/bitops/ffz.h> #include <asm-generic/bitops/sched.h> -#include <asm-generic/bitops/ffs.h> /* * hweightN: returns the hamming weight (i.e. the number * of bits set) of a N-bit word */ -#ifdef ULTRA_HAS_POPULATION_COUNT - -static inline unsigned int __arch_hweight64(unsigned long w) -{ - unsigned int res; - - __asm__ ("popc %1,%0" : "=r" (res) : "r" (w)); - return res; -} - -static inline unsigned int __arch_hweight32(unsigned int w) -{ - unsigned int res; - - __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xffffffff)); - return res; -} +unsigned long __arch_hweight64(__u64 w); +unsigned int __arch_hweight32(unsigned int w); +unsigned int __arch_hweight16(unsigned int w); +unsigned int __arch_hweight8(unsigned int w); -static inline unsigned int __arch_hweight16(unsigned int w) -{ - unsigned int res; - - __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xffff)); - return res; -} - -static inline unsigned int __arch_hweight8(unsigned int w) -{ - unsigned int res; - - __asm__ ("popc %1,%0" : "=r" (res) : "r" (w & 0xff)); - return res; -} - -#else - -#include <asm-generic/bitops/arch_hweight.h> - -#endif #include <asm-generic/bitops/const_hweight.h> #include <asm-generic/bitops/lock.h> #endif /* __KERNEL__ */ @@ -89,14 +54,9 @@ static inline unsigned int __arch_hweight8(unsigned int w) #ifdef __KERNEL__ -#include <asm-generic/bitops/ext2-non-atomic.h> - -#define ext2_set_bit_atomic(lock,nr,addr) \ - test_and_set_bit((nr) ^ 0x38,(unsigned long *)(addr)) -#define ext2_clear_bit_atomic(lock,nr,addr) \ - test_and_clear_bit((nr) ^ 0x38,(unsigned long *)(addr)) +#include <asm-generic/bitops/le.h> -#include <asm-generic/bitops/minix.h> +#include <asm-generic/bitops/ext2-atomic-setbit.h> #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/btext.h b/arch/sparc/include/asm/btext.h index 9b2bc6b6ed0..75a32b109e1 100644 --- a/arch/sparc/include/asm/btext.h +++ b/arch/sparc/include/asm/btext.h @@ -1,6 +1,6 @@ #ifndef _SPARC_BTEXT_H #define _SPARC_BTEXT_H -extern int btext_find_display(void); +int btext_find_display(void); #endif /* _SPARC_BTEXT_H */ diff --git a/arch/sparc/include/asm/btfixup.h b/arch/sparc/include/asm/btfixup.h deleted file mode 100644 index 797722cf69f..00000000000 --- a/arch/sparc/include/asm/btfixup.h +++ /dev/null @@ -1,208 +0,0 @@ -/* - * asm/btfixup.h: Macros for boot time linking. - * - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#ifndef _SPARC_BTFIXUP_H -#define _SPARC_BTFIXUP_H - -#include <linux/init.h> - -#ifndef __ASSEMBLY__ - -#ifdef MODULE -extern unsigned int ___illegal_use_of_BTFIXUP_SIMM13_in_module(void); -extern unsigned int ___illegal_use_of_BTFIXUP_SETHI_in_module(void); -extern unsigned int ___illegal_use_of_BTFIXUP_HALF_in_module(void); -extern unsigned int ___illegal_use_of_BTFIXUP_INT_in_module(void); - -#define BTFIXUP_SIMM13(__name) ___illegal_use_of_BTFIXUP_SIMM13_in_module() -#define BTFIXUP_HALF(__name) ___illegal_use_of_BTFIXUP_HALF_in_module() -#define BTFIXUP_SETHI(__name) ___illegal_use_of_BTFIXUP_SETHI_in_module() -#define BTFIXUP_INT(__name) ___illegal_use_of_BTFIXUP_INT_in_module() -#define BTFIXUP_BLACKBOX(__name) ___illegal_use_of_BTFIXUP_BLACKBOX_in_module - -#else - -#define BTFIXUP_SIMM13(__name) ___sf_##__name() -#define BTFIXUP_HALF(__name) ___af_##__name() -#define BTFIXUP_SETHI(__name) ___hf_##__name() -#define BTFIXUP_INT(__name) ((unsigned int)&___i_##__name) -/* This must be written in assembly and present in a sethi */ -#define BTFIXUP_BLACKBOX(__name) ___b_##__name -#endif /* MODULE */ - -/* Fixup call xx */ - -#define BTFIXUPDEF_CALL(__type, __name, __args...) \ - extern __type ___f_##__name(__args); \ - extern unsigned ___fs_##__name[3]; -#define BTFIXUPDEF_CALL_CONST(__type, __name, __args...) \ - extern __type ___f_##__name(__args) __attribute_const__; \ - extern unsigned ___fs_##__name[3]; -#define BTFIXUP_CALL(__name) ___f_##__name - -#define BTFIXUPDEF_BLACKBOX(__name) \ - extern unsigned ___bs_##__name[2]; - -/* Put bottom 13bits into some register variable */ - -#define BTFIXUPDEF_SIMM13(__name) \ - static inline unsigned int ___sf_##__name(void) __attribute_const__; \ - extern unsigned ___ss_##__name[2]; \ - static inline unsigned int ___sf_##__name(void) { \ - unsigned int ret; \ - __asm__ ("or %%g0, ___s_" #__name ", %0" : "=r"(ret)); \ - return ret; \ - } -#define BTFIXUPDEF_SIMM13_INIT(__name,__val) \ - static inline unsigned int ___sf_##__name(void) __attribute_const__; \ - extern unsigned ___ss_##__name[2]; \ - static inline unsigned int ___sf_##__name(void) { \ - unsigned int ret; \ - __asm__ ("or %%g0, ___s_" #__name "__btset_" #__val ", %0" : "=r"(ret));\ - return ret; \ - } - -/* Put either bottom 13 bits, or upper 22 bits into some register variable - * (depending on the value, this will lead into sethi FIX, reg; or - * mov FIX, reg; ) - */ - -#define BTFIXUPDEF_HALF(__name) \ - static inline unsigned int ___af_##__name(void) __attribute_const__; \ - extern unsigned ___as_##__name[2]; \ - static inline unsigned int ___af_##__name(void) { \ - unsigned int ret; \ - __asm__ ("or %%g0, ___a_" #__name ", %0" : "=r"(ret)); \ - return ret; \ - } -#define BTFIXUPDEF_HALF_INIT(__name,__val) \ - static inline unsigned int ___af_##__name(void) __attribute_const__; \ - extern unsigned ___as_##__name[2]; \ - static inline unsigned int ___af_##__name(void) { \ - unsigned int ret; \ - __asm__ ("or %%g0, ___a_" #__name "__btset_" #__val ", %0" : "=r"(ret));\ - return ret; \ - } - -/* Put upper 22 bits into some register variable */ - -#define BTFIXUPDEF_SETHI(__name) \ - static inline unsigned int ___hf_##__name(void) __attribute_const__; \ - extern unsigned ___hs_##__name[2]; \ - static inline unsigned int ___hf_##__name(void) { \ - unsigned int ret; \ - __asm__ ("sethi %%hi(___h_" #__name "), %0" : "=r"(ret)); \ - return ret; \ - } -#define BTFIXUPDEF_SETHI_INIT(__name,__val) \ - static inline unsigned int ___hf_##__name(void) __attribute_const__; \ - extern unsigned ___hs_##__name[2]; \ - static inline unsigned int ___hf_##__name(void) { \ - unsigned int ret; \ - __asm__ ("sethi %%hi(___h_" #__name "__btset_" #__val "), %0" : \ - "=r"(ret)); \ - return ret; \ - } - -/* Put a full 32bit integer into some register variable */ - -#define BTFIXUPDEF_INT(__name) \ - extern unsigned char ___i_##__name; \ - extern unsigned ___is_##__name[2]; - -#define BTFIXUPCALL_NORM 0x00000000 /* Always call */ -#define BTFIXUPCALL_NOP 0x01000000 /* Possibly optimize to nop */ -#define BTFIXUPCALL_RETINT(i) (0x90102000|((i) & 0x1fff)) /* Possibly optimize to mov i, %o0 */ -#define BTFIXUPCALL_ORINT(i) (0x90122000|((i) & 0x1fff)) /* Possibly optimize to or %o0, i, %o0 */ -#define BTFIXUPCALL_RETO0 0x01000000 /* Return first parameter, actually a nop */ -#define BTFIXUPCALL_ANDNINT(i) (0x902a2000|((i) & 0x1fff)) /* Possibly optimize to andn %o0, i, %o0 */ -#define BTFIXUPCALL_SWAPO0O1 0xd27a0000 /* Possibly optimize to swap [%o0],%o1 */ -#define BTFIXUPCALL_SWAPO0G0 0xc07a0000 /* Possibly optimize to swap [%o0],%g0 */ -#define BTFIXUPCALL_SWAPG1G2 0xc4784000 /* Possibly optimize to swap [%g1],%g2 */ -#define BTFIXUPCALL_STG0O0 0xc0220000 /* Possibly optimize to st %g0,[%o0] */ -#define BTFIXUPCALL_STO1O0 0xd2220000 /* Possibly optimize to st %o1,[%o0] */ - -#define BTFIXUPSET_CALL(__name, __addr, __insn) \ - do { \ - ___fs_##__name[0] |= 1; \ - ___fs_##__name[1] = (unsigned long)__addr; \ - ___fs_##__name[2] = __insn; \ - } while (0) - -#define BTFIXUPSET_BLACKBOX(__name, __func) \ - do { \ - ___bs_##__name[0] |= 1; \ - ___bs_##__name[1] = (unsigned long)__func; \ - } while (0) - -#define BTFIXUPCOPY_CALL(__name, __from) \ - do { \ - ___fs_##__name[0] |= 1; \ - ___fs_##__name[1] = ___fs_##__from[1]; \ - ___fs_##__name[2] = ___fs_##__from[2]; \ - } while (0) - -#define BTFIXUPSET_SIMM13(__name, __val) \ - do { \ - ___ss_##__name[0] |= 1; \ - ___ss_##__name[1] = (unsigned)__val; \ - } while (0) - -#define BTFIXUPCOPY_SIMM13(__name, __from) \ - do { \ - ___ss_##__name[0] |= 1; \ - ___ss_##__name[1] = ___ss_##__from[1]; \ - } while (0) - -#define BTFIXUPSET_HALF(__name, __val) \ - do { \ - ___as_##__name[0] |= 1; \ - ___as_##__name[1] = (unsigned)__val; \ - } while (0) - -#define BTFIXUPCOPY_HALF(__name, __from) \ - do { \ - ___as_##__name[0] |= 1; \ - ___as_##__name[1] = ___as_##__from[1]; \ - } while (0) - -#define BTFIXUPSET_SETHI(__name, __val) \ - do { \ - ___hs_##__name[0] |= 1; \ - ___hs_##__name[1] = (unsigned)__val; \ - } while (0) - -#define BTFIXUPCOPY_SETHI(__name, __from) \ - do { \ - ___hs_##__name[0] |= 1; \ - ___hs_##__name[1] = ___hs_##__from[1]; \ - } while (0) - -#define BTFIXUPSET_INT(__name, __val) \ - do { \ - ___is_##__name[0] |= 1; \ - ___is_##__name[1] = (unsigned)__val; \ - } while (0) - -#define BTFIXUPCOPY_INT(__name, __from) \ - do { \ - ___is_##__name[0] |= 1; \ - ___is_##__name[1] = ___is_##__from[1]; \ - } while (0) - -#define BTFIXUPVAL_CALL(__name) \ - ((unsigned long)___fs_##__name[1]) - -extern void btfixup(void); - -#else /* __ASSEMBLY__ */ - -#define BTFIXUP_SETHI(__name) %hi(___h_ ## __name) -#define BTFIXUP_SETHI_INIT(__name,__val) %hi(___h_ ## __name ## __btset_ ## __val) - -#endif /* __ASSEMBLY__ */ - -#endif /* !(_SPARC_BTFIXUP_H) */ diff --git a/arch/sparc/include/asm/bug.h b/arch/sparc/include/asm/bug.h index 8a59e5a8c21..eaa8f8d3812 100644 --- a/arch/sparc/include/asm/bug.h +++ b/arch/sparc/include/asm/bug.h @@ -5,7 +5,7 @@ #include <linux/compiler.h> #ifdef CONFIG_DEBUG_BUGVERBOSE -extern void do_BUG(const char *file, int line); +void do_BUG(const char *file, int line); #define BUG() do { \ do_BUG(__FILE__, __LINE__); \ __builtin_trap(); \ @@ -19,4 +19,7 @@ extern void do_BUG(const char *file, int line); #include <asm-generic/bug.h> +struct pt_regs; +void __noreturn die_if_kernel(char *str, struct pt_regs *regs); + #endif diff --git a/arch/sparc/include/asm/cache.h b/arch/sparc/include/asm/cache.h index 69358b590c9..5bb6991b485 100644 --- a/arch/sparc/include/asm/cache.h +++ b/arch/sparc/include/asm/cache.h @@ -22,118 +22,4 @@ #define __read_mostly __attribute__((__section__(".data..read_mostly"))) -#ifdef CONFIG_SPARC32 -#include <asm/asi.h> - -/* Direct access to the instruction cache is provided through and - * alternate address space. The IDC bit must be off in the ICCR on - * HyperSparcs for these accesses to work. The code below does not do - * any checking, the caller must do so. These routines are for - * diagnostics only, but could end up being useful. Use with care. - * Also, you are asking for trouble if you execute these in one of the - * three instructions following a %asr/%psr access or modification. - */ - -/* First, cache-tag access. */ -static inline unsigned int get_icache_tag(int setnum, int tagnum) -{ - unsigned int vaddr, retval; - - vaddr = ((setnum&1) << 12) | ((tagnum&0x7f) << 5); - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (vaddr), "i" (ASI_M_TXTC_TAG)); - return retval; -} - -static inline void put_icache_tag(int setnum, int tagnum, unsigned int entry) -{ - unsigned int vaddr; - - vaddr = ((setnum&1) << 12) | ((tagnum&0x7f) << 5); - __asm__ __volatile__("sta %0, [%1] %2\n\t" : : - "r" (entry), "r" (vaddr), "i" (ASI_M_TXTC_TAG) : - "memory"); -} - -/* Second cache-data access. The data is returned two-32bit quantities - * at a time. - */ -static inline void get_icache_data(int setnum, int tagnum, int subblock, - unsigned int *data) -{ - unsigned int value1, value2, vaddr; - - vaddr = ((setnum&0x1) << 12) | ((tagnum&0x7f) << 5) | - ((subblock&0x3) << 3); - __asm__ __volatile__("ldda [%2] %3, %%g2\n\t" - "or %%g0, %%g2, %0\n\t" - "or %%g0, %%g3, %1\n\t" : - "=r" (value1), "=r" (value2) : - "r" (vaddr), "i" (ASI_M_TXTC_DATA) : - "g2", "g3"); - data[0] = value1; data[1] = value2; -} - -static inline void put_icache_data(int setnum, int tagnum, int subblock, - unsigned int *data) -{ - unsigned int value1, value2, vaddr; - - vaddr = ((setnum&0x1) << 12) | ((tagnum&0x7f) << 5) | - ((subblock&0x3) << 3); - value1 = data[0]; value2 = data[1]; - __asm__ __volatile__("or %%g0, %0, %%g2\n\t" - "or %%g0, %1, %%g3\n\t" - "stda %%g2, [%2] %3\n\t" : : - "r" (value1), "r" (value2), - "r" (vaddr), "i" (ASI_M_TXTC_DATA) : - "g2", "g3", "memory" /* no joke */); -} - -/* Different types of flushes with the ICACHE. Some of the flushes - * affect both the ICACHE and the external cache. Others only clear - * the ICACHE entries on the cpu itself. V8's (most) allow - * granularity of flushes on the packet (element in line), whole line, - * and entire cache (ie. all lines) level. The ICACHE only flushes are - * ROSS HyperSparc specific and are in ross.h - */ - -/* Flushes which clear out both the on-chip and external caches */ -static inline void flush_ei_page(unsigned int addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_PAGE) : - "memory"); -} - -static inline void flush_ei_seg(unsigned int addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_SEG) : - "memory"); -} - -static inline void flush_ei_region(unsigned int addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_REGION) : - "memory"); -} - -static inline void flush_ei_ctx(unsigned int addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_CTX) : - "memory"); -} - -static inline void flush_ei_user(unsigned int addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_USER) : - "memory"); -} -#endif /* CONFIG_SPARC32 */ - #endif /* !(_SPARC_CACHE_H) */ diff --git a/arch/sparc/include/asm/cacheflush.h b/arch/sparc/include/asm/cacheflush.h index 049168087b1..f6c4839b838 100644 --- a/arch/sparc/include/asm/cacheflush.h +++ b/arch/sparc/include/asm/cacheflush.h @@ -1,5 +1,9 @@ #ifndef ___ASM_SPARC_CACHEFLUSH_H #define ___ASM_SPARC_CACHEFLUSH_H + +/* flush addr - to allow use of self-modifying code */ +#define flushi(addr) __asm__ __volatile__ ("flush %0" : : "r" (addr) : "memory") + #if defined(__sparc__) && defined(__arch64__) #include <asm/cacheflush_64.h> #else diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h index 2e468773f25..12164006181 100644 --- a/arch/sparc/include/asm/cacheflush_32.h +++ b/arch/sparc/include/asm/cacheflush_32.h @@ -1,56 +1,18 @@ #ifndef _SPARC_CACHEFLUSH_H #define _SPARC_CACHEFLUSH_H -#include <linux/mm.h> /* Common for other includes */ -// #include <linux/kernel.h> from pgalloc.h -// #include <linux/sched.h> from pgalloc.h - -// #include <asm/page.h> -#include <asm/btfixup.h> - -/* - * Fine grained cache flushing. - */ -#ifdef CONFIG_SMP - -BTFIXUPDEF_CALL(void, local_flush_cache_all, void) -BTFIXUPDEF_CALL(void, local_flush_cache_mm, struct mm_struct *) -BTFIXUPDEF_CALL(void, local_flush_cache_range, struct vm_area_struct *, unsigned long, unsigned long) -BTFIXUPDEF_CALL(void, local_flush_cache_page, struct vm_area_struct *, unsigned long) - -#define local_flush_cache_all() BTFIXUP_CALL(local_flush_cache_all)() -#define local_flush_cache_mm(mm) BTFIXUP_CALL(local_flush_cache_mm)(mm) -#define local_flush_cache_range(vma,start,end) BTFIXUP_CALL(local_flush_cache_range)(vma,start,end) -#define local_flush_cache_page(vma,addr) BTFIXUP_CALL(local_flush_cache_page)(vma,addr) - -BTFIXUPDEF_CALL(void, local_flush_page_to_ram, unsigned long) -BTFIXUPDEF_CALL(void, local_flush_sig_insns, struct mm_struct *, unsigned long) - -#define local_flush_page_to_ram(addr) BTFIXUP_CALL(local_flush_page_to_ram)(addr) -#define local_flush_sig_insns(mm,insn_addr) BTFIXUP_CALL(local_flush_sig_insns)(mm,insn_addr) - -extern void smp_flush_cache_all(void); -extern void smp_flush_cache_mm(struct mm_struct *mm); -extern void smp_flush_cache_range(struct vm_area_struct *vma, - unsigned long start, - unsigned long end); -extern void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page); - -extern void smp_flush_page_to_ram(unsigned long page); -extern void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); - -#endif /* CONFIG_SMP */ - -BTFIXUPDEF_CALL(void, flush_cache_all, void) -BTFIXUPDEF_CALL(void, flush_cache_mm, struct mm_struct *) -BTFIXUPDEF_CALL(void, flush_cache_range, struct vm_area_struct *, unsigned long, unsigned long) -BTFIXUPDEF_CALL(void, flush_cache_page, struct vm_area_struct *, unsigned long) - -#define flush_cache_all() BTFIXUP_CALL(flush_cache_all)() -#define flush_cache_mm(mm) BTFIXUP_CALL(flush_cache_mm)(mm) -#define flush_cache_dup_mm(mm) BTFIXUP_CALL(flush_cache_mm)(mm) -#define flush_cache_range(vma,start,end) BTFIXUP_CALL(flush_cache_range)(vma,start,end) -#define flush_cache_page(vma,addr,pfn) BTFIXUP_CALL(flush_cache_page)(vma,addr) +#include <asm/cachetlb_32.h> + +#define flush_cache_all() \ + sparc32_cachetlb_ops->cache_all() +#define flush_cache_mm(mm) \ + sparc32_cachetlb_ops->cache_mm(mm) +#define flush_cache_dup_mm(mm) \ + sparc32_cachetlb_ops->cache_mm(mm) +#define flush_cache_range(vma,start,end) \ + sparc32_cachetlb_ops->cache_range(vma, start, end) +#define flush_cache_page(vma,addr,pfn) \ + sparc32_cachetlb_ops->cache_page(vma, addr) #define flush_icache_range(start, end) do { } while (0) #define flush_icache_page(vma, pg) do { } while (0) @@ -67,13 +29,14 @@ BTFIXUPDEF_CALL(void, flush_cache_page, struct vm_area_struct *, unsigned long) memcpy(dst, src, len); \ } while (0) -BTFIXUPDEF_CALL(void, __flush_page_to_ram, unsigned long) -BTFIXUPDEF_CALL(void, flush_sig_insns, struct mm_struct *, unsigned long) +#define __flush_page_to_ram(addr) \ + sparc32_cachetlb_ops->page_to_ram(addr) +#define flush_sig_insns(mm,insn_addr) \ + sparc32_cachetlb_ops->sig_insns(mm, insn_addr) +#define flush_page_for_dma(addr) \ + sparc32_cachetlb_ops->page_for_dma(addr) -#define __flush_page_to_ram(addr) BTFIXUP_CALL(__flush_page_to_ram)(addr) -#define flush_sig_insns(mm,insn_addr) BTFIXUP_CALL(flush_sig_insns)(mm,insn_addr) - -extern void sparc_flush_page_to_ram(struct page *page); +void sparc_flush_page_to_ram(struct page *page); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 #define flush_dcache_page(page) sparc_flush_page_to_ram(page) @@ -83,4 +46,13 @@ extern void sparc_flush_page_to_ram(struct page *page); #define flush_cache_vmap(start, end) flush_cache_all() #define flush_cache_vunmap(start, end) flush_cache_all() +/* When a context switch happens we must flush all user windows so that + * the windows of the current process are flushed onto its stack. This + * way the windows are all clean for the next process and the stack + * frames are up to date. + */ +void flush_user_windows(void); +void kill_user_windows(void); +void flushw_all(void); + #endif /* _SPARC_CACHEFLUSH_H */ diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h index b95384033e8..38965379e35 100644 --- a/arch/sparc/include/asm/cacheflush_64.h +++ b/arch/sparc/include/asm/cacheflush_64.h @@ -8,6 +8,13 @@ #include <linux/mm.h> /* Cache flush operations. */ +#define flushw_all() __asm__ __volatile__("flushw") + +void __flushw_user(void); +#define flushw_user() __flushw_user() + +#define flush_user_windows flushw_user +#define flush_register_windows flushw_all /* These are the same regardless of whether this is an SMP kernel or not. */ #define flush_cache_mm(__mm) \ @@ -23,29 +30,29 @@ * use block commit stores (which invalidate icache lines) during * module load, so we need this. */ -extern void flush_icache_range(unsigned long start, unsigned long end); -extern void __flush_icache_page(unsigned long); +void flush_icache_range(unsigned long start, unsigned long end); +void __flush_icache_page(unsigned long); -extern void __flush_dcache_page(void *addr, int flush_icache); -extern void flush_dcache_page_impl(struct page *page); +void __flush_dcache_page(void *addr, int flush_icache); +void flush_dcache_page_impl(struct page *page); #ifdef CONFIG_SMP -extern void smp_flush_dcache_page_impl(struct page *page, int cpu); -extern void flush_dcache_page_all(struct mm_struct *mm, struct page *page); +void smp_flush_dcache_page_impl(struct page *page, int cpu); +void flush_dcache_page_all(struct mm_struct *mm, struct page *page); #else #define smp_flush_dcache_page_impl(page,cpu) flush_dcache_page_impl(page) #define flush_dcache_page_all(mm,page) flush_dcache_page_impl(page) #endif -extern void __flush_dcache_range(unsigned long start, unsigned long end); +void __flush_dcache_range(unsigned long start, unsigned long end); #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 -extern void flush_dcache_page(struct page *page); +void flush_dcache_page(struct page *page); #define flush_icache_page(vma, pg) do { } while(0) #define flush_icache_user_range(vma,pg,adr,len) do { } while (0) -extern void flush_ptrace_access(struct vm_area_struct *, struct page *, - unsigned long uaddr, void *kaddr, - unsigned long len, int write); +void flush_ptrace_access(struct vm_area_struct *, struct page *, + unsigned long uaddr, void *kaddr, + unsigned long len, int write); #define copy_to_user_page(vma, page, vaddr, dst, src, len) \ do { \ diff --git a/arch/sparc/include/asm/cachetlb_32.h b/arch/sparc/include/asm/cachetlb_32.h new file mode 100644 index 00000000000..efb19889a08 --- /dev/null +++ b/arch/sparc/include/asm/cachetlb_32.h @@ -0,0 +1,29 @@ +#ifndef _SPARC_CACHETLB_H +#define _SPARC_CACHETLB_H + +struct mm_struct; +struct vm_area_struct; + +struct sparc32_cachetlb_ops { + void (*cache_all)(void); + void (*cache_mm)(struct mm_struct *); + void (*cache_range)(struct vm_area_struct *, unsigned long, + unsigned long); + void (*cache_page)(struct vm_area_struct *, unsigned long); + + void (*tlb_all)(void); + void (*tlb_mm)(struct mm_struct *); + void (*tlb_range)(struct vm_area_struct *, unsigned long, + unsigned long); + void (*tlb_page)(struct vm_area_struct *, unsigned long); + + void (*page_to_ram)(unsigned long); + void (*sig_insns)(struct mm_struct *, unsigned long); + void (*page_for_dma)(unsigned long); +}; +extern const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +#ifdef CONFIG_SMP +extern const struct sparc32_cachetlb_ops *local_ops; +#endif + +#endif /* SPARC_CACHETLB_H */ diff --git a/arch/sparc/include/asm/checksum_32.h b/arch/sparc/include/asm/checksum_32.h index bdbda1453aa..426b2389a1c 100644 --- a/arch/sparc/include/asm/checksum_32.h +++ b/arch/sparc/include/asm/checksum_32.h @@ -29,7 +29,7 @@ * * it's best to have buff aligned on a 32-bit boundary */ -extern __wsum csum_partial(const void *buff, int len, __wsum sum); +__wsum csum_partial(const void *buff, int len, __wsum sum); /* the same as csum_partial, but copies from fs:src while it * checksums @@ -38,7 +38,7 @@ extern __wsum csum_partial(const void *buff, int len, __wsum sum); * better 64-bit) boundary */ -extern unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *); +unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *); static inline __wsum csum_partial_copy_nocheck(const void *src, void *dst, int len, __wsum sum) @@ -238,4 +238,16 @@ static inline __sum16 ip_compute_csum(const void *buff, int len) return csum_fold(csum_partial(buff, len, 0)); } +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + #endif /* !(__SPARC_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/checksum_64.h b/arch/sparc/include/asm/checksum_64.h index 019b9615e43..b8779a6a591 100644 --- a/arch/sparc/include/asm/checksum_64.h +++ b/arch/sparc/include/asm/checksum_64.h @@ -29,7 +29,7 @@ * * it's best to have buff aligned on a 32-bit boundary */ -extern __wsum csum_partial(const void * buff, int len, __wsum sum); +__wsum csum_partial(const void * buff, int len, __wsum sum); /* the same as csum_partial, but copies from user space while it * checksums @@ -37,12 +37,12 @@ extern __wsum csum_partial(const void * buff, int len, __wsum sum); * here even more important to align src and dst on a 32-bit (or even * better 64-bit) boundary */ -extern __wsum csum_partial_copy_nocheck(const void *src, void *dst, - int len, __wsum sum); +__wsum csum_partial_copy_nocheck(const void *src, void *dst, + int len, __wsum sum); -extern long __csum_partial_copy_from_user(const void __user *src, - void *dst, int len, - __wsum sum); +long __csum_partial_copy_from_user(const void __user *src, + void *dst, int len, + __wsum sum); static inline __wsum csum_partial_copy_from_user(const void __user *src, @@ -59,9 +59,9 @@ csum_partial_copy_from_user(const void __user *src, * Copy and checksum to user */ #define HAVE_CSUM_COPY_USER -extern long __csum_partial_copy_to_user(const void *src, - void __user *dst, int len, - __wsum sum); +long __csum_partial_copy_to_user(const void *src, + void __user *dst, int len, + __wsum sum); static inline __wsum csum_and_copy_to_user(const void *src, @@ -77,7 +77,7 @@ csum_and_copy_to_user(const void *src, /* ihl is always 5 or greater, almost always is 5, and iph is word aligned * the majority of the time. */ -extern __sum16 ip_fast_csum(const void *iph, unsigned int ihl); +__sum16 ip_fast_csum(const void *iph, unsigned int ihl); /* Fold a partial checksum without adding pseudo headers. */ static inline __sum16 csum_fold(__wsum sum) @@ -96,9 +96,9 @@ static inline __sum16 csum_fold(__wsum sum) } static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, - unsigned int len, - unsigned short proto, - __wsum sum) + unsigned int len, + unsigned short proto, + __wsum sum) { __asm__ __volatile__( " addcc %1, %0, %0\n" @@ -116,9 +116,9 @@ static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, * returns a 16-bit checksum, already complemented */ static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, - unsigned short len, - unsigned short proto, - __wsum sum) + unsigned short len, + unsigned short proto, + __wsum sum) { return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); } @@ -164,4 +164,16 @@ static inline __sum16 ip_compute_csum(const void *buff, int len) return csum_fold(csum_partial(buff, len, 0)); } +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + #endif /* !(__SPARC64_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/cmpxchg.h b/arch/sparc/include/asm/cmpxchg.h new file mode 100644 index 00000000000..9355893efa5 --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg.h @@ -0,0 +1,8 @@ +#ifndef ___ASM_SPARC_CMPXCHG_H +#define ___ASM_SPARC_CMPXCHG_H +#if defined(__sparc__) && defined(__arch64__) +#include <asm/cmpxchg_64.h> +#else +#include <asm/cmpxchg_32.h> +#endif +#endif diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h new file mode 100644 index 00000000000..32c29a133f9 --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -0,0 +1,85 @@ +/* 32-bit atomic xchg() and cmpxchg() definitions. + * + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) + * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com.au) + * Copyright (C) 2007 Kyle McMartin (kyle@parisc-linux.org) + * + * Additions by Keith M Wesolowski (wesolows@foobazco.org) based + * on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf <prumpf@tux.org>. + */ + +#ifndef __ARCH_SPARC_CMPXCHG__ +#define __ARCH_SPARC_CMPXCHG__ + +static inline unsigned long xchg_u32(__volatile__ unsigned long *m, unsigned long val) +{ + __asm__ __volatile__("swap [%2], %0" + : "=&r" (val) + : "0" (val), "r" (m) + : "memory"); + return val; +} + +void __xchg_called_with_bad_pointer(void); + +static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int size) +{ + switch (size) { + case 4: + return xchg_u32(ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + +#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) + +/* Emulate cmpxchg() the same way we emulate atomics, + * by hashing the object address and indexing into an array + * of spinlocks to get a bit of performance... + * + * See arch/sparc/lib/atomic32.c for implementation. + * + * Cribbed from <asm-parisc/atomic.h> + */ +#define __HAVE_ARCH_CMPXCHG 1 + +/* bug catcher for when unsupported size is used - won't link */ +void __cmpxchg_called_with_bad_pointer(void); +/* we only need to support cmpxchg of a u32 on sparc */ +unsigned long __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_); + +/* don't worry...optimizer will get rid of most of this */ +static inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) +{ + switch (size) { + case 4: + return __cmpxchg_u32((u32 *)ptr, (u32)old, (u32)new_); + default: + __cmpxchg_called_with_bad_pointer(); + break; + } + return old; +} + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr))); \ +}) + +#include <asm-generic/cmpxchg-local.h> + +/* + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make + * them available. + */ +#define cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ + (unsigned long)(n), sizeof(*(ptr)))) +#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) + +#endif /* __ARCH_SPARC_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/cmpxchg_64.h b/arch/sparc/include/asm/cmpxchg_64.h new file mode 100644 index 00000000000..0e1ed6cfbf6 --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg_64.h @@ -0,0 +1,146 @@ +/* 64-bit atomic xchg() and cmpxchg() definitions. + * + * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com) + */ + +#ifndef __ARCH_SPARC64_CMPXCHG__ +#define __ARCH_SPARC64_CMPXCHG__ + +static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val) +{ + unsigned long tmp1, tmp2; + + __asm__ __volatile__( +" mov %0, %1\n" +"1: lduw [%4], %2\n" +" cas [%4], %2, %0\n" +" cmp %2, %0\n" +" bne,a,pn %%icc, 1b\n" +" mov %1, %0\n" + : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) + : "0" (val), "r" (m) + : "cc", "memory"); + return val; +} + +static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long val) +{ + unsigned long tmp1, tmp2; + + __asm__ __volatile__( +" mov %0, %1\n" +"1: ldx [%4], %2\n" +" casx [%4], %2, %0\n" +" cmp %2, %0\n" +" bne,a,pn %%xcc, 1b\n" +" mov %1, %0\n" + : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) + : "0" (val), "r" (m) + : "cc", "memory"); + return val; +} + +#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) + +void __xchg_called_with_bad_pointer(void); + +static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, + int size) +{ + switch (size) { + case 4: + return xchg32(ptr, x); + case 8: + return xchg64(ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ + +#include <asm-generic/cmpxchg-local.h> + +#define __HAVE_ARCH_CMPXCHG 1 + +static inline unsigned long +__cmpxchg_u32(volatile int *m, int old, int new) +{ + __asm__ __volatile__("cas [%2], %3, %0" + : "=&r" (new) + : "0" (new), "r" (m), "r" (old) + : "memory"); + + return new; +} + +static inline unsigned long +__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) +{ + __asm__ __volatile__("casx [%2], %3, %0" + : "=&r" (new) + : "0" (new), "r" (m), "r" (old) + : "memory"); + + return new; +} + +/* This function doesn't exist, so you'll get a linker error + if something tries to do an invalid cmpxchg(). */ +void __cmpxchg_called_with_bad_pointer(void); + +static inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) +{ + switch (size) { + case 4: + return __cmpxchg_u32(ptr, old, new); + case 8: + return __cmpxchg_u64(ptr, old, new); + } + __cmpxchg_called_with_bad_pointer(); + return old; +} + +#define cmpxchg(ptr,o,n) \ + ({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr))); \ + }) + +/* + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make + * them available. + */ + +static inline unsigned long __cmpxchg_local(volatile void *ptr, + unsigned long old, + unsigned long new, int size) +{ + switch (size) { + case 4: + case 8: return __cmpxchg(ptr, old, new, size); + default: + return __cmpxchg_local_generic(ptr, old, new, size); + } + + return old; +} + +#define cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)))) +#define cmpxchg64_local(ptr, o, n) \ + ({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg_local((ptr), (o), (n)); \ + }) +#define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) + +#endif /* __ARCH_SPARC64_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/cmt.h b/arch/sparc/include/asm/cmt.h deleted file mode 100644 index 870db592857..00000000000 --- a/arch/sparc/include/asm/cmt.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef _SPARC64_CMT_H -#define _SPARC64_CMT_H - -/* cmt.h: Chip Multi-Threading register definitions - * - * Copyright (C) 2004 David S. Miller (davem@redhat.com) - */ - -/* ASI_CORE_ID - private */ -#define LP_ID 0x0000000000000010UL -#define LP_ID_MAX 0x00000000003f0000UL -#define LP_ID_ID 0x000000000000003fUL - -/* ASI_INTR_ID - private */ -#define LP_INTR_ID 0x0000000000000000UL -#define LP_INTR_ID_ID 0x00000000000003ffUL - -/* ASI_CESR_ID - private */ -#define CESR_ID 0x0000000000000040UL -#define CESR_ID_ID 0x00000000000000ffUL - -/* ASI_CORE_AVAILABLE - shared */ -#define LP_AVAIL 0x0000000000000000UL -#define LP_AVAIL_1 0x0000000000000002UL -#define LP_AVAIL_0 0x0000000000000001UL - -/* ASI_CORE_ENABLE_STATUS - shared */ -#define LP_ENAB_STAT 0x0000000000000010UL -#define LP_ENAB_STAT_1 0x0000000000000002UL -#define LP_ENAB_STAT_0 0x0000000000000001UL - -/* ASI_CORE_ENABLE - shared */ -#define LP_ENAB 0x0000000000000020UL -#define LP_ENAB_1 0x0000000000000002UL -#define LP_ENAB_0 0x0000000000000001UL - -/* ASI_CORE_RUNNING - shared */ -#define LP_RUNNING_RW 0x0000000000000050UL -#define LP_RUNNING_W1S 0x0000000000000060UL -#define LP_RUNNING_W1C 0x0000000000000068UL -#define LP_RUNNING_1 0x0000000000000002UL -#define LP_RUNNING_0 0x0000000000000001UL - -/* ASI_CORE_RUNNING_STAT - shared */ -#define LP_RUN_STAT 0x0000000000000058UL -#define LP_RUN_STAT_1 0x0000000000000002UL -#define LP_RUN_STAT_0 0x0000000000000001UL - -/* ASI_XIR_STEERING - shared */ -#define LP_XIR_STEER 0x0000000000000030UL -#define LP_XIR_STEER_1 0x0000000000000002UL -#define LP_XIR_STEER_0 0x0000000000000001UL - -/* ASI_CMT_ERROR_STEERING - shared */ -#define CMT_ER_STEER 0x0000000000000040UL -#define CMT_ER_STEER_1 0x0000000000000002UL -#define CMT_ER_STEER_0 0x0000000000000001UL - -#endif /* _SPARC64_CMT_H */ diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h index 6f57325bb88..830502fe62b 100644 --- a/arch/sparc/include/asm/compat.h +++ b/arch/sparc/include/asm/compat.h @@ -36,6 +36,7 @@ typedef s64 compat_s64; typedef u32 compat_uint_t; typedef u32 compat_ulong_t; typedef u64 compat_u64; +typedef u32 compat_uptr_t; struct compat_timespec { compat_time_t tv_sec; @@ -134,7 +135,8 @@ struct compat_statfs { compat_fsid_t f_fsid; int f_namelen; /* SunOS ignores this field. */ int f_frsize; - int f_spare[5]; + int f_flags; + int f_spare[4]; }; #define COMPAT_RLIM_INFINITY 0x7fffffff @@ -146,6 +148,65 @@ typedef u32 compat_old_sigset_t; typedef u32 compat_sigset_word; +typedef union compat_sigval { + compat_int_t sival_int; + compat_uptr_t sival_ptr; +} compat_sigval_t; + +#define SI_PAD_SIZE32 (128/sizeof(int) - 3) + +typedef struct compat_siginfo { + int si_signo; + int si_errno; + int si_code; + + union { + int _pad[SI_PAD_SIZE32]; + + /* kill() */ + struct { + compat_pid_t _pid; /* sender's pid */ + unsigned int _uid; /* sender's uid */ + } _kill; + + /* POSIX.1b timers */ + struct { + compat_timer_t _tid; /* timer id */ + int _overrun; /* overrun count */ + compat_sigval_t _sigval; /* same as below */ + int _sys_private; /* not to be passed to user */ + } _timer; + + /* POSIX.1b signals */ + struct { + compat_pid_t _pid; /* sender's pid */ + unsigned int _uid; /* sender's uid */ + compat_sigval_t _sigval; + } _rt; + + /* SIGCHLD */ + struct { + compat_pid_t _pid; /* which child */ + unsigned int _uid; /* sender's uid */ + int _status; /* exit code */ + compat_clock_t _utime; + compat_clock_t _stime; + } _sigchld; + + /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ + struct { + u32 _addr; /* faulting insn/memory ref. */ + int _trapno; + } _sigfault; + + /* SIGPOLL */ + struct { + int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ + int _fd; + } _sigpoll; + } _sifields; +} compat_siginfo_t; + #define COMPAT_OFF_T_MAX 0x7fffffff #define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL @@ -155,7 +216,6 @@ typedef u32 compat_sigset_word; * as pointers because the syscall entry code will have * appropriately converted them already. */ -typedef u32 compat_uptr_t; static inline void __user *compat_ptr(compat_uptr_t uptr) { @@ -172,9 +232,10 @@ static inline void __user *arch_compat_alloc_user_space(long len) struct pt_regs *regs = current_thread_info()->kregs; unsigned long usp = regs->u_regs[UREG_I6]; - if (!(test_thread_flag(TIF_32BIT))) + if (test_thread_64bit_stack(usp)) usp += STACK_BIAS; - else + + if (test_thread_flag(TIF_32BIT)) usp &= 0xffffffffUL; usp -= len; diff --git a/arch/sparc/include/asm/compat_signal.h b/arch/sparc/include/asm/compat_signal.h index b759eab9b51..9ed1f128b4d 100644 --- a/arch/sparc/include/asm/compat_signal.h +++ b/arch/sparc/include/asm/compat_signal.h @@ -18,12 +18,6 @@ struct __old_sigaction32 { unsigned int sa_flags; unsigned sa_restorer; /* not used by Linux/SPARC yet */ }; - -typedef struct sigaltstack32 { - u32 ss_sp; - int ss_flags; - compat_size_t ss_size; -} stack_t32; #endif #endif /* !(_COMPAT_SIGNAL_H) */ diff --git a/arch/sparc/include/asm/contregs.h b/arch/sparc/include/asm/contregs.h index 48fa8a4ef35..b8abdfcf555 100644 --- a/arch/sparc/include/asm/contregs.h +++ b/arch/sparc/include/asm/contregs.h @@ -7,28 +7,6 @@ * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) */ -/* 3=sun3 - 4=sun4 (as in sun4 sysmaint student book) - c=sun4c (according to davem) */ - -#define AC_IDPROM 0x00000000 /* 34 ID PROM, R/O, byte, 32 bytes */ -#define AC_PAGEMAP 0x10000000 /* 3 Pagemap R/W, long */ -#define AC_SEGMAP 0x20000000 /* 3 Segment map, byte */ -#define AC_CONTEXT 0x30000000 /* 34c current mmu-context */ -#define AC_SENABLE 0x40000000 /* 34c system dvma/cache/reset enable reg*/ -#define AC_UDVMA_ENB 0x50000000 /* 34 Not used on Sun boards, byte */ -#define AC_BUS_ERROR 0x60000000 /* 34 Not cleared on read, byte. */ -#define AC_SYNC_ERR 0x60000000 /* c fault type */ -#define AC_SYNC_VA 0x60000004 /* c fault virtual address */ -#define AC_ASYNC_ERR 0x60000008 /* c asynchronous fault type */ -#define AC_ASYNC_VA 0x6000000c /* c async fault virtual address */ -#define AC_LEDS 0x70000000 /* 34 Zero turns on LEDs, byte */ -#define AC_CACHETAGS 0x80000000 /* 34c direct access to the VAC tags */ -#define AC_CACHEDDATA 0x90000000 /* 3 c direct access to the VAC data */ -#define AC_UDVMA_MAP 0xD0000000 /* 4 Not used on Sun boards, byte */ -#define AC_VME_VECTOR 0xE0000000 /* 4 For non-Autovector VME, byte */ -#define AC_BOOT_SCC 0xF0000000 /* 34 bypass to access Zilog 8530. byte.*/ - /* s=Swift, h=Ross_HyperSPARC, v=TI_Viking, t=Tsunami, r=Ross_Cypress */ #define AC_M_PCR 0x0000 /* shv Processor Control Reg */ #define AC_M_CTPR 0x0100 /* shv Context Table Pointer Reg */ diff --git a/arch/sparc/include/asm/cpu_type.h b/arch/sparc/include/asm/cpu_type.h new file mode 100644 index 00000000000..84d7d83b808 --- /dev/null +++ b/arch/sparc/include/asm/cpu_type.h @@ -0,0 +1,28 @@ +#ifndef __ASM_CPU_TYPE_H +#define __ASM_CPU_TYPE_H + +/* + * Sparc (general) CPU types + */ +enum sparc_cpu { + sun4m = 0x00, + sun4d = 0x01, + sun4e = 0x02, + sun4u = 0x03, /* V8 ploos ploos */ + sun_unknown = 0x04, + ap1000 = 0x05, /* almost a sun4m */ + sparc_leon = 0x06, /* Leon SoC */ +}; + +#ifdef CONFIG_SPARC32 +extern enum sparc_cpu sparc_cpu_model; + +#define SUN4M_NCPUS 4 /* Architectural limit of sun4m. */ + +#else + +#define sparc_cpu_model sun4u + +#endif + +#endif /* __ASM_CPU_TYPE_H */ diff --git a/arch/sparc/include/asm/cpudata.h b/arch/sparc/include/asm/cpudata.h index b5976de7cac..128b56b0867 100644 --- a/arch/sparc/include/asm/cpudata.h +++ b/arch/sparc/include/asm/cpudata.h @@ -1,5 +1,15 @@ #ifndef ___ASM_SPARC_CPUDATA_H #define ___ASM_SPARC_CPUDATA_H + +#ifndef __ASSEMBLY__ + +#include <linux/threads.h> +#include <linux/percpu.h> + +extern const struct seq_operations cpuinfo_op; + +#endif /* !(__ASSEMBLY__) */ + #if defined(__sparc__) && defined(__arch64__) #include <asm/cpudata_64.h> #else diff --git a/arch/sparc/include/asm/cpudata_32.h b/arch/sparc/include/asm/cpudata_32.h index 31d48a0e32c..0300d94c25b 100644 --- a/arch/sparc/include/asm/cpudata_32.h +++ b/arch/sparc/include/asm/cpudata_32.h @@ -14,8 +14,11 @@ typedef struct { unsigned long udelay_val; unsigned long clock_tick; - unsigned int multiplier; unsigned int counter; +#ifdef CONFIG_SMP + unsigned int irq_resched_count; + unsigned int irq_call_count; +#endif int prom_node; int mid; int next; @@ -23,5 +26,6 @@ typedef struct { DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); #define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) +#define local_cpu_data() __get_cpu_var(__cpu_data) #endif /* _SPARC_CPUDATA_H */ diff --git a/arch/sparc/include/asm/cpudata_64.h b/arch/sparc/include/asm/cpudata_64.h index 050ef35b9dc..0e594076912 100644 --- a/arch/sparc/include/asm/cpudata_64.h +++ b/arch/sparc/include/asm/cpudata_64.h @@ -8,9 +8,6 @@ #ifndef __ASSEMBLY__ -#include <linux/percpu.h> -#include <linux/threads.h> - typedef struct { /* Dcache line 1 */ unsigned int __softirq_pending; /* must be 1st, see rtrap.S */ @@ -35,8 +32,6 @@ DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); #define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) #define local_cpu_data() __get_cpu_var(__cpu_data) -extern const struct seq_operations cpuinfo_op; - #endif /* !(__ASSEMBLY__) */ #include <asm/trap_block.h> diff --git a/arch/sparc/include/asm/cputime.h b/arch/sparc/include/asm/cputime.h deleted file mode 100644 index 1a642b81e01..00000000000 --- a/arch/sparc/include/asm/cputime.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __SPARC_CPUTIME_H -#define __SPARC_CPUTIME_H - -#include <asm-generic/cputime.h> - -#endif /* __SPARC_CPUTIME_H */ diff --git a/arch/sparc/include/asm/cypress.h b/arch/sparc/include/asm/cypress.h deleted file mode 100644 index 95e9772ea39..00000000000 --- a/arch/sparc/include/asm/cypress.h +++ /dev/null @@ -1,79 +0,0 @@ -/* - * cypress.h: Cypress module specific definitions and defines. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ - -#ifndef _SPARC_CYPRESS_H -#define _SPARC_CYPRESS_H - -/* Cypress chips have %psr 'impl' of '0001' and 'vers' of '0001'. */ - -/* The MMU control register fields on the Sparc Cypress 604/605 MMU's. - * - * --------------------------------------------------------------- - * |implvers| MCA | MCM |MV| MID |BM| C|RSV|MR|CM|CL|CE|RSV|NF|ME| - * --------------------------------------------------------------- - * 31 24 23-22 21-20 19 18-15 14 13 12 11 10 9 8 7-2 1 0 - * - * MCA: MultiChip Access -- Used for configuration of multiple - * CY7C604/605 cache units. - * MCM: MultiChip Mask -- Again, for multiple cache unit config. - * MV: MultiChip Valid -- Indicates MCM and MCA have valid settings. - * MID: ModuleID -- Unique processor ID for MBus transactions. (605 only) - * BM: Boot Mode -- 0 = not in boot mode, 1 = in boot mode - * C: Cacheable -- Indicates whether accesses are cacheable while - * the MMU is off. 0=no 1=yes - * MR: MemoryReflection -- Indicates whether the bus attached to the - * MBus supports memory reflection. 0=no 1=yes (605 only) - * CM: CacheMode -- Indicates whether the cache is operating in write - * through or copy-back mode. 0=write-through 1=copy-back - * CL: CacheLock -- Indicates if the entire cache is locked or not. - * 0=not-locked 1=locked (604 only) - * CE: CacheEnable -- Is the virtual cache on? 0=no 1=yes - * NF: NoFault -- Do faults generate traps? 0=yes 1=no - * ME: MmuEnable -- Is the MMU doing translations? 0=no 1=yes - */ - -#define CYPRESS_MCA 0x00c00000 -#define CYPRESS_MCM 0x00300000 -#define CYPRESS_MVALID 0x00080000 -#define CYPRESS_MIDMASK 0x00078000 /* Only on 605 */ -#define CYPRESS_BMODE 0x00004000 -#define CYPRESS_ACENABLE 0x00002000 -#define CYPRESS_MRFLCT 0x00000800 /* Only on 605 */ -#define CYPRESS_CMODE 0x00000400 -#define CYPRESS_CLOCK 0x00000200 /* Only on 604 */ -#define CYPRESS_CENABLE 0x00000100 -#define CYPRESS_NFAULT 0x00000002 -#define CYPRESS_MENABLE 0x00000001 - -static inline void cypress_flush_page(unsigned long page) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (page), "i" (ASI_M_FLUSH_PAGE)); -} - -static inline void cypress_flush_segment(unsigned long addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_SEG)); -} - -static inline void cypress_flush_region(unsigned long addr) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (addr), "i" (ASI_M_FLUSH_REGION)); -} - -static inline void cypress_flush_context(void) -{ - __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" : : - "i" (ASI_M_FLUSH_CTX)); -} - -/* XXX Displacement flushes for buggy chips and initial testing - * XXX go here. - */ - -#endif /* !(_SPARC_CYPRESS_H) */ diff --git a/arch/sparc/include/asm/delay_32.h b/arch/sparc/include/asm/delay_32.h index bc9aba2bead..3fb8ca144b4 100644 --- a/arch/sparc/include/asm/delay_32.h +++ b/arch/sparc/include/asm/delay_32.h @@ -20,8 +20,8 @@ static inline void __delay(unsigned long loops) } /* This is too messy with inline asm on the Sparc. */ -extern void __udelay(unsigned long usecs, unsigned long lpj); -extern void __ndelay(unsigned long nsecs, unsigned long lpj); +void __udelay(unsigned long usecs, unsigned long lpj); +void __ndelay(unsigned long nsecs, unsigned long lpj); #ifdef CONFIG_SMP #define __udelay_val cpu_data(smp_processor_id()).udelay_val diff --git a/arch/sparc/include/asm/delay_64.h b/arch/sparc/include/asm/delay_64.h index a77aa622d76..0ba5424856d 100644 --- a/arch/sparc/include/asm/delay_64.h +++ b/arch/sparc/include/asm/delay_64.h @@ -8,8 +8,8 @@ #ifndef __ASSEMBLY__ -extern void __delay(unsigned long loops); -extern void udelay(unsigned long usecs); +void __delay(unsigned long loops); +void udelay(unsigned long usecs); #define mdelay(n) udelay((n) * 1000) #endif /* !__ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/device.h b/arch/sparc/include/asm/device.h index daa6a8a5e9c..bb3f0b0c675 100644 --- a/arch/sparc/include/asm/device.h +++ b/arch/sparc/include/asm/device.h @@ -19,7 +19,7 @@ struct dev_archdata { int numa_node; }; -extern void of_propagate_archdata(struct platform_device *bus); +void of_propagate_archdata(struct platform_device *bus); struct pdev_archdata { struct resource resource[PROMREG_MAX]; diff --git a/arch/sparc/include/asm/div64.h b/arch/sparc/include/asm/div64.h deleted file mode 100644 index 6cd978cefb2..00000000000 --- a/arch/sparc/include/asm/div64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/div64.h> diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h index 8c0e4f7bb20..1ee02710b2d 100644 --- a/arch/sparc/include/asm/dma-mapping.h +++ b/arch/sparc/include/asm/dma-mapping.h @@ -7,18 +7,23 @@ #define DMA_ERROR_CODE (~(dma_addr_t)0x0) -extern int dma_supported(struct device *dev, u64 mask); +int dma_supported(struct device *dev, u64 mask); #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f) #define dma_free_noncoherent(d, s, v, h) dma_free_coherent(d, s, v, h) -extern struct dma_map_ops *dma_ops, pci32_dma_ops; +extern struct dma_map_ops *dma_ops; +extern struct dma_map_ops *leon_dma_ops; +extern struct dma_map_ops pci32_dma_ops; + extern struct bus_type pci_bus_type; static inline struct dma_map_ops *get_dma_ops(struct device *dev) { #if defined(CONFIG_SPARC32) && defined(CONFIG_PCI) - if (dev->bus == &pci_bus_type) + if (sparc_cpu_model == sparc_leon) + return leon_dma_ops; + else if (dev->bus == &pci_bus_type) return &pci32_dma_ops; #endif return dma_ops; @@ -26,28 +31,35 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev) #include <asm-generic/dma-mapping-common.h> -static inline void *dma_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t flag) +#define dma_alloc_coherent(d,s,h,f) dma_alloc_attrs(d,s,h,f,NULL) + +static inline void *dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + struct dma_attrs *attrs) { struct dma_map_ops *ops = get_dma_ops(dev); void *cpu_addr; - cpu_addr = ops->alloc_coherent(dev, size, dma_handle, flag); + cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); return cpu_addr; } -static inline void dma_free_coherent(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t dma_handle) +#define dma_free_coherent(d,s,c,h) dma_free_attrs(d,s,c,h,NULL) + +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, + struct dma_attrs *attrs) { struct dma_map_ops *ops = get_dma_ops(dev); debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - ops->free_coherent(dev, size, cpu_addr, dma_handle); + ops->free(dev, size, cpu_addr, dma_handle, attrs); } static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { + debug_dma_mapping_error(dev, dma_addr); return (dma_addr == DMA_ERROR_CODE); } diff --git a/arch/sparc/include/asm/dma.h b/arch/sparc/include/asm/dma.h index b554927bbaf..3d434ef5eae 100644 --- a/arch/sparc/include/asm/dma.h +++ b/arch/sparc/include/asm/dma.h @@ -92,27 +92,31 @@ extern int isa_dma_bridge_buggy; #ifdef CONFIG_SPARC32 /* Routines for data transfer buffers. */ -BTFIXUPDEF_CALL(char *, mmu_lockarea, char *, unsigned long) -BTFIXUPDEF_CALL(void, mmu_unlockarea, char *, unsigned long) - -#define mmu_lockarea(vaddr,len) BTFIXUP_CALL(mmu_lockarea)(vaddr,len) -#define mmu_unlockarea(vaddr,len) BTFIXUP_CALL(mmu_unlockarea)(vaddr,len) - -struct page; struct device; struct scatterlist; -/* These are implementations for sbus_map_sg/sbus_unmap_sg... collapse later */ -BTFIXUPDEF_CALL(__u32, mmu_get_scsi_one, struct device *, char *, unsigned long) -BTFIXUPDEF_CALL(void, mmu_get_scsi_sgl, struct device *, struct scatterlist *, int) -BTFIXUPDEF_CALL(void, mmu_release_scsi_one, struct device *, __u32, unsigned long) -BTFIXUPDEF_CALL(void, mmu_release_scsi_sgl, struct device *, struct scatterlist *, int) - -#define mmu_get_scsi_one(dev,vaddr,len) BTFIXUP_CALL(mmu_get_scsi_one)(dev,vaddr,len) -#define mmu_get_scsi_sgl(dev,sg,sz) BTFIXUP_CALL(mmu_get_scsi_sgl)(dev,sg,sz) -#define mmu_release_scsi_one(dev,vaddr,len) BTFIXUP_CALL(mmu_release_scsi_one)(dev,vaddr,len) -#define mmu_release_scsi_sgl(dev,sg,sz) BTFIXUP_CALL(mmu_release_scsi_sgl)(dev,sg,sz) - +struct sparc32_dma_ops { + __u32 (*get_scsi_one)(struct device *, char *, unsigned long); + void (*get_scsi_sgl)(struct device *, struct scatterlist *, int); + void (*release_scsi_one)(struct device *, __u32, unsigned long); + void (*release_scsi_sgl)(struct device *, struct scatterlist *,int); +#ifdef CONFIG_SBUS + int (*map_dma_area)(struct device *, dma_addr_t *, unsigned long, unsigned long, int); + void (*unmap_dma_area)(struct device *, unsigned long, int); +#endif +}; +extern const struct sparc32_dma_ops *sparc32_dma_ops; + +#define mmu_get_scsi_one(dev,vaddr,len) \ + sparc32_dma_ops->get_scsi_one(dev, vaddr, len) +#define mmu_get_scsi_sgl(dev,sg,sz) \ + sparc32_dma_ops->get_scsi_sgl(dev, sg, sz) +#define mmu_release_scsi_one(dev,vaddr,len) \ + sparc32_dma_ops->release_scsi_one(dev, vaddr,len) +#define mmu_release_scsi_sgl(dev,sg,sz) \ + sparc32_dma_ops->release_scsi_sgl(dev, sg, sz) + +#ifdef CONFIG_SBUS /* * mmu_map/unmap are provided by iommu/iounit; Invalid to call on IIep. * @@ -123,17 +127,17 @@ BTFIXUPDEF_CALL(void, mmu_release_scsi_sgl, struct device *, struct scatterlist * Second mapping is for device visible address, or "bus" address. * The bus address is returned at '*pba'. * - * These functions seem distinct, but are hard to split. On sun4c, - * at least for now, 'a' is equal to bus address, and retured in *pba. + * These functions seem distinct, but are hard to split. * On sun4m, page attributes depend on the CPU type, so we have to * know if we are mapping RAM or I/O, so it has to be an additional argument * to a separate mapping function for CPU visible mappings. */ -BTFIXUPDEF_CALL(int, mmu_map_dma_area, struct device *, dma_addr_t *, unsigned long, unsigned long, int len) -BTFIXUPDEF_CALL(void, mmu_unmap_dma_area, struct device *, unsigned long busa, int len) +#define sbus_map_dma_area(dev,pba,va,a,len) \ + sparc32_dma_ops->map_dma_area(dev, pba, va, a, len) +#define sbus_unmap_dma_area(dev,ba,len) \ + sparc32_dma_ops->unmap_dma_area(dev, ba, len) +#endif /* CONFIG_SBUS */ -#define mmu_map_dma_area(dev,pba,va,a,len) BTFIXUP_CALL(mmu_map_dma_area)(dev,pba,va,a,len) -#define mmu_unmap_dma_area(dev,ba,len) BTFIXUP_CALL(mmu_unmap_dma_area)(dev,ba,len) #endif #endif /* !(_ASM_SPARC_DMA_H) */ diff --git a/arch/sparc/include/asm/ebus_dma.h b/arch/sparc/include/asm/ebus_dma.h index f07a5b541c9..fcfb4948147 100644 --- a/arch/sparc/include/asm/ebus_dma.h +++ b/arch/sparc/include/asm/ebus_dma.h @@ -22,14 +22,14 @@ struct ebus_dma_info { unsigned char name[64]; }; -extern int ebus_dma_register(struct ebus_dma_info *p); -extern int ebus_dma_irq_enable(struct ebus_dma_info *p, int on); -extern void ebus_dma_unregister(struct ebus_dma_info *p); -extern int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, +int ebus_dma_register(struct ebus_dma_info *p); +int ebus_dma_irq_enable(struct ebus_dma_info *p, int on); +void ebus_dma_unregister(struct ebus_dma_info *p); +int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, size_t len); -extern void ebus_dma_prepare(struct ebus_dma_info *p, int write); -extern unsigned int ebus_dma_residue(struct ebus_dma_info *p); -extern unsigned int ebus_dma_addr(struct ebus_dma_info *p); -extern void ebus_dma_enable(struct ebus_dma_info *p, int on); +void ebus_dma_prepare(struct ebus_dma_info *p, int write); +unsigned int ebus_dma_residue(struct ebus_dma_info *p); +unsigned int ebus_dma_addr(struct ebus_dma_info *p); +void ebus_dma_enable(struct ebus_dma_info *p, int on); #endif /* __ASM_SPARC_EBUS_DMA_H */ diff --git a/arch/sparc/include/asm/elf_32.h b/arch/sparc/include/asm/elf_32.h index 4269ca6ad18..a24e41fcdde 100644 --- a/arch/sparc/include/asm/elf_32.h +++ b/arch/sparc/include/asm/elf_32.h @@ -118,16 +118,9 @@ typedef struct { instruction set this cpu supports. This can NOT be done in userspace on Sparc. */ -/* Sun4c has none of the capabilities, most sun4m's have them all. - * XXX This is gross, set some global variable at boot time. -DaveM - */ -#define ELF_HWCAP ((ARCH_SUN4C) ? 0 : \ - (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | \ - HWCAP_SPARC_SWAP | \ - ((srmmu_modtype != Cypress && \ - srmmu_modtype != Cypress_vE && \ - srmmu_modtype != Cypress_vD) ? \ - HWCAP_SPARC_MULDIV : 0))) +/* Most sun4m's have them all. */ +#define ELF_HWCAP (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | \ + HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV) /* This yields a string that ld.so will use to load implementation specific libraries for optimization. This is more specific in @@ -135,6 +128,4 @@ typedef struct { #define ELF_PLATFORM (NULL) -#define SET_PERSONALITY(ex) set_personality(PER_LINUX) - #endif /* !(__ASMSPARC_ELF_H) */ diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index e67880381b8..370ca1e71ff 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -59,15 +59,42 @@ #define R_SPARC_6 45 /* Bits present in AT_HWCAP, primarily for Sparc32. */ - -#define HWCAP_SPARC_FLUSH 1 /* CPU supports flush instruction. */ -#define HWCAP_SPARC_STBAR 2 -#define HWCAP_SPARC_SWAP 4 -#define HWCAP_SPARC_MULDIV 8 -#define HWCAP_SPARC_V9 16 -#define HWCAP_SPARC_ULTRA3 32 -#define HWCAP_SPARC_BLKINIT 64 -#define HWCAP_SPARC_N2 128 +#define HWCAP_SPARC_FLUSH 0x00000001 +#define HWCAP_SPARC_STBAR 0x00000002 +#define HWCAP_SPARC_SWAP 0x00000004 +#define HWCAP_SPARC_MULDIV 0x00000008 +#define HWCAP_SPARC_V9 0x00000010 +#define HWCAP_SPARC_ULTRA3 0x00000020 +#define HWCAP_SPARC_BLKINIT 0x00000040 +#define HWCAP_SPARC_N2 0x00000080 + +/* Solaris compatible AT_HWCAP bits. */ +#define AV_SPARC_MUL32 0x00000100 /* 32x32 multiply is efficient */ +#define AV_SPARC_DIV32 0x00000200 /* 32x32 divide is efficient */ +#define AV_SPARC_FSMULD 0x00000400 /* 'fsmuld' is efficient */ +#define AV_SPARC_V8PLUS 0x00000800 /* v9 insn available to 32bit */ +#define AV_SPARC_POPC 0x00001000 /* 'popc' is efficient */ +#define AV_SPARC_VIS 0x00002000 /* VIS insns available */ +#define AV_SPARC_VIS2 0x00004000 /* VIS2 insns available */ +#define AV_SPARC_ASI_BLK_INIT 0x00008000 /* block init ASIs available */ +#define AV_SPARC_FMAF 0x00010000 /* fused multiply-add */ +#define AV_SPARC_VIS3 0x00020000 /* VIS3 insns available */ +#define AV_SPARC_HPC 0x00040000 /* HPC insns available */ +#define AV_SPARC_RANDOM 0x00080000 /* 'random' insn available */ +#define AV_SPARC_TRANS 0x00100000 /* transaction insns available */ +#define AV_SPARC_FJFMAU 0x00200000 /* unfused multiply-add */ +#define AV_SPARC_IMA 0x00400000 /* integer multiply-add */ +#define AV_SPARC_ASI_CACHE_SPARING \ + 0x00800000 /* cache sparing ASIs available */ +#define AV_SPARC_PAUSE 0x01000000 /* PAUSE available */ +#define AV_SPARC_CBCOND 0x02000000 /* CBCOND insns available */ + +/* Solaris decided to enumerate every single crypto instruction type + * in the AT_HWCAP bits. This is wasteful, since if crypto is present, + * you still need to look in the CFR register to see if the opcode is + * really available. So we simply advertise only "crypto" support. + */ +#define HWCAP_SPARC_CRYPTO 0x04000000 /* CRYPTO insns available */ #define CORE_DUMP_USE_REGSET @@ -162,31 +189,8 @@ typedef struct { #define ELF_ET_DYN_BASE 0x0000010000000000UL #define COMPAT_ELF_ET_DYN_BASE 0x0000000070000000UL - -/* This yields a mask that user programs can use to figure out what - instruction set this cpu supports. */ - -/* On Ultra, we support all of the v8 capabilities. */ -static inline unsigned int sparc64_elf_hwcap(void) -{ - unsigned int cap = (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | - HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV | - HWCAP_SPARC_V9); - - if (tlb_type == cheetah || tlb_type == cheetah_plus) - cap |= HWCAP_SPARC_ULTRA3; - else if (tlb_type == hypervisor) { - if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || - sun4v_chip_type == SUN4V_CHIP_NIAGARA2) - cap |= HWCAP_SPARC_BLKINIT; - if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2) - cap |= HWCAP_SPARC_N2; - } - - return cap; -} - -#define ELF_HWCAP sparc64_elf_hwcap(); +extern unsigned long sparc64_elf_hwcap; +#define ELF_HWCAP sparc64_elf_hwcap /* This yields a string that ld.so will use to load implementation specific libraries for optimization. This is more specific in diff --git a/arch/sparc/include/asm/emergency-restart.h b/arch/sparc/include/asm/emergency-restart.h deleted file mode 100644 index 108d8c48e42..00000000000 --- a/arch/sparc/include/asm/emergency-restart.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _ASM_EMERGENCY_RESTART_H -#define _ASM_EMERGENCY_RESTART_H - -#include <asm-generic/emergency-restart.h> - -#endif /* _ASM_EMERGENCY_RESTART_H */ diff --git a/arch/sparc/include/asm/fbio.h b/arch/sparc/include/asm/fbio.h index 0a21da87f7d..1d9afe277e9 100644 --- a/arch/sparc/include/asm/fbio.h +++ b/arch/sparc/include/asm/fbio.h @@ -1,225 +1,10 @@ #ifndef __LINUX_FBIO_H #define __LINUX_FBIO_H -#include <linux/compiler.h> -#include <linux/types.h> +#include <uapi/asm/fbio.h> -/* Constants used for fbio SunOS compatibility */ -/* (C) 1996 Miguel de Icaza */ - -/* Frame buffer types */ -#define FBTYPE_NOTYPE -1 -#define FBTYPE_SUN1BW 0 /* mono */ -#define FBTYPE_SUN1COLOR 1 -#define FBTYPE_SUN2BW 2 -#define FBTYPE_SUN2COLOR 3 -#define FBTYPE_SUN2GP 4 -#define FBTYPE_SUN5COLOR 5 -#define FBTYPE_SUN3COLOR 6 -#define FBTYPE_MEMCOLOR 7 -#define FBTYPE_SUN4COLOR 8 - -#define FBTYPE_NOTSUN1 9 -#define FBTYPE_NOTSUN2 10 -#define FBTYPE_NOTSUN3 11 - -#define FBTYPE_SUNFAST_COLOR 12 /* cg6 */ -#define FBTYPE_SUNROP_COLOR 13 -#define FBTYPE_SUNFB_VIDEO 14 -#define FBTYPE_SUNGIFB 15 -#define FBTYPE_SUNGPLAS 16 -#define FBTYPE_SUNGP3 17 -#define FBTYPE_SUNGT 18 -#define FBTYPE_SUNLEO 19 /* zx Leo card */ -#define FBTYPE_MDICOLOR 20 /* cg14 */ -#define FBTYPE_TCXCOLOR 21 /* SUNW,tcx card */ - -#define FBTYPE_LASTPLUSONE 21 /* This is not last + 1 in fact... */ - -/* Does not seem to be listed in the Sun file either */ -#define FBTYPE_CREATOR 22 -#define FBTYPE_PCI_IGA1682 23 -#define FBTYPE_P9100COLOR 24 - -#define FBTYPE_PCI_GENERIC 1000 -#define FBTYPE_PCI_MACH64 1001 - -/* fbio ioctls */ -/* Returned by FBIOGTYPE */ -struct fbtype { - int fb_type; /* fb type, see above */ - int fb_height; /* pixels */ - int fb_width; /* pixels */ - int fb_depth; - int fb_cmsize; /* color map entries */ - int fb_size; /* fb size in bytes */ -}; -#define FBIOGTYPE _IOR('F', 0, struct fbtype) - -struct fbcmap { - int index; /* first element (0 origin) */ - int count; - unsigned char __user *red; - unsigned char __user *green; - unsigned char __user *blue; -}; - -#ifdef __KERNEL__ #define FBIOPUTCMAP_SPARC _IOW('F', 3, struct fbcmap) #define FBIOGETCMAP_SPARC _IOW('F', 4, struct fbcmap) -#else -#define FBIOPUTCMAP _IOW('F', 3, struct fbcmap) -#define FBIOGETCMAP _IOW('F', 4, struct fbcmap) -#endif - -/* # of device specific values */ -#define FB_ATTR_NDEVSPECIFIC 8 -/* # of possible emulations */ -#define FB_ATTR_NEMUTYPES 4 - -struct fbsattr { - int flags; - int emu_type; /* -1 if none */ - int dev_specific[FB_ATTR_NDEVSPECIFIC]; -}; - -struct fbgattr { - int real_type; /* real frame buffer type */ - int owner; /* unknown */ - struct fbtype fbtype; /* real frame buffer fbtype */ - struct fbsattr sattr; - int emu_types[FB_ATTR_NEMUTYPES]; /* supported emulations */ -}; -#define FBIOSATTR _IOW('F', 5, struct fbgattr) /* Unsupported: */ -#define FBIOGATTR _IOR('F', 6, struct fbgattr) /* supported */ - -#define FBIOSVIDEO _IOW('F', 7, int) -#define FBIOGVIDEO _IOR('F', 8, int) - -struct fbcursor { - short set; /* what to set, choose from the list above */ - short enable; /* cursor on/off */ - struct fbcurpos pos; /* cursor position */ - struct fbcurpos hot; /* cursor hot spot */ - struct fbcmap cmap; /* color map info */ - struct fbcurpos size; /* cursor bit map size */ - char __user *image; /* cursor image bits */ - char __user *mask; /* cursor mask bits */ -}; - -/* set/get cursor attributes/shape */ -#define FBIOSCURSOR _IOW('F', 24, struct fbcursor) -#define FBIOGCURSOR _IOWR('F', 25, struct fbcursor) - -/* set/get cursor position */ -#define FBIOSCURPOS _IOW('F', 26, struct fbcurpos) -#define FBIOGCURPOS _IOW('F', 27, struct fbcurpos) - -/* get max cursor size */ -#define FBIOGCURMAX _IOR('F', 28, struct fbcurpos) - -/* wid manipulation */ -struct fb_wid_alloc { -#define FB_WID_SHARED_8 0 -#define FB_WID_SHARED_24 1 -#define FB_WID_DBL_8 2 -#define FB_WID_DBL_24 3 - __u32 wa_type; - __s32 wa_index; /* Set on return */ - __u32 wa_count; -}; -struct fb_wid_item { - __u32 wi_type; - __s32 wi_index; - __u32 wi_attrs; - __u32 wi_values[32]; -}; -struct fb_wid_list { - __u32 wl_flags; - __u32 wl_count; - struct fb_wid_item *wl_list; -}; - -#define FBIO_WID_ALLOC _IOWR('F', 30, struct fb_wid_alloc) -#define FBIO_WID_FREE _IOW('F', 31, struct fb_wid_alloc) -#define FBIO_WID_PUT _IOW('F', 32, struct fb_wid_list) -#define FBIO_WID_GET _IOWR('F', 33, struct fb_wid_list) - -/* Creator ioctls */ -#define FFB_IOCTL ('F'<<8) -#define FFB_SYS_INFO (FFB_IOCTL|80) -#define FFB_CLUTREAD (FFB_IOCTL|81) -#define FFB_CLUTPOST (FFB_IOCTL|82) -#define FFB_SETDIAGMODE (FFB_IOCTL|83) -#define FFB_GETMONITORID (FFB_IOCTL|84) -#define FFB_GETVIDEOMODE (FFB_IOCTL|85) -#define FFB_SETVIDEOMODE (FFB_IOCTL|86) -#define FFB_SETSERVER (FFB_IOCTL|87) -#define FFB_SETOVCTL (FFB_IOCTL|88) -#define FFB_GETOVCTL (FFB_IOCTL|89) -#define FFB_GETSAXNUM (FFB_IOCTL|90) -#define FFB_FBDEBUG (FFB_IOCTL|91) - -/* Cg14 ioctls */ -#define MDI_IOCTL ('M'<<8) -#define MDI_RESET (MDI_IOCTL|1) -#define MDI_GET_CFGINFO (MDI_IOCTL|2) -#define MDI_SET_PIXELMODE (MDI_IOCTL|3) -# define MDI_32_PIX 32 -# define MDI_16_PIX 16 -# define MDI_8_PIX 8 - -struct mdi_cfginfo { - int mdi_ncluts; /* Number of implemented CLUTs in this MDI */ - int mdi_type; /* FBTYPE name */ - int mdi_height; /* height */ - int mdi_width; /* width */ - int mdi_size; /* available ram */ - int mdi_mode; /* 8bpp, 16bpp or 32bpp */ - int mdi_pixfreq; /* pixel clock (from PROM) */ -}; - -/* SparcLinux specific ioctl for the MDI, should be replaced for - * the SET_XLUT/SET_CLUTn ioctls instead - */ -#define MDI_CLEAR_XLUT (MDI_IOCTL|9) - -/* leo & ffb ioctls */ -struct fb_clut_alloc { - __u32 clutid; /* Set on return */ - __u32 flag; - __u32 index; -}; - -struct fb_clut { -#define FB_CLUT_WAIT 0x00000001 /* Not yet implemented */ - __u32 flag; - __u32 clutid; - __u32 offset; - __u32 count; - char * red; - char * green; - char * blue; -}; - -struct fb_clut32 { - __u32 flag; - __u32 clutid; - __u32 offset; - __u32 count; - __u32 red; - __u32 green; - __u32 blue; -}; - -#define LEO_CLUTALLOC _IOWR('L', 53, struct fb_clut_alloc) -#define LEO_CLUTFREE _IOW('L', 54, struct fb_clut_alloc) -#define LEO_CLUTREAD _IOW('L', 55, struct fb_clut) -#define LEO_CLUTPOST _IOW('L', 56, struct fb_clut) -#define LEO_SETGAMMA _IOW('L', 68, int) /* Not yet implemented */ -#define LEO_GETGAMMA _IOR('L', 69, int) /* Not yet implemented */ - -#ifdef __KERNEL__ /* Addresses on the fd of a cgsix that are mappable */ #define CG6_FBC 0x70000000 #define CG6_TEC 0x70001000 @@ -260,47 +45,6 @@ struct fb_clut32 { #define CG14_CLUT3 0x6000 /* Color Look Up Table */ #define CG14_AUTO 0xf000 -#endif /* KERNEL */ - -/* These are exported to userland for applications to use */ -/* Mappable offsets for the cg14: control registers */ -#define MDI_DIRECT_MAP 0x10000000 -#define MDI_CTLREG_MAP 0x20000000 -#define MDI_CURSOR_MAP 0x30000000 -#define MDI_SHDW_VRT_MAP 0x40000000 - -/* Mappable offsets for the cg14: frame buffer resolutions */ -/* 32 bits */ -#define MDI_CHUNKY_XBGR_MAP 0x50000000 -#define MDI_CHUNKY_BGR_MAP 0x60000000 - -/* 16 bits */ -#define MDI_PLANAR_X16_MAP 0x70000000 -#define MDI_PLANAR_C16_MAP 0x80000000 - -/* 8 bit is done as CG3 MMAP offset */ -/* 32 bits, planar */ -#define MDI_PLANAR_X32_MAP 0x90000000 -#define MDI_PLANAR_B32_MAP 0xa0000000 -#define MDI_PLANAR_G32_MAP 0xb0000000 -#define MDI_PLANAR_R32_MAP 0xc0000000 - -/* Mappable offsets on leo */ -#define LEO_SS0_MAP 0x00000000 -#define LEO_LC_SS0_USR_MAP 0x00800000 -#define LEO_LD_SS0_MAP 0x00801000 -#define LEO_LX_CURSOR_MAP 0x00802000 -#define LEO_SS1_MAP 0x00803000 -#define LEO_LC_SS1_USR_MAP 0x01003000 -#define LEO_LD_SS1_MAP 0x01004000 -#define LEO_UNK_MAP 0x01005000 -#define LEO_LX_KRN_MAP 0x01006000 -#define LEO_LC_SS0_KRN_MAP 0x01007000 -#define LEO_LC_SS1_KRN_MAP 0x01008000 -#define LEO_LD_GBL_MAP 0x01009000 -#define LEO_UNK2_MAP 0x0100a000 - -#ifdef __KERNEL__ struct fbcmap32 { int index; /* first element (0 origin) */ int count; @@ -325,6 +69,4 @@ struct fbcursor32 { #define FBIOSCURSOR32 _IOW('F', 24, struct fbcursor32) #define FBIOGCURSOR32 _IOW('F', 25, struct fbcursor32) -#endif - #endif /* __LINUX_FBIO_H */ diff --git a/arch/sparc/include/asm/fixmap.h b/arch/sparc/include/asm/fixmap.h deleted file mode 100644 index f18fc0755ad..00000000000 --- a/arch/sparc/include/asm/fixmap.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * fixmap.h: compile-time virtual memory allocation - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1998 Ingo Molnar - * - * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 - */ - -#ifndef _ASM_FIXMAP_H -#define _ASM_FIXMAP_H - -#include <linux/kernel.h> -#include <asm/page.h> -#ifdef CONFIG_HIGHMEM -#include <linux/threads.h> -#include <asm/kmap_types.h> -#endif - -/* - * Here we define all the compile-time 'special' virtual - * addresses. The point is to have a constant address at - * compile time, but to set the physical address only - * in the boot process. We allocate these special addresses - * from the top of unused virtual memory (0xfd000000 - 1 page) backwards. - * Also this lets us do fail-safe vmalloc(), we - * can guarantee that these special addresses and - * vmalloc()-ed addresses never overlap. - * - * these 'compile-time allocated' memory buffers are - * fixed-size 4k pages. (or larger if used with an increment - * highger than 1) use fixmap_set(idx,phys) to associate - * physical memory with fixmap indices. - * - * TLB entries of such buffers will not be flushed across - * task switches. - */ - -/* - * on UP currently we will have no trace of the fixmap mechanism, - * no page table allocations, etc. This might change in the - * future, say framebuffers for the console driver(s) could be - * fix-mapped? - */ -enum fixed_addresses { - FIX_HOLE, -#ifdef CONFIG_HIGHMEM - FIX_KMAP_BEGIN, - FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, -#endif - __end_of_fixed_addresses -}; - -extern void __set_fixmap (enum fixed_addresses idx, - unsigned long phys, pgprot_t flags); - -#define set_fixmap(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL) -/* - * Some hardware wants to get fixmapped without caching. - */ -#define set_fixmap_nocache(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) -/* - * used by vmalloc.c. - * - * Leave one empty page between IO pages at 0xfd000000 and - * the start of the fixmap. - */ -#define FIXADDR_TOP (0xfcfff000UL) -#define FIXADDR_SIZE ((__end_of_fixed_addresses) << PAGE_SHIFT) -#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) - -#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) -#define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) - -extern void __this_fixmap_does_not_exist(void); - -/* - * 'index to address' translation. If anyone tries to use the idx - * directly without tranlation, we catch the bug with a NULL-deference - * kernel oops. Illegal ranges of incoming indices are caught too. - */ -static inline unsigned long fix_to_virt(const unsigned int idx) -{ - /* - * this branch gets completely eliminated after inlining, - * except when someone tries to use fixaddr indices in an - * illegal way. (such as mixing up address types or using - * out-of-range indices). - * - * If it doesn't get removed, the linker will complain - * loudly with a reasonably clear error message.. - */ - if (idx >= __end_of_fixed_addresses) - __this_fixmap_does_not_exist(); - - return __fix_to_virt(idx); -} - -static inline unsigned long virt_to_fix(const unsigned long vaddr) -{ - BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START); - return __virt_to_fix(vaddr); -} - -#endif diff --git a/arch/sparc/include/asm/floppy_32.h b/arch/sparc/include/asm/floppy_32.h index 86666f70322..071b83e52f1 100644 --- a/arch/sparc/include/asm/floppy_32.h +++ b/arch/sparc/include/asm/floppy_32.h @@ -9,13 +9,12 @@ #include <linux/of.h> #include <linux/of_device.h> -#include <asm/page.h> #include <asm/pgtable.h> -#include <asm/system.h> #include <asm/idprom.h> -#include <asm/machines.h> #include <asm/oplib.h> #include <asm/auxio.h> +#include <asm/setup.h> +#include <asm/page.h> #include <asm/irq.h> /* We don't need no stinkin' I/O port allocation crap. */ @@ -51,7 +50,6 @@ struct sun_flpy_controller { /* You'll only ever find one controller on a SparcStation anyways. */ static struct sun_flpy_controller *sun_fdc = NULL; -extern volatile unsigned char *fdc_status; struct sun_floppy_ops { unsigned char (*fd_inb)(int port); @@ -104,25 +102,13 @@ static struct sun_floppy_ops sun_fdops; /* Routines unique to each controller type on a Sun. */ static void sun_set_dor(unsigned char value, int fdc_82077) { - if (sparc_cpu_model == sun4c) { - unsigned int bits = 0; - if (value & 0x10) - bits |= AUXIO_FLPY_DSEL; - if ((value & 0x80) == 0) - bits |= AUXIO_FLPY_EJCT; - set_auxio(bits, (~bits) & (AUXIO_FLPY_DSEL|AUXIO_FLPY_EJCT)); - } - if (fdc_82077) { + if (fdc_82077) sun_fdc->dor_82077 = value; - } } static unsigned char sun_read_dir(void) { - if (sparc_cpu_model == sun4c) - return (get_auxio() & AUXIO_FLPY_DCHG) ? 0x80 : 0; - else - return sun_fdc->dir_82077; + return sun_fdc->dir_82077; } static unsigned char sun_82072_fd_inb(int port) @@ -138,7 +124,7 @@ static unsigned char sun_82072_fd_inb(int port) return sun_fdc->data_82072; case 7: /* FD_DIR */ return sun_read_dir(); - }; + } panic("sun_82072_fd_inb: How did I get here?"); } @@ -161,7 +147,7 @@ static void sun_82072_fd_outb(unsigned char value, int port) case 4: /* FD_STATUS */ sun_fdc->status_82072 = value; break; - }; + } return; } @@ -186,7 +172,7 @@ static unsigned char sun_82077_fd_inb(int port) return sun_fdc->data_82077; case 7: /* FD_DIR */ return sun_read_dir(); - }; + } panic("sun_82077_fd_inb: How did I get here?"); } @@ -212,7 +198,7 @@ static void sun_82077_fd_outb(unsigned char value, int port) case 3: /* FD_TDR */ sun_fdc->tapectl_82077 = value; break; - }; + } return; } @@ -226,13 +212,6 @@ static void sun_82077_fd_outb(unsigned char value, int port) * underruns. If non-zero, doing_pdma encodes the direction of * the transfer for debugging. 1=read 2=write */ -extern char *pdma_vaddr; -extern unsigned long pdma_size; -extern volatile int doing_pdma; - -/* This is software state */ -extern char *pdma_base; -extern unsigned long pdma_areasize; /* Common routines to all controller types on the Sparc. */ static inline void virtual_dma_init(void) @@ -243,10 +222,7 @@ static inline void virtual_dma_init(void) static inline void sun_fd_disable_dma(void) { doing_pdma = 0; - if (pdma_base) { - mmu_unlockarea(pdma_base, pdma_areasize); - pdma_base = NULL; - } + pdma_base = NULL; } static inline void sun_fd_set_dma_mode(int mode) @@ -276,66 +252,60 @@ static inline void sun_fd_set_dma_count(int length) static inline void sun_fd_enable_dma(void) { - pdma_vaddr = mmu_lockarea(pdma_vaddr, pdma_size); pdma_base = pdma_vaddr; pdma_areasize = pdma_size; } -/* Our low-level entry point in arch/sparc/kernel/entry.S */ -extern int sparc_floppy_request_irq(int irq, unsigned long flags, - irq_handler_t irq_handler); +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); static int sun_fd_request_irq(void) { static int once = 0; - int error; - if(!once) { + if (!once) { once = 1; - error = sparc_floppy_request_irq(FLOPPY_IRQ, - IRQF_DISABLED, - floppy_interrupt); - return ((error == 0) ? 0 : -1); - } else return 0; + return sparc_floppy_request_irq(FLOPPY_IRQ, floppy_interrupt); + } else { + return 0; + } } static struct linux_prom_registers fd_regs[2]; static int sun_floppy_init(void) { + struct platform_device *op; + struct device_node *dp; + struct resource r; char state[128]; - phandle tnode, fd_node; + phandle fd_node; + phandle tnode; int num_regs; - struct resource r; use_virtual_dma = 1; - FLOPPY_IRQ = 11; /* Forget it if we aren't on a machine that could possibly * ever have a floppy drive. */ - if((sparc_cpu_model != sun4c && sparc_cpu_model != sun4m) || - ((idprom->id_machtype == (SM_SUN4C | SM_4C_SLC)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC)))) { + if (sparc_cpu_model != sun4m) { /* We certainly don't have a floppy controller. */ goto no_sun_fdc; } /* Well, try to find one. */ tnode = prom_getchild(prom_root_node); fd_node = prom_searchsiblings(tnode, "obio"); - if(fd_node != 0) { + if (fd_node != 0) { tnode = prom_getchild(fd_node); fd_node = prom_searchsiblings(tnode, "SUNW,fdtwo"); } else { fd_node = prom_searchsiblings(tnode, "fd"); } - if(fd_node == 0) { + if (fd_node == 0) { goto no_sun_fdc; } /* The sun4m lets us know if the controller is actually usable. */ - if(sparc_cpu_model == sun4m && - prom_getproperty(fd_node, "status", state, sizeof(state)) != -1) { + if (prom_getproperty(fd_node, "status", state, sizeof(state)) != -1) { if(!strcmp(state, "disabled")) { goto no_sun_fdc; } @@ -346,11 +316,31 @@ static int sun_floppy_init(void) memset(&r, 0, sizeof(r)); r.flags = fd_regs[0].which_io; r.start = fd_regs[0].phys_addr; - sun_fdc = (struct sun_flpy_controller *) - of_ioremap(&r, 0, fd_regs[0].reg_size, "floppy"); + sun_fdc = of_ioremap(&r, 0, fd_regs[0].reg_size, "floppy"); + + /* Look up irq in platform_device. + * We try "SUNW,fdtwo" and "fd" + */ + op = NULL; + for_each_node_by_name(dp, "SUNW,fdtwo") { + op = of_find_device_by_node(dp); + if (op) + break; + } + if (!op) { + for_each_node_by_name(dp, "fd") { + op = of_find_device_by_node(dp); + if (op) + break; + } + } + if (!op) + goto no_sun_fdc; + + FLOPPY_IRQ = op->archdata.irqs[0]; /* Last minute sanity check... */ - if(sun_fdc->status_82072 == 0xff) { + if (sun_fdc->status_82072 == 0xff) { sun_fdc = NULL; goto no_sun_fdc; } diff --git a/arch/sparc/include/asm/floppy_64.h b/arch/sparc/include/asm/floppy_64.h index 6597ce874d7..625756406a7 100644 --- a/arch/sparc/include/asm/floppy_64.h +++ b/arch/sparc/include/asm/floppy_64.h @@ -111,7 +111,7 @@ static unsigned char sun_82077_fd_inb(unsigned long port) case 7: /* FD_DIR */ /* XXX: Is DCL on 0x80 in sun4m? */ return sbus_readb(&sun_fdc->dir_82077); - }; + } panic("sun_82072_fd_inb: How did I get here?"); } @@ -135,7 +135,7 @@ static void sun_82077_fd_outb(unsigned char value, unsigned long port) case 4: /* FD_STATUS */ sbus_writeb(value, &sun_fdc->status_82077); break; - }; + } return; } @@ -161,10 +161,7 @@ unsigned long pdma_areasize; static void sun_fd_disable_dma(void) { doing_pdma = 0; - if (pdma_base) { - mmu_unlockarea(pdma_base, pdma_areasize); - pdma_base = NULL; - } + pdma_base = NULL; } static void sun_fd_set_dma_mode(int mode) @@ -194,7 +191,6 @@ static void sun_fd_set_dma_count(int length) static void sun_fd_enable_dma(void) { - pdma_vaddr = mmu_lockarea(pdma_vaddr, pdma_size); pdma_base = pdma_vaddr; pdma_areasize = pdma_size; } @@ -258,7 +254,7 @@ static int sun_fd_request_irq(void) once = 1; error = request_irq(FLOPPY_IRQ, sparc_floppy_irq, - IRQF_DISABLED, "floppy", NULL); + 0, "floppy", NULL); return ((error == 0) ? 0 : -1); } @@ -300,7 +296,7 @@ struct sun_pci_dma_op { static struct sun_pci_dma_op sun_pci_dma_current = { -1U, 0, 0, NULL}; static struct sun_pci_dma_op sun_pci_dma_pending = { -1U, 0, 0, NULL}; -extern irqreturn_t floppy_interrupt(int irq, void *dev_id); +irqreturn_t floppy_interrupt(int irq, void *dev_id); static unsigned char sun_pci_fd_inb(unsigned long port) { diff --git a/arch/sparc/include/asm/ftrace.h b/arch/sparc/include/asm/ftrace.h index b0f18e9893d..9ec94ad116f 100644 --- a/arch/sparc/include/asm/ftrace.h +++ b/arch/sparc/include/asm/ftrace.h @@ -6,7 +6,7 @@ #define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ #ifndef __ASSEMBLY__ -extern void _mcount(void); +void _mcount(void); #endif #endif @@ -22,4 +22,8 @@ struct dyn_arch_ftrace { }; #endif /* CONFIG_DYNAMIC_FTRACE */ +unsigned long prepare_ftrace_return(unsigned long parent, + unsigned long self_addr, + unsigned long frame_pointer); + #endif /* _ASM_SPARC64_FTRACE */ diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h index 47f95839dc6..4e899b0dabf 100644 --- a/arch/sparc/include/asm/futex_64.h +++ b/arch/sparc/include/asm/futex_64.h @@ -4,7 +4,6 @@ #include <linux/futex.h> #include <linux/uaccess.h> #include <asm/errno.h> -#include <asm/system.h> #define __futex_cas_op(insn, ret, oldval, uaddr, oparg) \ __asm__ __volatile__( \ @@ -30,7 +29,7 @@ : "r" (uaddr), "r" (oparg), "i" (-EFAULT) \ : "memory") -static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) +static inline int futex_atomic_op_inuser(int encoded_op, u32 __user *uaddr) { int op = (encoded_op >> 28) & 7; int cmp = (encoded_op >> 24) & 15; @@ -38,7 +37,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) int cmparg = (encoded_op << 20) >> 20; int oldval = 0, ret, tem; - if (unlikely(!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))) + if (unlikely(!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))) return -EFAULT; if (unlikely((((unsigned long) uaddr) & 0x3UL))) return -EINVAL; @@ -85,26 +84,30 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr) } static inline int -futex_atomic_cmpxchg_inatomic(int __user *uaddr, int oldval, int newval) +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) { + int ret = 0; + __asm__ __volatile__( - "\n1: casa [%3] %%asi, %2, %0\n" + "\n1: casa [%4] %%asi, %3, %1\n" "2:\n" " .section .fixup,#alloc,#execinstr\n" " .align 4\n" "3: sethi %%hi(2b), %0\n" " jmpl %0 + %%lo(2b), %%g0\n" - " mov %4, %0\n" + " mov %5, %0\n" " .previous\n" " .section __ex_table,\"a\"\n" " .align 4\n" " .word 1b, 3b\n" " .previous\n" - : "=r" (newval) - : "0" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT) + : "+r" (ret), "=r" (newval) + : "1" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT) : "memory"); - return newval; + *uval = newval; + return ret; } #endif /* !(_SPARC64_FUTEX_H) */ diff --git a/arch/sparc/include/asm/gpio.h b/arch/sparc/include/asm/gpio.h index a0e3ac0af59..b3799d88ffc 100644 --- a/arch/sparc/include/asm/gpio.h +++ b/arch/sparc/include/asm/gpio.h @@ -1,36 +1,4 @@ -#ifndef __ASM_SPARC_GPIO_H -#define __ASM_SPARC_GPIO_H - -#include <linux/errno.h> -#include <asm-generic/gpio.h> - -#ifdef CONFIG_GPIOLIB - -static inline int gpio_get_value(unsigned int gpio) -{ - return __gpio_get_value(gpio); -} - -static inline void gpio_set_value(unsigned int gpio, int value) -{ - __gpio_set_value(gpio, value); -} - -static inline int gpio_cansleep(unsigned int gpio) -{ - return __gpio_cansleep(gpio); -} - -static inline int gpio_to_irq(unsigned int gpio) -{ - return -ENOSYS; -} - -static inline int irq_to_gpio(unsigned int irq) -{ - return -EINVAL; -} - -#endif /* CONFIG_GPIOLIB */ - -#endif /* __ASM_SPARC_GPIO_H */ +#ifndef __LINUX_GPIO_H +#warning Include linux/gpio.h instead of asm/gpio.h +#include <linux/gpio.h> +#endif diff --git a/arch/sparc/include/asm/hardirq_32.h b/arch/sparc/include/asm/hardirq_32.h index 162007643cd..ee93923b7f8 100644 --- a/arch/sparc/include/asm/hardirq_32.h +++ b/arch/sparc/include/asm/hardirq_32.h @@ -7,7 +7,6 @@ #ifndef __SPARC_HARDIRQ_H #define __SPARC_HARDIRQ_H -#define HARDIRQ_BITS 8 #include <asm-generic/hardirq.h> #endif /* __SPARC_HARDIRQ_H */ diff --git a/arch/sparc/include/asm/hardirq_64.h b/arch/sparc/include/asm/hardirq_64.h index 7c29fd1a87a..f478ff1ddd0 100644 --- a/arch/sparc/include/asm/hardirq_64.h +++ b/arch/sparc/include/asm/hardirq_64.h @@ -14,6 +14,4 @@ void ack_bad_irq(unsigned int irq); -#define HARDIRQ_BITS 8 - #endif /* !(__SPARC64_HARDIRQ_H) */ diff --git a/arch/sparc/include/asm/head_32.h b/arch/sparc/include/asm/head_32.h index 7c35491a8b5..5f1dbe315bc 100644 --- a/arch/sparc/include/asm/head_32.h +++ b/arch/sparc/include/asm/head_32.h @@ -2,15 +2,8 @@ #define __SPARC_HEAD_H #define KERNBASE 0xf0000000 /* First address the kernel will eventually be */ -#define LOAD_ADDR 0x4000 /* prom jumps to us here unless this is elf /boot */ -#define SUN4C_SEGSZ (1 << 18) -#define SRMMU_L1_KBASE_OFFSET ((KERNBASE>>24)<<2) /* Used in boot remapping. */ -#define INTS_ENAB 0x01 /* entry.S uses this. */ - -#define SUN4_PROM_VECTOR 0xFFE81000 /* SUN4 PROM needs to be hardwired */ #define WRITE_PAUSE nop; nop; nop; /* Have to do this after %wim/%psr chg */ -#define NOP_INSN 0x01000000 /* Used to patch sparc_save_state */ /* Here are some trap goodies */ @@ -18,9 +11,7 @@ #define TRAP_ENTRY(type, label) \ rd %psr, %l0; b label; rd %wim, %l3; nop; -/* Data/text faults. Defaults to sun4c version at boot time. */ -#define SPARC_TFAULT rd %psr, %l0; rd %wim, %l3; b sun4c_fault; mov 1, %l7; -#define SPARC_DFAULT rd %psr, %l0; rd %wim, %l3; b sun4c_fault; mov 0, %l7; +/* Data/text faults */ #define SRMMU_TFAULT rd %psr, %l0; rd %wim, %l3; b srmmu_fault; mov 1, %l7; #define SRMMU_DFAULT rd %psr, %l0; rd %wim, %l3; b srmmu_fault; mov 0, %l7; @@ -64,15 +55,15 @@ /* The Get Condition Codes software trap for userland. */ #define GETCC_TRAP \ - b getcc_trap_handler; mov %psr, %l0; nop; nop; + b getcc_trap_handler; rd %psr, %l0; nop; nop; /* The Set Condition Codes software trap for userland. */ #define SETCC_TRAP \ - b setcc_trap_handler; mov %psr, %l0; nop; nop; + b setcc_trap_handler; rd %psr, %l0; nop; nop; /* The Get PSR software trap for userland. */ #define GETPSR_TRAP \ - mov %psr, %i0; jmp %l2; rett %l2 + 4; nop; + rd %psr, %i0; jmp %l2; rett %l2 + 4; nop; /* This is for hard interrupts from level 1-14, 15 is non-maskable (nmi) and * gets handled with another macro. @@ -80,16 +71,6 @@ #define TRAP_ENTRY_INTERRUPT(int_level) \ mov int_level, %l7; rd %psr, %l0; b real_irq_entry; rd %wim, %l3; -/* NMI's (Non Maskable Interrupts) are special, you can't keep them - * from coming in, and basically if you get one, the shows over. ;( - * On the sun4c they are usually asynchronous memory errors, on the - * the sun4m they could be either due to mem errors or a software - * initiated interrupt from the prom/kern on an SMP box saying "I - * command you to do CPU tricks, read your mailbox for more info." - */ -#define NMI_TRAP \ - rd %wim, %l3; b linux_trap_nmi_sun4c; mov %psr, %l0; nop; - /* Window overflows/underflows are special and we need to try to be as * efficient as possible here.... */ diff --git a/arch/sparc/include/asm/hibernate.h b/arch/sparc/include/asm/hibernate.h new file mode 100644 index 00000000000..2ec34f84224 --- /dev/null +++ b/arch/sparc/include/asm/hibernate.h @@ -0,0 +1,23 @@ +/* + * hibernate.h: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#ifndef ___SPARC_HIBERNATE_H +#define ___SPARC_HIBERNATE_H + +struct saved_context { + unsigned long fp; + unsigned long cwp; + unsigned long wstate; + + unsigned long tick; + unsigned long pstate; + + unsigned long g4; + unsigned long g5; + unsigned long g6; +}; + +#endif diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h index 3d7afbb7f4b..92ded294a4e 100644 --- a/arch/sparc/include/asm/highmem.h +++ b/arch/sparc/include/asm/highmem.h @@ -21,7 +21,6 @@ #ifdef __KERNEL__ #include <linux/interrupt.h> -#include <asm/fixmap.h> #include <asm/vaddrs.h> #include <asm/kmap_types.h> #include <asm/pgtable.h> @@ -29,11 +28,10 @@ /* declarations for highmem.c */ extern unsigned long highstart_pfn, highend_pfn; -extern pte_t *kmap_pte; extern pgprot_t kmap_prot; extern pte_t *pkmap_page_table; -extern void kmap_init(void) __init; +void kmap_init(void) __init; /* * Right now we initialize only a single pte table. It can be extended @@ -51,8 +49,8 @@ extern void kmap_init(void) __init; #define PKMAP_END (PKMAP_ADDR(LAST_PKMAP)) -extern void *kmap_high(struct page *page); -extern void kunmap_high(struct page *page); +void *kmap_high(struct page *page); +void kunmap_high(struct page *page); static inline void *kmap(struct page *page) { @@ -70,9 +68,8 @@ static inline void kunmap(struct page *page) kunmap_high(page); } -extern void *__kmap_atomic(struct page *page); -extern void __kunmap_atomic(void *kvaddr); -extern struct page *kmap_atomic_to_page(void *vaddr); +void *kmap_atomic(struct page *page); +void __kunmap_atomic(void *kvaddr); #define flush_cache_kmaps() flush_cache_all() diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h index 177061064ee..e4cab465b81 100644 --- a/arch/sparc/include/asm/hugetlb.h +++ b/arch/sparc/include/asm/hugetlb.h @@ -2,6 +2,7 @@ #define _ASM_SPARC64_HUGETLB_H #include <asm/page.h> +#include <asm-generic/hugetlb.h> void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, @@ -10,7 +11,9 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep); -void hugetlb_prefault_arch_hook(struct mm_struct *mm); +static inline void hugetlb_prefault_arch_hook(struct mm_struct *mm) +{ +} static inline int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, @@ -58,14 +61,20 @@ static inline pte_t huge_pte_wrprotect(pte_t pte) static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { - ptep_set_wrprotect(mm, addr, ptep); + pte_t old_pte = *ptep; + set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); } static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, pte_t pte, int dirty) { - return ptep_set_access_flags(vma, addr, ptep, pte, dirty); + int changed = !pte_same(*ptep, pte); + if (changed) { + set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + flush_tlb_page(vma, addr); + } + return changed; } static inline pte_t huge_ptep_get(pte_t *ptep) @@ -82,4 +91,8 @@ static inline void arch_release_hugepage(struct page *page) { } +static inline void arch_clear_hugepage_flags(struct page *page) +{ +} + #endif /* _ASM_SPARC64_HUGETLB_H */ diff --git a/arch/sparc/include/asm/hvtramp.h b/arch/sparc/include/asm/hvtramp.h index b2b9b947b3a..04b56f862bb 100644 --- a/arch/sparc/include/asm/hvtramp.h +++ b/arch/sparc/include/asm/hvtramp.h @@ -19,7 +19,7 @@ struct hvtramp_descr { struct hvtramp_mapping maps[1]; }; -extern void hv_cpu_startup(unsigned long hvdescr_pa); +void hv_cpu_startup(unsigned long hvdescr_pa); #endif diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index bafe5a631b6..94b39caea3e 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -98,7 +98,7 @@ #define HV_FAST_MACH_EXIT 0x00 #ifndef __ASSEMBLY__ -extern void sun4v_mach_exit(unsigned long exit_code); +void sun4v_mach_exit(unsigned long exit_code); #endif /* Domain services. */ @@ -127,9 +127,9 @@ extern void sun4v_mach_exit(unsigned long exit_code); #define HV_FAST_MACH_DESC 0x01 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_desc(unsigned long buffer_pa, - unsigned long buf_len, - unsigned long *real_buf_len); +unsigned long sun4v_mach_desc(unsigned long buffer_pa, + unsigned long buf_len, + unsigned long *real_buf_len); #endif /* mach_sir() @@ -148,7 +148,7 @@ extern unsigned long sun4v_mach_desc(unsigned long buffer_pa, #define HV_FAST_MACH_SIR 0x02 #ifndef __ASSEMBLY__ -extern void sun4v_mach_sir(void); +void sun4v_mach_sir(void); #endif /* mach_set_watchdog() @@ -204,8 +204,8 @@ extern void sun4v_mach_sir(void); #define HV_FAST_MACH_SET_WATCHDOG 0x05 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_set_watchdog(unsigned long timeout, - unsigned long *orig_timeout); +unsigned long sun4v_mach_set_watchdog(unsigned long timeout, + unsigned long *orig_timeout); #endif /* CPU services. @@ -250,10 +250,10 @@ extern unsigned long sun4v_mach_set_watchdog(unsigned long timeout, #define HV_FAST_CPU_START 0x10 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_start(unsigned long cpuid, - unsigned long pc, - unsigned long rtba, - unsigned long arg0); +unsigned long sun4v_cpu_start(unsigned long cpuid, + unsigned long pc, + unsigned long rtba, + unsigned long arg0); #endif /* cpu_stop() @@ -278,7 +278,7 @@ extern unsigned long sun4v_cpu_start(unsigned long cpuid, #define HV_FAST_CPU_STOP 0x11 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_stop(unsigned long cpuid); +unsigned long sun4v_cpu_stop(unsigned long cpuid); #endif /* cpu_yield() @@ -295,7 +295,7 @@ extern unsigned long sun4v_cpu_stop(unsigned long cpuid); #define HV_FAST_CPU_YIELD 0x12 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_yield(void); +unsigned long sun4v_cpu_yield(void); #endif /* cpu_qconf() @@ -341,9 +341,9 @@ extern unsigned long sun4v_cpu_yield(void); #define HV_CPU_QUEUE_NONRES_ERROR 0x3f #ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_qconf(unsigned long type, - unsigned long queue_paddr, - unsigned long num_queue_entries); +unsigned long sun4v_cpu_qconf(unsigned long type, + unsigned long queue_paddr, + unsigned long num_queue_entries); #endif /* cpu_qinfo() @@ -394,7 +394,9 @@ extern unsigned long sun4v_cpu_qconf(unsigned long type, #define HV_FAST_CPU_MONDO_SEND 0x42 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, unsigned long cpu_list_pa, unsigned long mondo_block_pa); +unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, + unsigned long cpu_list_pa, + unsigned long mondo_block_pa); #endif /* cpu_myid() @@ -425,7 +427,7 @@ extern unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, unsigned long #define HV_CPU_STATE_ERROR 0x03 #ifndef __ASSEMBLY__ -extern long sun4v_cpu_state(unsigned long cpuid); +long sun4v_cpu_state(unsigned long cpuid); #endif /* cpu_set_rtba() @@ -625,8 +627,8 @@ struct hv_fault_status { #define HV_FAST_MMU_TSB_CTX0 0x20 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, - unsigned long tsb_desc_ra); +unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, + unsigned long tsb_desc_ra); #endif /* mmu_tsb_ctxnon0() @@ -654,7 +656,7 @@ extern unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, * ARG3: mmu context * ARG4: flags (HV_MMU_{IMMU,DMMU}) * RET0: status - * ERRORS: EINVAL Invalid virutal address, context, or + * ERRORS: EINVAL Invalid virtual address, context, or * flags value * ENOTSUPPORTED ARG0 or ARG1 is non-zero * @@ -710,7 +712,7 @@ extern unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, #define HV_FAST_MMU_DEMAP_ALL 0x24 #ifndef __ASSEMBLY__ -extern void sun4v_mmu_demap_all(void); +void sun4v_mmu_demap_all(void); #endif /* mmu_map_perm_addr() @@ -721,7 +723,7 @@ extern void sun4v_mmu_demap_all(void); * ARG2: TTE * ARG3: flags (HV_MMU_{IMMU,DMMU}) * RET0: status - * ERRORS: EINVAL Invalid virutal address or flags value + * ERRORS: EINVAL Invalid virtual address or flags value * EBADPGSZ Invalid page size value * ENORADDR Invalid real address in TTE * ETOOMANY Too many mappings (max of 8 reached) @@ -740,10 +742,10 @@ extern void sun4v_mmu_demap_all(void); #define HV_FAST_MMU_MAP_PERM_ADDR 0x25 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, - unsigned long set_to_zero, - unsigned long tte, - unsigned long flags); +unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, + unsigned long set_to_zero, + unsigned long tte, + unsigned long flags); #endif /* mmu_fault_area_conf() @@ -800,7 +802,7 @@ extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, * ARG1: reserved, must be zero * ARG2: flags (HV_MMU_{IMMU,DMMU}) * RET0: status - * ERRORS: EINVAL Invalid virutal address or flags value + * ERRORS: EINVAL Invalid virtual address or flags value * ENOMAP Specified mapping was not found * * Demaps any permanent page mapping (established via @@ -945,7 +947,7 @@ extern unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, #define HV_FAST_TOD_GET 0x50 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_tod_get(unsigned long *time); +unsigned long sun4v_tod_get(unsigned long *time); #endif /* tod_set() @@ -962,7 +964,7 @@ extern unsigned long sun4v_tod_get(unsigned long *time); #define HV_FAST_TOD_SET 0x51 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_tod_set(unsigned long time); +unsigned long sun4v_tod_set(unsigned long time); #endif /* Console services */ @@ -1038,14 +1040,14 @@ extern unsigned long sun4v_tod_set(unsigned long time); #define HV_FAST_CONS_WRITE 0x63 #ifndef __ASSEMBLY__ -extern long sun4v_con_getchar(long *status); -extern long sun4v_con_putchar(long c); -extern long sun4v_con_read(unsigned long buffer, - unsigned long size, - unsigned long *bytes_read); -extern unsigned long sun4v_con_write(unsigned long buffer, - unsigned long size, - unsigned long *bytes_written); +long sun4v_con_getchar(long *status); +long sun4v_con_putchar(long c); +long sun4v_con_read(unsigned long buffer, + unsigned long size, + unsigned long *bytes_read); +unsigned long sun4v_con_write(unsigned long buffer, + unsigned long size, + unsigned long *bytes_written); #endif /* mach_set_soft_state() @@ -1080,8 +1082,8 @@ extern unsigned long sun4v_con_write(unsigned long buffer, #define HV_SOFT_STATE_TRANSITION 0x02 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, - unsigned long msg_string_ra); +unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, + unsigned long msg_string_ra); #endif /* mach_get_soft_state() @@ -1159,20 +1161,20 @@ extern unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, #define HV_FAST_SVC_CLRSTATUS 0x84 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_svc_send(unsigned long svc_id, - unsigned long buffer, - unsigned long buffer_size, - unsigned long *sent_bytes); -extern unsigned long sun4v_svc_recv(unsigned long svc_id, - unsigned long buffer, - unsigned long buffer_size, - unsigned long *recv_bytes); -extern unsigned long sun4v_svc_getstatus(unsigned long svc_id, - unsigned long *status_bits); -extern unsigned long sun4v_svc_setstatus(unsigned long svc_id, - unsigned long status_bits); -extern unsigned long sun4v_svc_clrstatus(unsigned long svc_id, - unsigned long status_bits); +unsigned long sun4v_svc_send(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *sent_bytes); +unsigned long sun4v_svc_recv(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *recv_bytes); +unsigned long sun4v_svc_getstatus(unsigned long svc_id, + unsigned long *status_bits); +unsigned long sun4v_svc_setstatus(unsigned long svc_id, + unsigned long status_bits); +unsigned long sun4v_svc_clrstatus(unsigned long svc_id, + unsigned long status_bits); #endif /* Trap trace services. @@ -1205,7 +1207,7 @@ struct hv_trap_trace_control { * structure contents. Attempts to do so will result in undefined * behavior for the guest. * - * Each trap trace buffer entry is layed out as follows: + * Each trap trace buffer entry is laid out as follows: */ #ifndef __ASSEMBLY__ struct hv_trap_trace_entry { @@ -1300,7 +1302,7 @@ struct hv_trap_trace_entry { * state in RET1. Future systems may define various flags for the * enable argument (ARG0), for the moment a guest should pass * "(uint64_t) -1" to enable, and "(uint64_t) 0" to disable all - * tracing - which will ensure future compatability. + * tracing - which will ensure future compatibility. */ #define HV_FAST_TTRACE_ENABLE 0x92 @@ -1458,8 +1460,8 @@ struct hv_trap_trace_entry { #define HV_FAST_INTR_DEVINO2SYSINO 0xa0 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_devino_to_sysino(unsigned long devhandle, - unsigned long devino); +unsigned long sun4v_devino_to_sysino(unsigned long devhandle, + unsigned long devino); #endif /* intr_getenabled() @@ -1476,7 +1478,7 @@ extern unsigned long sun4v_devino_to_sysino(unsigned long devhandle, #define HV_FAST_INTR_GETENABLED 0xa1 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_getenabled(unsigned long sysino); +unsigned long sun4v_intr_getenabled(unsigned long sysino); #endif /* intr_setenabled() @@ -1492,7 +1494,8 @@ extern unsigned long sun4v_intr_getenabled(unsigned long sysino); #define HV_FAST_INTR_SETENABLED 0xa2 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_setenabled(unsigned long sysino, unsigned long intr_enabled); +unsigned long sun4v_intr_setenabled(unsigned long sysino, + unsigned long intr_enabled); #endif /* intr_getstate() @@ -1508,7 +1511,7 @@ extern unsigned long sun4v_intr_setenabled(unsigned long sysino, unsigned long i #define HV_FAST_INTR_GETSTATE 0xa3 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_getstate(unsigned long sysino); +unsigned long sun4v_intr_getstate(unsigned long sysino); #endif /* intr_setstate() @@ -1528,7 +1531,7 @@ extern unsigned long sun4v_intr_getstate(unsigned long sysino); #define HV_FAST_INTR_SETSTATE 0xa4 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long intr_state); +unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long intr_state); #endif /* intr_gettarget() @@ -1546,7 +1549,7 @@ extern unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long int #define HV_FAST_INTR_GETTARGET 0xa5 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_gettarget(unsigned long sysino); +unsigned long sun4v_intr_gettarget(unsigned long sysino); #endif /* intr_settarget() @@ -1563,7 +1566,7 @@ extern unsigned long sun4v_intr_gettarget(unsigned long sysino); #define HV_FAST_INTR_SETTARGET 0xa6 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cpuid); +unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cpuid); #endif /* vintr_get_cookie() @@ -1647,30 +1650,30 @@ extern unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cp #define HV_FAST_VINTR_SET_TARGET 0xae #ifndef __ASSEMBLY__ -extern unsigned long sun4v_vintr_get_cookie(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *cookie); -extern unsigned long sun4v_vintr_set_cookie(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long cookie); -extern unsigned long sun4v_vintr_get_valid(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *valid); -extern unsigned long sun4v_vintr_set_valid(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long valid); -extern unsigned long sun4v_vintr_get_state(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *state); -extern unsigned long sun4v_vintr_set_state(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long state); -extern unsigned long sun4v_vintr_get_target(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long *cpuid); -extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, - unsigned long dev_ino, - unsigned long cpuid); +unsigned long sun4v_vintr_get_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cookie); +unsigned long sun4v_vintr_set_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cookie); +unsigned long sun4v_vintr_get_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *valid); +unsigned long sun4v_vintr_set_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long valid); +unsigned long sun4v_vintr_get_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *state); +unsigned long sun4v_vintr_set_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long state); +unsigned long sun4v_vintr_get_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cpuid); +unsigned long sun4v_vintr_set_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cpuid); #endif /* PCI IO services. @@ -1880,7 +1883,7 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, * pci_device, at pci_config_offset from the beginning of the device's * configuration space. If there was no error, RET1 is set to zero and * RET2 is set to the data read. Insignificant bits in RET2 are not - * guarenteed to have any specific value and therefore must be ignored. + * guaranteed to have any specific value and therefore must be ignored. * * The data returned in RET2 is size based byte swapped. * @@ -1941,9 +1944,9 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, * and return the actual data read in RET2. The data returned is size based * byte swapped. * - * Non-significant bits in RET2 are not guarenteed to have any specific value + * Non-significant bits in RET2 are not guaranteed to have any specific value * and therefore must be ignored. If RET1 is returned as non-zero, the data - * value is not guarenteed to have any specific value and should be ignored. + * value is not guaranteed to have any specific value and should be ignored. * * The caller must have permission to read from the given devhandle, real * address, which must be an IO address. The argument real address must be a @@ -2456,9 +2459,9 @@ extern unsigned long sun4v_vintr_set_target(unsigned long dev_handle, * * As receive queue configuration causes a reset of the queue's head and * tail pointers there is no way for a gues to determine how many entries - * have been received between a preceeding ldc_get_rx_state() API call + * have been received between a preceding ldc_get_rx_state() API call * and the completion of the configuration operation. It should be noted - * that datagram delivery is not guarenteed via domain channels anyway, + * that datagram delivery is not guaranteed via domain channels anyway, * and therefore any higher protocol should be resilient to datagram * loss if necessary. However, to overcome this specific race potential * it is recommended, for example, that a higher level protocol be employed @@ -2627,50 +2630,50 @@ struct ldc_mtable_entry { #define HV_FAST_LDC_REVOKE 0xef #ifndef __ASSEMBLY__ -extern unsigned long sun4v_ldc_tx_qconf(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_tx_qinfo(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_tx_get_state(unsigned long channel, - unsigned long *head_off, - unsigned long *tail_off, - unsigned long *chan_state); -extern unsigned long sun4v_ldc_tx_set_qtail(unsigned long channel, - unsigned long tail_off); -extern unsigned long sun4v_ldc_rx_qconf(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_rx_qinfo(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_rx_get_state(unsigned long channel, - unsigned long *head_off, - unsigned long *tail_off, - unsigned long *chan_state); -extern unsigned long sun4v_ldc_rx_set_qhead(unsigned long channel, - unsigned long head_off); -extern unsigned long sun4v_ldc_set_map_table(unsigned long channel, - unsigned long ra, - unsigned long num_entries); -extern unsigned long sun4v_ldc_get_map_table(unsigned long channel, - unsigned long *ra, - unsigned long *num_entries); -extern unsigned long sun4v_ldc_copy(unsigned long channel, - unsigned long dir_code, - unsigned long tgt_raddr, - unsigned long lcl_raddr, - unsigned long len, - unsigned long *actual_len); -extern unsigned long sun4v_ldc_mapin(unsigned long channel, - unsigned long cookie, - unsigned long *ra, - unsigned long *perm); -extern unsigned long sun4v_ldc_unmap(unsigned long ra); -extern unsigned long sun4v_ldc_revoke(unsigned long channel, - unsigned long cookie, - unsigned long mte_cookie); +unsigned long sun4v_ldc_tx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_tx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_tx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_tx_set_qtail(unsigned long channel, + unsigned long tail_off); +unsigned long sun4v_ldc_rx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_rx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_rx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_rx_set_qhead(unsigned long channel, + unsigned long head_off); +unsigned long sun4v_ldc_set_map_table(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_get_map_table(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_copy(unsigned long channel, + unsigned long dir_code, + unsigned long tgt_raddr, + unsigned long lcl_raddr, + unsigned long len, + unsigned long *actual_len); +unsigned long sun4v_ldc_mapin(unsigned long channel, + unsigned long cookie, + unsigned long *ra, + unsigned long *perm); +unsigned long sun4v_ldc_unmap(unsigned long ra); +unsigned long sun4v_ldc_revoke(unsigned long channel, + unsigned long cookie, + unsigned long mte_cookie); #endif /* Performance counter services. */ @@ -2727,14 +2730,14 @@ extern unsigned long sun4v_ldc_revoke(unsigned long channel, #define HV_FAST_N2_SET_PERFREG 0x105 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_niagara_getperf(unsigned long reg, - unsigned long *val); -extern unsigned long sun4v_niagara_setperf(unsigned long reg, - unsigned long val); -extern unsigned long sun4v_niagara2_getperf(unsigned long reg, - unsigned long *val); -extern unsigned long sun4v_niagara2_setperf(unsigned long reg, - unsigned long val); +unsigned long sun4v_niagara_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara_setperf(unsigned long reg, + unsigned long val); +unsigned long sun4v_niagara2_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara2_setperf(unsigned long reg, + unsigned long val); #endif /* MMU statistics services. @@ -2829,8 +2832,8 @@ struct hv_mmu_statistics { #define HV_FAST_MMUSTAT_INFO 0x103 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_mmustat_conf(unsigned long ra, unsigned long *orig_ra); -extern unsigned long sun4v_mmustat_info(unsigned long *ra); +unsigned long sun4v_mmustat_conf(unsigned long ra, unsigned long *orig_ra); +unsigned long sun4v_mmustat_info(unsigned long *ra); #endif /* NCS crypto services */ @@ -2919,14 +2922,31 @@ struct hv_ncs_qtail_update_arg { #define HV_FAST_NCS_REQUEST 0x110 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_ncs_request(unsigned long request, - unsigned long arg_ra, - unsigned long arg_size); +unsigned long sun4v_ncs_request(unsigned long request, + unsigned long arg_ra, + unsigned long arg_size); #endif #define HV_FAST_FIRE_GET_PERFREG 0x120 #define HV_FAST_FIRE_SET_PERFREG 0x121 +#define HV_FAST_REBOOT_DATA_SET 0x172 + +#ifndef __ASSEMBLY__ +unsigned long sun4v_reboot_data_set(unsigned long ra, + unsigned long len); +#endif + +#define HV_FAST_VT_GET_PERFREG 0x184 +#define HV_FAST_VT_SET_PERFREG 0x185 + +#ifndef __ASSEMBLY__ +unsigned long sun4v_vt_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + /* Function numbers for HV_CORE_TRAP. */ #define HV_CORE_SET_VER 0x00 #define HV_CORE_PUTCHAR 0x01 @@ -2940,34 +2960,42 @@ extern unsigned long sun4v_ncs_request(unsigned long request, #define HV_GRP_CORE 0x0001 #define HV_GRP_INTR 0x0002 #define HV_GRP_SOFT_STATE 0x0003 +#define HV_GRP_TM 0x0080 #define HV_GRP_PCI 0x0100 #define HV_GRP_LDOM 0x0101 #define HV_GRP_SVC_CHAN 0x0102 #define HV_GRP_NCS 0x0103 #define HV_GRP_RNG 0x0104 +#define HV_GRP_PBOOT 0x0105 +#define HV_GRP_TPM 0x0107 +#define HV_GRP_SDIO 0x0108 +#define HV_GRP_SDIO_ERR 0x0109 +#define HV_GRP_REBOOT_DATA 0x0110 #define HV_GRP_NIAG_PERF 0x0200 #define HV_GRP_FIRE_PERF 0x0201 #define HV_GRP_N2_CPU 0x0202 #define HV_GRP_NIU 0x0204 #define HV_GRP_VF_CPU 0x0205 +#define HV_GRP_KT_CPU 0x0209 +#define HV_GRP_VT_CPU 0x020c #define HV_GRP_DIAG 0x0300 #ifndef __ASSEMBLY__ -extern unsigned long sun4v_get_version(unsigned long group, - unsigned long *major, - unsigned long *minor); -extern unsigned long sun4v_set_version(unsigned long group, - unsigned long major, - unsigned long minor, - unsigned long *actual_minor); - -extern int sun4v_hvapi_register(unsigned long group, unsigned long major, - unsigned long *minor); -extern void sun4v_hvapi_unregister(unsigned long group); -extern int sun4v_hvapi_get(unsigned long group, - unsigned long *major, - unsigned long *minor); -extern void sun4v_hvapi_init(void); +unsigned long sun4v_get_version(unsigned long group, + unsigned long *major, + unsigned long *minor); +unsigned long sun4v_set_version(unsigned long group, + unsigned long major, + unsigned long minor, + unsigned long *actual_minor); + +int sun4v_hvapi_register(unsigned long group, unsigned long major, + unsigned long *minor); +void sun4v_hvapi_unregister(unsigned long group); +int sun4v_hvapi_get(unsigned long group, + unsigned long *major, + unsigned long *minor); +void sun4v_hvapi_init(void); #endif #endif /* !(_SPARC64_HYPERVISOR_H) */ diff --git a/arch/sparc/include/asm/idprom.h b/arch/sparc/include/asm/idprom.h index 6976aa2439c..3793f7f91c4 100644 --- a/arch/sparc/include/asm/idprom.h +++ b/arch/sparc/include/asm/idprom.h @@ -20,6 +20,6 @@ struct idprom { }; extern struct idprom *idprom; -extern void idprom_init(void); +void idprom_init(void); #endif /* !(_SPARC_IDPROM_H) */ diff --git a/arch/sparc/include/asm/io-unit.h b/arch/sparc/include/asm/io-unit.h index 01ab2f613e9..04a9701e720 100644 --- a/arch/sparc/include/asm/io-unit.h +++ b/arch/sparc/include/asm/io-unit.h @@ -43,7 +43,7 @@ struct iounit_struct { unsigned long bmap[(IOUNIT_DMA_SIZE >> (PAGE_SHIFT + 3)) / sizeof(unsigned long)]; spinlock_t lock; - iopte_t *page_table; + iopte_t __iomem *page_table; unsigned long rotor[3]; unsigned long limit[4]; }; diff --git a/arch/sparc/include/asm/io.h b/arch/sparc/include/asm/io.h index a34b2994937..f6902cf3cbe 100644 --- a/arch/sparc/include/asm/io.h +++ b/arch/sparc/include/asm/io.h @@ -5,4 +5,17 @@ #else #include <asm/io_32.h> #endif + +/* + * Defines used for both SPARC32 and SPARC64 + */ + +/* Big endian versions of memory read/write routines */ +#define readb_be(__addr) __raw_readb(__addr) +#define readw_be(__addr) __raw_readw(__addr) +#define readl_be(__addr) __raw_readl(__addr) +#define writeb_be(__b, __addr) __raw_writeb(__b, __addr) +#define writel_be(__w, __addr) __raw_writel(__w, __addr) +#define writew_be(__l, __addr) __raw_writew(__l, __addr) + #endif diff --git a/arch/sparc/include/asm/io_32.h b/arch/sparc/include/asm/io_32.h index c2ced21c9dc..9f532902627 100644 --- a/arch/sparc/include/asm/io_32.h +++ b/arch/sparc/include/asm/io_32.h @@ -2,191 +2,94 @@ #define __SPARC_IO_H #include <linux/kernel.h> -#include <linux/types.h> #include <linux/ioport.h> /* struct resource */ -#include <asm/page.h> /* IO address mapping routines need this */ -#include <asm/system.h> - -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) - -static inline u32 flip_dword (u32 l) -{ - return ((l&0xff)<<24) | (((l>>8)&0xff)<<16) | (((l>>16)&0xff)<<8)| ((l>>24)&0xff); -} - -static inline u16 flip_word (u16 w) -{ - return ((w&0xff) << 8) | ((w>>8)&0xff); -} - -#define mmiowb() - -/* - * Memory mapped I/O to PCI - */ - -static inline u8 __raw_readb(const volatile void __iomem *addr) -{ - return *(__force volatile u8 *)addr; -} - -static inline u16 __raw_readw(const volatile void __iomem *addr) -{ - return *(__force volatile u16 *)addr; -} - -static inline u32 __raw_readl(const volatile void __iomem *addr) -{ - return *(__force volatile u32 *)addr; -} +#define readb_relaxed(__addr) readb(__addr) +#define readw_relaxed(__addr) readw(__addr) +#define readl_relaxed(__addr) readl(__addr) -static inline void __raw_writeb(u8 b, volatile void __iomem *addr) -{ - *(__force volatile u8 *)addr = b; -} +#define IO_SPACE_LIMIT 0xffffffff -static inline void __raw_writew(u16 w, volatile void __iomem *addr) -{ - *(__force volatile u16 *)addr = w; -} +#define memset_io(d,c,sz) _memset_io(d,c,sz) +#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) +#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) -static inline void __raw_writel(u32 l, volatile void __iomem *addr) -{ - *(__force volatile u32 *)addr = l; -} +#include <asm-generic/io.h> -static inline u8 __readb(const volatile void __iomem *addr) +static inline void _memset_io(volatile void __iomem *dst, + int c, __kernel_size_t n) { - return *(__force volatile u8 *)addr; -} + volatile void __iomem *d = dst; -static inline u16 __readw(const volatile void __iomem *addr) -{ - return flip_word(*(__force volatile u16 *)addr); + while (n--) { + writeb(c, d); + d++; + } } -static inline u32 __readl(const volatile void __iomem *addr) +static inline void _memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) { - return flip_dword(*(__force volatile u32 *)addr); -} + char *d = dst; -static inline void __writeb(u8 b, volatile void __iomem *addr) -{ - *(__force volatile u8 *)addr = b; + while (n--) { + char tmp = readb(src); + *d++ = tmp; + src++; + } } -static inline void __writew(u16 w, volatile void __iomem *addr) +static inline void _memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) { - *(__force volatile u16 *)addr = flip_word(w); -} + const char *s = src; + volatile void __iomem *d = dst; -static inline void __writel(u32 l, volatile void __iomem *addr) -{ - *(__force volatile u32 *)addr = flip_dword(l); + while (n--) { + char tmp = *s++; + writeb(tmp, d); + d++; + } } -#define readb(__addr) __readb(__addr) -#define readw(__addr) __readw(__addr) -#define readl(__addr) __readl(__addr) -#define readb_relaxed(__addr) readb(__addr) -#define readw_relaxed(__addr) readw(__addr) -#define readl_relaxed(__addr) readl(__addr) - -#define writeb(__b, __addr) __writeb((__b),(__addr)) -#define writew(__w, __addr) __writew((__w),(__addr)) -#define writel(__l, __addr) __writel((__l),(__addr)) - -/* - * I/O space operations - * - * Arrangement on a Sun is somewhat complicated. - * - * First of all, we want to use standard Linux drivers - * for keyboard, PC serial, etc. These drivers think - * they access I/O space and use inb/outb. - * On the other hand, EBus bridge accepts PCI *memory* - * cycles and converts them into ISA *I/O* cycles. - * Ergo, we want inb & outb to generate PCI memory cycles. - * - * If we want to issue PCI *I/O* cycles, we do this - * with a low 64K fixed window in PCIC. This window gets - * mapped somewhere into virtual kernel space and we - * can use inb/outb again. - */ -#define inb_local(__addr) __readb((void __iomem *)(unsigned long)(__addr)) -#define inb(__addr) __readb((void __iomem *)(unsigned long)(__addr)) -#define inw(__addr) __readw((void __iomem *)(unsigned long)(__addr)) -#define inl(__addr) __readl((void __iomem *)(unsigned long)(__addr)) - -#define outb_local(__b, __addr) __writeb(__b, (void __iomem *)(unsigned long)(__addr)) -#define outb(__b, __addr) __writeb(__b, (void __iomem *)(unsigned long)(__addr)) -#define outw(__w, __addr) __writew(__w, (void __iomem *)(unsigned long)(__addr)) -#define outl(__l, __addr) __writel(__l, (void __iomem *)(unsigned long)(__addr)) - -#define inb_p(__addr) inb(__addr) -#define outb_p(__b, __addr) outb(__b, __addr) -#define inw_p(__addr) inw(__addr) -#define outw_p(__w, __addr) outw(__w, __addr) -#define inl_p(__addr) inl(__addr) -#define outl_p(__l, __addr) outl(__l, __addr) - -void outsb(unsigned long addr, const void *src, unsigned long cnt); -void outsw(unsigned long addr, const void *src, unsigned long cnt); -void outsl(unsigned long addr, const void *src, unsigned long cnt); -void insb(unsigned long addr, void *dst, unsigned long count); -void insw(unsigned long addr, void *dst, unsigned long count); -void insl(unsigned long addr, void *dst, unsigned long count); - -#define IO_SPACE_LIMIT 0xffffffff - /* * SBus accessors. * * SBus has only one, memory mapped, I/O space. * We do not need to flip bytes for SBus of course. */ -static inline u8 _sbus_readb(const volatile void __iomem *addr) +static inline u8 sbus_readb(const volatile void __iomem *addr) { return *(__force volatile u8 *)addr; } -static inline u16 _sbus_readw(const volatile void __iomem *addr) +static inline u16 sbus_readw(const volatile void __iomem *addr) { return *(__force volatile u16 *)addr; } -static inline u32 _sbus_readl(const volatile void __iomem *addr) +static inline u32 sbus_readl(const volatile void __iomem *addr) { return *(__force volatile u32 *)addr; } -static inline void _sbus_writeb(u8 b, volatile void __iomem *addr) +static inline void sbus_writeb(u8 b, volatile void __iomem *addr) { *(__force volatile u8 *)addr = b; } -static inline void _sbus_writew(u16 w, volatile void __iomem *addr) +static inline void sbus_writew(u16 w, volatile void __iomem *addr) { *(__force volatile u16 *)addr = w; } -static inline void _sbus_writel(u32 l, volatile void __iomem *addr) +static inline void sbus_writel(u32 l, volatile void __iomem *addr) { *(__force volatile u32 *)addr = l; } -/* - * The only reason for #define's is to hide casts to unsigned long. - */ -#define sbus_readb(__addr) _sbus_readb(__addr) -#define sbus_readw(__addr) _sbus_readw(__addr) -#define sbus_readl(__addr) _sbus_readl(__addr) -#define sbus_writeb(__b, __addr) _sbus_writeb(__b, __addr) -#define sbus_writew(__w, __addr) _sbus_writew(__w, __addr) -#define sbus_writel(__l, __addr) _sbus_writel(__l, __addr) - -static inline void sbus_memset_io(volatile void __iomem *__dst, int c, __kernel_size_t n) +static inline void sbus_memset_io(volatile void __iomem *__dst, int c, + __kernel_size_t n) { while(n--) { sbus_writeb(c, __dst); @@ -194,22 +97,9 @@ static inline void sbus_memset_io(volatile void __iomem *__dst, int c, __kernel_ } } -static inline void -_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) -{ - volatile void __iomem *d = dst; - - while (n--) { - writeb(c, d); - d++; - } -} - -#define memset_io(d,c,sz) _memset_io(d,c,sz) - -static inline void -_sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, - __kernel_size_t n) +static inline void sbus_memcpy_fromio(void *dst, + const volatile void __iomem *src, + __kernel_size_t n) { char *d = dst; @@ -220,25 +110,9 @@ _sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, } } -#define sbus_memcpy_fromio(d, s, sz) _sbus_memcpy_fromio(d, s, sz) - -static inline void -_memcpy_fromio(void *dst, const volatile void __iomem *src, __kernel_size_t n) -{ - char *d = dst; - - while (n--) { - char tmp = readb(src); - *d++ = tmp; - src++; - } -} - -#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) - -static inline void -_sbus_memcpy_toio(volatile void __iomem *dst, const void *src, - __kernel_size_t n) +static inline void sbus_memcpy_toio(volatile void __iomem *dst, + const void *src, + __kernel_size_t n) { const char *s = src; volatile void __iomem *d = dst; @@ -250,82 +124,26 @@ _sbus_memcpy_toio(volatile void __iomem *dst, const void *src, } } -#define sbus_memcpy_toio(d, s, sz) _sbus_memcpy_toio(d, s, sz) - -static inline void -_memcpy_toio(volatile void __iomem *dst, const void *src, __kernel_size_t n) -{ - const char *s = src; - volatile void __iomem *d = dst; - - while (n--) { - char tmp = *s++; - writeb(tmp, d); - d++; - } -} - -#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) - #ifdef __KERNEL__ /* * Bus number may be embedded in the higher bits of the physical address. * This is why we have no bus number argument to ioremap(). */ -extern void __iomem *ioremap(unsigned long offset, unsigned long size); +void __iomem *ioremap(unsigned long offset, unsigned long size); #define ioremap_nocache(X,Y) ioremap((X),(Y)) #define ioremap_wc(X,Y) ioremap((X),(Y)) -extern void iounmap(volatile void __iomem *addr); - -#define ioread8(X) readb(X) -#define ioread16(X) readw(X) -#define ioread16be(X) __raw_readw(X) -#define ioread32(X) readl(X) -#define ioread32be(X) __raw_readl(X) -#define iowrite8(val,X) writeb(val,X) -#define iowrite16(val,X) writew(val,X) -#define iowrite16be(val,X) __raw_writew(val,X) -#define iowrite32(val,X) writel(val,X) -#define iowrite32be(val,X) __raw_writel(val,X) - -static inline void ioread8_rep(void __iomem *port, void *buf, unsigned long count) -{ - insb((unsigned long __force)port, buf, count); -} -static inline void ioread16_rep(void __iomem *port, void *buf, unsigned long count) -{ - insw((unsigned long __force)port, buf, count); -} - -static inline void ioread32_rep(void __iomem *port, void *buf, unsigned long count) -{ - insl((unsigned long __force)port, buf, count); -} - -static inline void iowrite8_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsb((unsigned long __force)port, buf, count); -} - -static inline void iowrite16_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsw((unsigned long __force)port, buf, count); -} - -static inline void iowrite32_rep(void __iomem *port, const void *buf, unsigned long count) -{ - outsl((unsigned long __force)port, buf, count); -} +void iounmap(volatile void __iomem *addr); /* Create a virtual mapping cookie for an IO port range */ -extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -extern void ioport_unmap(void __iomem *); +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ struct pci_dev; -extern void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long max); -extern void pci_iounmap(struct pci_dev *dev, void __iomem *); +void pci_iounmap(struct pci_dev *dev, void __iomem *); + + /* * At the moment, we do not use CMOS_READ anywhere outside of rtc.c, @@ -344,21 +162,11 @@ static inline int sbus_can_burst64(void) return 0; /* actually, sparc_cpu_model==sun4d */ } struct device; -extern void sbus_set_sbus64(struct device *, int); +void sbus_set_sbus64(struct device *, int); #endif #define __ARCH_HAS_NO_PAGE_ZERO_MAPPED 1 -/* - * Convert a physical pointer to a virtual kernel pointer for /dev/mem - * access - */ -#define xlate_dev_mem_ptr(p) __va(p) - -/* - * Convert a virtual cached pointer to an uncached pointer - */ -#define xlate_dev_kmem_ptr(p) p #endif /* !(__SPARC_IO_H) */ diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h index 9c8965415f0..05381c3a422 100644 --- a/arch/sparc/include/asm/io_64.h +++ b/arch/sparc/include/asm/io_64.h @@ -6,8 +6,8 @@ #include <linux/types.h> #include <asm/page.h> /* IO address mapping routines need this */ -#include <asm/system.h> #include <asm/asi.h> +#include <asm-generic/pci_iomap.h> /* PC crapola... */ #define __SLOW_DOWN_IO do { } while (0) @@ -15,7 +15,6 @@ /* BIO layer definitions. */ extern unsigned long kern_base, kern_size; -#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) static inline u8 _inb(unsigned long addr) { @@ -91,12 +90,12 @@ static inline void _outl(u32 l, unsigned long addr) #define inl_p(__addr) inl(__addr) #define outl_p(__l, __addr) outl(__l, __addr) -extern void outsb(unsigned long, const void *, unsigned long); -extern void outsw(unsigned long, const void *, unsigned long); -extern void outsl(unsigned long, const void *, unsigned long); -extern void insb(unsigned long, void *, unsigned long); -extern void insw(unsigned long, void *, unsigned long); -extern void insl(unsigned long, void *, unsigned long); +void outsb(unsigned long, const void *, unsigned long); +void outsw(unsigned long, const void *, unsigned long); +void outsl(unsigned long, const void *, unsigned long); +void insb(unsigned long, void *, unsigned long); +void insw(unsigned long, void *, unsigned long); +void insl(unsigned long, void *, unsigned long); static inline void ioread8_rep(void __iomem *port, void *buf, unsigned long count) { @@ -509,13 +508,12 @@ static inline void iounmap(volatile void __iomem *addr) #define iowrite32be(val,X) __raw_writel(val,X) /* Create a virtual mapping cookie for an IO port range */ -extern void __iomem *ioport_map(unsigned long port, unsigned int nr); -extern void ioport_unmap(void __iomem *); +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ struct pci_dev; -extern void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long max); -extern void pci_iounmap(struct pci_dev *dev, void __iomem *); +void pci_iounmap(struct pci_dev *dev, void __iomem *); static inline int sbus_can_dma_64bit(void) { @@ -526,7 +524,7 @@ static inline int sbus_can_burst64(void) return 1; } struct device; -extern void sbus_set_sbus64(struct device *, int); +void sbus_set_sbus64(struct device *, int); /* * Convert a physical pointer to a virtual kernel pointer for /dev/mem diff --git a/arch/sparc/include/asm/ioctls.h b/arch/sparc/include/asm/ioctls.h index ed3807b96bb..77413b7e3a1 100644 --- a/arch/sparc/include/asm/ioctls.h +++ b/arch/sparc/include/asm/ioctls.h @@ -1,122 +1,8 @@ #ifndef _ASM_SPARC_IOCTLS_H #define _ASM_SPARC_IOCTLS_H -#include <asm/ioctl.h> +#include <uapi/asm/ioctls.h> -/* Big T */ -#define TCGETA _IOR('T', 1, struct termio) -#define TCSETA _IOW('T', 2, struct termio) -#define TCSETAW _IOW('T', 3, struct termio) -#define TCSETAF _IOW('T', 4, struct termio) -#define TCSBRK _IO('T', 5) -#define TCXONC _IO('T', 6) -#define TCFLSH _IO('T', 7) -#define TCGETS _IOR('T', 8, struct termios) -#define TCSETS _IOW('T', 9, struct termios) -#define TCSETSW _IOW('T', 10, struct termios) -#define TCSETSF _IOW('T', 11, struct termios) -#define TCGETS2 _IOR('T', 12, struct termios2) -#define TCSETS2 _IOW('T', 13, struct termios2) -#define TCSETSW2 _IOW('T', 14, struct termios2) -#define TCSETSF2 _IOW('T', 15, struct termios2) -#define TIOCGDEV _IOR('T',0x32, unsigned int) /* Get primary device node of /dev/console */ - -/* Note that all the ioctls that are not available in Linux have a - * double underscore on the front to: a) avoid some programs to - * think we support some ioctls under Linux (autoconfiguration stuff) - */ -/* Little t */ -#define TIOCGETD _IOR('t', 0, int) -#define TIOCSETD _IOW('t', 1, int) -#define __TIOCHPCL _IO('t', 2) /* SunOS Specific */ -#define __TIOCMODG _IOR('t', 3, int) /* SunOS Specific */ -#define __TIOCMODS _IOW('t', 4, int) /* SunOS Specific */ -#define __TIOCGETP _IOR('t', 8, struct sgttyb) /* SunOS Specific */ -#define __TIOCSETP _IOW('t', 9, struct sgttyb) /* SunOS Specific */ -#define __TIOCSETN _IOW('t', 10, struct sgttyb) /* SunOS Specific */ -#define TIOCEXCL _IO('t', 13) -#define TIOCNXCL _IO('t', 14) -#define __TIOCFLUSH _IOW('t', 16, int) /* SunOS Specific */ -#define __TIOCSETC _IOW('t', 17, struct tchars) /* SunOS Specific */ -#define __TIOCGETC _IOR('t', 18, struct tchars) /* SunOS Specific */ -#define __TIOCTCNTL _IOW('t', 32, int) /* SunOS Specific */ -#define __TIOCSIGNAL _IOW('t', 33, int) /* SunOS Specific */ -#define __TIOCSETX _IOW('t', 34, int) /* SunOS Specific */ -#define __TIOCGETX _IOR('t', 35, int) /* SunOS Specific */ -#define TIOCCONS _IO('t', 36) -#define TIOCGSOFTCAR _IOR('t', 100, int) -#define TIOCSSOFTCAR _IOW('t', 101, int) -#define __TIOCUCNTL _IOW('t', 102, int) /* SunOS Specific */ -#define TIOCSWINSZ _IOW('t', 103, struct winsize) -#define TIOCGWINSZ _IOR('t', 104, struct winsize) -#define __TIOCREMOTE _IOW('t', 105, int) /* SunOS Specific */ -#define TIOCMGET _IOR('t', 106, int) -#define TIOCMBIC _IOW('t', 107, int) -#define TIOCMBIS _IOW('t', 108, int) -#define TIOCMSET _IOW('t', 109, int) -#define TIOCSTART _IO('t', 110) -#define TIOCSTOP _IO('t', 111) -#define TIOCPKT _IOW('t', 112, int) -#define TIOCNOTTY _IO('t', 113) -#define TIOCSTI _IOW('t', 114, char) -#define TIOCOUTQ _IOR('t', 115, int) -#define __TIOCGLTC _IOR('t', 116, struct ltchars) /* SunOS Specific */ -#define __TIOCSLTC _IOW('t', 117, struct ltchars) /* SunOS Specific */ -/* 118 is the non-posix setpgrp tty ioctl */ -/* 119 is the non-posix getpgrp tty ioctl */ -#define __TIOCCDTR _IO('t', 120) /* SunOS Specific */ -#define __TIOCSDTR _IO('t', 121) /* SunOS Specific */ -#define TIOCCBRK _IO('t', 122) -#define TIOCSBRK _IO('t', 123) -#define __TIOCLGET _IOW('t', 124, int) /* SunOS Specific */ -#define __TIOCLSET _IOW('t', 125, int) /* SunOS Specific */ -#define __TIOCLBIC _IOW('t', 126, int) /* SunOS Specific */ -#define __TIOCLBIS _IOW('t', 127, int) /* SunOS Specific */ -#define __TIOCISPACE _IOR('t', 128, int) /* SunOS Specific */ -#define __TIOCISIZE _IOR('t', 129, int) /* SunOS Specific */ -#define TIOCSPGRP _IOW('t', 130, int) -#define TIOCGPGRP _IOR('t', 131, int) -#define TIOCSCTTY _IO('t', 132) -#define TIOCGSID _IOR('t', 133, int) -/* Get minor device of a pty master's FD -- Solaris equiv is ISPTM */ -#define TIOCGPTN _IOR('t', 134, unsigned int) /* Get Pty Number */ -#define TIOCSPTLCK _IOW('t', 135, int) /* Lock/unlock PTY */ -#define TIOCSIG _IOW('t', 136, int) /* Generate signal on Pty slave */ - -/* Little f */ -#define FIOCLEX _IO('f', 1) -#define FIONCLEX _IO('f', 2) -#define FIOASYNC _IOW('f', 125, int) -#define FIONBIO _IOW('f', 126, int) -#define FIONREAD _IOR('f', 127, int) -#define TIOCINQ FIONREAD -#define FIOQSIZE _IOR('f', 128, loff_t) - -/* SCARY Rutgers local SunOS kernel hackery, perhaps I will support it - * someday. This is completely bogus, I know... - */ -#define __TCGETSTAT _IO('T', 200) /* Rutgers specific */ -#define __TCSETSTAT _IO('T', 201) /* Rutgers specific */ - -/* Linux specific, no SunOS equivalent. */ -#define TIOCLINUX 0x541C -#define TIOCGSERIAL 0x541E -#define TIOCSSERIAL 0x541F -#define TCSBRKP 0x5425 -#define TIOCSERCONFIG 0x5453 -#define TIOCSERGWILD 0x5454 -#define TIOCSERSWILD 0x5455 -#define TIOCGLCKTRMIOS 0x5456 -#define TIOCSLCKTRMIOS 0x5457 -#define TIOCSERGSTRUCT 0x5458 /* For debugging only */ -#define TIOCSERGETLSR 0x5459 /* Get line status register */ -#define TIOCSERGETMULTI 0x545A /* Get multiport config */ -#define TIOCSERSETMULTI 0x545B /* Set multiport config */ -#define TIOCMIWAIT 0x545C /* Wait for change on serial input line(s) */ -#define TIOCGICOUNT 0x545D /* Read serial port inline interrupt counts */ - -/* Kernel definitions */ -#ifdef __KERNEL__ #define TIOCGETC __TIOCGETC #define TIOCGETP __TIOCGETP #define TIOCGLTC __TIOCGLTC @@ -124,16 +10,4 @@ #define TIOCSETP __TIOCSETP #define TIOCSETN __TIOCSETN #define TIOCSETC __TIOCSETC -#endif - -/* Used for packet mode */ -#define TIOCPKT_DATA 0 -#define TIOCPKT_FLUSHREAD 1 -#define TIOCPKT_FLUSHWRITE 2 -#define TIOCPKT_STOP 4 -#define TIOCPKT_START 8 -#define TIOCPKT_NOSTOP 16 -#define TIOCPKT_DOSTOP 32 -#define TIOCPKT_IOCTL 64 - #endif /* !(_ASM_SPARC_IOCTLS_H) */ diff --git a/arch/sparc/include/asm/iommu_32.h b/arch/sparc/include/asm/iommu_32.h index 70c589c05a1..f6c066b52fd 100644 --- a/arch/sparc/include/asm/iommu_32.h +++ b/arch/sparc/include/asm/iommu_32.h @@ -99,7 +99,7 @@ struct iommu_regs { #define IOPTE_WAZ 0x00000001 /* Write as zeros */ struct iommu_struct { - struct iommu_regs *regs; + struct iommu_regs __iomem *regs; iopte_t *page_table; /* For convenience */ unsigned long start; /* First managed virtual address */ @@ -108,14 +108,14 @@ struct iommu_struct { struct bit_map usemap; }; -static inline void iommu_invalidate(struct iommu_regs *regs) +static inline void iommu_invalidate(struct iommu_regs __iomem *regs) { - regs->tlbflush = 0; + sbus_writel(0, ®s->tlbflush); } -static inline void iommu_invalidate_page(struct iommu_regs *regs, unsigned long ba) +static inline void iommu_invalidate_page(struct iommu_regs __iomem *regs, unsigned long ba) { - regs->pageflush = (ba & PAGE_MASK); + sbus_writel(ba & PAGE_MASK, ®s->pageflush); } #endif /* !(_SPARC_IOMMU_H) */ diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h index caf798b5619..2b9321ab064 100644 --- a/arch/sparc/include/asm/iommu_64.h +++ b/arch/sparc/include/asm/iommu_64.h @@ -58,8 +58,8 @@ struct strbuf { volatile unsigned long __flushflag_buf[(64+(64-1)) / sizeof(long)]; }; -extern int iommu_table_init(struct iommu *iommu, int tsbsize, - u32 dma_offset, u32 dma_addr_mask, - int numa_node); +int iommu_table_init(struct iommu *iommu, int tsbsize, + u32 dma_offset, u32 dma_addr_mask, + int numa_node); #endif /* !(_SPARC64_IOMMU_H) */ diff --git a/arch/sparc/include/asm/irq_32.h b/arch/sparc/include/asm/irq_32.h index cbf4801deaa..eecd3d8442c 100644 --- a/arch/sparc/include/asm/irq_32.h +++ b/arch/sparc/include/asm/irq_32.h @@ -6,11 +6,19 @@ #ifndef _SPARC_IRQ_H #define _SPARC_IRQ_H -#define NR_IRQS 16 +/* Allocated number of logical irq numbers. + * sun4d boxes (ss2000e) should be OK with ~32. + * Be on the safe side and make room for 64 + */ +#define NR_IRQS 64 #include <linux/interrupt.h> #define irq_canonicalize(irq) (irq) -extern void __init init_IRQ(void); +void __init init_IRQ(void); +void __init sun4d_init_sbi_irq(void); + +#define NO_IRQ 0xffffffff + #endif diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index a0b443cb3c1..91d21938130 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -33,38 +33,38 @@ /* The largest number of unique interrupt sources we support. * If this needs to ever be larger than 255, you need to change - * the type of ino_bucket->virt_irq as appropriate. + * the type of ino_bucket->irq as appropriate. * - * ino_bucket->virt_irq allocation is made during {sun4v_,}build_irq(). + * ino_bucket->irq allocation is made during {sun4v_,}build_irq(). */ #define NR_IRQS 255 -extern void irq_install_pre_handler(int virt_irq, - void (*func)(unsigned int, void *, void *), - void *arg1, void *arg2); +void irq_install_pre_handler(int irq, + void (*func)(unsigned int, void *, void *), + void *arg1, void *arg2); #define irq_canonicalize(irq) (irq) -extern unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap); -extern unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino); -extern unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino); -extern unsigned int sun4v_build_msi(u32 devhandle, unsigned int *virt_irq_p, - unsigned int msi_devino_start, - unsigned int msi_devino_end); -extern void sun4v_destroy_msi(unsigned int virt_irq); -extern unsigned int sun4u_build_msi(u32 portid, unsigned int *virt_irq_p, - unsigned int msi_devino_start, - unsigned int msi_devino_end, - unsigned long imap_base, - unsigned long iclr_base); -extern void sun4u_destroy_msi(unsigned int virt_irq); - -extern unsigned char virt_irq_alloc(unsigned int dev_handle, - unsigned int dev_ino); +unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap); +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_msi(u32 devhandle, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end); +void sun4v_destroy_msi(unsigned int irq); +unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end, + unsigned long imap_base, + unsigned long iclr_base); +void sun4u_destroy_msi(unsigned int irq); + +unsigned char irq_alloc(unsigned int dev_handle, + unsigned int dev_ino); #ifdef CONFIG_PCI_MSI -extern void virt_irq_free(unsigned int virt_irq); +void irq_free(unsigned int irq); #endif -extern void __init init_IRQ(void); -extern void fixup_irqs(void); +void __init init_IRQ(void); +void fixup_irqs(void); static inline void set_softint(unsigned long bits) { @@ -89,12 +89,13 @@ static inline unsigned long get_softint(void) return retval; } -void arch_trigger_all_cpu_backtrace(void); +void arch_trigger_all_cpu_backtrace(bool); #define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; #define __ARCH_HAS_DO_SOFTIRQ -#define ARCH_HAS_NMI_WATCHDOG + +#define NO_IRQ 0xffffffff #endif diff --git a/arch/sparc/include/asm/irq_regs.h b/arch/sparc/include/asm/irq_regs.h deleted file mode 100644 index 3dd9c0b7027..00000000000 --- a/arch/sparc/include/asm/irq_regs.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/irq_regs.h> diff --git a/arch/sparc/include/asm/irqflags_32.h b/arch/sparc/include/asm/irqflags_32.h index d4d0711de0f..71cc284f55c 100644 --- a/arch/sparc/include/asm/irqflags_32.h +++ b/arch/sparc/include/asm/irqflags_32.h @@ -13,12 +13,13 @@ #ifndef __ASSEMBLY__ #include <linux/types.h> +#include <asm/psr.h> -extern void arch_local_irq_restore(unsigned long); -extern unsigned long arch_local_irq_save(void); -extern void arch_local_irq_enable(void); +void arch_local_irq_restore(unsigned long); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable(void); -static inline unsigned long arch_local_save_flags(void) +static inline notrace unsigned long arch_local_save_flags(void) { unsigned long flags; @@ -26,17 +27,17 @@ static inline unsigned long arch_local_save_flags(void) return flags; } -static inline void arch_local_irq_disable(void) +static inline notrace void arch_local_irq_disable(void) { arch_local_irq_save(); } -static inline bool arch_irqs_disabled_flags(unsigned long flags) +static inline notrace bool arch_irqs_disabled_flags(unsigned long flags) { return (flags & PSR_PIL) != 0; } -static inline bool arch_irqs_disabled(void) +static inline notrace bool arch_irqs_disabled(void) { return arch_irqs_disabled_flags(arch_local_save_flags()); } diff --git a/arch/sparc/include/asm/irqflags_64.h b/arch/sparc/include/asm/irqflags_64.h index aab969c82c2..23cd27f6beb 100644 --- a/arch/sparc/include/asm/irqflags_64.h +++ b/arch/sparc/include/asm/irqflags_64.h @@ -14,7 +14,7 @@ #ifndef __ASSEMBLY__ -static inline unsigned long arch_local_save_flags(void) +static inline notrace unsigned long arch_local_save_flags(void) { unsigned long flags; @@ -26,7 +26,7 @@ static inline unsigned long arch_local_save_flags(void) return flags; } -static inline void arch_local_irq_restore(unsigned long flags) +static inline notrace void arch_local_irq_restore(unsigned long flags) { __asm__ __volatile__( "wrpr %0, %%pil" @@ -36,7 +36,7 @@ static inline void arch_local_irq_restore(unsigned long flags) ); } -static inline void arch_local_irq_disable(void) +static inline notrace void arch_local_irq_disable(void) { __asm__ __volatile__( "wrpr %0, %%pil" @@ -46,7 +46,7 @@ static inline void arch_local_irq_disable(void) ); } -static inline void arch_local_irq_enable(void) +static inline notrace void arch_local_irq_enable(void) { __asm__ __volatile__( "wrpr 0, %%pil" @@ -56,17 +56,17 @@ static inline void arch_local_irq_enable(void) ); } -static inline int arch_irqs_disabled_flags(unsigned long flags) +static inline notrace int arch_irqs_disabled_flags(unsigned long flags) { return (flags > 0); } -static inline int arch_irqs_disabled(void) +static inline notrace int arch_irqs_disabled(void) { return arch_irqs_disabled_flags(arch_local_save_flags()); } -static inline unsigned long arch_local_irq_save(void) +static inline notrace unsigned long arch_local_irq_save(void) { unsigned long flags, tmp; diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h index 427d4684e0d..ec2e2e2aba7 100644 --- a/arch/sparc/include/asm/jump_label.h +++ b/arch/sparc/include/asm/jump_label.h @@ -7,17 +7,20 @@ #define JUMP_LABEL_NOP_SIZE 4 -#define JUMP_LABEL(key, label) \ - do { \ - asm goto("1:\n\t" \ - "nop\n\t" \ - "nop\n\t" \ - ".pushsection __jump_table, \"a\"\n\t"\ - ".align 4\n\t" \ - ".word 1b, %l[" #label "], %c0\n\t" \ - ".popsection \n\t" \ - : : "i" (key) : : label);\ - } while (0) +static __always_inline bool arch_static_branch(struct static_key *key) +{ + asm_volatile_goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (key) : : l_yes); + return false; +l_yes: + return true; +} #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/kdebug_64.h b/arch/sparc/include/asm/kdebug_64.h index feb3578e12c..04465de8f3b 100644 --- a/arch/sparc/include/asm/kdebug_64.h +++ b/arch/sparc/include/asm/kdebug_64.h @@ -3,7 +3,7 @@ struct pt_regs; -extern void bad_trap(struct pt_regs *, long); +void bad_trap(struct pt_regs *, long); /* Grossly misnamed. */ enum die_val { diff --git a/arch/sparc/include/asm/kgdb.h b/arch/sparc/include/asm/kgdb.h index b6ef301d05b..47366af7a58 100644 --- a/arch/sparc/include/asm/kgdb.h +++ b/arch/sparc/include/asm/kgdb.h @@ -28,9 +28,12 @@ enum regnames { #define NUMREGBYTES ((GDB_CSR + 1) * 4) #else #define NUMREGBYTES ((GDB_Y + 1) * 8) + +struct pt_regs; +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs); #endif -extern void arch_kgdb_breakpoint(void); +void arch_kgdb_breakpoint(void); #define BREAK_INSTR_SIZE 4 #define CACHE_FLUSH_IS_SAFE 1 diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h index 5879d71afda..a145d798e11 100644 --- a/arch/sparc/include/asm/kprobes.h +++ b/arch/sparc/include/asm/kprobes.h @@ -43,7 +43,9 @@ struct kprobe_ctlblk { struct prev_kprobe prev_kprobe; }; -extern int kprobe_exceptions_notify(struct notifier_block *self, - unsigned long val, void *data); -extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +int kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data); +int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, + struct pt_regs *regs); #endif /* _SPARC64_KPROBES_H */ diff --git a/arch/sparc/include/asm/ldc.h b/arch/sparc/include/asm/ldc.h index bdb524a7b81..c8c67f621f4 100644 --- a/arch/sparc/include/asm/ldc.h +++ b/arch/sparc/include/asm/ldc.h @@ -4,9 +4,9 @@ #include <asm/hypervisor.h> extern int ldom_domaining_enabled; -extern void ldom_set_var(const char *var, const char *value); -extern void ldom_reboot(const char *boot_command); -extern void ldom_power_off(void); +void ldom_set_var(const char *var, const char *value); +void ldom_reboot(const char *boot_command); +void ldom_power_off(void); /* The event handler will be evoked when link state changes * or data becomes available on the receive side. @@ -51,30 +51,30 @@ struct ldc_channel_config { struct ldc_channel; /* Allocate state for a channel. */ -extern struct ldc_channel *ldc_alloc(unsigned long id, - const struct ldc_channel_config *cfgp, - void *event_arg); +struct ldc_channel *ldc_alloc(unsigned long id, + const struct ldc_channel_config *cfgp, + void *event_arg); /* Shut down and free state for a channel. */ -extern void ldc_free(struct ldc_channel *lp); +void ldc_free(struct ldc_channel *lp); /* Register TX and RX queues of the link with the hypervisor. */ -extern int ldc_bind(struct ldc_channel *lp, const char *name); +int ldc_bind(struct ldc_channel *lp, const char *name); /* For non-RAW protocols we need to complete a handshake before * communication can proceed. ldc_connect() does that, if the * handshake completes successfully, an LDC_EVENT_UP event will * be sent up to the driver. */ -extern int ldc_connect(struct ldc_channel *lp); -extern int ldc_disconnect(struct ldc_channel *lp); +int ldc_connect(struct ldc_channel *lp); +int ldc_disconnect(struct ldc_channel *lp); -extern int ldc_state(struct ldc_channel *lp); +int ldc_state(struct ldc_channel *lp); /* Read and write operations. Only valid when the link is up. */ -extern int ldc_write(struct ldc_channel *lp, const void *buf, - unsigned int size); -extern int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size); +int ldc_write(struct ldc_channel *lp, const void *buf, + unsigned int size); +int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size); #define LDC_MAP_SHADOW 0x01 #define LDC_MAP_DIRECT 0x02 @@ -92,22 +92,22 @@ struct ldc_trans_cookie { }; struct scatterlist; -extern int ldc_map_sg(struct ldc_channel *lp, - struct scatterlist *sg, int num_sg, - struct ldc_trans_cookie *cookies, int ncookies, - unsigned int map_perm); +int ldc_map_sg(struct ldc_channel *lp, + struct scatterlist *sg, int num_sg, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); -extern int ldc_map_single(struct ldc_channel *lp, - void *buf, unsigned int len, - struct ldc_trans_cookie *cookies, int ncookies, - unsigned int map_perm); +int ldc_map_single(struct ldc_channel *lp, + void *buf, unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); -extern void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, - int ncookies); +void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, + int ncookies); -extern int ldc_copy(struct ldc_channel *lp, int copy_dir, - void *buf, unsigned int len, unsigned long offset, - struct ldc_trans_cookie *cookies, int ncookies); +int ldc_copy(struct ldc_channel *lp, int copy_dir, + void *buf, unsigned int len, unsigned long offset, + struct ldc_trans_cookie *cookies, int ncookies); static inline int ldc_get_dring_entry(struct ldc_channel *lp, void *buf, unsigned int len, @@ -127,12 +127,12 @@ static inline int ldc_put_dring_entry(struct ldc_channel *lp, return ldc_copy(lp, LDC_COPY_OUT, buf, len, offset, cookies, ncookies); } -extern void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, - struct ldc_trans_cookie *cookies, - int *ncookies, unsigned int map_perm); +void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, + struct ldc_trans_cookie *cookies, + int *ncookies, unsigned int map_perm); -extern void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, - unsigned int len, - struct ldc_trans_cookie *cookies, int ncookies); +void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, + unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies); #endif /* _SPARC64_LDC_H */ diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h index 8580d1764f9..204771cd74a 100644 --- a/arch/sparc/include/asm/leon.h +++ b/arch/sparc/include/asm/leon.h @@ -8,21 +8,6 @@ #ifndef LEON_H_INCLUDE #define LEON_H_INCLUDE -#ifdef CONFIG_SPARC_LEON - -#define ASI_LEON_NOCACHE 0x01 - -#define ASI_LEON_DCACHE_MISS 0x1 - -#define ASI_LEON_CACHEREGS 0x02 -#define ASI_LEON_IFLUSH 0x10 -#define ASI_LEON_DFLUSH 0x11 - -#define ASI_LEON_MMUFLUSH 0x18 -#define ASI_LEON_MMUREGS 0x19 -#define ASI_LEON_BYPASS 0x1c -#define ASI_LEON_FLUSH_PAGE 0x10 - /* mmu register access, ASI_LEON_MMUREGS */ #define LEON_CNR_CTRL 0x000 #define LEON_CNR_CTXP 0x100 @@ -52,57 +37,11 @@ #define LEON_DIAGF_VALID 0x2000 #define LEON_DIAGF_VALID_SHIFT 13 -/* - * Interrupt Sources - * - * The interrupt source numbers directly map to the trap type and to - * the bits used in the Interrupt Clear, Interrupt Force, Interrupt Mask, - * and the Interrupt Pending Registers. - */ -#define LEON_INTERRUPT_CORRECTABLE_MEMORY_ERROR 1 -#define LEON_INTERRUPT_UART_1_RX_TX 2 -#define LEON_INTERRUPT_UART_0_RX_TX 3 -#define LEON_INTERRUPT_EXTERNAL_0 4 -#define LEON_INTERRUPT_EXTERNAL_1 5 -#define LEON_INTERRUPT_EXTERNAL_2 6 -#define LEON_INTERRUPT_EXTERNAL_3 7 -#define LEON_INTERRUPT_TIMER1 8 -#define LEON_INTERRUPT_TIMER2 9 -#define LEON_INTERRUPT_EMPTY1 10 -#define LEON_INTERRUPT_EMPTY2 11 -#define LEON_INTERRUPT_OPEN_ETH 12 -#define LEON_INTERRUPT_EMPTY4 13 -#define LEON_INTERRUPT_EMPTY5 14 -#define LEON_INTERRUPT_EMPTY6 15 - /* irq masks */ #define LEON_HARD_INT(x) (1 << (x)) /* irq 0-15 */ #define LEON_IRQMASK_R 0x0000fffe /* bit 15- 1 of lregs.irqmask */ #define LEON_IRQPRIO_R 0xfffe0000 /* bit 31-17 of lregs.irqmask */ -/* leon uart register definitions */ -#define LEON_OFF_UDATA 0x0 -#define LEON_OFF_USTAT 0x4 -#define LEON_OFF_UCTRL 0x8 -#define LEON_OFF_USCAL 0xc - -#define LEON_UCTRL_RE 0x01 -#define LEON_UCTRL_TE 0x02 -#define LEON_UCTRL_RI 0x04 -#define LEON_UCTRL_TI 0x08 -#define LEON_UCTRL_PS 0x10 -#define LEON_UCTRL_PE 0x20 -#define LEON_UCTRL_FL 0x40 -#define LEON_UCTRL_LB 0x80 - -#define LEON_USTAT_DR 0x01 -#define LEON_USTAT_TS 0x02 -#define LEON_USTAT_TH 0x04 -#define LEON_USTAT_BR 0x08 -#define LEON_USTAT_OV 0x10 -#define LEON_USTAT_PE 0x20 -#define LEON_USTAT_FE 0x40 - #define LEON_MCFG2_SRAMDIS 0x00002000 #define LEON_MCFG2_SDRAMEN 0x00004000 #define LEON_MCFG2_SRAMBANKSZ 0x00001e00 /* [12-9] */ @@ -112,8 +51,6 @@ #define LEON_TCNT0_MASK 0x7fffff -#define LEON_USTAT_ERROR (LEON_USTAT_OV | LEON_USTAT_PE | LEON_USTAT_FE) -/* no break yet */ #define ASI_LEON3_SYSCTRL 0x02 #define ASI_LEON3_SYSCTRL_ICFG 0x08 @@ -123,15 +60,6 @@ #ifndef __ASSEMBLY__ -/* do a virtual address read without cache */ -static inline unsigned long leon_readnobuffer_reg(unsigned long paddr) -{ - unsigned long retval; - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r"(retval) : "r"(paddr), "i"(ASI_LEON_NOCACHE)); - return retval; -} - /* do a physical address bypass write, i.e. for 0x80000000 */ static inline void leon_store_reg(unsigned long paddr, unsigned long value) { @@ -148,47 +76,14 @@ static inline unsigned long leon_load_reg(unsigned long paddr) return retval; } -static inline void leon_srmmu_disabletlb(void) -{ - unsigned int retval; - __asm__ __volatile__("lda [%%g0] %2, %0\n\t" : "=r"(retval) : "r"(0), - "i"(ASI_LEON_MMUREGS)); - retval |= LEON_CNR_CTRL_TLBDIS; - __asm__ __volatile__("sta %0, [%%g0] %2\n\t" : : "r"(retval), "r"(0), - "i"(ASI_LEON_MMUREGS) : "memory"); -} - -static inline void leon_srmmu_enabletlb(void) -{ - unsigned int retval; - __asm__ __volatile__("lda [%%g0] %2, %0\n\t" : "=r"(retval) : "r"(0), - "i"(ASI_LEON_MMUREGS)); - retval = retval & ~LEON_CNR_CTRL_TLBDIS; - __asm__ __volatile__("sta %0, [%%g0] %2\n\t" : : "r"(retval), "r"(0), - "i"(ASI_LEON_MMUREGS) : "memory"); -} - /* macro access for leon_load_reg() and leon_store_reg() */ #define LEON3_BYPASS_LOAD_PA(x) (leon_load_reg((unsigned long)(x))) #define LEON3_BYPASS_STORE_PA(x, v) (leon_store_reg((unsigned long)(x), (unsigned long)(v))) -#define LEON3_BYPASS_ANDIN_PA(x, v) LEON3_BYPASS_STORE_PA(x, LEON3_BYPASS_LOAD_PA(x) & v) -#define LEON3_BYPASS_ORIN_PA(x, v) LEON3_BYPASS_STORE_PA(x, LEON3_BYPASS_LOAD_PA(x) | v) #define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x)) #define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v)) -#define LEON_REGLOAD_PA(x) leon_load_reg((unsigned long)(x)+LEON_PREGS) -#define LEON_REGSTORE_PA(x, v) leon_store_reg((unsigned long)(x)+LEON_PREGS, (unsigned long)(v)) -#define LEON_REGSTORE_OR_PA(x, v) LEON_REGSTORE_PA(x, LEON_REGLOAD_PA(x) | (unsigned long)(v)) -#define LEON_REGSTORE_AND_PA(x, v) LEON_REGSTORE_PA(x, LEON_REGLOAD_PA(x) & (unsigned long)(v)) - -/* macro access for leon_readnobuffer_reg() */ -#define LEON_BYPASSCACHE_LOAD_VA(x) leon_readnobuffer_reg((unsigned long)(x)) -extern void sparc_leon_eirq_register(int eirq); -extern void leon_init(void); -extern void leon_switch_mm(void); -extern void leon_init_IRQ(void); - -extern unsigned long last_valid_pfn; +void leon_switch_mm(void); +void leon_init_IRQ(void); static inline unsigned long sparc_leon3_get_dcachecfg(void) { @@ -213,7 +108,7 @@ static inline int sparc_leon3_snooping_enabled(void) { u32 cctrl; __asm__ __volatile__("lda [%%g0] 2, %0\n\t" : "=r"(cctrl)); - return (cctrl >> 23) & 1; + return ((cctrl >> 23) & 1) && ((cctrl >> 17) & 1); }; static inline void sparc_leon3_disable_cache(void) @@ -239,8 +134,8 @@ static inline int sparc_leon3_cpuid(void) #endif /*!__ASSEMBLY__*/ #ifdef CONFIG_SMP -# define LEON3_IRQ_RESCHEDULE 13 -# define LEON3_IRQ_TICKER (leon_percpu_timer_dev[0].irq) +# define LEON3_IRQ_IPI_DEFAULT 13 +# define LEON3_IRQ_TICKER (leon3_gptimer_irq) # define LEON3_IRQ_CROSS_CALL 15 #endif @@ -292,9 +187,6 @@ static inline int sparc_leon3_cpuid(void) #error cannot determine LEON_PAGE_SIZE_LEON #endif -#define PAGE_MIN_SHIFT (12) -#define PAGE_MIN_SIZE (1UL << PAGE_MIN_SHIFT) - #define LEON3_XCCR_SETS_MASK 0x07000000UL #define LEON3_XCCR_SSIZE_MASK 0x00f00000UL @@ -302,23 +194,16 @@ static inline int sparc_leon3_cpuid(void) #define LEON2_CFG_SSIZE_MASK 0x00007000UL #ifndef __ASSEMBLY__ -extern unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr); -extern void leon_flush_icache_all(void); -extern void leon_flush_dcache_all(void); -extern void leon_flush_cache_all(void); -extern void leon_flush_tlb_all(void); -extern int leon_flush_during_switch; -extern int leon_flush_needed(void); - struct vm_area_struct; -extern void leon_flush_icache_all(void); -extern void leon_flush_dcache_all(void); -extern void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); -extern void leon_flush_cache_all(void); -extern void leon_flush_tlb_all(void); + +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr); +void leon_flush_icache_all(void); +void leon_flush_dcache_all(void); +void leon_flush_cache_all(void); +void leon_flush_tlb_all(void); extern int leon_flush_during_switch; -extern int leon_flush_needed(void); -extern void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); +int leon_flush_needed(void); +void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); /* struct that hold LEON3 cache configuration registers */ struct leon3_cacheregs { @@ -327,64 +212,41 @@ struct leon3_cacheregs { unsigned long dccr; /* 0x0c - Data Cache Configuration Register */ }; -/* struct that hold LEON2 cache configuration register - * & configuration register - */ -struct leon2_cacheregs { - unsigned long ccr, cfg; -}; - -#ifdef __KERNEL__ - +#include <linux/irq.h> #include <linux/interrupt.h> struct device_node; -extern int sparc_leon_eirq_get(int eirq, int cpu); -extern irqreturn_t sparc_leon_eirq_isr(int dummy, void *dev_id); -extern void sparc_leon_eirq_register(int eirq); -extern void leon_clear_clock_irq(void); -extern void leon_load_profile_irq(int cpu, unsigned int limit); -extern void leon_init_timers(irq_handler_t counter_fn); -extern void leon_clear_clock_irq(void); -extern void leon_load_profile_irq(int cpu, unsigned int limit); -extern void leon_trans_init(struct device_node *dp); -extern void leon_node_init(struct device_node *dp, struct device_node ***nextp); -extern void leon_init_IRQ(void); -extern void leon_init(void); -extern unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr); -extern void init_leon(void); -extern void poke_leonsparc(void); -extern void leon3_getCacheRegs(struct leon3_cacheregs *regs); -extern int leon_flush_needed(void); -extern void leon_switch_mm(void); -extern int srmmu_swprobe_trace; +struct task_struct; +unsigned int leon_build_device_irq(unsigned int real_irq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_update_virq_handling(unsigned int virq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_init_timers(void); +void leon_trans_init(struct device_node *dp); +void leon_node_init(struct device_node *dp, struct device_node ***nextp); +void init_leon(void); +void poke_leonsparc(void); +void leon3_getCacheRegs(struct leon3_cacheregs *regs); +extern int leon3_ticker_irq; #ifdef CONFIG_SMP -extern int leon_smp_nrcpus(void); -extern void leon_clear_profile_irq(int cpu); -extern void leon_smp_done(void); -extern void leon_boot_cpus(void); -extern int leon_boot_one_cpu(int i); +int leon_smp_nrcpus(void); +void leon_clear_profile_irq(int cpu); +void leon_smp_done(void); +void leon_boot_cpus(void); +int leon_boot_one_cpu(int i, struct task_struct *); void leon_init_smp(void); -extern void cpu_probe(void); -extern void cpu_idle(void); -extern void init_IRQ(void); -extern void cpu_panic(void); -extern int __leon_processor_id(void); void leon_enable_irq_cpu(unsigned int irq_nr, unsigned int cpu); +irqreturn_t leon_percpu_timer_interrupt(int irq, void *unused); -extern unsigned int real_irq_entry[], smpleon_ticker[]; -extern unsigned int patchme_maybe_smp_msg[]; -extern unsigned long trapbase_cpu1[]; -extern unsigned long trapbase_cpu2[]; -extern unsigned long trapbase_cpu3[]; -extern unsigned int t_nmi[], linux_trap_ipi15_leon[]; -extern unsigned int linux_trap_ipi15_sun4m[]; +extern unsigned int smpleon_ipi[]; +extern unsigned int linux_trap_ipi15_leon[]; +extern int leon_ipi_irq; #endif /* CONFIG_SMP */ -#endif /* __KERNEL__ */ - #endif /* __ASSEMBLY__ */ /* macros used in leon_mm.c */ @@ -392,18 +254,4 @@ extern unsigned int linux_trap_ipi15_sun4m[]; #define _pfn_valid(pfn) ((pfn < last_valid_pfn) && (pfn >= PFN(phys_base))) #define _SRMMU_PTE_PMASK_LEON 0xffffffff -#else /* defined(CONFIG_SPARC_LEON) */ - -/* nop definitions for !LEON case */ -#define leon_init() do {} while (0) -#define leon_switch_mm() do {} while (0) -#define leon_init_IRQ() do {} while (0) -#define init_leon() do {} while (0) -#define leon_smp_done() do {} while (0) -#define leon_boot_cpus() do {} while (0) -#define leon_boot_one_cpu(i) 1 -#define leon_init_smp() do {} while (0) - -#endif /* !defined(CONFIG_SPARC_LEON) */ - #endif diff --git a/arch/sparc/include/asm/leon_amba.h b/arch/sparc/include/asm/leon_amba.h index 263c719e96f..24ec48c3ff9 100644 --- a/arch/sparc/include/asm/leon_amba.h +++ b/arch/sparc/include/asm/leon_amba.h @@ -47,6 +47,7 @@ struct amba_prom_registers { #define LEON3_GPTIMER_LD 4 #define LEON3_GPTIMER_IRQEN 8 #define LEON3_GPTIMER_SEPIRQ 8 +#define LEON3_GPTIMER_TIMERS 0x7 #define LEON23_REG_TIMER_CONTROL_EN 0x00000001 /* 1 = enable counting */ /* 0 = hold scalar and counter */ @@ -87,8 +88,6 @@ struct amba_prom_registers { #define LEON3_GPTIMER_CONFIG_NRTIMERS(c) ((c)->config & 0x7) #define LEON3_GPTIMER_CTRL_ISPENDING(r) (((r)&LEON3_GPTIMER_CTRL_PENDING) ? 1 : 0) -#ifdef CONFIG_SPARC_LEON - #ifndef __ASSEMBLY__ struct leon3_irqctrl_regs_map { @@ -180,6 +179,7 @@ struct amba_ahb_device { struct device_node; void _amba_init(struct device_node *dp, struct device_node ***nextp); +extern unsigned long amba_system_id; extern struct leon3_irqctrl_regs_map *leon3_irqctrl_regs; extern struct leon3_gptimer_regs_map *leon3_gptimer_regs; extern struct amba_apb_device leon_percpu_timer_dev[16]; @@ -254,10 +254,13 @@ extern unsigned int sparc_leon_eirq; #define GAISLER_L2C 0xffe /* internal device: leon2compat */ #define GAISLER_PLUGPLAY 0xfff /* internal device: plug & play configarea */ +/* Chip IDs */ +#define AEROFLEX_UT699 0x0699 +#define LEON4_NEXTREME1 0x0102 +#define GAISLER_GR712RC 0x0712 + #define amba_vendor(x) (((x) >> 24) & 0xff) #define amba_device(x) (((x) >> 12) & 0xfff) -#endif /* !defined(CONFIG_SPARC_LEON) */ - #endif diff --git a/arch/sparc/include/asm/leon_pci.h b/arch/sparc/include/asm/leon_pci.h new file mode 100644 index 00000000000..049d067ed8b --- /dev/null +++ b/arch/sparc/include/asm/leon_pci.h @@ -0,0 +1,22 @@ +/* + * asm/leon_pci.h + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + */ + +#ifndef _ASM_LEON_PCI_H_ +#define _ASM_LEON_PCI_H_ + +/* PCI related definitions */ +struct leon_pci_info { + struct pci_ops *ops; + struct resource io_space; + struct resource mem_space; + struct resource busn; + int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); +}; + +void leon_pci_init(struct platform_device *ofdev, + struct leon_pci_info *info); + +#endif /* _ASM_LEON_PCI_H_ */ diff --git a/arch/sparc/include/asm/linkage.h b/arch/sparc/include/asm/linkage.h deleted file mode 100644 index 291c2d01c44..00000000000 --- a/arch/sparc/include/asm/linkage.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __ASM_LINKAGE_H -#define __ASM_LINKAGE_H - -/* Nothing to see here... */ - -#endif diff --git a/arch/sparc/include/asm/local.h b/arch/sparc/include/asm/local.h deleted file mode 100644 index bc80815a435..00000000000 --- a/arch/sparc/include/asm/local.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _SPARC_LOCAL_H -#define _SPARC_LOCAL_H - -#include <asm-generic/local.h> - -#endif diff --git a/arch/sparc/include/asm/local64.h b/arch/sparc/include/asm/local64.h deleted file mode 100644 index 36c93b5cc23..00000000000 --- a/arch/sparc/include/asm/local64.h +++ /dev/null @@ -1 +0,0 @@ -#include <asm-generic/local64.h> diff --git a/arch/sparc/include/asm/machines.h b/arch/sparc/include/asm/machines.h index cd9c099567e..fd6ddb05d1b 100644 --- a/arch/sparc/include/asm/machines.h +++ b/arch/sparc/include/asm/machines.h @@ -12,11 +12,6 @@ struct Sun_Machine_Models { unsigned char id_machtype; }; -/* Current number of machines we know about that has an IDPROM - * machtype entry including one entry for the 0x80 OBP machines. - */ -#define NUM_SUN_MACHINES 16 - /* The machine type in the idprom area looks like this: * * --------------- @@ -24,36 +19,20 @@ struct Sun_Machine_Models { * --------------- * 7 4 3 0 * - * The ARCH field determines the architecture line (sun4, sun4c, etc). + * The ARCH field determines the architecture line (sun4m, etc). * The MACH field determines the machine make within that architecture. */ #define SM_ARCH_MASK 0xf0 -#define SM_SUN4 0x20 #define M_LEON 0x30 -#define SM_SUN4C 0x50 #define SM_SUN4M 0x70 #define SM_SUN4M_OBP 0x80 #define SM_TYP_MASK 0x0f -/* Sun4 machines */ -#define SM_4_260 0x01 /* Sun 4/200 series */ -#define SM_4_110 0x02 /* Sun 4/100 series */ -#define SM_4_330 0x03 /* Sun 4/300 series */ -#define SM_4_470 0x04 /* Sun 4/400 series */ /* Leon machines */ #define M_LEON3_SOC 0x02 /* Leon3 SoC */ -/* Sun4c machines Full Name - PROM NAME */ -#define SM_4C_SS1 0x01 /* Sun4c SparcStation 1 - Sun 4/60 */ -#define SM_4C_IPC 0x02 /* Sun4c SparcStation IPC - Sun 4/40 */ -#define SM_4C_SS1PLUS 0x03 /* Sun4c SparcStation 1+ - Sun 4/65 */ -#define SM_4C_SLC 0x04 /* Sun4c SparcStation SLC - Sun 4/20 */ -#define SM_4C_SS2 0x05 /* Sun4c SparcStation 2 - Sun 4/75 */ -#define SM_4C_ELC 0x06 /* Sun4c SparcStation ELC - Sun 4/25 */ -#define SM_4C_IPX 0x07 /* Sun4c SparcStation IPX - Sun 4/50 */ - /* Sun4m machines, these predate the OpenBoot. These values only mean * something if the value in the ARCH field is SM_SUN4M, if it is * SM_SUN4M_OBP then you have the following situation: diff --git a/arch/sparc/include/asm/mbus.h b/arch/sparc/include/asm/mbus.h index 69f07a022ee..14128bcc582 100644 --- a/arch/sparc/include/asm/mbus.h +++ b/arch/sparc/include/asm/mbus.h @@ -8,14 +8,10 @@ #define _SPARC_MBUS_H #include <asm/ross.h> /* HyperSparc stuff */ -#include <asm/cypress.h> /* Cypress Chips */ #include <asm/viking.h> /* Ugh, bug city... */ enum mbus_module { HyperSparc = 0, - Cypress = 1, - Cypress_vE = 2, - Cypress_vD = 3, Swift_ok = 4, Swift_bad_c = 5, Swift_lots_o_bugs = 6, diff --git a/arch/sparc/include/asm/mc146818rtc.h b/arch/sparc/include/asm/mc146818rtc.h index 67ed9e3a023..d8e72f37dc4 100644 --- a/arch/sparc/include/asm/mc146818rtc.h +++ b/arch/sparc/include/asm/mc146818rtc.h @@ -1,5 +1,10 @@ #ifndef ___ASM_SPARC_MC146818RTC_H #define ___ASM_SPARC_MC146818RTC_H + +#include <linux/spinlock.h> + +extern spinlock_t rtc_lock; + #if defined(__sparc__) && defined(__arch64__) #include <asm/mc146818rtc_64.h> #else diff --git a/arch/sparc/include/asm/mdesc.h b/arch/sparc/include/asm/mdesc.h index 9faa046713f..aebeb88f70d 100644 --- a/arch/sparc/include/asm/mdesc.h +++ b/arch/sparc/include/asm/mdesc.h @@ -12,13 +12,13 @@ struct mdesc_handle; * the first argument to all of the operational calls that work * on mdescs. */ -extern struct mdesc_handle *mdesc_grab(void); -extern void mdesc_release(struct mdesc_handle *); +struct mdesc_handle *mdesc_grab(void); +void mdesc_release(struct mdesc_handle *); #define MDESC_NODE_NULL (~(u64)0) -extern u64 mdesc_node_by_name(struct mdesc_handle *handle, - u64 from_node, const char *name); +u64 mdesc_node_by_name(struct mdesc_handle *handle, + u64 from_node, const char *name); #define mdesc_for_each_node_by_name(__hdl, __node, __name) \ for (__node = mdesc_node_by_name(__hdl, MDESC_NODE_NULL, __name); \ (__node) != MDESC_NODE_NULL; \ @@ -34,9 +34,9 @@ extern u64 mdesc_node_by_name(struct mdesc_handle *handle, * * These same rules apply to mdesc_node_name(). */ -extern const void *mdesc_get_property(struct mdesc_handle *handle, - u64 node, const char *name, int *lenp); -extern const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); +const void *mdesc_get_property(struct mdesc_handle *handle, + u64 node, const char *name, int *lenp); +const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); /* MD arc iteration, the standard sequence is: * @@ -50,16 +50,16 @@ extern const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); #define MDESC_ARC_TYPE_FWD "fwd" #define MDESC_ARC_TYPE_BACK "back" -extern u64 mdesc_next_arc(struct mdesc_handle *handle, u64 from, - const char *arc_type); +u64 mdesc_next_arc(struct mdesc_handle *handle, u64 from, + const char *arc_type); #define mdesc_for_each_arc(__arc, __hdl, __node, __type) \ for (__arc = mdesc_next_arc(__hdl, __node, __type); \ (__arc) != MDESC_NODE_NULL; \ __arc = mdesc_next_arc(__hdl, __arc, __type)) -extern u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc); +u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc); -extern void mdesc_update(void); +void mdesc_update(void); struct mdesc_notifier_client { void (*add)(struct mdesc_handle *handle, u64 node); @@ -69,11 +69,12 @@ struct mdesc_notifier_client { struct mdesc_notifier_client *next; }; -extern void mdesc_register_notifier(struct mdesc_notifier_client *client); +void mdesc_register_notifier(struct mdesc_notifier_client *client); -extern void mdesc_fill_in_cpu_data(cpumask_t *mask); -extern void mdesc_populate_present_mask(cpumask_t *mask); +void mdesc_fill_in_cpu_data(cpumask_t *mask); +void mdesc_populate_present_mask(cpumask_t *mask); +void mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask); -extern void sun4v_mdesc_init(void); +void sun4v_mdesc_init(void); #endif diff --git a/arch/sparc/include/asm/memblock.h b/arch/sparc/include/asm/memblock.h deleted file mode 100644 index c67b047ef85..00000000000 --- a/arch/sparc/include/asm/memblock.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _SPARC64_MEMBLOCK_H -#define _SPARC64_MEMBLOCK_H - -#include <asm/oplib.h> - -#define MEMBLOCK_DBG(fmt...) prom_printf(fmt) - -#endif /* !(_SPARC64_MEMBLOCK_H) */ diff --git a/arch/sparc/include/asm/memreg.h b/arch/sparc/include/asm/memreg.h deleted file mode 100644 index 845ad2b3918..00000000000 --- a/arch/sparc/include/asm/memreg.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef _SPARC_MEMREG_H -#define _SPARC_MEMREG_H -/* memreg.h: Definitions of the values found in the synchronous - * and asynchronous memory error registers when a fault - * occurs on the sun4c. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ - -/* First the synchronous error codes, these are usually just - * normal page faults. - */ - -#define SUN4C_SYNC_WDRESET 0x0001 /* watchdog reset */ -#define SUN4C_SYNC_SIZE 0x0002 /* bad access size? whuz this? */ -#define SUN4C_SYNC_PARITY 0x0008 /* bad ram chips caused a parity error */ -#define SUN4C_SYNC_SBUS 0x0010 /* the SBUS had some problems... */ -#define SUN4C_SYNC_NOMEM 0x0020 /* translation to non-existent ram */ -#define SUN4C_SYNC_PROT 0x0040 /* access violated pte protections */ -#define SUN4C_SYNC_NPRESENT 0x0080 /* pte said that page was not present */ -#define SUN4C_SYNC_BADWRITE 0x8000 /* while writing something went bogus */ - -#define SUN4C_SYNC_BOLIXED \ - (SUN4C_SYNC_WDRESET | SUN4C_SYNC_SIZE | SUN4C_SYNC_SBUS | \ - SUN4C_SYNC_NOMEM | SUN4C_SYNC_PARITY) - -/* Now the asynchronous error codes, these are almost always produced - * by the cache writing things back to memory and getting a bad translation. - * Bad DVMA transactions can cause these faults too. - */ - -#define SUN4C_ASYNC_BADDVMA 0x0010 /* error during DVMA access */ -#define SUN4C_ASYNC_NOMEM 0x0020 /* write back pointed to bad phys addr */ -#define SUN4C_ASYNC_BADWB 0x0080 /* write back points to non-present page */ - -/* Memory parity error register with associated bit constants. */ -#ifndef __ASSEMBLY__ -extern __volatile__ unsigned long __iomem *sun4c_memerr_reg; -#endif - -#define SUN4C_MPE_ERROR 0x80 /* Parity error detected. (ro) */ -#define SUN4C_MPE_MULTI 0x40 /* Multiple parity errors detected. (ro) */ -#define SUN4C_MPE_TEST 0x20 /* Write inverse parity. (rw) */ -#define SUN4C_MPE_CHECK 0x10 /* Enable parity checking. (rw) */ -#define SUN4C_MPE_ERR00 0x08 /* Parity error in bits 0-7. (ro) */ -#define SUN4C_MPE_ERR08 0x04 /* Parity error in bits 8-15. (ro) */ -#define SUN4C_MPE_ERR16 0x02 /* Parity error in bits 16-23. (ro) */ -#define SUN4C_MPE_ERR24 0x01 /* Parity error in bits 24-31. (ro) */ -#define SUN4C_MPE_ERRS 0x0F /* Bit mask for the error bits. (ro) */ - -#endif /* !(_SPARC_MEMREG_H) */ diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h index c3029ad6619..59bb5938d85 100644 --- a/arch/sparc/include/asm/mman.h +++ b/arch/sparc/include/asm/mman.h @@ -1,33 +1,10 @@ #ifndef __SPARC_MMAN_H__ #define __SPARC_MMAN_H__ -#include <asm-generic/mman-common.h> +#include <uapi/asm/mman.h> -/* SunOS'ified... */ - -#define MAP_RENAME MAP_ANONYMOUS /* In SunOS terminology */ -#define MAP_NORESERVE 0x40 /* don't reserve swap pages */ -#define MAP_INHERIT 0x80 /* SunOS doesn't do this, but... */ -#define MAP_LOCKED 0x100 /* lock the mapping */ -#define _MAP_NEW 0x80000000 /* Binary compatibility is fun... */ - -#define MAP_GROWSDOWN 0x0200 /* stack-like segment */ -#define MAP_DENYWRITE 0x0800 /* ETXTBSY */ -#define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ - -#define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ -#define MCL_FUTURE 0x4000 /* lock all additions to address space */ - -#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ -#define MAP_NONBLOCK 0x10000 /* do not block on IO */ -#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ -#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ - -#ifdef __KERNEL__ #ifndef __ASSEMBLY__ #define arch_mmap_check(addr,len,flags) sparc_mmap_check(addr,len) int sparc_mmap_check(unsigned long addr, unsigned long len); #endif -#endif - #endif /* __SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/asm/mmu_32.h b/arch/sparc/include/asm/mmu_32.h index ccd36d26615..6f056e535cf 100644 --- a/arch/sparc/include/asm/mmu_32.h +++ b/arch/sparc/include/asm/mmu_32.h @@ -4,4 +4,7 @@ /* Default "unsigned long" context */ typedef unsigned long mm_context_t; +/* mm/srmmu.c */ +extern ctxd_t *srmmu_ctx_table_phys; + #endif diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index 9067dc50053..70067ce184b 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h @@ -30,22 +30,8 @@ #define CTX_PGSZ_MASK ((CTX_PGSZ_BITS << CTX_PGSZ0_SHIFT) | \ (CTX_PGSZ_BITS << CTX_PGSZ1_SHIFT)) -#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB) #define CTX_PGSZ_BASE CTX_PGSZ_8KB -#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB) -#define CTX_PGSZ_BASE CTX_PGSZ_64KB -#else -#error No page size specified in kernel configuration -#endif - -#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) -#define CTX_PGSZ_HUGE CTX_PGSZ_4MB -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define CTX_PGSZ_HUGE CTX_PGSZ_512KB -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define CTX_PGSZ_HUGE CTX_PGSZ_64KB -#endif - +#define CTX_PGSZ_HUGE CTX_PGSZ_4MB #define CTX_PGSZ_KERN CTX_PGSZ_4MB /* Thus, when running on UltraSPARC-III+ and later, we use the following @@ -81,9 +67,9 @@ struct tsb { unsigned long pte; } __attribute__((aligned(TSB_ENTRY_ALIGNMENT))); -extern void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); -extern void tsb_flush(unsigned long ent, unsigned long tag); -extern void tsb_init(struct tsb *tsb, unsigned long size); +void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); +void tsb_flush(unsigned long ent, unsigned long tag); +void tsb_init(struct tsb *tsb, unsigned long size); struct tsb_config { struct tsb *tsb; @@ -96,7 +82,7 @@ struct tsb_config { #define MM_TSB_BASE 0 -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define MM_TSB_HUGE 1 #define MM_NUM_TSBS 2 #else diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h index 671a997b9e6..2df2a9be8f6 100644 --- a/arch/sparc/include/asm/mmu_context_32.h +++ b/arch/sparc/include/asm/mmu_context_32.h @@ -1,8 +1,6 @@ #ifndef __SPARC_MMU_CONTEXT_H #define __SPARC_MMU_CONTEXT_H -#include <asm/btfixup.h> - #ifndef __ASSEMBLY__ #include <asm-generic/mm_hooks.h> @@ -11,26 +9,21 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { } -/* - * Initialize a new mmu context. This is invoked when a new +/* Initialize a new mmu context. This is invoked when a new * address space instance (unique or shared) is instantiated. */ -#define init_new_context(tsk, mm) (((mm)->context = NO_CONTEXT), 0) +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); -/* - * Destroy a dead context. This occurs when mmput drops the +/* Destroy a dead context. This occurs when mmput drops the * mm_users count to zero, the mmaps have been released, and * all the page tables have been flushed. Our job is to destroy * any remaining processor-specific state. */ -BTFIXUPDEF_CALL(void, destroy_context, struct mm_struct *) - -#define destroy_context(mm) BTFIXUP_CALL(destroy_context)(mm) +void destroy_context(struct mm_struct *mm); /* Switch the current MM context. */ -BTFIXUPDEF_CALL(void, switch_mm, struct mm_struct *, struct mm_struct *, struct task_struct *) - -#define switch_mm(old_mm, mm, tsk) BTFIXUP_CALL(switch_mm)(old_mm, mm, tsk) +void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, + struct task_struct *tsk); #define deactivate_mm(tsk,mm) do { } while (0) diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h index 666a73fef28..b84be675e50 100644 --- a/arch/sparc/include/asm/mmu_context_64.h +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -6,7 +6,6 @@ #ifndef __ASSEMBLY__ #include <linux/spinlock.h> -#include <asm/system.h> #include <asm/spitfire.h> #include <asm-generic/mm_hooks.h> @@ -18,26 +17,26 @@ extern spinlock_t ctx_alloc_lock; extern unsigned long tlb_context_cache; extern unsigned long mmu_context_bmap[]; -extern void get_new_mmu_context(struct mm_struct *mm); +void get_new_mmu_context(struct mm_struct *mm); #ifdef CONFIG_SMP -extern void smp_new_mmu_context_version(void); +void smp_new_mmu_context_version(void); #else #define smp_new_mmu_context_version() do { } while (0) #endif -extern int init_new_context(struct task_struct *tsk, struct mm_struct *mm); -extern void destroy_context(struct mm_struct *mm); +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); +void destroy_context(struct mm_struct *mm); -extern void __tsb_context_switch(unsigned long pgd_pa, - struct tsb_config *tsb_base, - struct tsb_config *tsb_huge, - unsigned long tsb_descr_pa); +void __tsb_context_switch(unsigned long pgd_pa, + struct tsb_config *tsb_base, + struct tsb_config *tsb_huge, + unsigned long tsb_descr_pa); static inline void tsb_context_switch(struct mm_struct *mm) { __tsb_context_switch(__pa(mm->pgd), &mm->context.tsb_block[0], -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) (mm->context.tsb_block[1].tsb ? &mm->context.tsb_block[1] : NULL) @@ -47,9 +46,11 @@ static inline void tsb_context_switch(struct mm_struct *mm) , __pa(&mm->context.tsb_descr[0])); } -extern void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long mm_rss); +void tsb_grow(struct mm_struct *mm, + unsigned long tsb_index, + unsigned long mm_rss); #ifdef CONFIG_SMP -extern void smp_tsb_sync(struct mm_struct *mm); +void smp_tsb_sync(struct mm_struct *mm); #else #define smp_tsb_sync(__mm) do { } while (0) #endif @@ -67,9 +68,9 @@ extern void smp_tsb_sync(struct mm_struct *mm); : "r" (CTX_HWBITS((__mm)->context)), \ "r" (SECONDARY_CONTEXT), "i" (ASI_DMMU), "i" (ASI_MMU)) -extern void __flush_tlb_mm(unsigned long, unsigned long); +void __flush_tlb_mm(unsigned long, unsigned long); -/* Switch the current MM context. Interrupts are disabled. */ +/* Switch the current MM context. */ static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk) { unsigned long ctx_valid, flags; diff --git a/arch/sparc/include/asm/mmzone.h b/arch/sparc/include/asm/mmzone.h index e8c648741ed..99d9b9f577b 100644 --- a/arch/sparc/include/asm/mmzone.h +++ b/arch/sparc/include/asm/mmzone.h @@ -8,8 +8,6 @@ extern struct pglist_data *node_data[]; #define NODE_DATA(nid) (node_data[nid]) -#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) -#define node_end_pfn(nid) (NODE_DATA(nid)->node_end_pfn) extern int numa_cpu_lookup_table[]; extern cpumask_t numa_cpumask_lookup_table[]; diff --git a/arch/sparc/include/asm/module.h b/arch/sparc/include/asm/module.h deleted file mode 100644 index ff8e02d8033..00000000000 --- a/arch/sparc/include/asm/module.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef __SPARC_MODULE_H -#define __SPARC_MODULE_H -struct mod_arch_specific { }; - -/* - * Use some preprocessor magic to define the correct symbol - * for sparc32 and sparc64. - * Elf_Addr becomes Elf32_Addr for sparc32 and Elf64_Addr for sparc64 - */ -#define ___ELF(a, b, c) a##b##c -#define __ELF(a, b, c) ___ELF(a, b, c) -#define _Elf(t) __ELF(Elf, CONFIG_BITS, t) -#define _ELF(t) __ELF(ELF, CONFIG_BITS, t) - -#define Elf_Shdr _Elf(_Shdr) -#define Elf_Sym _Elf(_Sym) -#define Elf_Ehdr _Elf(_Ehdr) -#define Elf_Rela _Elf(_Rela) -#define Elf_Addr _Elf(_Addr) - -#define ELF_R_SYM _ELF(_R_SYM) -#define ELF_R_TYPE _ELF(_R_TYPE) - -#endif /* __SPARC_MODULE_H */ diff --git a/arch/sparc/include/asm/mpmbox.h b/arch/sparc/include/asm/mpmbox.h deleted file mode 100644 index f8423039b24..00000000000 --- a/arch/sparc/include/asm/mpmbox.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * mpmbox.h: Interface and defines for the OpenProm mailbox - * facilities for MP machines under Linux. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ - -#ifndef _SPARC_MPMBOX_H -#define _SPARC_MPMBOX_H - -/* The prom allocates, for each CPU on the machine an unsigned - * byte in physical ram. You probe the device tree prom nodes - * for these values. The purpose of this byte is to be able to - * pass messages from one cpu to another. - */ - -/* These are the main message types we have to look for in our - * Cpu mailboxes, based upon these values we decide what course - * of action to take. - */ - -/* The CPU is executing code in the kernel. */ -#define MAILBOX_ISRUNNING 0xf0 - -/* Another CPU called romvec->pv_exit(), you should call - * prom_stopcpu() when you see this in your mailbox. - */ -#define MAILBOX_EXIT 0xfb - -/* Another CPU called romvec->pv_enter(), you should call - * prom_cpuidle() when this is seen. - */ -#define MAILBOX_GOSPIN 0xfc - -/* Another CPU has hit a breakpoint either into kadb or the prom - * itself. Just like MAILBOX_GOSPIN, you should call prom_cpuidle() - * at this point. - */ -#define MAILBOX_BPT_SPIN 0xfd - -/* Oh geese, some other nitwit got a damn watchdog reset. The party's - * over so go call prom_stopcpu(). - */ -#define MAILBOX_WDOG_STOP 0xfe - -#ifndef __ASSEMBLY__ - -/* Handy macro's to determine a cpu's state. */ - -/* Is the cpu still in Power On Self Test? */ -#define MBOX_POST_P(letter) ((letter) >= 0x00 && (letter) <= 0x7f) - -/* Is the cpu at the 'ok' prompt of the PROM? */ -#define MBOX_PROMPROMPT_P(letter) ((letter) >= 0x80 && (letter) <= 0x8f) - -/* Is the cpu spinning in the PROM? */ -#define MBOX_PROMSPIN_P(letter) ((letter) >= 0x90 && (letter) <= 0xef) - -/* Sanity check... This is junk mail, throw it out. */ -#define MBOX_BOGON_P(letter) ((letter) >= 0xf1 && (letter) <= 0xfa) - -/* Is the cpu actively running an application/kernel-code? */ -#define MBOX_RUNNING_P(letter) ((letter) == MAILBOX_ISRUNNING) - -#endif /* !(__ASSEMBLY__) */ - -#endif /* !(_SPARC_MPMBOX_H) */ diff --git a/arch/sparc/include/asm/mutex.h b/arch/sparc/include/asm/mutex.h deleted file mode 100644 index 458c1f7fbc1..00000000000 --- a/arch/sparc/include/asm/mutex.h +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Pull in the generic implementation for the mutex fastpath. - * - * TODO: implement optimized primitives instead, or leave the generic - * implementation in place, or pick the atomic_xchg() based generic - * implementation. (see asm-generic/mutex-xchg.h for details) - */ - -#include <asm-generic/mutex-dec.h> diff --git a/arch/sparc/include/asm/nmi.h b/arch/sparc/include/asm/nmi.h index 72e6500e7ab..26ad2b2607c 100644 --- a/arch/sparc/include/asm/nmi.h +++ b/arch/sparc/include/asm/nmi.h @@ -1,13 +1,13 @@ #ifndef __NMI_H #define __NMI_H -extern int __init nmi_init(void); -extern void perfctr_irq(int irq, struct pt_regs *regs); -extern void nmi_adjust_hz(unsigned int new_hz); +int __init nmi_init(void); +void perfctr_irq(int irq, struct pt_regs *regs); +void nmi_adjust_hz(unsigned int new_hz); extern atomic_t nmi_active; -extern void start_nmi_watchdog(void *unused); -extern void stop_nmi_watchdog(void *unused); +void start_nmi_watchdog(void *unused); +void stop_nmi_watchdog(void *unused); #endif /* __NMI_H */ diff --git a/arch/sparc/include/asm/ns87303.h b/arch/sparc/include/asm/ns87303.h index 686defe6aaa..6b947ee0f6a 100644 --- a/arch/sparc/include/asm/ns87303.h +++ b/arch/sparc/include/asm/ns87303.h @@ -37,7 +37,7 @@ /* Power and Test Register (PTR) bits */ #define PTR_LPTB_IRQ7 0x08 #define PTR_LEVEL_IRQ 0x80 /* When not ECP/EPP: Use level IRQ */ -#define PTR_LPT_REG_DIR 0x80 /* When ECP/EPP: LPT CTR controlls direction */ +#define PTR_LPT_REG_DIR 0x80 /* When ECP/EPP: LPT CTR controls direction */ /* of the parallel port */ /* Function Control Register (FCR) bits */ @@ -79,7 +79,6 @@ #include <linux/spinlock.h> -#include <asm/system.h> #include <asm/io.h> extern spinlock_t ns87303_lock; diff --git a/arch/sparc/include/asm/obio.h b/arch/sparc/include/asm/obio.h index 4ade0c8a2c7..910c1d9af1f 100644 --- a/arch/sparc/include/asm/obio.h +++ b/arch/sparc/include/asm/obio.h @@ -220,19 +220,6 @@ static inline void cc_set_igen(unsigned gen) "i" (ASI_M_MXCC)); } -/* +-------+-------------+-----------+------------------------------------+ - * | bcast | devid | sid | levels mask | - * +-------+-------------+-----------+------------------------------------+ - * 31 30 23 22 15 14 0 - */ -#define IGEN_MESSAGE(bcast, devid, sid, levels) \ - (((bcast) << 31) | ((devid) << 23) | ((sid) << 15) | (levels)) - -static inline void sun4d_send_ipi(int cpu, int level) -{ - cc_set_igen(IGEN_MESSAGE(0, cpu << 3, 6 + ((level >> 1) & 7), 1 << (level - 1))); -} - #endif /* !__ASSEMBLY__ */ #endif /* !(_SPARC_OBIO_H) */ diff --git a/arch/sparc/include/asm/oplib_32.h b/arch/sparc/include/asm/oplib_32.h index 71e5e9aeb67..56a09b9d7b1 100644 --- a/arch/sparc/include/asm/oplib_32.h +++ b/arch/sparc/include/asm/oplib_32.h @@ -43,28 +43,28 @@ extern struct linux_nodeops *prom_nodeops; /* You must call prom_init() before using any of the library services, * preferably as early as possible. Pass it the romvec pointer. */ -extern void prom_init(struct linux_romvec *rom_ptr); +void prom_init(struct linux_romvec *rom_ptr); /* Boot argument acquisition, returns the boot command line string. */ -extern char *prom_getbootargs(void); +char *prom_getbootargs(void); /* Miscellaneous routines, don't really fit in any category per se. */ /* Reboot the machine with the command line passed. */ -extern void prom_reboot(char *boot_command); +void prom_reboot(char *boot_command); /* Evaluate the forth string passed. */ -extern void prom_feval(char *forth_string); +void prom_feval(char *forth_string); /* Enter the prom, with possibility of continuation with the 'go' * command in newer proms. */ -extern void prom_cmdline(void); +void prom_cmdline(void); /* Enter the prom, with no chance of continuation for the stand-alone * which calls this. */ -extern void __noreturn prom_halt(void); +void __noreturn prom_halt(void); /* Set the PROM 'sync' callback function to the passed function pointer. * When the user gives the 'sync' command at the prom prompt while the @@ -73,45 +73,37 @@ extern void __noreturn prom_halt(void); * XXX The arguments are different on V0 vs. V2->higher proms, grrr! XXX */ typedef void (*sync_func_t)(void); -extern void prom_setsync(sync_func_t func_ptr); +void prom_setsync(sync_func_t func_ptr); /* Acquire the IDPROM of the root node in the prom device tree. This * gets passed a buffer where you would like it stuffed. The return value * is the format type of this idprom or 0xff on error. */ -extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); /* Get the prom major version. */ -extern int prom_version(void); +int prom_version(void); /* Get the prom plugin revision. */ -extern int prom_getrev(void); +int prom_getrev(void); /* Get the prom firmware revision. */ -extern int prom_getprev(void); +int prom_getprev(void); /* Write a buffer of characters to the console. */ -extern void prom_console_write_buf(const char *buf, int len); +void prom_console_write_buf(const char *buf, int len); /* Prom's internal routines, don't use in kernel/boot code. */ -extern void prom_printf(const char *fmt, ...); -extern void prom_write(const char *buf, unsigned int len); +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); /* Multiprocessor operations... */ /* Start the CPU with the given device tree node, context table, and context * at the passed program counter. */ -extern int prom_startcpu(int cpunode, struct linux_prom_registers *context_table, - int context, char *program_counter); - -/* Sun4/sun4c specific memory-management startup hook. */ - -/* Map the passed segment in the given context at the passed - * virtual address. - */ -extern void prom_putsegment(int context, unsigned long virt_addr, - int physical_segment); +int prom_startcpu(int cpunode, struct linux_prom_registers *context_table, + int context, char *program_counter); /* Initialize the memory lists based upon the prom version. */ void prom_meminit(void); @@ -119,65 +111,65 @@ void prom_meminit(void); /* PROM device tree traversal functions... */ /* Get the child node of the given node, or zero if no child exists. */ -extern phandle prom_getchild(phandle parent_node); +phandle prom_getchild(phandle parent_node); /* Get the next sibling node of the given node, or zero if no further * siblings exist. */ -extern phandle prom_getsibling(phandle node); +phandle prom_getsibling(phandle node); /* Get the length, at the passed node, of the given property type. * Returns -1 on error (ie. no such property at this node). */ -extern int prom_getproplen(phandle thisnode, const char *property); +int prom_getproplen(phandle thisnode, const char *property); /* Fetch the requested property using the given buffer. Returns * the number of bytes the prom put into your buffer or -1 on error. */ -extern int __must_check prom_getproperty(phandle thisnode, const char *property, - char *prop_buffer, int propbuf_size); +int __must_check prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); /* Acquire an integer property. */ -extern int prom_getint(phandle node, char *property); +int prom_getint(phandle node, char *property); /* Acquire an integer property, with a default value. */ -extern int prom_getintdefault(phandle node, char *property, int defval); +int prom_getintdefault(phandle node, char *property, int defval); /* Acquire a boolean property, 0=FALSE 1=TRUE. */ -extern int prom_getbool(phandle node, char *prop); +int prom_getbool(phandle node, char *prop); /* Acquire a string property, null string on error. */ -extern void prom_getstring(phandle node, char *prop, char *buf, int bufsize); +void prom_getstring(phandle node, char *prop, char *buf, int bufsize); /* Search all siblings starting at the passed node for "name" matching * the given string. Returns the node on success, zero on failure. */ -extern phandle prom_searchsiblings(phandle node_start, char *name); +phandle prom_searchsiblings(phandle node_start, char *name); /* Returns the next property after the passed property for the given * node. Returns null string on failure. */ -extern char *prom_nextprop(phandle node, char *prev_property, char *buffer); +char *prom_nextprop(phandle node, char *prev_property, char *buffer); /* Returns phandle of the path specified */ -extern phandle prom_finddevice(char *name); +phandle prom_finddevice(char *name); /* Set the indicated property at the given node with the passed value. * Returns the number of bytes of your value that the prom took. */ -extern int prom_setprop(phandle node, const char *prop_name, char *prop_value, - int value_size); +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); -extern phandle prom_inst2pkg(int); +phandle prom_inst2pkg(int); /* Dorking with Bus ranges... */ /* Apply promlib probes OBIO ranges to registers. */ -extern void prom_apply_obio_ranges(struct linux_prom_registers *obioregs, int nregs); +void prom_apply_obio_ranges(struct linux_prom_registers *obioregs, int nregs); /* Apply ranges of any prom node (and optionally parent node as well) to registers. */ -extern void prom_apply_generic_ranges(phandle node, phandle parent, - struct linux_prom_registers *sbusregs, int nregs); +void prom_apply_generic_ranges(phandle node, phandle parent, + struct linux_prom_registers *sbusregs, int nregs); void prom_ranges_init(void); diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h index 97a90475c31..f34682430fc 100644 --- a/arch/sparc/include/asm/oplib_64.h +++ b/arch/sparc/include/asm/oplib_64.h @@ -62,100 +62,100 @@ struct linux_mem_p1275 { /* You must call prom_init() before using any of the library services, * preferably as early as possible. Pass it the romvec pointer. */ -extern void prom_init(void *cif_handler, void *cif_stack); +void prom_init(void *cif_handler, void *cif_stack); /* Boot argument acquisition, returns the boot command line string. */ -extern char *prom_getbootargs(void); +char *prom_getbootargs(void); /* Miscellaneous routines, don't really fit in any category per se. */ /* Reboot the machine with the command line passed. */ -extern void prom_reboot(const char *boot_command); +void prom_reboot(const char *boot_command); /* Evaluate the forth string passed. */ -extern void prom_feval(const char *forth_string); +void prom_feval(const char *forth_string); /* Enter the prom, with possibility of continuation with the 'go' * command in newer proms. */ -extern void prom_cmdline(void); +void prom_cmdline(void); /* Enter the prom, with no chance of continuation for the stand-alone * which calls this. */ -extern void prom_halt(void) __attribute__ ((noreturn)); +void prom_halt(void) __attribute__ ((noreturn)); /* Halt and power-off the machine. */ -extern void prom_halt_power_off(void) __attribute__ ((noreturn)); +void prom_halt_power_off(void) __attribute__ ((noreturn)); /* Acquire the IDPROM of the root node in the prom device tree. This * gets passed a buffer where you would like it stuffed. The return value * is the format type of this idprom or 0xff on error. */ -extern unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); /* Write a buffer of characters to the console. */ -extern void prom_console_write_buf(const char *buf, int len); +void prom_console_write_buf(const char *buf, int len); /* Prom's internal routines, don't use in kernel/boot code. */ -extern void prom_printf(const char *fmt, ...); -extern void prom_write(const char *buf, unsigned int len); +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); /* Multiprocessor operations... */ #ifdef CONFIG_SMP /* Start the CPU with the given device tree node at the passed program * counter with the given arg passed in via register %o0. */ -extern void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg); +void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg); /* Start the CPU with the given cpu ID at the passed program * counter with the given arg passed in via register %o0. */ -extern void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg); +void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg); /* Stop the CPU with the given cpu ID. */ -extern void prom_stopcpu_cpuid(int cpuid); +void prom_stopcpu_cpuid(int cpuid); /* Stop the current CPU. */ -extern void prom_stopself(void); +void prom_stopself(void); /* Idle the current CPU. */ -extern void prom_idleself(void); +void prom_idleself(void); /* Resume the CPU with the passed device tree node. */ -extern void prom_resumecpu(int cpunode); +void prom_resumecpu(int cpunode); #endif /* Power management interfaces. */ /* Put the current CPU to sleep. */ -extern void prom_sleepself(void); +void prom_sleepself(void); /* Put the entire system to sleep. */ -extern int prom_sleepsystem(void); +int prom_sleepsystem(void); /* Initiate a wakeup event. */ -extern int prom_wakeupsystem(void); +int prom_wakeupsystem(void); /* MMU and memory related OBP interfaces. */ /* Get unique string identifying SIMM at given physical address. */ -extern int prom_getunumber(int syndrome_code, - unsigned long phys_addr, - char *buf, int buflen); +int prom_getunumber(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen); /* Retain physical memory to the caller across soft resets. */ -extern int prom_retain(const char *name, unsigned long size, - unsigned long align, unsigned long *paddr); +int prom_retain(const char *name, unsigned long size, + unsigned long align, unsigned long *paddr); /* Load explicit I/D TLB entries into the calling processor. */ -extern long prom_itlb_load(unsigned long index, - unsigned long tte_data, - unsigned long vaddr); +long prom_itlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); -extern long prom_dtlb_load(unsigned long index, - unsigned long tte_data, - unsigned long vaddr); +long prom_dtlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); /* Map/Unmap client program address ranges. First the format of * the mapping mode argument. @@ -170,81 +170,81 @@ extern long prom_dtlb_load(unsigned long index, #define PROM_MAP_IE 0x0100 /* Invert-Endianness */ #define PROM_MAP_DEFAULT (PROM_MAP_WRITE | PROM_MAP_READ | PROM_MAP_EXEC | PROM_MAP_CACHED) -extern int prom_map(int mode, unsigned long size, - unsigned long vaddr, unsigned long paddr); -extern void prom_unmap(unsigned long size, unsigned long vaddr); +int prom_map(int mode, unsigned long size, + unsigned long vaddr, unsigned long paddr); +void prom_unmap(unsigned long size, unsigned long vaddr); /* PROM device tree traversal functions... */ /* Get the child node of the given node, or zero if no child exists. */ -extern phandle prom_getchild(phandle parent_node); +phandle prom_getchild(phandle parent_node); /* Get the next sibling node of the given node, or zero if no further * siblings exist. */ -extern phandle prom_getsibling(phandle node); +phandle prom_getsibling(phandle node); /* Get the length, at the passed node, of the given property type. * Returns -1 on error (ie. no such property at this node). */ -extern int prom_getproplen(phandle thisnode, const char *property); +int prom_getproplen(phandle thisnode, const char *property); /* Fetch the requested property using the given buffer. Returns * the number of bytes the prom put into your buffer or -1 on error. */ -extern int prom_getproperty(phandle thisnode, const char *property, - char *prop_buffer, int propbuf_size); +int prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); /* Acquire an integer property. */ -extern int prom_getint(phandle node, const char *property); +int prom_getint(phandle node, const char *property); /* Acquire an integer property, with a default value. */ -extern int prom_getintdefault(phandle node, const char *property, int defval); +int prom_getintdefault(phandle node, const char *property, int defval); /* Acquire a boolean property, 0=FALSE 1=TRUE. */ -extern int prom_getbool(phandle node, const char *prop); +int prom_getbool(phandle node, const char *prop); /* Acquire a string property, null string on error. */ -extern void prom_getstring(phandle node, const char *prop, char *buf, - int bufsize); +void prom_getstring(phandle node, const char *prop, char *buf, + int bufsize); /* Does the passed node have the given "name"? YES=1 NO=0 */ -extern int prom_nodematch(phandle thisnode, const char *name); +int prom_nodematch(phandle thisnode, const char *name); /* Search all siblings starting at the passed node for "name" matching * the given string. Returns the node on success, zero on failure. */ -extern phandle prom_searchsiblings(phandle node_start, const char *name); +phandle prom_searchsiblings(phandle node_start, const char *name); /* Return the first property type, as a string, for the given node. * Returns a null string on error. Buffer should be at least 32B long. */ -extern char *prom_firstprop(phandle node, char *buffer); +char *prom_firstprop(phandle node, char *buffer); /* Returns the next property after the passed property for the given * node. Returns null string on failure. Buffer should be at least 32B long. */ -extern char *prom_nextprop(phandle node, const char *prev_property, char *buf); +char *prom_nextprop(phandle node, const char *prev_property, char *buf); /* Returns 1 if the specified node has given property. */ -extern int prom_node_has_property(phandle node, const char *property); +int prom_node_has_property(phandle node, const char *property); /* Returns phandle of the path specified */ -extern phandle prom_finddevice(const char *name); +phandle prom_finddevice(const char *name); /* Set the indicated property at the given node with the passed value. * Returns the number of bytes of your value that the prom took. */ -extern int prom_setprop(phandle node, const char *prop_name, char *prop_value, - int value_size); +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); -extern phandle prom_inst2pkg(int); -extern void prom_sun4v_guest_soft_state(void); +phandle prom_inst2pkg(int); +void prom_sun4v_guest_soft_state(void); -extern int prom_ihandle2path(int handle, char *buffer, int bufsize); +int prom_ihandle2path(int handle, char *buffer, int bufsize); /* Client interface level routines. */ -extern void p1275_cmd_direct(unsigned long *); +void p1275_cmd_direct(unsigned long *); #endif /* !(__SPARC64_OPLIB_H) */ diff --git a/arch/sparc/include/asm/page.h b/arch/sparc/include/asm/page.h index f21de034902..1be2fdec626 100644 --- a/arch/sparc/include/asm/page.h +++ b/arch/sparc/include/asm/page.h @@ -1,5 +1,8 @@ #ifndef ___ASM_SPARC_PAGE_H #define ___ASM_SPARC_PAGE_H + +#define page_to_phys(page) (page_to_pfn(page) << PAGE_SHIFT) + #if defined(__sparc__) && defined(__arch64__) #include <asm/page_64.h> #else diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h index 156707b0f18..f82a1f36b65 100644 --- a/arch/sparc/include/asm/page_32.h +++ b/arch/sparc/include/asm/page_32.h @@ -8,18 +8,12 @@ #ifndef _SPARC_PAGE_H #define _SPARC_PAGE_H -#define PAGE_SHIFT 12 +#include <linux/const.h> -#ifndef __ASSEMBLY__ -/* I have my suspicions... -DaveM */ -#define PAGE_SIZE (1UL << PAGE_SHIFT) -#else -#define PAGE_SIZE (1 << PAGE_SHIFT) -#endif +#define PAGE_SHIFT 12 +#define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) -#include <asm/btfixup.h> - #ifndef __ASSEMBLY__ #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) @@ -49,12 +43,6 @@ struct sparc_phys_banks { extern struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; -/* Cache alias structure. Entry is valid if context != -1. */ -struct cache_palias { - unsigned long vaddr; - int context; -}; - /* passing structs on the Sparc slow us down tremendously... */ /* #define STRICT_MM_TYPECHECKS */ @@ -119,11 +107,7 @@ typedef unsigned long iopgprot_t; typedef struct page *pgtable_t; -extern unsigned long sparc_unmapped_base; - -BTFIXUPDEF_SETHI(sparc_unmapped_base) - -#define TASK_UNMAPPED_BASE BTFIXUP_SETHI(sparc_unmapped_base) +#define TASK_UNMAPPED_BASE 0x50000000 #else /* !(__ASSEMBLY__) */ diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index f0d09b40103..bf109984a03 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -3,13 +3,7 @@ #include <linux/const.h> -#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB) #define PAGE_SHIFT 13 -#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB) -#define PAGE_SHIFT 16 -#else -#error No page size specified in kernel configuration -#endif #define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) @@ -21,15 +15,12 @@ #define DCACHE_ALIASING_POSSIBLE #endif -#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) -#define HPAGE_SHIFT 22 -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define HPAGE_SHIFT 19 -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define HPAGE_SHIFT 16 -#endif +#define HPAGE_SHIFT 23 +#define REAL_HPAGE_SHIFT 22 + +#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) #define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) @@ -38,14 +29,19 @@ #ifndef __ASSEMBLY__ +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +struct pt_regs; +void hugetlb_setup(struct pt_regs *regs); +#endif + #define WANT_PAGE_VIRTUAL -extern void _clear_page(void *page); +void _clear_page(void *page); #define clear_page(X) _clear_page((void *)(X)) struct page; -extern void clear_user_page(void *addr, unsigned long vaddr, struct page *page); +void clear_user_page(void *addr, unsigned long vaddr, struct page *page); #define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) -extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); +void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); /* Unlike sparc32, sparc64's parameter passing API is more * sane in that structures which as small enough are passed @@ -60,8 +56,8 @@ extern void copy_user_page(void *to, void *from, unsigned long vaddr, struct pag /* These are used to make use of C type-checking.. */ typedef struct { unsigned long pte; } pte_t; typedef struct { unsigned long iopte; } iopte_t; -typedef struct { unsigned int pmd; } pmd_t; -typedef struct { unsigned int pgd; } pgd_t; +typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pgd; } pgd_t; typedef struct { unsigned long pgprot; } pgprot_t; #define pte_val(x) ((x).pte) @@ -80,8 +76,8 @@ typedef struct { unsigned long pgprot; } pgprot_t; /* .. while these make it easier on the compiler */ typedef unsigned long pte_t; typedef unsigned long iopte_t; -typedef unsigned int pmd_t; -typedef unsigned int pgd_t; +typedef unsigned long pmd_t; +typedef unsigned long pgd_t; typedef unsigned long pgprot_t; #define pte_val(x) (x) @@ -98,20 +94,46 @@ typedef unsigned long pgprot_t; #endif /* (STRICT_MM_TYPECHECKS) */ -typedef struct page *pgtable_t; +typedef pte_t *pgtable_t; + +/* These two values define the virtual address space range in which we + * must forbid 64-bit user processes from making mappings. It used to + * represent precisely the virtual address space hole present in most + * early sparc64 chips including UltraSPARC-I. But now it also is + * further constrained by the limits of our page tables, which is + * 43-bits of virtual address. + */ +#define SPARC64_VA_HOLE_TOP _AC(0xfffffc0000000000,UL) +#define SPARC64_VA_HOLE_BOTTOM _AC(0x0000040000000000,UL) + +/* The next two defines specify the actual exclusion region we + * enforce, wherein we use a 4GB red zone on each side of the VA hole. + */ +#define VA_EXCLUDE_START (SPARC64_VA_HOLE_BOTTOM - (1UL << 32UL)) +#define VA_EXCLUDE_END (SPARC64_VA_HOLE_TOP + (1UL << 32UL)) #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ - (_AC(0x0000000070000000,UL)) : \ - (_AC(0xfffff80000000000,UL) + (1UL << 32UL))) + _AC(0x0000000070000000,UL) : \ + VA_EXCLUDE_END) #include <asm-generic/memory_model.h> +#define PAGE_OFFSET_BY_BITS(X) (-(_AC(1,UL) << (X))) +extern unsigned long PAGE_OFFSET; + #endif /* !(__ASSEMBLY__) */ -/* We used to stick this into a hard-coded global register (%g4) - * but that does not make sense anymore. +/* The maximum number of physical memory address bits we support, this + * is used to size various tables used to manage kernel TLB misses and + * also the sparsemem code. + */ +#define MAX_PHYS_ADDRESS_BITS 47 + +/* These two shift counts are used when indexing sparc64_valid_addr_bitmap + * and kpte_linear_bitmap. */ -#define PAGE_OFFSET _AC(0xFFFFF80000000000,UL) +#define ILOG2_4MB 22 +#define ILOG2_256MB 28 #ifndef __ASSEMBLY__ diff --git a/arch/sparc/include/asm/parport.h b/arch/sparc/include/asm/parport.h index aa4c82648d8..c55291e5b83 100644 --- a/arch/sparc/include/asm/parport.h +++ b/arch/sparc/include/asm/parport.h @@ -103,7 +103,7 @@ static inline unsigned int get_dma_residue(unsigned int dmanr) return ebus_dma_residue(&sparc_ebus_dmas[dmanr].info); } -static int __devinit ecpp_probe(struct platform_device *op, const struct of_device_id *match) +static int ecpp_probe(struct platform_device *op) { unsigned long base = op->resource[0].start; unsigned long config = op->resource[1].start; @@ -192,7 +192,7 @@ out_err: return err; } -static int __devexit ecpp_remove(struct platform_device *op) +static int ecpp_remove(struct platform_device *op) { struct parport *p = dev_get_drvdata(&op->dev); int slot = p->dma; @@ -235,19 +235,19 @@ static const struct of_device_id ecpp_match[] = { {}, }; -static struct of_platform_driver ecpp_driver = { +static struct platform_driver ecpp_driver = { .driver = { .name = "ecpp", .owner = THIS_MODULE, .of_match_table = ecpp_match, }, .probe = ecpp_probe, - .remove = __devexit_p(ecpp_remove), + .remove = ecpp_remove, }; static int parport_pc_find_nonpci_ports(int autoirq, int autodma) { - return of_register_platform_driver(&ecpp_driver); + return platform_driver_register(&ecpp_driver); } #endif /* !(_ASM_SPARC64_PARPORT_H */ diff --git a/arch/sparc/include/asm/pci_32.h b/arch/sparc/include/asm/pci_32.h index 332ac9ab36b..53e9b4987db 100644 --- a/arch/sparc/include/asm/pci_32.h +++ b/arch/sparc/include/asm/pci_32.h @@ -16,16 +16,6 @@ #define PCI_IRQ_NONE 0xffffffff -static inline void pcibios_set_master(struct pci_dev *dev) -{ - /* No special bus mastering setup handling */ -} - -static inline void pcibios_penalize_isa_irq(int irq, int active) -{ - /* We don't do dynamic PCI IRQ allocation */ -} - /* Dynamic DMA mapping stuff. */ #define PCI_DMA_BUS_IS_PHYS (0) @@ -42,12 +32,25 @@ static inline void pci_dma_burst_advice(struct pci_dev *pdev, } #endif -struct device_node; -extern struct device_node *pci_device_to_OF_node(struct pci_dev *pdev); - #endif /* __KERNEL__ */ +#ifndef CONFIG_LEON_PCI /* generic pci stuff */ #include <asm-generic/pci.h> +#else +/* + * On LEON PCI Memory space is mapped 1:1 with physical address space. + * + * I/O space is located at low 64Kbytes in PCI I/O space. The I/O addresses + * are converted into CPU addresses to virtual addresses that are mapped with + * MMU to the PCI Host PCI I/O space window which are translated to the low + * 64Kbytes by the Host controller. + */ + +static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) +{ + return PCI_IRQ_NONE; +} +#endif #endif /* __SPARC_PCI_H */ diff --git a/arch/sparc/include/asm/pci_64.h b/arch/sparc/include/asm/pci_64.h index 948b686ec08..bd00a622616 100644 --- a/arch/sparc/include/asm/pci_64.h +++ b/arch/sparc/include/asm/pci_64.h @@ -16,16 +16,6 @@ #define PCI_IRQ_NONE 0xffffffff -static inline void pcibios_set_master(struct pci_dev *dev) -{ - /* No special bus mastering setup handling */ -} - -static inline void pcibios_penalize_isa_irq(int irq, int active) -{ - /* We don't do dynamic PCI IRQ allocation */ -} - /* The PCI address space does not equal the physical memory * address space. The networking and block device layers use * this boolean for bounce buffer decisions. @@ -62,7 +52,7 @@ static inline void pci_dma_burst_advice(struct pci_dev *pdev, /* Return the index of the PCI controller for device PDEV. */ -extern int pci_domain_nr(struct pci_bus *bus); +int pci_domain_nr(struct pci_bus *bus); static inline int pci_proc_domain(struct pci_bus *bus) { return 1; @@ -74,30 +64,19 @@ static inline int pci_proc_domain(struct pci_bus *bus) #define HAVE_ARCH_PCI_GET_UNMAPPED_AREA #define get_pci_unmapped_area get_fb_unmapped_area -extern int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state, - int write_combine); - -extern void -pcibios_resource_to_bus(struct pci_dev *dev, struct pci_bus_region *region, - struct resource *res); - -extern void -pcibios_bus_to_resource(struct pci_dev *dev, struct resource *res, - struct pci_bus_region *region); +int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, + enum pci_mmap_state mmap_state, + int write_combine); static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) { return PCI_IRQ_NONE; } -struct device_node; -extern struct device_node *pci_device_to_OF_node(struct pci_dev *pdev); - #define HAVE_ARCH_PCI_RESOURCE_TO_USER -extern void pci_resource_to_user(const struct pci_dev *dev, int bar, - const struct resource *rsrc, - resource_size_t *start, resource_size_t *end); +void pci_resource_to_user(const struct pci_dev *dev, int bar, + const struct resource *rsrc, + resource_size_t *start, resource_size_t *end); #endif /* __KERNEL__ */ #endif /* __SPARC64_PCI_H */ diff --git a/arch/sparc/include/asm/pcic.h b/arch/sparc/include/asm/pcic.h index f20ef562b26..f4170679259 100644 --- a/arch/sparc/include/asm/pcic.h +++ b/arch/sparc/include/asm/pcic.h @@ -29,11 +29,17 @@ struct linux_pcic { int pcic_imdim; }; -extern int pcic_probe(void); -/* Erm... MJ redefined pcibios_present() so that it does not work early. */ -extern int pcic_present(void); -extern void sun4m_pci_init_IRQ(void); - +#ifdef CONFIG_PCIC_PCI +int pcic_present(void); +int pcic_probe(void); +void pci_time_init(void); +void sun4m_pci_init_IRQ(void); +#else +static inline int pcic_present(void) { return 0; } +static inline int pcic_probe(void) { return 0; } +static inline void pci_time_init(void) {} +static inline void sun4m_pci_init_IRQ(void) {} +#endif #endif /* Size of PCI I/O space which we relocate. */ diff --git a/arch/sparc/include/asm/pcr.h b/arch/sparc/include/asm/pcr.h index 843e4faf6a5..cdf800c3326 100644 --- a/arch/sparc/include/asm/pcr.h +++ b/arch/sparc/include/asm/pcr.h @@ -2,13 +2,18 @@ #define __PCR_H struct pcr_ops { - u64 (*read)(void); - void (*write)(u64); + u64 (*read_pcr)(unsigned long); + void (*write_pcr)(unsigned long, u64); + u64 (*read_pic)(unsigned long); + void (*write_pic)(unsigned long, u64); + u64 (*nmi_picl_value)(unsigned int nmi_hz); + u64 pcr_nmi_enable; + u64 pcr_nmi_disable; }; extern const struct pcr_ops *pcr_ops; -extern void deferred_pcr_work_irq(int irq, struct pt_regs *regs); -extern void schedule_deferred_pcr_work(void); +void deferred_pcr_work_irq(int irq, struct pt_regs *regs); +void schedule_deferred_pcr_work(void); #define PCR_PIC_PRIV 0x00000001 /* PIC access is privileged */ #define PCR_STRACE 0x00000002 /* Trace supervisor events */ @@ -27,22 +32,19 @@ extern void schedule_deferred_pcr_work(void); #define PCR_N2_SL1_SHIFT 27 #define PCR_N2_OV1 0x80000000 -extern unsigned int picl_shift; - -/* In order to commonize as much of the implementation as - * possible, we use PICH as our counter. Mostly this is - * to accomodate Niagara-1 which can only count insn cycles - * in PICH. - */ -static inline u64 picl_value(unsigned int nmi_hz) -{ - u32 delta = local_cpu_data().clock_tick / (nmi_hz << picl_shift); - - return ((u64)((0 - delta) & 0xffffffff)) << 32; -} - -extern u64 pcr_enable; - -extern int pcr_arch_init(void); +#define PCR_N4_OV 0x00000001 /* PIC overflow */ +#define PCR_N4_TOE 0x00000002 /* Trap On Event */ +#define PCR_N4_UTRACE 0x00000004 /* Trace user events */ +#define PCR_N4_STRACE 0x00000008 /* Trace supervisor events */ +#define PCR_N4_HTRACE 0x00000010 /* Trace hypervisor events */ +#define PCR_N4_MASK 0x000007e0 /* Event mask */ +#define PCR_N4_MASK_SHIFT 5 +#define PCR_N4_SL 0x0000f800 /* Event Select */ +#define PCR_N4_SL_SHIFT 11 +#define PCR_N4_PICNPT 0x00010000 /* PIC non-privileged trap */ +#define PCR_N4_PICNHT 0x00020000 /* PIC non-hypervisor trap */ +#define PCR_N4_NTC 0x00040000 /* Next-To-Commit wrap */ + +int pcr_arch_init(void); #endif /* __PCR_H */ diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index ca2b34456c4..a3890da9442 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -4,65 +4,76 @@ #include <linux/kernel.h> #include <linux/sched.h> +#include <asm/pgtsrmmu.h> +#include <asm/pgtable.h> +#include <asm/vaddrs.h> #include <asm/page.h> -#include <asm/btfixup.h> struct page; -extern struct pgtable_cache_struct { - unsigned long *pgd_cache; - unsigned long *pte_cache; - unsigned long pgtable_cache_sz; - unsigned long pgd_cache_sz; -} pgt_quicklists; -#define pgd_quicklist (pgt_quicklists.pgd_cache) -#define pmd_quicklist ((unsigned long *)0) -#define pte_quicklist (pgt_quicklists.pte_cache) -#define pgtable_cache_size (pgt_quicklists.pgtable_cache_sz) -#define pgd_cache_size (pgt_quicklists.pgd_cache_sz) +void *srmmu_get_nocache(int size, int align); +void srmmu_free_nocache(void *addr, int size); -extern void check_pgt_cache(void); -BTFIXUPDEF_CALL(void, do_check_pgt_cache, int, int) -#define do_check_pgt_cache(low,high) BTFIXUP_CALL(do_check_pgt_cache)(low,high) +extern struct resource sparc_iomap; -BTFIXUPDEF_CALL(pgd_t *, get_pgd_fast, void) -#define get_pgd_fast() BTFIXUP_CALL(get_pgd_fast)() +#define check_pgt_cache() do { } while (0) -BTFIXUPDEF_CALL(void, free_pgd_fast, pgd_t *) -#define free_pgd_fast(pgd) BTFIXUP_CALL(free_pgd_fast)(pgd) +pgd_t *get_pgd_fast(void); +static inline void free_pgd_fast(pgd_t *pgd) +{ + srmmu_free_nocache(pgd, SRMMU_PGD_TABLE_SIZE); +} #define pgd_free(mm, pgd) free_pgd_fast(pgd) #define pgd_alloc(mm) get_pgd_fast() -BTFIXUPDEF_CALL(void, pgd_set, pgd_t *, pmd_t *) -#define pgd_set(pgdp,pmdp) BTFIXUP_CALL(pgd_set)(pgdp,pmdp) +static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) +{ + unsigned long pa = __nocache_pa((unsigned long)pmdp); + + set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (pa >> 4))); +} + #define pgd_populate(MM, PGD, PMD) pgd_set(PGD, PMD) -BTFIXUPDEF_CALL(pmd_t *, pmd_alloc_one, struct mm_struct *, unsigned long) -#define pmd_alloc_one(mm, address) BTFIXUP_CALL(pmd_alloc_one)(mm, address) +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, + unsigned long address) +{ + return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); +} -BTFIXUPDEF_CALL(void, free_pmd_fast, pmd_t *) -#define free_pmd_fast(pmd) BTFIXUP_CALL(free_pmd_fast)(pmd) +static inline void free_pmd_fast(pmd_t * pmd) +{ + srmmu_free_nocache(pmd, SRMMU_PMD_TABLE_SIZE); +} #define pmd_free(mm, pmd) free_pmd_fast(pmd) #define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) -BTFIXUPDEF_CALL(void, pmd_populate, pmd_t *, struct page *) -#define pmd_populate(MM, PMD, PTE) BTFIXUP_CALL(pmd_populate)(PMD, PTE) +void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep); #define pmd_pgtable(pmd) pmd_page(pmd) -BTFIXUPDEF_CALL(void, pmd_set, pmd_t *, pte_t *) -#define pmd_populate_kernel(MM, PMD, PTE) BTFIXUP_CALL(pmd_set)(PMD, PTE) -BTFIXUPDEF_CALL(pgtable_t , pte_alloc_one, struct mm_struct *, unsigned long) -#define pte_alloc_one(mm, address) BTFIXUP_CALL(pte_alloc_one)(mm, address) -BTFIXUPDEF_CALL(pte_t *, pte_alloc_one_kernel, struct mm_struct *, unsigned long) -#define pte_alloc_one_kernel(mm, addr) BTFIXUP_CALL(pte_alloc_one_kernel)(mm, addr) +void pmd_set(pmd_t *pmdp, pte_t *ptep); +#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) + +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address); + +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long address) +{ + return srmmu_get_nocache(PTE_SIZE, PTE_SIZE); +} + + +static inline void free_pte_fast(pte_t *pte) +{ + srmmu_free_nocache(pte, PTE_SIZE); +} -BTFIXUPDEF_CALL(void, free_pte_fast, pte_t *) -#define pte_free_kernel(mm, pte) BTFIXUP_CALL(free_pte_fast)(pte) +#define pte_free_kernel(mm, pte) free_pte_fast(pte) -BTFIXUPDEF_CALL(void, pte_free, pgtable_t ) -#define pte_free(mm, pte) BTFIXUP_CALL(pte_free)(pte) +void pte_free(struct mm_struct * mm, pgtable_t pte); #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #endif /* _SPARC_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 5bdfa2c6e40..39a7ac49b00 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -5,7 +5,6 @@ #include <linux/sched.h> #include <linux/mm.h> #include <linux/slab.h> -#include <linux/quicklist.h> #include <asm/spitfire.h> #include <asm/cpudata.h> @@ -14,68 +13,82 @@ /* Page table allocation/freeing. */ +extern struct kmem_cache *pgtable_cache; + static inline pgd_t *pgd_alloc(struct mm_struct *mm) { - return quicklist_alloc(0, GFP_KERNEL, NULL); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { - quicklist_free(0, NULL, pgd); + kmem_cache_free(pgtable_cache, pgd); } #define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return quicklist_alloc(0, GFP_KERNEL, NULL); + return kmem_cache_alloc(pgtable_cache, + GFP_KERNEL|__GFP_REPEAT); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) { - quicklist_free(0, NULL, pmd); + kmem_cache_free(pgtable_cache, pmd); } -static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, - unsigned long address) -{ - return quicklist_alloc(0, GFP_KERNEL, NULL); -} +pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long address); +pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address); +void pte_free_kernel(struct mm_struct *mm, pte_t *pte); +void pte_free(struct mm_struct *mm, pgtable_t ptepage); + +#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(MM, PMD, PTE) +#define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) +#define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD)) + +#define check_pgt_cache() do { } while (0) + +void pgtable_free(void *table, bool is_page); + +#ifdef CONFIG_SMP -static inline pgtable_t pte_alloc_one(struct mm_struct *mm, - unsigned long address) +struct mmu_gather; +void tlb_remove_table(struct mmu_gather *, void *); + +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) { - struct page *page; - void *pg; - - pg = quicklist_alloc(0, GFP_KERNEL, NULL); - if (!pg) - return NULL; - page = virt_to_page(pg); - pgtable_page_ctor(page); - return page; + unsigned long pgf = (unsigned long)table; + if (is_page) + pgf |= 0x1UL; + tlb_remove_table(tlb, (void *)pgf); } -static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +static inline void __tlb_remove_table(void *_table) { - quicklist_free(0, NULL, pte); -} + void *table = (void *)((unsigned long)_table & ~0x1UL); + bool is_page = false; -static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) + if ((unsigned long)_table & 0x1UL) + is_page = true; + pgtable_free(table, is_page); +} +#else /* CONFIG_SMP */ +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) { - pgtable_page_dtor(ptepage); - quicklist_free_page(0, NULL, ptepage); + pgtable_free(table, is_page); } +#endif /* !CONFIG_SMP */ - -#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE) -#define pmd_populate(MM,PMD,PTE_PAGE) \ - pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE)) -#define pmd_pgtable(pmd) pmd_page(pmd) - -static inline void check_pgt_cache(void) +static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, + unsigned long address) { - quicklist_trim(0, NULL, 25, 16); + pgtable_free_tlb(tlb, pte, true); } +#define __pmd_free_tlb(tlb, pmd, addr) \ + pgtable_free_tlb(tlb, pmd, false) + #endif /* _SPARC64_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index 303bd4dc829..b9b91ae19fe 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -8,109 +8,77 @@ * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/const.h> + #ifndef __ASSEMBLY__ #include <asm-generic/4level-fixup.h> #include <linux/spinlock.h> #include <linux/swap.h> #include <asm/types.h> -#include <asm/pgtsun4c.h> #include <asm/pgtsrmmu.h> -#include <asm/vac-ops.h> +#include <asm/vaddrs.h> #include <asm/oplib.h> -#include <asm/btfixup.h> -#include <asm/system.h> +#include <asm/cpu_type.h> struct vm_area_struct; struct page; -extern void load_mmu(void); -extern unsigned long calc_highpages(void); - -BTFIXUPDEF_SIMM13(pgdir_shift) -BTFIXUPDEF_SETHI(pgdir_size) -BTFIXUPDEF_SETHI(pgdir_mask) - -BTFIXUPDEF_SIMM13(ptrs_per_pmd) -BTFIXUPDEF_SIMM13(ptrs_per_pgd) -BTFIXUPDEF_SIMM13(user_ptrs_per_pgd) +void load_mmu(void); +unsigned long calc_highpages(void); +unsigned long __init bootmem_init(unsigned long *pages_avail); #define pte_ERROR(e) __builtin_trap() #define pmd_ERROR(e) __builtin_trap() #define pgd_ERROR(e) __builtin_trap() -BTFIXUPDEF_INT(page_none) -BTFIXUPDEF_INT(page_copy) -BTFIXUPDEF_INT(page_readonly) -BTFIXUPDEF_INT(page_kernel) - -#define PMD_SHIFT SUN4C_PMD_SHIFT +#define PMD_SHIFT 22 #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) #define PMD_ALIGN(__addr) (((__addr) + ~PMD_MASK) & PMD_MASK) -#define PGDIR_SHIFT BTFIXUP_SIMM13(pgdir_shift) -#define PGDIR_SIZE BTFIXUP_SETHI(pgdir_size) -#define PGDIR_MASK BTFIXUP_SETHI(pgdir_mask) +#define PGDIR_SHIFT SRMMU_PGDIR_SHIFT +#define PGDIR_SIZE SRMMU_PGDIR_SIZE +#define PGDIR_MASK SRMMU_PGDIR_MASK #define PTRS_PER_PTE 1024 -#define PTRS_PER_PMD BTFIXUP_SIMM13(ptrs_per_pmd) -#define PTRS_PER_PGD BTFIXUP_SIMM13(ptrs_per_pgd) -#define USER_PTRS_PER_PGD BTFIXUP_SIMM13(user_ptrs_per_pgd) +#define PTRS_PER_PMD SRMMU_PTRS_PER_PMD +#define PTRS_PER_PGD SRMMU_PTRS_PER_PGD +#define USER_PTRS_PER_PGD PAGE_OFFSET / SRMMU_PGDIR_SIZE #define FIRST_USER_ADDRESS 0 #define PTE_SIZE (PTRS_PER_PTE*4) -#define PAGE_NONE __pgprot(BTFIXUP_INT(page_none)) -extern pgprot_t PAGE_SHARED; -#define PAGE_COPY __pgprot(BTFIXUP_INT(page_copy)) -#define PAGE_READONLY __pgprot(BTFIXUP_INT(page_readonly)) +#define PAGE_NONE SRMMU_PAGE_NONE +#define PAGE_SHARED SRMMU_PAGE_SHARED +#define PAGE_COPY SRMMU_PAGE_COPY +#define PAGE_READONLY SRMMU_PAGE_RDONLY +#define PAGE_KERNEL SRMMU_PAGE_KERNEL -extern unsigned long page_kernel; +/* Top-level page directory - dummy used by init-mm. + * srmmu.c will assign the real one (which is dynamically sized) */ +#define swapper_pg_dir NULL -#ifdef MODULE -#define PAGE_KERNEL page_kernel -#else -#define PAGE_KERNEL __pgprot(BTFIXUP_INT(page_kernel)) -#endif - -/* Top-level page directory */ -extern pgd_t swapper_pg_dir[1024]; - -extern void paging_init(void); - -/* Page table for 0-4MB for everybody, on the Sparc this - * holds the same as on the i386. - */ -extern pte_t pg0[1024]; -extern pte_t pg1[1024]; -extern pte_t pg2[1024]; -extern pte_t pg3[1024]; +void paging_init(void); extern unsigned long ptr_in_current_pgd; -/* Here is a trick, since mmap.c need the initializer elements for - * protection_map[] to be constant at compile time, I set the following - * to all zeros. I set it to the real values after I link in the - * appropriate MMU page table routines at boot time. - */ -#define __P000 __pgprot(0) -#define __P001 __pgprot(0) -#define __P010 __pgprot(0) -#define __P011 __pgprot(0) -#define __P100 __pgprot(0) -#define __P101 __pgprot(0) -#define __P110 __pgprot(0) -#define __P111 __pgprot(0) - -#define __S000 __pgprot(0) -#define __S001 __pgprot(0) -#define __S010 __pgprot(0) -#define __S011 __pgprot(0) -#define __S100 __pgprot(0) -#define __S101 __pgprot(0) -#define __S110 __pgprot(0) -#define __S111 __pgprot(0) - -extern int num_contexts; +/* xwr */ +#define __P000 PAGE_NONE +#define __P001 PAGE_READONLY +#define __P010 PAGE_COPY +#define __P011 PAGE_COPY +#define __P100 PAGE_READONLY +#define __P101 PAGE_READONLY +#define __P110 PAGE_COPY +#define __P111 PAGE_COPY + +#define __S000 PAGE_NONE +#define __S001 PAGE_READONLY +#define __S010 PAGE_SHARED +#define __S011 PAGE_SHARED +#define __S100 PAGE_READONLY +#define __S101 PAGE_READONLY +#define __S110 PAGE_SHARED +#define __S111 PAGE_SHARED /* First physical page can be anywhere, the following is needed so that * va-->pa and vice versa conversions work properly without performance @@ -120,97 +88,145 @@ extern unsigned long phys_base; extern unsigned long pfn_base; /* - * BAD_PAGETABLE is used when we need a bogus page-table, while - * BAD_PAGE is used for a bogus page. - * * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -extern pte_t * __bad_pagetable(void); -extern pte_t __bad_page(void); extern unsigned long empty_zero_page; -#define BAD_PAGETABLE __bad_pagetable() -#define BAD_PAGE __bad_page() #define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page)) /* + * In general all page table modifications should use the V8 atomic + * swap instruction. This insures the mmu and the cpu are in sync + * with respect to ref/mod bits in the page tables. */ -BTFIXUPDEF_CALL_CONST(struct page *, pmd_page, pmd_t) -BTFIXUPDEF_CALL_CONST(unsigned long, pgd_page_vaddr, pgd_t) +static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value) +{ + __asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr)); + return value; +} -#define pmd_page(pmd) BTFIXUP_CALL(pmd_page)(pmd) -#define pgd_page_vaddr(pgd) BTFIXUP_CALL(pgd_page_vaddr)(pgd) +/* Certain architectures need to do special things when pte's + * within a page table are directly modified. Thus, the following + * hook is made available. + */ + +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + srmmu_swap((unsigned long *)ptep, pte_val(pteval)); +} + +#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) + +static inline int srmmu_device_memory(unsigned long x) +{ + return ((x & 0xF0000000) != 0); +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + if (srmmu_device_memory(pmd_val(pmd))) + BUG(); + return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); +} + +static inline unsigned long pgd_page_vaddr(pgd_t pgd) +{ + if (srmmu_device_memory(pgd_val(pgd))) { + return ~0; + } else { + unsigned long v = pgd_val(pgd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); + } +} -BTFIXUPDEF_CALL_CONST(int, pte_present, pte_t) -BTFIXUPDEF_CALL(void, pte_clear, pte_t *) +static inline int pte_present(pte_t pte) +{ + return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); +} static inline int pte_none(pte_t pte) { return !pte_val(pte); } -#define pte_present(pte) BTFIXUP_CALL(pte_present)(pte) -#define pte_clear(mm,addr,pte) BTFIXUP_CALL(pte_clear)(pte) +static inline void __pte_clear(pte_t *ptep) +{ + set_pte(ptep, __pte(0)); +} + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + __pte_clear(ptep); +} + +static inline int pmd_bad(pmd_t pmd) +{ + return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; +} -BTFIXUPDEF_CALL_CONST(int, pmd_bad, pmd_t) -BTFIXUPDEF_CALL_CONST(int, pmd_present, pmd_t) -BTFIXUPDEF_CALL(void, pmd_clear, pmd_t *) +static inline int pmd_present(pmd_t pmd) +{ + return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); +} static inline int pmd_none(pmd_t pmd) { return !pmd_val(pmd); } -#define pmd_bad(pmd) BTFIXUP_CALL(pmd_bad)(pmd) -#define pmd_present(pmd) BTFIXUP_CALL(pmd_present)(pmd) -#define pmd_clear(pmd) BTFIXUP_CALL(pmd_clear)(pmd) +static inline void pmd_clear(pmd_t *pmdp) +{ + int i; + for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) + set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); +} -BTFIXUPDEF_CALL_CONST(int, pgd_none, pgd_t) -BTFIXUPDEF_CALL_CONST(int, pgd_bad, pgd_t) -BTFIXUPDEF_CALL_CONST(int, pgd_present, pgd_t) -BTFIXUPDEF_CALL(void, pgd_clear, pgd_t *) +static inline int pgd_none(pgd_t pgd) +{ + return !(pgd_val(pgd) & 0xFFFFFFF); +} -#define pgd_none(pgd) BTFIXUP_CALL(pgd_none)(pgd) -#define pgd_bad(pgd) BTFIXUP_CALL(pgd_bad)(pgd) -#define pgd_present(pgd) BTFIXUP_CALL(pgd_present)(pgd) -#define pgd_clear(pgd) BTFIXUP_CALL(pgd_clear)(pgd) +static inline int pgd_bad(pgd_t pgd) +{ + return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; +} + +static inline int pgd_present(pgd_t pgd) +{ + return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); +} + +static inline void pgd_clear(pgd_t *pgdp) +{ + set_pte((pte_t *)pgdp, __pte(0)); +} /* * The following only work if pte_present() is true. * Undefined behaviour if not.. */ -BTFIXUPDEF_HALF(pte_writei) -BTFIXUPDEF_HALF(pte_dirtyi) -BTFIXUPDEF_HALF(pte_youngi) - -static int pte_write(pte_t pte) __attribute_const__; static inline int pte_write(pte_t pte) { - return pte_val(pte) & BTFIXUP_HALF(pte_writei); + return pte_val(pte) & SRMMU_WRITE; } -static int pte_dirty(pte_t pte) __attribute_const__; static inline int pte_dirty(pte_t pte) { - return pte_val(pte) & BTFIXUP_HALF(pte_dirtyi); + return pte_val(pte) & SRMMU_DIRTY; } -static int pte_young(pte_t pte) __attribute_const__; static inline int pte_young(pte_t pte) { - return pte_val(pte) & BTFIXUP_HALF(pte_youngi); + return pte_val(pte) & SRMMU_REF; } /* * The following only work if pte_present() is not true. */ -BTFIXUPDEF_HALF(pte_filei) - -static int pte_file(pte_t pte) __attribute_const__; static inline int pte_file(pte_t pte) { - return pte_val(pte) & BTFIXUP_HALF(pte_filei); + return pte_val(pte) & SRMMU_FILE; } static inline int pte_special(pte_t pte) @@ -218,68 +234,85 @@ static inline int pte_special(pte_t pte) return 0; } -/* - */ -BTFIXUPDEF_HALF(pte_wrprotecti) -BTFIXUPDEF_HALF(pte_mkcleani) -BTFIXUPDEF_HALF(pte_mkoldi) - -static pte_t pte_wrprotect(pte_t pte) __attribute_const__; static inline pte_t pte_wrprotect(pte_t pte) { - return __pte(pte_val(pte) & ~BTFIXUP_HALF(pte_wrprotecti)); + return __pte(pte_val(pte) & ~SRMMU_WRITE); } -static pte_t pte_mkclean(pte_t pte) __attribute_const__; static inline pte_t pte_mkclean(pte_t pte) { - return __pte(pte_val(pte) & ~BTFIXUP_HALF(pte_mkcleani)); + return __pte(pte_val(pte) & ~SRMMU_DIRTY); } -static pte_t pte_mkold(pte_t pte) __attribute_const__; static inline pte_t pte_mkold(pte_t pte) { - return __pte(pte_val(pte) & ~BTFIXUP_HALF(pte_mkoldi)); + return __pte(pte_val(pte) & ~SRMMU_REF); } -BTFIXUPDEF_CALL_CONST(pte_t, pte_mkwrite, pte_t) -BTFIXUPDEF_CALL_CONST(pte_t, pte_mkdirty, pte_t) -BTFIXUPDEF_CALL_CONST(pte_t, pte_mkyoung, pte_t) +static inline pte_t pte_mkwrite(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_WRITE); +} -#define pte_mkwrite(pte) BTFIXUP_CALL(pte_mkwrite)(pte) -#define pte_mkdirty(pte) BTFIXUP_CALL(pte_mkdirty)(pte) -#define pte_mkyoung(pte) BTFIXUP_CALL(pte_mkyoung)(pte) +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_REF); +} #define pte_mkspecial(pte) (pte) #define pfn_pte(pfn, prot) mk_pte(pfn_to_page(pfn), prot) -BTFIXUPDEF_CALL(unsigned long, pte_pfn, pte_t) -#define pte_pfn(pte) BTFIXUP_CALL(pte_pfn)(pte) +static inline unsigned long pte_pfn(pte_t pte) +{ + if (srmmu_device_memory(pte_val(pte))) { + /* Just return something that will cause + * pfn_valid() to return false. This makes + * copy_one_pte() to just directly copy to + * PTE over. + */ + return ~0UL; + } + return (pte_val(pte) & SRMMU_PTE_PMASK) >> (PAGE_SHIFT-4); +} + #define pte_page(pte) pfn_to_page(pte_pfn(pte)) /* * Conversion functions: convert a page and protection to a page entry, * and a page entry and page directory to the page they refer to. */ -BTFIXUPDEF_CALL_CONST(pte_t, mk_pte, struct page *, pgprot_t) - -BTFIXUPDEF_CALL_CONST(pte_t, mk_pte_phys, unsigned long, pgprot_t) -BTFIXUPDEF_CALL_CONST(pte_t, mk_pte_io, unsigned long, pgprot_t, int) -BTFIXUPDEF_CALL_CONST(pgprot_t, pgprot_noncached, pgprot_t) +static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) +{ + return __pte((page_to_pfn(page) << (PAGE_SHIFT-4)) | pgprot_val(pgprot)); +} -#define mk_pte(page,pgprot) BTFIXUP_CALL(mk_pte)(page,pgprot) -#define mk_pte_phys(page,pgprot) BTFIXUP_CALL(mk_pte_phys)(page,pgprot) -#define mk_pte_io(page,pgprot,space) BTFIXUP_CALL(mk_pte_io)(page,pgprot,space) +static inline pte_t mk_pte_phys(unsigned long page, pgprot_t pgprot) +{ + return __pte(((page) >> 4) | pgprot_val(pgprot)); +} -#define pgprot_noncached(pgprot) BTFIXUP_CALL(pgprot_noncached)(pgprot) +static inline pte_t mk_pte_io(unsigned long page, pgprot_t pgprot, int space) +{ + return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); +} -BTFIXUPDEF_INT(pte_modify_mask) +#define pgprot_noncached pgprot_noncached +static inline pgprot_t pgprot_noncached(pgprot_t prot) +{ + prot &= ~__pgprot(SRMMU_CACHE); + return prot; +} static pte_t pte_modify(pte_t pte, pgprot_t newprot) __attribute_const__; static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - return __pte((pte_val(pte) & BTFIXUP_INT(pte_modify_mask)) | + return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); } @@ -292,114 +325,76 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pgd_offset_k(address) pgd_offset(&init_mm, address) /* Find an entry in the second-level page table.. */ -BTFIXUPDEF_CALL(pmd_t *, pmd_offset, pgd_t *, unsigned long) -#define pmd_offset(dir,addr) BTFIXUP_CALL(pmd_offset)(dir,addr) +static inline pmd_t *pmd_offset(pgd_t * dir, unsigned long address) +{ + return (pmd_t *) pgd_page_vaddr(*dir) + + ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); +} /* Find an entry in the third-level page table.. */ -BTFIXUPDEF_CALL(pte_t *, pte_offset_kernel, pmd_t *, unsigned long) -#define pte_offset_kernel(dir,addr) BTFIXUP_CALL(pte_offset_kernel)(dir,addr) +pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address); /* - * This shortcut works on sun4m (and sun4d) because the nocache area is static, - * and sun4c is guaranteed to have no highmem anyway. + * This shortcut works on sun4m (and sun4d) because the nocache area is static. */ #define pte_offset_map(d, a) pte_offset_kernel(d,a) #define pte_unmap(pte) do{}while(0) -/* Certain architectures need to do special things when pte's - * within a page table are directly modified. Thus, the following - * hook is made available. - */ - -BTFIXUPDEF_CALL(void, set_pte, pte_t *, pte_t) - -#define set_pte(ptep,pteval) BTFIXUP_CALL(set_pte)(ptep,pteval) -#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval) - struct seq_file; -BTFIXUPDEF_CALL(void, mmu_info, struct seq_file *) - -#define mmu_info(p) BTFIXUP_CALL(mmu_info)(p) +void mmu_info(struct seq_file *m); /* Fault handler stuff... */ #define FAULT_CODE_PROT 0x1 #define FAULT_CODE_WRITE 0x2 #define FAULT_CODE_USER 0x4 -BTFIXUPDEF_CALL(void, update_mmu_cache, struct vm_area_struct *, unsigned long, pte_t *) - -#define update_mmu_cache(vma,addr,ptep) BTFIXUP_CALL(update_mmu_cache)(vma,addr,ptep) - -BTFIXUPDEF_CALL(void, sparc_mapiorange, unsigned int, unsigned long, - unsigned long, unsigned int) -BTFIXUPDEF_CALL(void, sparc_unmapiorange, unsigned long, unsigned int) -#define sparc_mapiorange(bus,pa,va,len) BTFIXUP_CALL(sparc_mapiorange)(bus,pa,va,len) -#define sparc_unmapiorange(va,len) BTFIXUP_CALL(sparc_unmapiorange)(va,len) +#define update_mmu_cache(vma, address, ptep) do { } while (0) -extern int invalid_segment; +void srmmu_mapiorange(unsigned int bus, unsigned long xpa, + unsigned long xva, unsigned int len); +void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len); /* Encode and de-code a swap entry */ -BTFIXUPDEF_CALL(unsigned long, __swp_type, swp_entry_t) -BTFIXUPDEF_CALL(unsigned long, __swp_offset, swp_entry_t) -BTFIXUPDEF_CALL(swp_entry_t, __swp_entry, unsigned long, unsigned long) +static inline unsigned long __swp_type(swp_entry_t entry) +{ + return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK; +} + +static inline unsigned long __swp_offset(swp_entry_t entry) +{ + return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK; +} -#define __swp_type(__x) BTFIXUP_CALL(__swp_type)(__x) -#define __swp_offset(__x) BTFIXUP_CALL(__swp_offset)(__x) -#define __swp_entry(__type,__off) BTFIXUP_CALL(__swp_entry)(__type,__off) +static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) +{ + return (swp_entry_t) { + (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT + | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT }; +} #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) /* file-offset-in-pte helpers */ -BTFIXUPDEF_CALL(unsigned long, pte_to_pgoff, pte_t pte); -BTFIXUPDEF_CALL(pte_t, pgoff_to_pte, unsigned long pgoff); +static inline unsigned long pte_to_pgoff(pte_t pte) +{ + return pte_val(pte) >> SRMMU_PTE_FILE_SHIFT; +} -#define pte_to_pgoff(pte) BTFIXUP_CALL(pte_to_pgoff)(pte) -#define pgoff_to_pte(off) BTFIXUP_CALL(pgoff_to_pte)(off) +static inline pte_t pgoff_to_pte(unsigned long pgoff) +{ + return __pte((pgoff << SRMMU_PTE_FILE_SHIFT) | SRMMU_FILE); +} /* * This is made a constant because mm/fremap.c required a constant. - * Note that layout of these bits is different between sun4c.c and srmmu.c. */ #define PTE_FILE_MAX_BITS 24 -/* - */ -struct ctx_list { - struct ctx_list *next; - struct ctx_list *prev; - unsigned int ctx_number; - struct mm_struct *ctx_mm; -}; - -extern struct ctx_list *ctx_list_pool; /* Dynamically allocated */ -extern struct ctx_list ctx_free; /* Head of free list */ -extern struct ctx_list ctx_used; /* Head of used contexts list */ - -#define NO_CONTEXT -1 - -static inline void remove_from_ctx_list(struct ctx_list *entry) -{ - entry->next->prev = entry->prev; - entry->prev->next = entry->next; -} - -static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) -{ - entry->next = head; - (entry->prev = head->prev)->next = entry; - head->prev = entry; -} -#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) -#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) - static inline unsigned long __get_phys (unsigned long addr) { switch (sparc_cpu_model){ - case sun4: - case sun4c: - return sun4c_get_pte (addr) << PAGE_SHIFT; case sun4m: case sun4d: return ((srmmu_get_pte (addr) & 0xffffff00) << 4); @@ -412,9 +407,6 @@ static inline int __get_iospace (unsigned long addr) { switch (sparc_cpu_model){ - case sun4: - case sun4c: - return -1; /* Don't check iospace on sun4c */ case sun4m: case sun4d: return (srmmu_get_pte (addr) >> 28); @@ -429,10 +421,6 @@ extern unsigned long *sparc_valid_addr_bitmap; #define kern_addr_valid(addr) \ (test_bit(__pa((unsigned long)(addr))>>20, sparc_valid_addr_bitmap)) -extern int io_remap_pfn_range(struct vm_area_struct *vma, - unsigned long from, unsigned long pfn, - unsigned long size, pgprot_t prot); - /* * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in * its high 4 bits. These macros/functions put it there or get it from there. @@ -441,6 +429,23 @@ extern int io_remap_pfn_range(struct vm_area_struct *vma, #define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) #define GET_PFN(pfn) (pfn & 0x0fffffffUL) +int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long, + unsigned long, pgprot_t); + +static inline int io_remap_pfn_range(struct vm_area_struct *vma, + unsigned long from, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + unsigned long long offset, space, phys_base; + + offset = ((unsigned long long) GET_PFN(pfn)) << PAGE_SHIFT; + space = GET_IOSPACE(pfn); + phys_base = offset | (space << 32ULL); + + return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); +} +#define io_remap_pfn_range io_remap_pfn_range + #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ ({ \ @@ -449,17 +454,15 @@ extern int io_remap_pfn_range(struct vm_area_struct *vma, set_pte_at((__vma)->vm_mm, (__address), __ptep, __entry); \ flush_tlb_page(__vma, __address); \ } \ - (sparc_cpu_model == sun4c) || __changed; \ + __changed; \ }) #include <asm-generic/pgtable.h> #endif /* !(__ASSEMBLY__) */ -#define VMALLOC_START 0xfe600000 -/* XXX Alter this when I get around to fixing sun4c - Anton */ -#define VMALLOC_END 0xffc00000 - +#define VMALLOC_START _AC(0xfe600000,UL) +#define VMALLOC_END _AC(0xffc00000,UL) /* We provide our own get_unmapped_area to cope with VA holes for userland */ #define HAVE_ARCH_UNMAPPED_AREA diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index f8dddb7045b..3770bf5c6e1 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -12,20 +12,20 @@ * the SpitFire page tables. */ -#include <asm-generic/pgtable-nopud.h> - #include <linux/compiler.h> #include <linux/const.h> #include <asm/types.h> #include <asm/spitfire.h> #include <asm/asi.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/processor.h> +#include <asm-generic/pgtable-nopud.h> + /* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB). * The page copy blockops can use 0x6000000 to 0x8000000. - * The TSB is mapped in the 0x8000000 to 0xa000000 range. + * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range. + * The 4M TSB is mapped in the 0x8400000 to 0x8800000 range. * The PROM resides in an area spanning 0xf0000000 to 0x100000000. * The vmalloc area spans 0x100000000 to 0x200000000. * Since modules need to be in the lowest 32-bits of the address space, @@ -34,7 +34,8 @@ * 0x400000000. */ #define TLBTEMP_BASE _AC(0x0000000006000000,UL) -#define TSBMAP_BASE _AC(0x0000000008000000,UL) +#define TSBMAP_8K_BASE _AC(0x0000000008000000,UL) +#define TSBMAP_4M_BASE _AC(0x0000000008400000,UL) #define MODULES_VADDR _AC(0x0000000010000000,UL) #define MODULES_LEN _AC(0x00000000e0000000,UL) #define MODULES_END _AC(0x00000000f0000000,UL) @@ -46,38 +47,49 @@ #define vmemmap ((struct page *)VMEMMAP_BASE) -/* XXX All of this needs to be rethought so we can take advantage - * XXX cheetah's full 64-bit virtual address space, ie. no more hole - * XXX in the middle like on spitfire. -DaveM - */ -/* - * Given a virtual address, the lowest PAGE_SHIFT bits determine offset - * into the page; the next higher PAGE_SHIFT-3 bits determine the pte# - * in the proper pagetable (the -3 is from the 8 byte ptes, and each page - * table is a single page long). The next higher PMD_BITS determine pmd# - * in the proper pmdtable (where we must have PMD_BITS <= (PAGE_SHIFT-2) - * since the pmd entries are 4 bytes, and each pmd page is a single page - * long). Finally, the higher few bits determine pgde#. - */ - /* PMD_SHIFT determines the size of the area a second-level page * table can map */ #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3)) #define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -#define PMD_BITS (PAGE_SHIFT - 2) +#define PMD_BITS (PAGE_SHIFT - 3) /* PGDIR_SHIFT determines what a third-level page table entry can map */ #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3) + PMD_BITS) #define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define PGDIR_BITS (PAGE_SHIFT - 2) +#define PGDIR_BITS (PAGE_SHIFT - 3) + +#if (PGDIR_SHIFT + PGDIR_BITS) != 43 +#error Page table parameters do not cover virtual address space properly. +#endif + +#if (PMD_SHIFT != HPAGE_SHIFT) +#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. +#endif #ifndef __ASSEMBLY__ #include <linux/sched.h> +extern unsigned long sparc64_valid_addr_bitmap[]; + +/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ +static inline bool __kern_addr_valid(unsigned long paddr) +{ + if ((paddr >> MAX_PHYS_ADDRESS_BITS) != 0UL) + return false; + return test_bit(paddr >> ILOG2_4MB, sparc64_valid_addr_bitmap); +} + +static inline bool kern_addr_valid(unsigned long addr) +{ + unsigned long paddr = __pa(addr); + + return __kern_addr_valid(paddr); +} + /* Entries per page directory level. */ #define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) #define PTRS_PER_PMD (1UL << PMD_BITS) @@ -86,15 +98,23 @@ /* Kernel has a separate 44bit address space. */ #define FIRST_USER_ADDRESS 0 -#define pte_ERROR(e) __builtin_trap() -#define pmd_ERROR(e) __builtin_trap() -#define pgd_ERROR(e) __builtin_trap() +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) #endif /* !(__ASSEMBLY__) */ /* PTE bits which are the same in SUN4U and SUN4V format. */ #define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ #define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ +#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ + +/* Advertise support for _PAGE_SPECIAL */ +#define __HAVE_ARCH_PTE_SPECIAL /* SUN4U pte bits... */ #define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ @@ -104,6 +124,8 @@ #define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */ #define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ #define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ +#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */ #define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ #define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ #define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ @@ -133,6 +155,8 @@ #define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */ #define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ #define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ +#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */ #define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ #define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ #define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ @@ -155,26 +179,15 @@ #define _PAGE_SZ8K_4V _AC(0x0000000000000000,UL) /* 8K Page */ #define _PAGE_SZALL_4V _AC(0x0000000000000007,UL) /* All pgsz bits */ -#if PAGE_SHIFT == 13 #define _PAGE_SZBITS_4U _PAGE_SZ8K_4U #define _PAGE_SZBITS_4V _PAGE_SZ8K_4V -#elif PAGE_SHIFT == 16 -#define _PAGE_SZBITS_4U _PAGE_SZ64K_4U -#define _PAGE_SZBITS_4V _PAGE_SZ64K_4V -#else -#error Wrong PAGE_SHIFT specified + +#if REAL_HPAGE_SHIFT != 22 +#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up #endif -#if defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) #define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U #define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define _PAGE_SZHUGE_4U _PAGE_SZ512K_4U -#define _PAGE_SZHUGE_4V _PAGE_SZ512K_4V -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define _PAGE_SZHUGE_4U _PAGE_SZ64K_4U -#define _PAGE_SZHUGE_4V _PAGE_SZ64K_4V -#endif /* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */ #define __P000 __pgprot(0) @@ -197,9 +210,9 @@ #ifndef __ASSEMBLY__ -extern pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long); +pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long); -extern unsigned long pte_sz_bits(unsigned long size); +unsigned long pte_sz_bits(unsigned long size); extern pgprot_t PAGE_KERNEL; extern pgprot_t PAGE_KERNEL_LOCKED; @@ -213,7 +226,6 @@ extern unsigned long _PAGE_CACHE; extern unsigned long pg_iobits; extern unsigned long _PAGE_ALL_SZ_BITS; -extern unsigned long _PAGE_SZBITS; extern struct page *mem_map_zero; #define ZERO_PAGE(vaddr) (mem_map_zero) @@ -226,25 +238,22 @@ extern struct page *mem_map_zero; static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { unsigned long paddr = pfn << PAGE_SHIFT; - unsigned long sz_bits; - - sz_bits = 0UL; - if (_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL) { - __asm__ __volatile__( - "\n661: sethi %%uhi(%1), %0\n" - " sllx %0, 32, %0\n" - " .section .sun4v_2insn_patch, \"ax\"\n" - " .word 661b\n" - " mov %2, %0\n" - " nop\n" - " .previous\n" - : "=r" (sz_bits) - : "i" (_PAGE_SZBITS_4U), "i" (_PAGE_SZBITS_4V)); - } - return __pte(paddr | sz_bits | pgprot_val(prot)); + + BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL); + return __pte(paddr | pgprot_val(prot)); } #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) +{ + pte_t pte = pfn_pte(page_nr, pgprot); + + return __pmd(pte_val(pte)); +} +#define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) +#endif + /* This one can be done with two shifts. */ static inline unsigned long pte_pfn(pte_t pte) { @@ -271,8 +280,8 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) { unsigned long mask, tmp; - /* SUN4U: 0x600307ffffffecb8 (negated == 0x9ffcf80000001347) - * SUN4V: 0x30ffffffffffee17 (negated == 0xcf000000000011e8) + /* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7) + * SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8) * * Even if we use negation tricks the result is still a 6 * instruction sequence, so don't try to play fancy and just @@ -281,6 +290,7 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) * Note: We encode this into 3 sun4v 2-insn patch sequences. */ + BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL); __asm__ __volatile__( "\n661: sethi %%uhi(%2), %1\n" " sethi %%hi(%2), %0\n" @@ -301,15 +311,26 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t prot) " .previous\n" : "=r" (mask), "=r" (tmp) : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | - _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U | - _PAGE_SZBITS_4U), + _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U), "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | - _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V | - _PAGE_SZBITS_4V)); + _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V)); return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_modify(pte, newprot); + + return __pmd(pte_val(pte)); +} +#endif + static inline pte_t pgoff_to_pte(unsigned long off) { off <<= PAGE_SHIFT; @@ -350,7 +371,7 @@ static inline pgprot_t pgprot_noncached(pgprot_t prot) */ #define pgprot_noncached pgprot_noncached -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) static inline pte_t pte_mkhuge(pte_t pte) { unsigned long mask; @@ -368,6 +389,17 @@ static inline pte_t pte_mkhuge(pte_t pte) return __pte(pte_val(pte) | mask); } +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkhuge(pte); + pte_val(pte) |= _PAGE_PMD_HUGE; + + return __pmd(pte_val(pte)); +} +#endif #endif static inline pte_t pte_mkdirty(pte_t pte) @@ -502,6 +534,7 @@ static inline pte_t pte_mkyoung(pte_t pte) static inline pte_t pte_mkspecial(pte_t pte) { + pte_val(pte) |= _PAGE_SPECIAL; return pte; } @@ -607,29 +640,181 @@ static inline unsigned long pte_present(pte_t pte) return val; } -static inline int pte_special(pte_t pte) +#define pte_accessible pte_accessible +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) +{ + return pte_val(a) & _PAGE_VALID; +} + +static inline unsigned long pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +static inline unsigned long pmd_large(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline unsigned long pmd_young(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_young(pte); +} + +static inline unsigned long pmd_write(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_write(pte); +} + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_pfn(pte); +} + +static inline unsigned long pmd_trans_huge(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} + +static inline unsigned long pmd_trans_splitting(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pmd_trans_huge(pmd) && pte_special(pte); +} + +#define has_transparent_hugepage() 1 + +static inline pmd_t pmd_mkold(pmd_t pmd) { - return 0; + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkold(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_wrprotect(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkdirty(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkyoung(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkwrite(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkwrite(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mksplitting(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkspecial(pte); + + return __pmd(pte_val(pte)); +} + +static inline pgprot_t pmd_pgprot(pmd_t entry) +{ + unsigned long val = pmd_val(entry); + + return __pgprot(val); +} +#endif + +static inline int pmd_present(pmd_t pmd) +{ + return pmd_val(pmd) != 0UL; +} + +#define pmd_none(pmd) (!pmd_val(pmd)) + +/* pmd_bad() is only called on non-trans-huge PMDs. Our encoding is + * very simple, it's just the physical address. PTE tables are of + * size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and + * the top bits outside of the range of any physical address size we + * support are clear as well. We also validate the physical itself. + */ +#define pmd_bad(pmd) ((pmd_val(pmd) & ~PAGE_MASK) || \ + !__kern_addr_valid(pmd_val(pmd))) + +#define pud_none(pud) (!pud_val(pud)) + +#define pud_bad(pud) ((pud_val(pud) & ~PAGE_MASK) || \ + !__kern_addr_valid(pud_val(pud))) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); +#else +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} +#endif + +static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) +{ + unsigned long val = __pa((unsigned long) (ptep)); + + pmd_val(*pmdp) = val; } -#define pmd_set(pmdp, ptep) \ - (pmd_val(*(pmdp)) = (__pa((unsigned long) (ptep)) >> 11UL)) #define pud_set(pudp, pmdp) \ - (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)) >> 11UL)) -#define __pmd_page(pmd) \ - ((unsigned long) __va((((unsigned long)pmd_val(pmd))<<11UL))) + (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)))) +static inline unsigned long __pmd_page(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + unsigned long pfn; + + pfn = pte_pfn(pte); + + return ((unsigned long) __va(pfn << PAGE_SHIFT)); +} #define pmd_page(pmd) virt_to_page((void *)__pmd_page(pmd)) #define pud_page_vaddr(pud) \ - ((unsigned long) __va((((unsigned long)pud_val(pud))<<11UL))) + ((unsigned long) __va(pud_val(pud))) #define pud_page(pud) virt_to_page((void *)pud_page_vaddr(pud)) -#define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_bad(pmd) (0) -#define pmd_present(pmd) (pmd_val(pmd) != 0U) -#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0U) -#define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (0) +#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) #define pud_present(pud) (pud_val(pud) != 0U) -#define pud_clear(pudp) (pud_val(*(pudp)) = 0U) +#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) /* Same in both SUN4V and SUN4U. */ #define pte_none(pte) (!pte_val(pte)) @@ -655,9 +840,21 @@ static inline int pte_special(pte_t pte) #define pte_unmap(pte) do { } while (0) /* Actual page table PTE updates. */ -extern void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t orig); +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm); + +#define __HAVE_ARCH_PMDP_GET_AND_CLEAR +static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, + unsigned long addr, + pmd_t *pmdp) +{ + pmd_t pmd = *pmdp; + set_pmd_at(mm, addr, pmdp, __pmd(0UL)); + return pmd; +} -static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int fullmm) { pte_t orig = *ptep; @@ -669,13 +866,20 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *p * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U * and SUN4V pte layout, so this inline test is fine. */ - if (likely(mm != &init_mm) && (pte_val(orig) & _PAGE_VALID)) - tlb_batch_add(mm, addr, ptep, orig); + if (likely(mm != &init_mm) && pte_accessible(mm, orig)) + tlb_batch_add(mm, addr, ptep, orig, fullmm); } +#define set_pte_at(mm,addr,ptep,pte) \ + __set_pte_at((mm), (addr), (ptep), (pte), 0) + #define pte_clear(mm,addr,ptep) \ set_pte_at((mm), (addr), (ptep), __pte(0UL)) +#define __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL +#define pte_clear_not_present_full(mm,addr,ptep,fullmm) \ + __set_pte_at((mm), (addr), (ptep), __pte(0UL), (fullmm)) + #ifdef DCACHE_ALIASING_POSSIBLE #define __HAVE_ARCH_MOVE_PTE #define move_pte(pte, prot, old_addr, new_addr) \ @@ -693,18 +897,32 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *p }) #endif -extern pgd_t swapper_pg_dir[2048]; -extern pmd_t swapper_low_pmd_dir[2048]; +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; +extern pmd_t swapper_low_pmd_dir[PTRS_PER_PMD]; -extern void paging_init(void); -extern unsigned long find_ecache_flush_span(unsigned long size); +void paging_init(void); +unsigned long find_ecache_flush_span(unsigned long size); -/* These do nothing with the way I have things setup. */ -#define mmu_lockarea(vaddr, len) (vaddr) -#define mmu_unlockarea(vaddr, len) do { } while(0) +struct seq_file; +void mmu_info(struct seq_file *); struct vm_area_struct; -extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); +void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd); + +#define __HAVE_ARCH_PMDP_INVALIDATE +extern void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); + +#define __HAVE_ARCH_PGTABLE_DEPOSIT +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); + +#define __HAVE_ARCH_PGTABLE_WITHDRAW +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +#endif /* Encode and de-code a swap entry */ #define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0xffUL) @@ -719,28 +937,12 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t *); #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) /* File offset in PTE support. */ -extern unsigned long pte_file(pte_t); +unsigned long pte_file(pte_t); #define pte_to_pgoff(pte) (pte_val(pte) >> PAGE_SHIFT) -extern pte_t pgoff_to_pte(unsigned long); +pte_t pgoff_to_pte(unsigned long); #define PTE_FILE_MAX_BITS (64UL - PAGE_SHIFT - 1UL) -extern unsigned long sparc64_valid_addr_bitmap[]; - -/* Needs to be defined here and not in linux/mm.h, as it is arch dependent */ -static inline bool kern_addr_valid(unsigned long addr) -{ - unsigned long paddr = __pa(addr); - - if ((paddr >> 41UL) != 0UL) - return false; - return test_bit(paddr >> 22, sparc64_valid_addr_bitmap); -} - -extern int page_in_phys_avail(unsigned long paddr); - -extern int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, - unsigned long pfn, - unsigned long size, pgprot_t prot); +int page_in_phys_avail(unsigned long paddr); /* * For sparc32&64, the pfn in io_remap_pfn_range() carries <iospace> in @@ -750,6 +952,24 @@ extern int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, #define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) #define GET_PFN(pfn) (pfn & 0x0fffffffffffffffUL) +int remap_pfn_range(struct vm_area_struct *, unsigned long, unsigned long, + unsigned long, pgprot_t); + +static inline int io_remap_pfn_range(struct vm_area_struct *vma, + unsigned long from, unsigned long pfn, + unsigned long size, pgprot_t prot) +{ + unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; + int space = GET_IOSPACE(pfn); + unsigned long phys_base; + + phys_base = offset | (((unsigned long) space) << 32UL); + + return remap_pfn_range(vma, from, phys_base >> PAGE_SHIFT, size, prot); +} +#define io_remap_pfn_range io_remap_pfn_range + +#include <asm/tlbflush.h> #include <asm-generic/pgtable.h> /* We provide our own get_unmapped_area to cope with VA holes and @@ -761,20 +981,20 @@ extern int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, /* We provide a special get_unmapped_area for framebuffer mmaps to try and use * the largest alignment possible such that larget PTEs can be used. */ -extern unsigned long get_fb_unmapped_area(struct file *filp, unsigned long, - unsigned long, unsigned long, - unsigned long); +unsigned long get_fb_unmapped_area(struct file *filp, unsigned long, + unsigned long, unsigned long, + unsigned long); #define HAVE_ARCH_FB_UNMAPPED_AREA -extern void pgtable_cache_init(void); -extern void sun4v_register_fault_status(void); -extern void sun4v_ktsb_register(void); -extern void __init cheetah_ecache_flush_init(void); -extern void sun4v_patch_tlb_handlers(void); +void pgtable_cache_init(void); +void sun4v_register_fault_status(void); +void sun4v_ktsb_register(void); +void __init cheetah_ecache_flush_init(void); +void sun4v_patch_tlb_handlers(void); extern unsigned long cmdline_memory_size; -extern asmlinkage void do_sparc64_fault(struct pt_regs *regs); +asmlinkage void do_sparc64_fault(struct pt_regs *regs); #endif /* !(__ASSEMBLY__) */ diff --git a/arch/sparc/include/asm/pgtsrmmu.h b/arch/sparc/include/asm/pgtsrmmu.h index 1407c07bdad..79da17866fa 100644 --- a/arch/sparc/include/asm/pgtsrmmu.h +++ b/arch/sparc/include/asm/pgtsrmmu.h @@ -139,6 +139,7 @@ restore %g0, %g0, %g0; #ifndef __ASSEMBLY__ +extern unsigned long last_valid_pfn; /* This makes sense. Honest it does - Anton */ /* XXX Yes but it's ugly as sin. FIXME. -KMW */ @@ -148,78 +149,13 @@ extern void *srmmu_nocache_pool; #define __nocache_fix(VADDR) __va(__nocache_pa(VADDR)) /* Accessing the MMU control register. */ -static inline unsigned int srmmu_get_mmureg(void) -{ - unsigned int retval; - __asm__ __volatile__("lda [%%g0] %1, %0\n\t" : - "=r" (retval) : - "i" (ASI_M_MMUREGS)); - return retval; -} - -static inline void srmmu_set_mmureg(unsigned long regval) -{ - __asm__ __volatile__("sta %0, [%%g0] %1\n\t" : : - "r" (regval), "i" (ASI_M_MMUREGS) : "memory"); - -} - -static inline void srmmu_set_ctable_ptr(unsigned long paddr) -{ - paddr = ((paddr >> 4) & SRMMU_CTX_PMASK); - __asm__ __volatile__("sta %0, [%1] %2\n\t" : : - "r" (paddr), "r" (SRMMU_CTXTBL_PTR), - "i" (ASI_M_MMUREGS) : - "memory"); -} - -static inline unsigned long srmmu_get_ctable_ptr(void) -{ - unsigned int retval; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (SRMMU_CTXTBL_PTR), - "i" (ASI_M_MMUREGS)); - return (retval & SRMMU_CTX_PMASK) << 4; -} - -static inline void srmmu_set_context(int context) -{ - __asm__ __volatile__("sta %0, [%1] %2\n\t" : : - "r" (context), "r" (SRMMU_CTX_REG), - "i" (ASI_M_MMUREGS) : "memory"); -} - -static inline int srmmu_get_context(void) -{ - register int retval; - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (SRMMU_CTX_REG), - "i" (ASI_M_MMUREGS)); - return retval; -} - -static inline unsigned int srmmu_get_fstatus(void) -{ - unsigned int retval; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (SRMMU_FAULT_STATUS), "i" (ASI_M_MMUREGS)); - return retval; -} - -static inline unsigned int srmmu_get_faddr(void) -{ - unsigned int retval; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (SRMMU_FAULT_ADDR), "i" (ASI_M_MMUREGS)); - return retval; -} +unsigned int srmmu_get_mmureg(void); +void srmmu_set_mmureg(unsigned long regval); +void srmmu_set_ctable_ptr(unsigned long paddr); +void srmmu_set_context(int context); +int srmmu_get_context(void); +unsigned int srmmu_get_fstatus(void); +unsigned int srmmu_get_faddr(void); /* This is guaranteed on all SRMMU's. */ static inline void srmmu_flush_whole_tlb(void) @@ -230,59 +166,6 @@ static inline void srmmu_flush_whole_tlb(void) } -/* These flush types are not available on all chips... */ -static inline void srmmu_flush_tlb_ctx(void) -{ - __asm__ __volatile__("sta %%g0, [%0] %1\n\t": : - "r" (0x300), /* Flush TLB ctx.. */ - "i" (ASI_M_FLUSH_PROBE) : "memory"); - -} - -static inline void srmmu_flush_tlb_region(unsigned long addr) -{ - addr &= SRMMU_PGDIR_MASK; - __asm__ __volatile__("sta %%g0, [%0] %1\n\t": : - "r" (addr | 0x200), /* Flush TLB region.. */ - "i" (ASI_M_FLUSH_PROBE) : "memory"); - -} - - -static inline void srmmu_flush_tlb_segment(unsigned long addr) -{ - addr &= SRMMU_REAL_PMD_MASK; - __asm__ __volatile__("sta %%g0, [%0] %1\n\t": : - "r" (addr | 0x100), /* Flush TLB segment.. */ - "i" (ASI_M_FLUSH_PROBE) : "memory"); - -} - -static inline void srmmu_flush_tlb_page(unsigned long page) -{ - page &= PAGE_MASK; - __asm__ __volatile__("sta %%g0, [%0] %1\n\t": : - "r" (page), /* Flush TLB page.. */ - "i" (ASI_M_FLUSH_PROBE) : "memory"); - -} - -#ifndef CONFIG_SPARC_LEON -static inline unsigned long srmmu_hwprobe(unsigned long vaddr) -{ - unsigned long retval; - - vaddr &= PAGE_MASK; - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (retval) : - "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); - - return retval; -} -#else -#define srmmu_hwprobe(addr) (srmmu_swprobe(addr, 0) & SRMMU_PTE_PMASK) -#endif - static inline int srmmu_get_pte (unsigned long addr) { @@ -294,9 +177,6 @@ srmmu_get_pte (unsigned long addr) return entry; } -extern unsigned long (*srmmu_read_physical)(unsigned long paddr); -extern void (*srmmu_write_physical)(unsigned long paddr, unsigned long word); - #endif /* !(__ASSEMBLY__) */ #endif /* !(_SPARC_PGTSRMMU_H) */ diff --git a/arch/sparc/include/asm/pgtsun4.h b/arch/sparc/include/asm/pgtsun4.h deleted file mode 100644 index 5a0d661fb82..00000000000 --- a/arch/sparc/include/asm/pgtsun4.h +++ /dev/null @@ -1,171 +0,0 @@ -/* - * pgtsun4.h: Sun4 specific pgtable.h defines and code. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ -#ifndef _SPARC_PGTSUN4C_H -#define _SPARC_PGTSUN4C_H - -#include <asm/contregs.h> - -/* PMD_SHIFT determines the size of the area a second-level page table can map */ -#define SUN4C_PMD_SHIFT 23 - -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define SUN4C_PGDIR_SHIFT 23 -#define SUN4C_PGDIR_SIZE (1UL << SUN4C_PGDIR_SHIFT) -#define SUN4C_PGDIR_MASK (~(SUN4C_PGDIR_SIZE-1)) -#define SUN4C_PGDIR_ALIGN(addr) (((addr)+SUN4C_PGDIR_SIZE-1)&SUN4C_PGDIR_MASK) - -/* To represent how the sun4c mmu really lays things out. */ -#define SUN4C_REAL_PGDIR_SHIFT 18 -#define SUN4C_REAL_PGDIR_SIZE (1UL << SUN4C_REAL_PGDIR_SHIFT) -#define SUN4C_REAL_PGDIR_MASK (~(SUN4C_REAL_PGDIR_SIZE-1)) -#define SUN4C_REAL_PGDIR_ALIGN(addr) (((addr)+SUN4C_REAL_PGDIR_SIZE-1)&SUN4C_REAL_PGDIR_MASK) - -/* 19 bit PFN on sun4 */ -#define SUN4C_PFN_MASK 0x7ffff - -/* Don't increase these unless the structures in sun4c.c are fixed */ -#define SUN4C_MAX_SEGMAPS 256 -#define SUN4C_MAX_CONTEXTS 16 - -/* - * To be efficient, and not have to worry about allocating such - * a huge pgd, we make the kernel sun4c tables each hold 1024 - * entries and the pgd similarly just like the i386 tables. - */ -#define SUN4C_PTRS_PER_PTE 1024 -#define SUN4C_PTRS_PER_PMD 1 -#define SUN4C_PTRS_PER_PGD 1024 - -/* - * Sparc SUN4C pte fields. - */ -#define _SUN4C_PAGE_VALID 0x80000000 -#define _SUN4C_PAGE_SILENT_READ 0x80000000 /* synonym */ -#define _SUN4C_PAGE_DIRTY 0x40000000 -#define _SUN4C_PAGE_SILENT_WRITE 0x40000000 /* synonym */ -#define _SUN4C_PAGE_PRIV 0x20000000 /* privileged page */ -#define _SUN4C_PAGE_NOCACHE 0x10000000 /* non-cacheable page */ -#define _SUN4C_PAGE_PRESENT 0x08000000 /* implemented in software */ -#define _SUN4C_PAGE_IO 0x04000000 /* I/O page */ -#define _SUN4C_PAGE_FILE 0x02000000 /* implemented in software */ -#define _SUN4C_PAGE_READ 0x00800000 /* implemented in software */ -#define _SUN4C_PAGE_WRITE 0x00400000 /* implemented in software */ -#define _SUN4C_PAGE_ACCESSED 0x00200000 /* implemented in software */ -#define _SUN4C_PAGE_MODIFIED 0x00100000 /* implemented in software */ - -#define _SUN4C_READABLE (_SUN4C_PAGE_READ|_SUN4C_PAGE_SILENT_READ|\ - _SUN4C_PAGE_ACCESSED) -#define _SUN4C_WRITEABLE (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE|\ - _SUN4C_PAGE_MODIFIED) - -#define _SUN4C_PAGE_CHG_MASK (0xffff|_SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_MODIFIED) - -#define SUN4C_PAGE_NONE __pgprot(_SUN4C_PAGE_PRESENT) -#define SUN4C_PAGE_SHARED __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE|\ - _SUN4C_PAGE_WRITE) -#define SUN4C_PAGE_COPY __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE) -#define SUN4C_PAGE_READONLY __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE) -#define SUN4C_PAGE_KERNEL __pgprot(_SUN4C_READABLE|_SUN4C_WRITEABLE|\ - _SUN4C_PAGE_DIRTY|_SUN4C_PAGE_PRIV) - -/* SUN4C swap entry encoding - * - * We use 5 bits for the type and 19 for the offset. This gives us - * 32 swapfiles of 4GB each. Encoding looks like: - * - * RRRRRRRRooooooooooooooooooottttt - * fedcba9876543210fedcba9876543210 - * - * The top 8 bits are reserved for protection and status bits, especially - * FILE and PRESENT. - */ -#define SUN4C_SWP_TYPE_MASK 0x1f -#define SUN4C_SWP_OFF_MASK 0x7ffff -#define SUN4C_SWP_OFF_SHIFT 5 - -#ifndef __ASSEMBLY__ - -static inline unsigned long sun4c_get_synchronous_error(void) -{ - unsigned long sync_err; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (sync_err) : - "r" (AC_SYNC_ERR), "i" (ASI_CONTROL)); - return sync_err; -} - -static inline unsigned long sun4c_get_synchronous_address(void) -{ - unsigned long sync_addr; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (sync_addr) : - "r" (AC_SYNC_VA), "i" (ASI_CONTROL)); - return sync_addr; -} - -/* SUN4 pte, segmap, and context manipulation */ -static inline unsigned long sun4c_get_segmap(unsigned long addr) -{ - register unsigned long entry; - - __asm__ __volatile__("\n\tlduha [%1] %2, %0\n\t" : - "=r" (entry) : - "r" (addr), "i" (ASI_SEGMAP)); - return entry; -} - -static inline void sun4c_put_segmap(unsigned long addr, unsigned long entry) -{ - __asm__ __volatile__("\n\tstha %1, [%0] %2; nop; nop; nop;\n\t" : : - "r" (addr), "r" (entry), - "i" (ASI_SEGMAP) - : "memory"); -} - -static inline unsigned long sun4c_get_pte(unsigned long addr) -{ - register unsigned long entry; - - __asm__ __volatile__("\n\tlda [%1] %2, %0\n\t" : - "=r" (entry) : - "r" (addr), "i" (ASI_PTE)); - return entry; -} - -static inline void sun4c_put_pte(unsigned long addr, unsigned long entry) -{ - __asm__ __volatile__("\n\tsta %1, [%0] %2; nop; nop; nop;\n\t" : : - "r" (addr), - "r" ((entry & ~(_SUN4C_PAGE_PRESENT))), "i" (ASI_PTE) - : "memory"); -} - -static inline int sun4c_get_context(void) -{ - register int ctx; - - __asm__ __volatile__("\n\tlduba [%1] %2, %0\n\t" : - "=r" (ctx) : - "r" (AC_CONTEXT), "i" (ASI_CONTROL)); - - return ctx; -} - -static inline int sun4c_set_context(int ctx) -{ - __asm__ __volatile__("\n\tstba %0, [%1] %2; nop; nop; nop;\n\t" : : - "r" (ctx), "r" (AC_CONTEXT), "i" (ASI_CONTROL) - : "memory"); - - return ctx; -} - -#endif /* !(__ASSEMBLY__) */ - -#endif /* !(_SPARC_PGTSUN4_H) */ diff --git a/arch/sparc/include/asm/pgtsun4c.h b/arch/sparc/include/asm/pgtsun4c.h deleted file mode 100644 index aeb25e91217..00000000000 --- a/arch/sparc/include/asm/pgtsun4c.h +++ /dev/null @@ -1,172 +0,0 @@ -/* - * pgtsun4c.h: Sun4c specific pgtable.h defines and code. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ -#ifndef _SPARC_PGTSUN4C_H -#define _SPARC_PGTSUN4C_H - -#include <asm/contregs.h> - -/* PMD_SHIFT determines the size of the area a second-level page table can map */ -#define SUN4C_PMD_SHIFT 22 - -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define SUN4C_PGDIR_SHIFT 22 -#define SUN4C_PGDIR_SIZE (1UL << SUN4C_PGDIR_SHIFT) -#define SUN4C_PGDIR_MASK (~(SUN4C_PGDIR_SIZE-1)) -#define SUN4C_PGDIR_ALIGN(addr) (((addr)+SUN4C_PGDIR_SIZE-1)&SUN4C_PGDIR_MASK) - -/* To represent how the sun4c mmu really lays things out. */ -#define SUN4C_REAL_PGDIR_SHIFT 18 -#define SUN4C_REAL_PGDIR_SIZE (1UL << SUN4C_REAL_PGDIR_SHIFT) -#define SUN4C_REAL_PGDIR_MASK (~(SUN4C_REAL_PGDIR_SIZE-1)) -#define SUN4C_REAL_PGDIR_ALIGN(addr) (((addr)+SUN4C_REAL_PGDIR_SIZE-1)&SUN4C_REAL_PGDIR_MASK) - -/* 16 bit PFN on sun4c */ -#define SUN4C_PFN_MASK 0xffff - -/* Don't increase these unless the structures in sun4c.c are fixed */ -#define SUN4C_MAX_SEGMAPS 256 -#define SUN4C_MAX_CONTEXTS 16 - -/* - * To be efficient, and not have to worry about allocating such - * a huge pgd, we make the kernel sun4c tables each hold 1024 - * entries and the pgd similarly just like the i386 tables. - */ -#define SUN4C_PTRS_PER_PTE 1024 -#define SUN4C_PTRS_PER_PMD 1 -#define SUN4C_PTRS_PER_PGD 1024 - -/* - * Sparc SUN4C pte fields. - */ -#define _SUN4C_PAGE_VALID 0x80000000 -#define _SUN4C_PAGE_SILENT_READ 0x80000000 /* synonym */ -#define _SUN4C_PAGE_DIRTY 0x40000000 -#define _SUN4C_PAGE_SILENT_WRITE 0x40000000 /* synonym */ -#define _SUN4C_PAGE_PRIV 0x20000000 /* privileged page */ -#define _SUN4C_PAGE_NOCACHE 0x10000000 /* non-cacheable page */ -#define _SUN4C_PAGE_PRESENT 0x08000000 /* implemented in software */ -#define _SUN4C_PAGE_IO 0x04000000 /* I/O page */ -#define _SUN4C_PAGE_FILE 0x02000000 /* implemented in software */ -#define _SUN4C_PAGE_READ 0x00800000 /* implemented in software */ -#define _SUN4C_PAGE_WRITE 0x00400000 /* implemented in software */ -#define _SUN4C_PAGE_ACCESSED 0x00200000 /* implemented in software */ -#define _SUN4C_PAGE_MODIFIED 0x00100000 /* implemented in software */ - -#define _SUN4C_READABLE (_SUN4C_PAGE_READ|_SUN4C_PAGE_SILENT_READ|\ - _SUN4C_PAGE_ACCESSED) -#define _SUN4C_WRITEABLE (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE|\ - _SUN4C_PAGE_MODIFIED) - -#define _SUN4C_PAGE_CHG_MASK (0xffff|_SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_MODIFIED) - -#define SUN4C_PAGE_NONE __pgprot(_SUN4C_PAGE_PRESENT) -#define SUN4C_PAGE_SHARED __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE|\ - _SUN4C_PAGE_WRITE) -#define SUN4C_PAGE_COPY __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE) -#define SUN4C_PAGE_READONLY __pgprot(_SUN4C_PAGE_PRESENT|_SUN4C_READABLE) -#define SUN4C_PAGE_KERNEL __pgprot(_SUN4C_READABLE|_SUN4C_WRITEABLE|\ - _SUN4C_PAGE_DIRTY|_SUN4C_PAGE_PRIV) - -/* SUN4C swap entry encoding - * - * We use 5 bits for the type and 19 for the offset. This gives us - * 32 swapfiles of 4GB each. Encoding looks like: - * - * RRRRRRRRooooooooooooooooooottttt - * fedcba9876543210fedcba9876543210 - * - * The top 8 bits are reserved for protection and status bits, especially - * FILE and PRESENT. - */ -#define SUN4C_SWP_TYPE_MASK 0x1f -#define SUN4C_SWP_OFF_MASK 0x7ffff -#define SUN4C_SWP_OFF_SHIFT 5 - -#ifndef __ASSEMBLY__ - -static inline unsigned long sun4c_get_synchronous_error(void) -{ - unsigned long sync_err; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (sync_err) : - "r" (AC_SYNC_ERR), "i" (ASI_CONTROL)); - return sync_err; -} - -static inline unsigned long sun4c_get_synchronous_address(void) -{ - unsigned long sync_addr; - - __asm__ __volatile__("lda [%1] %2, %0\n\t" : - "=r" (sync_addr) : - "r" (AC_SYNC_VA), "i" (ASI_CONTROL)); - return sync_addr; -} - -/* SUN4C pte, segmap, and context manipulation */ -static inline unsigned long sun4c_get_segmap(unsigned long addr) -{ - register unsigned long entry; - - __asm__ __volatile__("\n\tlduba [%1] %2, %0\n\t" : - "=r" (entry) : - "r" (addr), "i" (ASI_SEGMAP)); - - return entry; -} - -static inline void sun4c_put_segmap(unsigned long addr, unsigned long entry) -{ - - __asm__ __volatile__("\n\tstba %1, [%0] %2; nop; nop; nop;\n\t" : : - "r" (addr), "r" (entry), - "i" (ASI_SEGMAP) - : "memory"); -} - -static inline unsigned long sun4c_get_pte(unsigned long addr) -{ - register unsigned long entry; - - __asm__ __volatile__("\n\tlda [%1] %2, %0\n\t" : - "=r" (entry) : - "r" (addr), "i" (ASI_PTE)); - return entry; -} - -static inline void sun4c_put_pte(unsigned long addr, unsigned long entry) -{ - __asm__ __volatile__("\n\tsta %1, [%0] %2; nop; nop; nop;\n\t" : : - "r" (addr), - "r" ((entry & ~(_SUN4C_PAGE_PRESENT))), "i" (ASI_PTE) - : "memory"); -} - -static inline int sun4c_get_context(void) -{ - register int ctx; - - __asm__ __volatile__("\n\tlduba [%1] %2, %0\n\t" : - "=r" (ctx) : - "r" (AC_CONTEXT), "i" (ASI_CONTROL)); - - return ctx; -} - -static inline int sun4c_set_context(int ctx) -{ - __asm__ __volatile__("\n\tstba %0, [%1] %2; nop; nop; nop;\n\t" : : - "r" (ctx), "r" (AC_CONTEXT), "i" (ASI_CONTROL) - : "memory"); - - return ctx; -} - -#endif /* !(__ASSEMBLY__) */ - -#endif /* !(_SPARC_PGTSUN4C_H) */ diff --git a/arch/sparc/include/asm/posix_types.h b/arch/sparc/include/asm/posix_types.h deleted file mode 100644 index 98d6ebb922f..00000000000 --- a/arch/sparc/include/asm/posix_types.h +++ /dev/null @@ -1,155 +0,0 @@ -/* - * This file is generally used by user-level software, so you need to - * be a little careful about namespace pollution etc. Also, we cannot - * assume GCC is being used. - */ - -#ifndef __SPARC_POSIX_TYPES_H -#define __SPARC_POSIX_TYPES_H - -#if defined(__sparc__) && defined(__arch64__) -/* sparc 64 bit */ -typedef unsigned long __kernel_size_t; -typedef long __kernel_ssize_t; -typedef long __kernel_ptrdiff_t; -typedef long __kernel_time_t; -typedef long __kernel_clock_t; -typedef int __kernel_pid_t; -typedef int __kernel_ipc_pid_t; -typedef unsigned int __kernel_uid_t; -typedef unsigned int __kernel_gid_t; -typedef unsigned long __kernel_ino_t; -typedef unsigned int __kernel_mode_t; -typedef unsigned short __kernel_umode_t; -typedef unsigned int __kernel_nlink_t; -typedef int __kernel_daddr_t; -typedef long __kernel_off_t; -typedef char * __kernel_caddr_t; -typedef unsigned short __kernel_uid16_t; -typedef unsigned short __kernel_gid16_t; -typedef int __kernel_clockid_t; -typedef int __kernel_timer_t; - -typedef unsigned short __kernel_old_uid_t; -typedef unsigned short __kernel_old_gid_t; -typedef __kernel_uid_t __kernel_uid32_t; -typedef __kernel_gid_t __kernel_gid32_t; - -typedef unsigned int __kernel_old_dev_t; - -/* Note this piece of asymmetry from the v9 ABI. */ -typedef int __kernel_suseconds_t; - -#else -/* sparc 32 bit */ - -typedef unsigned int __kernel_size_t; -typedef int __kernel_ssize_t; -typedef long int __kernel_ptrdiff_t; -typedef long __kernel_time_t; -typedef long __kernel_suseconds_t; -typedef long __kernel_clock_t; -typedef int __kernel_pid_t; -typedef unsigned short __kernel_ipc_pid_t; -typedef unsigned short __kernel_uid_t; -typedef unsigned short __kernel_gid_t; -typedef unsigned long __kernel_ino_t; -typedef unsigned short __kernel_mode_t; -typedef unsigned short __kernel_umode_t; -typedef short __kernel_nlink_t; -typedef long __kernel_daddr_t; -typedef long __kernel_off_t; -typedef char * __kernel_caddr_t; -typedef unsigned short __kernel_uid16_t; -typedef unsigned short __kernel_gid16_t; -typedef unsigned int __kernel_uid32_t; -typedef unsigned int __kernel_gid32_t; -typedef unsigned short __kernel_old_uid_t; -typedef unsigned short __kernel_old_gid_t; -typedef unsigned short __kernel_old_dev_t; -typedef int __kernel_clockid_t; -typedef int __kernel_timer_t; - -#endif /* defined(__sparc__) && defined(__arch64__) */ - -#ifdef __GNUC__ -typedef long long __kernel_loff_t; -#endif - -typedef struct { - int val[2]; -} __kernel_fsid_t; - -#ifdef __KERNEL__ - -#undef __FD_SET -static inline void __FD_SET(unsigned long fd, __kernel_fd_set *fdsetp) -{ - unsigned long _tmp = fd / __NFDBITS; - unsigned long _rem = fd % __NFDBITS; - fdsetp->fds_bits[_tmp] |= (1UL<<_rem); -} - -#undef __FD_CLR -static inline void __FD_CLR(unsigned long fd, __kernel_fd_set *fdsetp) -{ - unsigned long _tmp = fd / __NFDBITS; - unsigned long _rem = fd % __NFDBITS; - fdsetp->fds_bits[_tmp] &= ~(1UL<<_rem); -} - -#undef __FD_ISSET -static inline int __FD_ISSET(unsigned long fd, __const__ __kernel_fd_set *p) -{ - unsigned long _tmp = fd / __NFDBITS; - unsigned long _rem = fd % __NFDBITS; - return (p->fds_bits[_tmp] & (1UL<<_rem)) != 0; -} - -/* - * This will unroll the loop for the normal constant cases (8 or 32 longs, - * for 256 and 1024-bit fd_sets respectively) - */ -#undef __FD_ZERO -static inline void __FD_ZERO(__kernel_fd_set *p) -{ - unsigned long *tmp = p->fds_bits; - int i; - - if (__builtin_constant_p(__FDSET_LONGS)) { - switch (__FDSET_LONGS) { - case 32: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; - tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; - tmp[16] = 0; tmp[17] = 0; tmp[18] = 0; tmp[19] = 0; - tmp[20] = 0; tmp[21] = 0; tmp[22] = 0; tmp[23] = 0; - tmp[24] = 0; tmp[25] = 0; tmp[26] = 0; tmp[27] = 0; - tmp[28] = 0; tmp[29] = 0; tmp[30] = 0; tmp[31] = 0; - return; - case 16: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - tmp[ 8] = 0; tmp[ 9] = 0; tmp[10] = 0; tmp[11] = 0; - tmp[12] = 0; tmp[13] = 0; tmp[14] = 0; tmp[15] = 0; - return; - case 8: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - tmp[ 4] = 0; tmp[ 5] = 0; tmp[ 6] = 0; tmp[ 7] = 0; - return; - case 4: - tmp[ 0] = 0; tmp[ 1] = 0; tmp[ 2] = 0; tmp[ 3] = 0; - return; - } - } - i = __FDSET_LONGS; - while (i) { - i--; - *tmp = 0; - tmp++; - } -} - -#endif /* __KERNEL__ */ -#endif /* __SPARC_POSIX_TYPES_H */ diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h index 9da9646bf6c..2fe99e66e76 100644 --- a/arch/sparc/include/asm/processor.h +++ b/arch/sparc/include/asm/processor.h @@ -5,4 +5,7 @@ #else #include <asm/processor_32.h> #endif + +#define nop() __asm__ __volatile__ ("nop") + #endif diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index 09521c6a5ed..a564817bbc2 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -16,7 +16,6 @@ #include <asm/ptrace.h> #include <asm/head.h> #include <asm/signal.h> -#include <asm/btfixup.h> #include <asm/page.h> /* @@ -75,7 +74,7 @@ struct thread_struct { } /* Return saved PC of a blocked thread. */ -extern unsigned long thread_saved_pc(struct task_struct *t); +unsigned long thread_saved_pc(struct task_struct *t); /* Do necessary setup to start up a newly executed thread. */ static inline void start_thread(struct pt_regs * regs, unsigned long pc, @@ -107,12 +106,8 @@ static inline void start_thread(struct pt_regs * regs, unsigned long pc, /* Free all resources held by a thread. */ #define release_thread(tsk) do { } while(0) -extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); -/* Prepare to copy thread state - unlazy all lazy status */ -#define prepare_to_copy(tsk) do { } while (0) - -extern unsigned long get_wchan(struct task_struct *); +unsigned long get_wchan(struct task_struct *); #define task_pt_regs(tsk) ((tsk)->thread.kregs) #define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc) @@ -121,8 +116,10 @@ extern unsigned long get_wchan(struct task_struct *); #ifdef __KERNEL__ extern struct task_struct *last_task_used_math; +int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); #define cpu_relax() barrier() +extern void (*sparc_idle)(void); #endif diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 59fcebb8f44..7028fe1a7c0 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -39,7 +39,9 @@ #define TASK_SIZE_OF(tsk) \ (test_tsk_thread_flag(tsk,TIF_32BIT) ? \ (1UL << 32UL) : ((unsigned long)-VPTE_SIZE)) -#define TASK_SIZE TASK_SIZE_OF(current) +#define TASK_SIZE \ + (test_thread_flag(TIF_32BIT) ? \ + (1UL << 32UL) : ((unsigned long)-VPTE_SIZE)) #ifdef __KERNEL__ #define STACK_TOP32 ((1UL << 32UL) - PAGE_SIZE) @@ -89,10 +91,11 @@ struct thread_struct { #ifndef __ASSEMBLY__ #include <linux/types.h> +#include <asm/fpumacro.h> /* Return saved PC of a blocked thread. */ struct task_struct; -extern unsigned long thread_saved_pc(struct task_struct *); +unsigned long thread_saved_pc(struct task_struct *); /* On Uniprocessor, even in RMO processes see TSO semantics */ #ifdef CONFIG_SMP @@ -138,6 +141,10 @@ do { \ : \ : "r" (regs), "r" (sp - sizeof(struct reg_window) - STACK_BIAS), \ "i" ((const unsigned long)(&((struct pt_regs *)0)->u_regs[0]))); \ + fprs_write(0); \ + current_thread_info()->xfsr[0] = 0; \ + current_thread_info()->fpsaved[0] = 0; \ + regs->tstate &= ~TSTATE_PEF; \ } while (0) #define start_thread32(regs, pc, sp) \ @@ -178,23 +185,37 @@ do { \ : \ : "r" (regs), "r" (sp - sizeof(struct reg_window32)), \ "i" ((const unsigned long)(&((struct pt_regs *)0)->u_regs[0]))); \ + fprs_write(0); \ + current_thread_info()->xfsr[0] = 0; \ + current_thread_info()->fpsaved[0] = 0; \ + regs->tstate &= ~TSTATE_PEF; \ } while (0) /* Free all resources held by a thread. */ #define release_thread(tsk) do { } while (0) -/* Prepare to copy thread state - unlazy all lazy status */ -#define prepare_to_copy(tsk) do { } while (0) - -extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags); - -extern unsigned long get_wchan(struct task_struct *task); +unsigned long get_wchan(struct task_struct *task); #define task_pt_regs(tsk) (task_thread_info(tsk)->kregs) #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) -#define cpu_relax() barrier() +/* Please see the commentary in asm/backoff.h for a description of + * what these instructions are doing and how they have been choosen. + * To make a long story short, we are trying to yield the current cpu + * strand during busy loops. + */ +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ".section .pause_3insn_patch,\"ax\"\n\t"\ + ".word 99b\n\t" \ + "wr %%g0, 128, %%asr27\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".previous" \ + ::: "memory") /* Prefetch support. This is tuned for UltraSPARC-III and later. * UltraSPARC-I will treat these as nops, and UltraSPARC-II has @@ -232,6 +253,8 @@ static inline void prefetchw(const void *x) #define HAVE_ARCH_PICK_MMAP_LAYOUT +int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap); + #endif /* !(__ASSEMBLY__) */ #endif /* !(__ASM_SPARC64_PROCESSOR_H) */ diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h index 56bbaadef64..d955c8df62d 100644 --- a/arch/sparc/include/asm/prom.h +++ b/arch/sparc/include/asm/prom.h @@ -21,7 +21,8 @@ #include <linux/of_pdt.h> #include <linux/proc_fs.h> #include <linux/mutex.h> -#include <asm/atomic.h> +#include <linux/atomic.h> +#include <linux/irqdomain.h> #define OF_ROOT_NODE_ADDR_CELLS_DEFAULT 2 #define OF_ROOT_NODE_SIZE_CELLS_DEFAULT 1 @@ -35,41 +36,28 @@ struct of_irq_controller { void *data; }; -extern struct device_node *of_find_node_by_cpuid(int cpuid); -extern int of_set_property(struct device_node *node, const char *name, void *val, int len); +struct device_node *of_find_node_by_cpuid(int cpuid); +int of_set_property(struct device_node *node, const char *name, void *val, int len); extern struct mutex of_set_property_mutex; -extern int of_getintprop_default(struct device_node *np, - const char *name, +int of_getintprop_default(struct device_node *np, + const char *name, int def); -extern int of_find_in_proplist(const char *list, const char *match, int len); -#ifdef CONFIG_NUMA -extern int of_node_to_nid(struct device_node *dp); -#define of_node_to_nid of_node_to_nid -#endif +int of_find_in_proplist(const char *list, const char *match, int len); -extern void prom_build_devicetree(void); -extern void of_populate_present_mask(void); -extern void of_fill_in_cpu_data(void); +void prom_build_devicetree(void); +void of_populate_present_mask(void); +void of_fill_in_cpu_data(void); struct resource; -extern void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); -extern void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); - -/* These routines are here to provide compatibility with how powerpc - * handles IRQ mapping for OF device nodes. We precompute and permanently - * register them in the platform_device objects, whereas powerpc computes them - * on request. - */ -static inline void irq_dispose_mapping(unsigned int virq) -{ -} +void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); +void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); extern struct device_node *of_console_device; extern char *of_console_path; extern char *of_console_options; -extern void irq_trans_init(struct device_node *dp); -extern char *build_path_component(struct device_node *dp); +void irq_trans_init(struct device_node *dp); +char *build_path_component(struct device_node *dp); #endif /* __KERNEL__ */ #endif /* _SPARC_PROM_H */ diff --git a/arch/sparc/include/asm/psr.h b/arch/sparc/include/asm/psr.h index b8c0e5f0a66..e71eb57945e 100644 --- a/arch/sparc/include/asm/psr.h +++ b/arch/sparc/include/asm/psr.h @@ -7,35 +7,11 @@ * * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) */ - #ifndef __LINUX_SPARC_PSR_H #define __LINUX_SPARC_PSR_H -/* The Sparc PSR fields are laid out as the following: - * - * ------------------------------------------------------------------------ - * | impl | vers | icc | resv | EC | EF | PIL | S | PS | ET | CWP | - * | 31-28 | 27-24 | 23-20 | 19-14 | 13 | 12 | 11-8 | 7 | 6 | 5 | 4-0 | - * ------------------------------------------------------------------------ - */ -#define PSR_CWP 0x0000001f /* current window pointer */ -#define PSR_ET 0x00000020 /* enable traps field */ -#define PSR_PS 0x00000040 /* previous privilege level */ -#define PSR_S 0x00000080 /* current privilege level */ -#define PSR_PIL 0x00000f00 /* processor interrupt level */ -#define PSR_EF 0x00001000 /* enable floating point */ -#define PSR_EC 0x00002000 /* enable co-processor */ -#define PSR_SYSCALL 0x00004000 /* inside of a syscall */ -#define PSR_LE 0x00008000 /* SuperSparcII little-endian */ -#define PSR_ICC 0x00f00000 /* integer condition codes */ -#define PSR_C 0x00100000 /* carry bit */ -#define PSR_V 0x00200000 /* overflow bit */ -#define PSR_Z 0x00400000 /* zero bit */ -#define PSR_N 0x00800000 /* negative bit */ -#define PSR_VERS 0x0f000000 /* cpu-version field */ -#define PSR_IMPL 0xf0000000 /* cpu-implementation field */ +#include <uapi/asm/psr.h> -#ifdef __KERNEL__ #ifndef __ASSEMBLY__ /* Get the %psr register. */ @@ -88,6 +64,4 @@ static inline unsigned int get_fsr(void) #endif /* !(__ASSEMBLY__) */ -#endif /* (__KERNEL__) */ - #endif /* !(__LINUX_SPARC_PSR_H) */ diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h index 30b0b797dc0..bac6a946ee0 100644 --- a/arch/sparc/include/asm/ptrace.h +++ b/arch/sparc/include/asm/ptrace.h @@ -1,169 +1,13 @@ #ifndef __SPARC_PTRACE_H #define __SPARC_PTRACE_H -#if defined(__sparc__) && defined(__arch64__) -/* 64 bit sparc */ -#include <asm/pstate.h> - -/* This struct defines the way the registers are stored on the - * stack during a system call and basically all traps. - */ - -/* This magic value must have the low 9 bits clear, - * as that is where we encode the %tt value, see below. - */ -#define PT_REGS_MAGIC 0x57ac6c00 - -#ifndef __ASSEMBLY__ - -#include <linux/types.h> - -struct pt_regs { - unsigned long u_regs[16]; /* globals and ins */ - unsigned long tstate; - unsigned long tpc; - unsigned long tnpc; - unsigned int y; - - /* We encode a magic number, PT_REGS_MAGIC, along - * with the %tt (trap type) register value at trap - * entry time. The magic number allows us to identify - * accurately a trap stack frame in the stack - * unwinder, and the %tt value allows us to test - * things like "in a system call" etc. for an arbitray - * process. - * - * The PT_REGS_MAGIC is choosen such that it can be - * loaded completely using just a sethi instruction. - */ - unsigned int magic; -}; - -struct pt_regs32 { - unsigned int psr; - unsigned int pc; - unsigned int npc; - unsigned int y; - unsigned int u_regs[16]; /* globals and ins */ -}; - -/* A V9 register window */ -struct reg_window { - unsigned long locals[8]; - unsigned long ins[8]; -}; - -/* A 32-bit register window. */ -struct reg_window32 { - unsigned int locals[8]; - unsigned int ins[8]; -}; - -/* A V9 Sparc stack frame */ -struct sparc_stackf { - unsigned long locals[8]; - unsigned long ins[6]; - struct sparc_stackf *fp; - unsigned long callers_pc; - char *structptr; - unsigned long xargs[6]; - unsigned long xxargs[1]; -}; - -/* A 32-bit Sparc stack frame */ -struct sparc_stackf32 { - unsigned int locals[8]; - unsigned int ins[6]; - unsigned int fp; - unsigned int callers_pc; - unsigned int structptr; - unsigned int xargs[6]; - unsigned int xxargs[1]; -}; - -struct sparc_trapf { - unsigned long locals[8]; - unsigned long ins[8]; - unsigned long _unused; - struct pt_regs *regs; -}; -#endif /* (!__ASSEMBLY__) */ -#else -/* 32 bit sparc */ - -#include <asm/psr.h> - -/* This struct defines the way the registers are stored on the - * stack during a system call and basically all traps. - */ -#ifndef __ASSEMBLY__ - -struct pt_regs { - unsigned long psr; - unsigned long pc; - unsigned long npc; - unsigned long y; - unsigned long u_regs[16]; /* globals and ins */ -}; - -/* A 32-bit register window. */ -struct reg_window32 { - unsigned long locals[8]; - unsigned long ins[8]; -}; - -/* A Sparc stack frame */ -struct sparc_stackf { - unsigned long locals[8]; - unsigned long ins[6]; - struct sparc_stackf *fp; - unsigned long callers_pc; - char *structptr; - unsigned long xargs[6]; - unsigned long xxargs[1]; -}; -#endif /* (!__ASSEMBLY__) */ - -#endif /* (defined(__sparc__) && defined(__arch64__))*/ - -#ifndef __ASSEMBLY__ - -#define TRACEREG_SZ sizeof(struct pt_regs) -#define STACKFRAME_SZ sizeof(struct sparc_stackf) - -#define TRACEREG32_SZ sizeof(struct pt_regs32) -#define STACKFRAME32_SZ sizeof(struct sparc_stackf32) - -#endif /* (!__ASSEMBLY__) */ - -#define UREG_G0 0 -#define UREG_G1 1 -#define UREG_G2 2 -#define UREG_G3 3 -#define UREG_G4 4 -#define UREG_G5 5 -#define UREG_G6 6 -#define UREG_G7 7 -#define UREG_I0 8 -#define UREG_I1 9 -#define UREG_I2 10 -#define UREG_I3 11 -#define UREG_I4 12 -#define UREG_I5 13 -#define UREG_I6 14 -#define UREG_I7 15 -#define UREG_FP UREG_I6 -#define UREG_RETPC UREG_I7 +#include <uapi/asm/ptrace.h> #if defined(__sparc__) && defined(__arch64__) -/* 64 bit sparc */ - #ifndef __ASSEMBLY__ -#ifdef __KERNEL__ - #include <linux/threads.h> -#include <asm/system.h> +#include <asm/switch_to.h> static inline int pt_regs_trap_type(struct pt_regs *regs) { @@ -188,6 +32,9 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) #define arch_ptrace_stop(exit_code, info) \ synchronize_user_stack() +#define current_pt_regs() \ + ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1) + struct global_reg_snapshot { unsigned long tstate; unsigned long tpc; @@ -198,41 +45,43 @@ struct global_reg_snapshot { struct thread_info *thread; unsigned long pad1; }; -extern struct global_reg_snapshot global_reg_snapshot[NR_CPUS]; -#define force_successful_syscall_return() \ -do { current_thread_info()->syscall_noerror = 1; \ -} while (0) +struct global_pmu_snapshot { + unsigned long pcr[4]; + unsigned long pic[4]; +}; + +union global_cpu_snapshot { + struct global_reg_snapshot reg; + struct global_pmu_snapshot pmu; +}; + +extern union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; + +#define force_successful_syscall_return() set_thread_noerror(1) #define user_mode(regs) (!((regs)->tstate & TSTATE_PRIV)) #define instruction_pointer(regs) ((regs)->tpc) +#define instruction_pointer_set(regs, val) ((regs)->tpc = (val)) #define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) -#define regs_return_value(regs) ((regs)->u_regs[UREG_I0]) +static inline int is_syscall_success(struct pt_regs *regs) +{ + return !(regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY)); +} + +static inline long regs_return_value(struct pt_regs *regs) +{ + return regs->u_regs[UREG_I0]; +} #ifdef CONFIG_SMP -extern unsigned long profile_pc(struct pt_regs *); +unsigned long profile_pc(struct pt_regs *); #else #define profile_pc(regs) instruction_pointer(regs) #endif -extern void show_regs(struct pt_regs *); -#endif /* (__KERNEL__) */ - #else /* __ASSEMBLY__ */ -/* For assembly code. */ -#define TRACEREG_SZ 0xa0 -#define STACKFRAME_SZ 0xc0 - -#define TRACEREG32_SZ 0x50 -#define STACKFRAME32_SZ 0x60 #endif /* __ASSEMBLY__ */ - #else /* (defined(__sparc__) && defined(__arch64__)) */ - -/* 32 bit sparc */ - #ifndef __ASSEMBLY__ - -#ifdef __KERNEL__ - -#include <asm/system.h> +#include <asm/switch_to.h> static inline bool pt_regs_is_syscall(struct pt_regs *regs) { @@ -252,163 +101,17 @@ static inline bool pt_regs_clear_syscall(struct pt_regs *regs) #define arch_ptrace_stop(exit_code, info) \ synchronize_user_stack() +#define current_pt_regs() \ + ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1) + #define user_mode(regs) (!((regs)->psr & PSR_PS)) #define instruction_pointer(regs) ((regs)->pc) #define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) unsigned long profile_pc(struct pt_regs *); -extern void show_regs(struct pt_regs *); -#endif /* (__KERNEL__) */ - #else /* (!__ASSEMBLY__) */ -/* For assembly code. */ -#define TRACEREG_SZ 0x50 -#define STACKFRAME_SZ 0x60 #endif /* (!__ASSEMBLY__) */ - #endif /* (defined(__sparc__) && defined(__arch64__)) */ - -#ifdef __KERNEL__ #define STACK_BIAS 2047 -#endif - -/* These are for pt_regs. */ -#define PT_V9_G0 0x00 -#define PT_V9_G1 0x08 -#define PT_V9_G2 0x10 -#define PT_V9_G3 0x18 -#define PT_V9_G4 0x20 -#define PT_V9_G5 0x28 -#define PT_V9_G6 0x30 -#define PT_V9_G7 0x38 -#define PT_V9_I0 0x40 -#define PT_V9_I1 0x48 -#define PT_V9_I2 0x50 -#define PT_V9_I3 0x58 -#define PT_V9_I4 0x60 -#define PT_V9_I5 0x68 -#define PT_V9_I6 0x70 -#define PT_V9_FP PT_V9_I6 -#define PT_V9_I7 0x78 -#define PT_V9_TSTATE 0x80 -#define PT_V9_TPC 0x88 -#define PT_V9_TNPC 0x90 -#define PT_V9_Y 0x98 -#define PT_V9_MAGIC 0x9c -#define PT_TSTATE PT_V9_TSTATE -#define PT_TPC PT_V9_TPC -#define PT_TNPC PT_V9_TNPC - -/* These for pt_regs32. */ -#define PT_PSR 0x0 -#define PT_PC 0x4 -#define PT_NPC 0x8 -#define PT_Y 0xc -#define PT_G0 0x10 -#define PT_WIM PT_G0 -#define PT_G1 0x14 -#define PT_G2 0x18 -#define PT_G3 0x1c -#define PT_G4 0x20 -#define PT_G5 0x24 -#define PT_G6 0x28 -#define PT_G7 0x2c -#define PT_I0 0x30 -#define PT_I1 0x34 -#define PT_I2 0x38 -#define PT_I3 0x3c -#define PT_I4 0x40 -#define PT_I5 0x44 -#define PT_I6 0x48 -#define PT_FP PT_I6 -#define PT_I7 0x4c - -/* Reg_window offsets */ -#define RW_V9_L0 0x00 -#define RW_V9_L1 0x08 -#define RW_V9_L2 0x10 -#define RW_V9_L3 0x18 -#define RW_V9_L4 0x20 -#define RW_V9_L5 0x28 -#define RW_V9_L6 0x30 -#define RW_V9_L7 0x38 -#define RW_V9_I0 0x40 -#define RW_V9_I1 0x48 -#define RW_V9_I2 0x50 -#define RW_V9_I3 0x58 -#define RW_V9_I4 0x60 -#define RW_V9_I5 0x68 -#define RW_V9_I6 0x70 -#define RW_V9_I7 0x78 - -#define RW_L0 0x00 -#define RW_L1 0x04 -#define RW_L2 0x08 -#define RW_L3 0x0c -#define RW_L4 0x10 -#define RW_L5 0x14 -#define RW_L6 0x18 -#define RW_L7 0x1c -#define RW_I0 0x20 -#define RW_I1 0x24 -#define RW_I2 0x28 -#define RW_I3 0x2c -#define RW_I4 0x30 -#define RW_I5 0x34 -#define RW_I6 0x38 -#define RW_I7 0x3c - -/* Stack_frame offsets */ -#define SF_V9_L0 0x00 -#define SF_V9_L1 0x08 -#define SF_V9_L2 0x10 -#define SF_V9_L3 0x18 -#define SF_V9_L4 0x20 -#define SF_V9_L5 0x28 -#define SF_V9_L6 0x30 -#define SF_V9_L7 0x38 -#define SF_V9_I0 0x40 -#define SF_V9_I1 0x48 -#define SF_V9_I2 0x50 -#define SF_V9_I3 0x58 -#define SF_V9_I4 0x60 -#define SF_V9_I5 0x68 -#define SF_V9_FP 0x70 -#define SF_V9_PC 0x78 -#define SF_V9_RETP 0x80 -#define SF_V9_XARG0 0x88 -#define SF_V9_XARG1 0x90 -#define SF_V9_XARG2 0x98 -#define SF_V9_XARG3 0xa0 -#define SF_V9_XARG4 0xa8 -#define SF_V9_XARG5 0xb0 -#define SF_V9_XXARG 0xb8 - -#define SF_L0 0x00 -#define SF_L1 0x04 -#define SF_L2 0x08 -#define SF_L3 0x0c -#define SF_L4 0x10 -#define SF_L5 0x14 -#define SF_L6 0x18 -#define SF_L7 0x1c -#define SF_I0 0x20 -#define SF_I1 0x24 -#define SF_I2 0x28 -#define SF_I3 0x2c -#define SF_I4 0x30 -#define SF_I5 0x34 -#define SF_FP 0x38 -#define SF_PC 0x3c -#define SF_RETP 0x40 -#define SF_XARG0 0x44 -#define SF_XARG1 0x48 -#define SF_XARG2 0x4c -#define SF_XARG3 0x50 -#define SF_XARG4 0x54 -#define SF_XARG5 0x58 -#define SF_XXARG 0x5c - -#ifdef __KERNEL__ /* global_reg_snapshot offsets */ #define GR_SNAP_TSTATE 0x00 @@ -420,29 +123,4 @@ extern void show_regs(struct pt_regs *); #define GR_SNAP_THREAD 0x30 #define GR_SNAP_PAD1 0x38 -#endif /* __KERNEL__ */ - -/* Stuff for the ptrace system call */ -#define PTRACE_SPARC_DETACH 11 -#define PTRACE_GETREGS 12 -#define PTRACE_SETREGS 13 -#define PTRACE_GETFPREGS 14 -#define PTRACE_SETFPREGS 15 -#define PTRACE_READDATA 16 -#define PTRACE_WRITEDATA 17 -#define PTRACE_READTEXT 18 -#define PTRACE_WRITETEXT 19 -#define PTRACE_GETFPAREGS 20 -#define PTRACE_SETFPAREGS 21 - -/* There are for debugging 64-bit processes, either from a 32 or 64 bit - * parent. Thus their complements are for debugging 32-bit processes only. - */ - -#define PTRACE_GETREGS64 22 -#define PTRACE_SETREGS64 23 -/* PTRACE_SYSCALL is 24 */ -#define PTRACE_GETFPREGS64 25 -#define PTRACE_SETFPREGS64 26 - #endif /* !(__SPARC_PTRACE_H) */ diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h index a2b4302869b..069bf4d663a 100644 --- a/arch/sparc/include/asm/rwsem.h +++ b/arch/sparc/include/asm/rwsem.h @@ -13,53 +13,12 @@ #ifdef __KERNEL__ -#include <linux/list.h> -#include <linux/spinlock.h> - -struct rwsem_waiter; - -struct rw_semaphore { - signed long count; #define RWSEM_UNLOCKED_VALUE 0x00000000L #define RWSEM_ACTIVE_BIAS 0x00000001L #define RWSEM_ACTIVE_MASK 0xffffffffL #define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1) #define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS #define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS) - spinlock_t wait_lock; - struct list_head wait_list; -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lockdep_map dep_map; -#endif -}; - -#ifdef CONFIG_DEBUG_LOCK_ALLOC -# define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname } -#else -# define __RWSEM_DEP_MAP_INIT(lockname) -#endif - -#define __RWSEM_INITIALIZER(name) \ -{ RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \ - LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) } - -#define DECLARE_RWSEM(name) \ - struct rw_semaphore name = __RWSEM_INITIALIZER(name) - -extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem); -extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem); - -extern void __init_rwsem(struct rw_semaphore *sem, const char *name, - struct lock_class_key *key); - -#define init_rwsem(sem) \ -do { \ - static struct lock_class_key __key; \ - \ - __init_rwsem((sem), #sem, &__key); \ -} while (0) /* * lock for reading @@ -160,11 +119,6 @@ static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem) return atomic64_add_return(delta, (atomic64_t *)(&sem->count)); } -static inline int rwsem_is_locked(struct rw_semaphore *sem) -{ - return (sem->count != 0); -} - #endif /* __KERNEL__ */ #endif /* _SPARC64_RWSEM_H */ diff --git a/arch/sparc/include/asm/sections.h b/arch/sparc/include/asm/sections.h index 0b0553bbd8a..f300d1a9b2b 100644 --- a/arch/sparc/include/asm/sections.h +++ b/arch/sparc/include/asm/sections.h @@ -7,4 +7,7 @@ /* sparc entry point */ extern char _start[]; +extern char __leon_1insn_patch[]; +extern char __leon_1insn_patch_end[]; + #endif diff --git a/arch/sparc/include/asm/serial.h b/arch/sparc/include/asm/serial.h deleted file mode 100644 index f90d61c2805..00000000000 --- a/arch/sparc/include/asm/serial.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef __SPARC_SERIAL_H -#define __SPARC_SERIAL_H - -#define BASE_BAUD ( 1843200 / 16 ) - -#endif /* __SPARC_SERIAL_H */ diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h index 2643c62f4ac..f5fffd84d0d 100644 --- a/arch/sparc/include/asm/setup.h +++ b/arch/sparc/include/asm/setup.h @@ -1,14 +1,66 @@ /* * Just a place holder. */ - #ifndef _SPARC_SETUP_H #define _SPARC_SETUP_H -#if defined(__sparc__) && defined(__arch64__) -# define COMMAND_LINE_SIZE 2048 -#else -# define COMMAND_LINE_SIZE 256 +#include <linux/interrupt.h> + +#include <uapi/asm/setup.h> + +extern char reboot_command[]; + +#ifdef CONFIG_SPARC32 +/* The CPU that was used for booting + * Only sun4d + leon may have boot_cpu_id != 0 + */ +extern unsigned char boot_cpu_id; + +extern unsigned long empty_zero_page; + +extern int serial_console; +static inline int con_is_present(void) +{ + return serial_console ? 0 : 1; +} + +/* from irq_32.c */ +extern volatile unsigned char *fdc_status; +extern char *pdma_vaddr; +extern unsigned long pdma_size; +extern volatile int doing_pdma; + +/* This is software state */ +extern char *pdma_base; +extern unsigned long pdma_areasize; + +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); + +/* setup_32.c */ +extern unsigned long cmdline_memory_size; + +/* devices.c */ +void __init device_scan(void); + +/* unaligned_32.c */ +unsigned long safe_compute_effective_address(struct pt_regs *, unsigned int); + #endif +#ifdef CONFIG_SPARC64 +/* unaligned_64.c */ +int handle_ldf_stq(u32 insn, struct pt_regs *regs); +void handle_ld_nf(u32 insn, struct pt_regs *regs); + +/* init_64.c */ +extern atomic_t dcpage_flushes; +extern atomic_t dcpage_flushes_xcall; + +extern int sysctl_tsb_ratio; +#endif + +void sun_do_break(void); +extern int stop_a_enabled; +extern int scons_pwroff; + #endif /* _SPARC_SETUP_H */ diff --git a/arch/sparc/include/asm/sfp-machine_32.h b/arch/sparc/include/asm/sfp-machine_32.h index 01d9c3b5a73..838c9d58f3b 100644 --- a/arch/sparc/include/asm/sfp-machine_32.h +++ b/arch/sparc/include/asm/sfp-machine_32.h @@ -79,9 +79,9 @@ __asm__ ("addcc %r7,%8,%2\n\t" \ "addxcc %r5,%6,%1\n\t" \ "addx %r3,%4,%0\n" \ - : "=r" ((USItype)(r2)), \ - "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x2)), \ "rI" ((USItype)(y2)), \ "%rJ" ((USItype)(x1)), \ @@ -94,9 +94,9 @@ __asm__ ("subcc %r7,%8,%2\n\t" \ "subxcc %r5,%6,%1\n\t" \ "subx %r3,%4,%0\n" \ - : "=r" ((USItype)(r2)), \ - "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x2)), \ "rI" ((USItype)(y2)), \ "%rJ" ((USItype)(x1)), \ @@ -115,8 +115,8 @@ "addxcc %r6,%7,%0\n\t" \ "addxcc %r4,%5,%%g2\n\t" \ "addx %r2,%3,%%g1\n\t" \ - : "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x3)), \ "rI" ((USItype)(y3)), \ "%rJ" ((USItype)(x2)), \ @@ -140,8 +140,8 @@ "subxcc %r6,%7,%0\n\t" \ "subxcc %r4,%5,%%g2\n\t" \ "subx %r2,%3,%%g1\n\t" \ - : "=&r" ((USItype)(r1)), \ - "=&r" ((USItype)(r0)) \ + : "=&r" (r1), \ + "=&r" (r0) \ : "%rJ" ((USItype)(x3)), \ "rI" ((USItype)(y3)), \ "%rJ" ((USItype)(x2)), \ @@ -164,10 +164,10 @@ "addxcc %2,%%g0,%2\n\t" \ "addxcc %1,%%g0,%1\n\t" \ "addx %0,%%g0,%0\n\t" \ - : "=&r" ((USItype)(x3)), \ - "=&r" ((USItype)(x2)), \ - "=&r" ((USItype)(x1)), \ - "=&r" ((USItype)(x0)) \ + : "=&r" (x3), \ + "=&r" (x2), \ + "=&r" (x1), \ + "=&r" (x0) \ : "rI" ((USItype)(i)), \ "0" ((USItype)(x3)), \ "1" ((USItype)(x2)), \ diff --git a/arch/sparc/include/asm/shmparam_32.h b/arch/sparc/include/asm/shmparam_32.h index 59a1243c12f..142825c8d3a 100644 --- a/arch/sparc/include/asm/shmparam_32.h +++ b/arch/sparc/include/asm/shmparam_32.h @@ -4,8 +4,6 @@ #define __ARCH_FORCE_SHMLBA 1 extern int vac_cache_size; -#define SHMLBA (vac_cache_size ? vac_cache_size : \ - (sparc_cpu_model == sun4c ? (64 * 1024) : \ - (sparc_cpu_model == sun4 ? (128 * 1024) : PAGE_SIZE))) +#define SHMLBA (vac_cache_size ? vac_cache_size : PAGE_SIZE) #endif /* _ASMSPARC_SHMPARAM_H */ diff --git a/arch/sparc/include/asm/sigcontext.h b/arch/sparc/include/asm/sigcontext.h index a1607d18035..fc2df1e892c 100644 --- a/arch/sparc/include/asm/sigcontext.h +++ b/arch/sparc/include/asm/sigcontext.h @@ -1,8 +1,8 @@ #ifndef __SPARC_SIGCONTEXT_H #define __SPARC_SIGCONTEXT_H -#ifdef __KERNEL__ #include <asm/ptrace.h> +#include <uapi/asm/sigcontext.h> #ifndef __ASSEMBLY__ @@ -45,6 +45,19 @@ typedef struct { int si_mask; } __siginfo32_t; +#define __SIGC_MAXWIN 7 + +typedef struct { + unsigned long locals[8]; + unsigned long ins[8]; +} __siginfo_reg_window; + +typedef struct { + int wsaved; + __siginfo_reg_window reg_window[__SIGC_MAXWIN]; + unsigned long rwbuf_stkptrs[__SIGC_MAXWIN]; +} __siginfo_rwin_t; + #ifdef CONFIG_SPARC64 typedef struct { unsigned int si_float_regs [64]; @@ -73,6 +86,7 @@ struct sigcontext { unsigned long ss_size; } sigc_stack; unsigned long sigc_mask; + __siginfo_rwin_t * sigc_rwin_save; }; #else @@ -91,6 +105,4 @@ typedef struct { #endif /* !(__ASSEMBLY__) */ -#endif /* (__KERNEL__) */ - #endif /* !(__SPARC_SIGCONTEXT_H) */ diff --git a/arch/sparc/include/asm/siginfo.h b/arch/sparc/include/asm/siginfo.h index 988e5d8ed11..48c34c19f81 100644 --- a/arch/sparc/include/asm/siginfo.h +++ b/arch/sparc/include/asm/siginfo.h @@ -1,22 +1,8 @@ #ifndef __SPARC_SIGINFO_H #define __SPARC_SIGINFO_H -#if defined(__sparc__) && defined(__arch64__) +#include <uapi/asm/siginfo.h> -#define SI_PAD_SIZE32 ((SI_MAX_SIZE/sizeof(int)) - 3) -#define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) -#define __ARCH_SI_BAND_T int - -#endif /* defined(__sparc__) && defined(__arch64__) */ - - -#define __ARCH_SI_TRAPNO - -#include <asm-generic/siginfo.h> - -#ifdef __KERNEL__ - -#include <linux/compat.h> #ifdef CONFIG_COMPAT @@ -24,14 +10,4 @@ struct compat_siginfo; #endif /* CONFIG_COMPAT */ -#endif /* __KERNEL__ */ - -#define SI_NOINFO 32767 /* no information in siginfo_t */ - -/* - * SIGEMT si_codes - */ -#define EMT_TAGOVF (__SI_FAULT|1) /* tag overflow */ -#define NSIGEMT 1 - #endif /* !(__SPARC_SIGINFO_H) */ diff --git a/arch/sparc/include/asm/signal.h b/arch/sparc/include/asm/signal.h index e49b828a247..c33ce3f2ba8 100644 --- a/arch/sparc/include/asm/signal.h +++ b/arch/sparc/include/asm/signal.h @@ -1,167 +1,13 @@ #ifndef __SPARC_SIGNAL_H #define __SPARC_SIGNAL_H -#include <asm/sigcontext.h> -#include <linux/compiler.h> - -#ifdef __KERNEL__ #ifndef __ASSEMBLY__ #include <linux/personality.h> #include <linux/types.h> #endif -#endif - -/* On the Sparc the signal handlers get passed a 'sub-signal' code - * for certain signal types, which we document here. - */ -#define SIGHUP 1 -#define SIGINT 2 -#define SIGQUIT 3 -#define SIGILL 4 -#define SUBSIG_STACK 0 -#define SUBSIG_ILLINST 2 -#define SUBSIG_PRIVINST 3 -#define SUBSIG_BADTRAP(t) (0x80 + (t)) - -#define SIGTRAP 5 -#define SIGABRT 6 -#define SIGIOT 6 - -#define SIGEMT 7 -#define SUBSIG_TAG 10 - -#define SIGFPE 8 -#define SUBSIG_FPDISABLED 0x400 -#define SUBSIG_FPERROR 0x404 -#define SUBSIG_FPINTOVFL 0x001 -#define SUBSIG_FPSTSIG 0x002 -#define SUBSIG_IDIVZERO 0x014 -#define SUBSIG_FPINEXACT 0x0c4 -#define SUBSIG_FPDIVZERO 0x0c8 -#define SUBSIG_FPUNFLOW 0x0cc -#define SUBSIG_FPOPERROR 0x0d0 -#define SUBSIG_FPOVFLOW 0x0d4 - -#define SIGKILL 9 -#define SIGBUS 10 -#define SUBSIG_BUSTIMEOUT 1 -#define SUBSIG_ALIGNMENT 2 -#define SUBSIG_MISCERROR 5 - -#define SIGSEGV 11 -#define SUBSIG_NOMAPPING 3 -#define SUBSIG_PROTECTION 4 -#define SUBSIG_SEGERROR 5 - -#define SIGSYS 12 - -#define SIGPIPE 13 -#define SIGALRM 14 -#define SIGTERM 15 -#define SIGURG 16 - -/* SunOS values which deviate from the Linux/i386 ones */ -#define SIGSTOP 17 -#define SIGTSTP 18 -#define SIGCONT 19 -#define SIGCHLD 20 -#define SIGTTIN 21 -#define SIGTTOU 22 -#define SIGIO 23 -#define SIGPOLL SIGIO /* SysV name for SIGIO */ -#define SIGXCPU 24 -#define SIGXFSZ 25 -#define SIGVTALRM 26 -#define SIGPROF 27 -#define SIGWINCH 28 -#define SIGLOST 29 -#define SIGPWR SIGLOST -#define SIGUSR1 30 -#define SIGUSR2 31 - -/* Most things should be clean enough to redefine this at will, if care - is taken to make libc match. */ - -#define __OLD_NSIG 32 -#define __NEW_NSIG 64 -#ifdef __arch64__ -#define _NSIG_BPW 64 -#else -#define _NSIG_BPW 32 -#endif -#define _NSIG_WORDS (__NEW_NSIG / _NSIG_BPW) - -#define SIGRTMIN 32 -#define SIGRTMAX __NEW_NSIG - -#if defined(__KERNEL__) || defined(__WANT_POSIX1B_SIGNALS__) -#define _NSIG __NEW_NSIG -#define __new_sigset_t sigset_t -#define __new_sigaction sigaction -#define __new_sigaction32 sigaction32 -#define __old_sigset_t old_sigset_t -#define __old_sigaction old_sigaction -#define __old_sigaction32 old_sigaction32 -#else -#define _NSIG __OLD_NSIG -#define NSIG _NSIG -#define __old_sigset_t sigset_t -#define __old_sigaction sigaction -#define __old_sigaction32 sigaction32 -#endif +#include <uapi/asm/signal.h> #ifndef __ASSEMBLY__ - -typedef unsigned long __old_sigset_t; /* at least 32 bits */ - -typedef struct { - unsigned long sig[_NSIG_WORDS]; -} __new_sigset_t; - -/* A SunOS sigstack */ -struct sigstack { - /* XXX 32-bit pointers pinhead XXX */ - char *the_stack; - int cur_status; -}; - -/* Sigvec flags */ -#define _SV_SSTACK 1u /* This signal handler should use sig-stack */ -#define _SV_INTR 2u /* Sig return should not restart system call */ -#define _SV_RESET 4u /* Set handler to SIG_DFL upon taken signal */ -#define _SV_IGNCHILD 8u /* Do not send SIGCHLD */ - -/* - * sa_flags values: SA_STACK is not currently supported, but will allow the - * usage of signal stacks by using the (now obsolete) sa_restorer field in - * the sigaction structure as a stack pointer. This is now possible due to - * the changes in signal handling. LBT 010493. - * SA_RESTART flag to get restarting signals (which were the default long ago) - */ -#define SA_NOCLDSTOP _SV_IGNCHILD -#define SA_STACK _SV_SSTACK -#define SA_ONSTACK _SV_SSTACK -#define SA_RESTART _SV_INTR -#define SA_ONESHOT _SV_RESET -#define SA_NOMASK 0x20u -#define SA_NOCLDWAIT 0x100u -#define SA_SIGINFO 0x200u - - -#define SIG_BLOCK 0x01 /* for blocking signals */ -#define SIG_UNBLOCK 0x02 /* for unblocking signals */ -#define SIG_SETMASK 0x04 /* for setting the signal mask */ - -/* - * sigaltstack controls - */ -#define SS_ONSTACK 1 -#define SS_DISABLE 2 - -#define MINSIGSTKSZ 4096 -#define SIGSTKSZ 16384 - -#ifdef __KERNEL__ /* * DJHR * SA_STATIC_ALLOC is used for the sparc32 system to indicate that this @@ -174,41 +20,9 @@ struct sigstack { * */ #define SA_STATIC_ALLOC 0x8000 -#endif - -#include <asm-generic/signal-defs.h> - -struct __new_sigaction { - __sighandler_t sa_handler; - unsigned long sa_flags; - __sigrestore_t sa_restorer; /* not used by Linux/SPARC yet */ - __new_sigset_t sa_mask; -}; - -struct __old_sigaction { - __sighandler_t sa_handler; - __old_sigset_t sa_mask; - unsigned long sa_flags; - void (*sa_restorer)(void); /* not used by Linux/SPARC yet */ -}; -typedef struct sigaltstack { - void __user *ss_sp; - int ss_flags; - size_t ss_size; -} stack_t; - -#ifdef __KERNEL__ - -struct k_sigaction { - struct __new_sigaction sa; - void __user *ka_restorer; -}; - -#define ptrace_signal_deliver(regs, cookie) do { } while (0) - -#endif /* !(__KERNEL__) */ +#define __ARCH_HAS_KA_RESTORER +#define __ARCH_HAS_SA_RESTORER #endif /* !(__ASSEMBLY__) */ - #endif /* !(__SPARC_SIGNAL_H) */ diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h index 841905c1021..7c24e08a88d 100644 --- a/arch/sparc/include/asm/smp_32.h +++ b/arch/sparc/include/asm/smp_32.h @@ -8,7 +8,6 @@ #include <linux/threads.h> #include <asm/head.h> -#include <asm/btfixup.h> #ifndef __ASSEMBLY__ @@ -22,17 +21,22 @@ #include <asm/ptrace.h> #include <asm/asi.h> -#include <asm/atomic.h> +#include <linux/atomic.h> /* * Private routines/data */ extern unsigned char boot_cpu_id; +extern volatile unsigned long cpu_callin_map[NR_CPUS]; +extern cpumask_t smp_commenced_mask; +extern struct linux_prom_registers smp_penguin_ctable; typedef void (*smpfunc_t)(unsigned long, unsigned long, unsigned long, unsigned long, unsigned long); +void cpu_panic(void); + /* * General functions that each host system must provide. */ @@ -41,120 +45,66 @@ void sun4m_init_smp(void); void sun4d_init_smp(void); void smp_callin(void); -void smp_boot_cpus(void); void smp_store_cpu_info(int); +void smp_resched_interrupt(void); +void smp_call_function_single_interrupt(void); +void smp_call_function_interrupt(void); + struct seq_file; void smp_bogo(struct seq_file *); void smp_info(struct seq_file *); -BTFIXUPDEF_CALL(void, smp_cross_call, smpfunc_t, cpumask_t, unsigned long, unsigned long, unsigned long, unsigned long) -BTFIXUPDEF_CALL(int, __hard_smp_processor_id, void) -BTFIXUPDEF_BLACKBOX(hard_smp_processor_id) -BTFIXUPDEF_BLACKBOX(load_current) - -#define smp_cross_call(func,mask,arg1,arg2,arg3,arg4) BTFIXUP_CALL(smp_cross_call)(func,mask,arg1,arg2,arg3,arg4) - -static inline void xc0(smpfunc_t func) { smp_cross_call(func, cpu_online_map, 0, 0, 0, 0); } -static inline void xc1(smpfunc_t func, unsigned long arg1) -{ smp_cross_call(func, cpu_online_map, arg1, 0, 0, 0); } -static inline void xc2(smpfunc_t func, unsigned long arg1, unsigned long arg2) -{ smp_cross_call(func, cpu_online_map, arg1, arg2, 0, 0); } -static inline void xc3(smpfunc_t func, unsigned long arg1, unsigned long arg2, - unsigned long arg3) -{ smp_cross_call(func, cpu_online_map, arg1, arg2, arg3, 0); } -static inline void xc4(smpfunc_t func, unsigned long arg1, unsigned long arg2, - unsigned long arg3, unsigned long arg4) -{ smp_cross_call(func, cpu_online_map, arg1, arg2, arg3, arg4); } - -static inline int smp_call_function(void (*func)(void *info), void *info, int wait) +struct sparc32_ipi_ops { + void (*cross_call)(smpfunc_t func, cpumask_t mask, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4); + void (*resched)(int cpu); + void (*single)(int cpu); + void (*mask_one)(int cpu); +}; +extern const struct sparc32_ipi_ops *sparc32_ipi_ops; + +static inline void xc0(smpfunc_t func) { - xc1((smpfunc_t)func, (unsigned long)info); - return 0; + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, 0, 0, 0, 0); } -static inline int smp_call_function_single(int cpuid, void (*func) (void *info), - void *info, int wait) +static inline void xc1(smpfunc_t func, unsigned long arg1) { - smp_cross_call((smpfunc_t)func, cpumask_of_cpu(cpuid), - (unsigned long) info, 0, 0, 0); - return 0; + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, 0, 0, 0); } - -static inline int cpu_logical_map(int cpu) +static inline void xc2(smpfunc_t func, unsigned long arg1, unsigned long arg2) { - return cpu; + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, 0, 0); } -static inline int hard_smp4m_processor_id(void) +static inline void xc3(smpfunc_t func, unsigned long arg1, unsigned long arg2, + unsigned long arg3) { - int cpuid; - - __asm__ __volatile__("rd %%tbr, %0\n\t" - "srl %0, 12, %0\n\t" - "and %0, 3, %0\n\t" : - "=&r" (cpuid)); - return cpuid; + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, + arg1, arg2, arg3, 0); } -static inline int hard_smp4d_processor_id(void) +static inline void xc4(smpfunc_t func, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4) { - int cpuid; - - __asm__ __volatile__("lda [%%g0] %1, %0\n\t" : - "=&r" (cpuid) : "i" (ASI_M_VIKING_TMP1)); - return cpuid; + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, + arg1, arg2, arg3, arg4); } -extern inline int hard_smpleon_processor_id(void) -{ - int cpuid; - __asm__ __volatile__("rd %%asr17,%0\n\t" - "srl %0,28,%0" : - "=&r" (cpuid) : ); - return cpuid; -} +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); -#ifndef MODULE -static inline int hard_smp_processor_id(void) +static inline int cpu_logical_map(int cpu) { - int cpuid; - - /* Black box - sun4m - __asm__ __volatile__("rd %%tbr, %0\n\t" - "srl %0, 12, %0\n\t" - "and %0, 3, %0\n\t" : - "=&r" (cpuid)); - - sun4d - __asm__ __volatile__("lda [%g0] ASI_M_VIKING_TMP1, %0\n\t" - "nop; nop" : - "=&r" (cpuid)); - See btfixup.h and btfixupprep.c to understand how a blackbox works. - */ - __asm__ __volatile__("sethi %%hi(___b_hard_smp_processor_id), %0\n\t" - "sethi %%hi(boot_cpu_id), %0\n\t" - "ldub [%0 + %%lo(boot_cpu_id)], %0\n\t" : - "=&r" (cpuid)); - return cpuid; + return cpu; } -#else -static inline int hard_smp_processor_id(void) -{ - int cpuid; - __asm__ __volatile__("mov %%o7, %%g1\n\t" - "call ___f___hard_smp_processor_id\n\t" - " nop\n\t" - "mov %%g2, %0\n\t" : "=r"(cpuid) : : "g1", "g2"); - return cpuid; -} -#endif +int hard_smp_processor_id(void); #define raw_smp_processor_id() (current_thread_info()->cpu) -#define prof_multiplier(__cpu) cpu_data(__cpu).multiplier -#define prof_counter(__cpu) cpu_data(__cpu).counter - void smp_setup_cpu_possible_map(void); #endif /* !(__ASSEMBLY__) */ diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index f49e11cd4de..26d9e772686 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -27,31 +27,41 @@ */ #include <linux/bitops.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/percpu.h> DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); extern cpumask_t cpu_core_map[NR_CPUS]; -extern int sparc64_multi_core; -extern void arch_send_call_function_single_ipi(int cpu); -extern void arch_send_call_function_ipi_mask(const struct cpumask *mask); +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); /* * General functions that each host system must provide. */ -extern int hard_smp_processor_id(void); +int hard_smp_processor_id(void); #define raw_smp_processor_id() (current_thread_info()->cpu) -extern void smp_fill_in_sib_core_maps(void); -extern void cpu_play_dead(void); +void smp_fill_in_sib_core_maps(void); +void cpu_play_dead(void); -extern void smp_fetch_global_regs(void); +void smp_fetch_global_regs(void); +void smp_fetch_global_pmu(void); + +struct seq_file; +void smp_bogo(struct seq_file *); +void smp_info(struct seq_file *); + +void smp_callin(void); +void cpu_panic(void); +void smp_synchronize_tick_client(void); +void smp_capture(void); +void smp_release(void); #ifdef CONFIG_HOTPLUG_CPU -extern int __cpu_disable(void); -extern void __cpu_die(unsigned int cpu); +int __cpu_disable(void); +void __cpu_die(unsigned int cpu); #endif #endif /* !(__ASSEMBLY__) */ @@ -61,6 +71,7 @@ extern void __cpu_die(unsigned int cpu); #define hard_smp_processor_id() 0 #define smp_fill_in_sib_core_maps() do { } while (0) #define smp_fetch_global_regs() do { } while (0) +#define smp_fetch_global_pmu() do { } while (0) #endif /* !(CONFIG_SMP) */ diff --git a/arch/sparc/include/asm/smpprim.h b/arch/sparc/include/asm/smpprim.h deleted file mode 100644 index eb849d862c6..00000000000 --- a/arch/sparc/include/asm/smpprim.h +++ /dev/null @@ -1,54 +0,0 @@ -/* - * smpprim.h: SMP locking primitives on the Sparc - * - * God knows we won't be actually using this code for some time - * but I thought I'd write it since I knew how. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ - -#ifndef __SPARC_SMPPRIM_H -#define __SPARC_SMPPRIM_H - -/* Test and set the unsigned byte at ADDR to 1. Returns the previous - * value. On the Sparc we use the ldstub instruction since it is - * atomic. - */ - -static inline __volatile__ char test_and_set(void *addr) -{ - char state = 0; - - __asm__ __volatile__("ldstub [%0], %1 ! test_and_set\n\t" - "=r" (addr), "=r" (state) : - "0" (addr), "1" (state) : "memory"); - - return state; -} - -/* Initialize a spin-lock. */ -static inline __volatile__ smp_initlock(void *spinlock) -{ - /* Unset the lock. */ - *((unsigned char *) spinlock) = 0; - - return; -} - -/* This routine spins until it acquires the lock at ADDR. */ -static inline __volatile__ smp_lock(void *addr) -{ - while(test_and_set(addr) == 0xff) - ; - - /* We now have the lock */ - return; -} - -/* This routine releases the lock at ADDR. */ -static inline __volatile__ smp_unlock(void *addr) -{ - *((unsigned char *) addr) = 0; -} - -#endif /* !(__SPARC_SMPPRIM_H) */ diff --git a/arch/sparc/include/asm/sparsemem.h b/arch/sparc/include/asm/sparsemem.h index b99d4e4b6d2..e5e1752d5d7 100644 --- a/arch/sparc/include/asm/sparsemem.h +++ b/arch/sparc/include/asm/sparsemem.h @@ -3,9 +3,11 @@ #ifdef __KERNEL__ +#include <asm/page.h> + #define SECTION_SIZE_BITS 30 -#define MAX_PHYSADDR_BITS 42 -#define MAX_PHYSMEM_BITS 42 +#define MAX_PHYSADDR_BITS MAX_PHYS_ADDRESS_BITS +#define MAX_PHYSMEM_BITS MAX_PHYS_ADDRESS_BITS #endif /* !(__KERNEL__) */ diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h index 7f9b9dba38a..bcc98fc3528 100644 --- a/arch/sparc/include/asm/spinlock_32.h +++ b/arch/sparc/include/asm/spinlock_32.h @@ -9,6 +9,7 @@ #ifndef __ASSEMBLY__ #include <asm/psr.h> +#include <asm/processor.h> /* for cpu_relax */ #define arch_spin_is_locked(lock) (*((volatile unsigned char *)(lock)) != 0) @@ -130,6 +131,15 @@ static inline void arch_write_lock(arch_rwlock_t *rw) *(volatile __u32 *)&lp->lock = ~0U; } +static void inline arch_write_unlock(arch_rwlock_t *lock) +{ + __asm__ __volatile__( +" st %%g0, [%0]" + : /* no outputs */ + : "r" (lock) + : "memory"); +} + static inline int arch_write_trylock(arch_rwlock_t *rw) { unsigned int val; @@ -174,8 +184,6 @@ static inline int __arch_read_trylock(arch_rwlock_t *rw) res; \ }) -#define arch_write_unlock(rw) do { (rw)->lock = 0; } while(0) - #define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock) #define arch_read_lock_flags(rw, flags) arch_read_lock(rw) #define arch_write_lock_flags(rw, flags) arch_write_lock(rw) diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h index 073936a8b27..96891769497 100644 --- a/arch/sparc/include/asm/spinlock_64.h +++ b/arch/sparc/include/asm/spinlock_64.h @@ -210,14 +210,8 @@ static int inline arch_write_trylock(arch_rwlock_t *lock) return result; } -#define arch_read_lock(p) arch_read_lock(p) #define arch_read_lock_flags(p, f) arch_read_lock(p) -#define arch_read_trylock(p) arch_read_trylock(p) -#define arch_read_unlock(p) arch_read_unlock(p) -#define arch_write_lock(p) arch_write_lock(p) #define arch_write_lock_flags(p, f) arch_write_lock(p) -#define arch_write_unlock(p) arch_write_unlock(p) -#define arch_write_trylock(p) arch_write_trylock(p) #define arch_read_can_lock(rw) (!((rw)->lock & 0x80000000UL)) #define arch_write_can_lock(rw) (!(rw)->lock) diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h index f0d0c40c44d..3fc58691dbd 100644 --- a/arch/sparc/include/asm/spitfire.h +++ b/arch/sparc/include/asm/spitfire.h @@ -42,6 +42,10 @@ #define SUN4V_CHIP_INVALID 0x00 #define SUN4V_CHIP_NIAGARA1 0x01 #define SUN4V_CHIP_NIAGARA2 0x02 +#define SUN4V_CHIP_NIAGARA3 0x03 +#define SUN4V_CHIP_NIAGARA4 0x04 +#define SUN4V_CHIP_NIAGARA5 0x05 +#define SUN4V_CHIP_SPARC64X 0x8a #define SUN4V_CHIP_UNKNOWN 0xff #ifndef __ASSEMBLY__ @@ -58,7 +62,7 @@ extern enum ultra_tlb_layout tlb_type; extern int sun4v_chip_type; extern int cheetah_pcache_forced_on; -extern void cheetah_enable_pcache(void); +void cheetah_enable_pcache(void); #define sparc64_highest_locked_tlbent() \ (tlb_type == spitfire ? \ diff --git a/arch/sparc/include/asm/stacktrace.h b/arch/sparc/include/asm/stacktrace.h index 6cee39adf6d..c30d066f304 100644 --- a/arch/sparc/include/asm/stacktrace.h +++ b/arch/sparc/include/asm/stacktrace.h @@ -1,6 +1,6 @@ #ifndef _SPARC64_STACKTRACE_H #define _SPARC64_STACKTRACE_H -extern void stack_trace_flush(void); +void stack_trace_flush(void); #endif /* _SPARC64_STACKTRACE_H */ diff --git a/arch/sparc/include/asm/starfire.h b/arch/sparc/include/asm/starfire.h index d56ce60a599..c100dc27a0a 100644 --- a/arch/sparc/include/asm/starfire.h +++ b/arch/sparc/include/asm/starfire.h @@ -11,10 +11,10 @@ extern int this_is_starfire; -extern void check_if_starfire(void); -extern int starfire_hard_smp_processor_id(void); -extern void starfire_hookup(int); -extern unsigned int starfire_translate(unsigned long imap, unsigned int upaid); +void check_if_starfire(void); +int starfire_hard_smp_processor_id(void); +void starfire_hookup(int); +unsigned int starfire_translate(unsigned long imap, unsigned int upaid); #endif #endif diff --git a/arch/sparc/include/asm/string_32.h b/arch/sparc/include/asm/string_32.h index edf196ee4ef..69974e92461 100644 --- a/arch/sparc/include/asm/string_32.h +++ b/arch/sparc/include/asm/string_32.h @@ -15,7 +15,7 @@ #ifdef __KERNEL__ -extern void __memmove(void *,const void *,__kernel_size_t); +void __memmove(void *,const void *,__kernel_size_t); #ifndef EXPORT_SYMTAB_STROPS @@ -40,8 +40,8 @@ extern void __memmove(void *,const void *,__kernel_size_t); #undef memscan #define memscan(__arg0, __char, __arg2) \ ({ \ - extern void *__memscan_zero(void *, size_t); \ - extern void *__memscan_generic(void *, int, size_t); \ + void *__memscan_zero(void *, size_t); \ + void *__memscan_generic(void *, int, size_t); \ void *__retval, *__addr = (__arg0); \ size_t __size = (__arg2); \ \ @@ -54,75 +54,14 @@ extern void __memmove(void *,const void *,__kernel_size_t); }) #define __HAVE_ARCH_MEMCMP -extern int memcmp(const void *,const void *,__kernel_size_t); +int memcmp(const void *,const void *,__kernel_size_t); /* Now the str*() stuff... */ #define __HAVE_ARCH_STRLEN -extern __kernel_size_t strlen(const char *); +__kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP - -extern int __strncmp(const char *, const char *, __kernel_size_t); - -static inline int __constant_strncmp(const char *src, const char *dest, __kernel_size_t count) -{ - register int retval; - switch(count) { - case 0: return 0; - case 1: return (src[0] - dest[0]); - case 2: retval = (src[0] - dest[0]); - if(!retval && src[0]) - retval = (src[1] - dest[1]); - return retval; - case 3: retval = (src[0] - dest[0]); - if(!retval && src[0]) { - retval = (src[1] - dest[1]); - if(!retval && src[1]) - retval = (src[2] - dest[2]); - } - return retval; - case 4: retval = (src[0] - dest[0]); - if(!retval && src[0]) { - retval = (src[1] - dest[1]); - if(!retval && src[1]) { - retval = (src[2] - dest[2]); - if (!retval && src[2]) - retval = (src[3] - dest[3]); - } - } - return retval; - case 5: retval = (src[0] - dest[0]); - if(!retval && src[0]) { - retval = (src[1] - dest[1]); - if(!retval && src[1]) { - retval = (src[2] - dest[2]); - if (!retval && src[2]) { - retval = (src[3] - dest[3]); - if (!retval && src[3]) - retval = (src[4] - dest[4]); - } - } - } - return retval; - default: - retval = (src[0] - dest[0]); - if(!retval && src[0]) { - retval = (src[1] - dest[1]); - if(!retval && src[1]) { - retval = (src[2] - dest[2]); - if(!retval && src[2]) - retval = __strncmp(src+3,dest+3,count-3); - } - } - return retval; - } -} - -#undef strncmp -#define strncmp(__arg0, __arg1, __arg2) \ -(__builtin_constant_p(__arg2) ? \ - __constant_strncmp(__arg0, __arg1, __arg2) : \ - __strncmp(__arg0, __arg1, __arg2)) +int strncmp(const char *, const char *, __kernel_size_t); #endif /* !EXPORT_SYMTAB_STROPS */ diff --git a/arch/sparc/include/asm/string_64.h b/arch/sparc/include/asm/string_64.h index 9623bc21315..5936b8ff3c0 100644 --- a/arch/sparc/include/asm/string_64.h +++ b/arch/sparc/include/asm/string_64.h @@ -19,7 +19,7 @@ /* First the mem*() things. */ #define __HAVE_ARCH_MEMMOVE -extern void *memmove(void *, const void *, __kernel_size_t); +void *memmove(void *, const void *, __kernel_size_t); #define __HAVE_ARCH_MEMCPY #define memcpy(t, f, n) __builtin_memcpy(t, f, n) @@ -32,8 +32,8 @@ extern void *memmove(void *, const void *, __kernel_size_t); #undef memscan #define memscan(__arg0, __char, __arg2) \ ({ \ - extern void *__memscan_zero(void *, size_t); \ - extern void *__memscan_generic(void *, int, size_t); \ + void *__memscan_zero(void *, size_t); \ + void *__memscan_generic(void *, int, size_t); \ void *__retval, *__addr = (__arg0); \ size_t __size = (__arg2); \ \ @@ -46,14 +46,14 @@ extern void *memmove(void *, const void *, __kernel_size_t); }) #define __HAVE_ARCH_MEMCMP -extern int memcmp(const void *,const void *,__kernel_size_t); +int memcmp(const void *,const void *,__kernel_size_t); /* Now the str*() stuff... */ #define __HAVE_ARCH_STRLEN -extern __kernel_size_t strlen(const char *); +__kernel_size_t strlen(const char *); #define __HAVE_ARCH_STRNCMP -extern int strncmp(const char *, const char *, __kernel_size_t); +int strncmp(const char *, const char *, __kernel_size_t); #endif /* !EXPORT_SYMTAB_STROPS */ diff --git a/arch/sparc/include/asm/switch_to.h b/arch/sparc/include/asm/switch_to.h new file mode 100644 index 00000000000..2dc4fa5c6f8 --- /dev/null +++ b/arch/sparc/include/asm/switch_to.h @@ -0,0 +1,8 @@ +#ifndef ___ASM_SPARC_SWITCH_TO_H +#define ___ASM_SPARC_SWITCH_TO_H +#if defined(__sparc__) && defined(__arch64__) +#include <asm/switch_to_64.h> +#else +#include <asm/switch_to_32.h> +#endif +#endif diff --git a/arch/sparc/include/asm/switch_to_32.h b/arch/sparc/include/asm/switch_to_32.h new file mode 100644 index 00000000000..16f10374feb --- /dev/null +++ b/arch/sparc/include/asm/switch_to_32.h @@ -0,0 +1,106 @@ +#ifndef __SPARC_SWITCH_TO_H +#define __SPARC_SWITCH_TO_H + +#include <asm/smp.h> + +extern struct thread_info *current_set[NR_CPUS]; + +/* + * Flush windows so that the VM switch which follows + * would not pull the stack from under us. + * + * SWITCH_ENTER and SWITH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) + * XXX WTF is the above comment? Found in late teen 2.4.x. + */ +#ifdef CONFIG_SMP +#define SWITCH_ENTER(prv) \ + do { \ + if (test_tsk_thread_flag(prv, TIF_USEDFPU)) { \ + put_psr(get_psr() | PSR_EF); \ + fpsave(&(prv)->thread.float_regs[0], &(prv)->thread.fsr, \ + &(prv)->thread.fpqueue[0], &(prv)->thread.fpqdepth); \ + clear_tsk_thread_flag(prv, TIF_USEDFPU); \ + (prv)->thread.kregs->psr &= ~PSR_EF; \ + } \ + } while(0) + +#define SWITCH_DO_LAZY_FPU(next) /* */ +#else +#define SWITCH_ENTER(prv) /* */ +#define SWITCH_DO_LAZY_FPU(nxt) \ + do { \ + if (last_task_used_math != (nxt)) \ + (nxt)->thread.kregs->psr&=~PSR_EF; \ + } while(0) +#endif + +#define prepare_arch_switch(next) do { \ + __asm__ __volatile__( \ + ".globl\tflush_patch_switch\nflush_patch_switch:\n\t" \ + "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ + "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ + "save %sp, -0x40, %sp\n\t" \ + "restore; restore; restore; restore; restore; restore; restore"); \ +} while(0) + + /* Much care has gone into this code, do not touch it. + * + * We need to loadup regs l0/l1 for the newly forked child + * case because the trap return path relies on those registers + * holding certain values, gcc is told that they are clobbered. + * Gcc needs registers for 3 values in and 1 value out, so we + * clobber every non-fixed-usage register besides l2/l3/o4/o5. -DaveM + * + * Hey Dave, that do not touch sign is too much of an incentive + * - Anton & Pete + */ +#define switch_to(prev, next, last) do { \ + SWITCH_ENTER(prev); \ + SWITCH_DO_LAZY_FPU(next); \ + cpumask_set_cpu(smp_processor_id(), mm_cpumask(next->active_mm)); \ + __asm__ __volatile__( \ + "sethi %%hi(here - 0x8), %%o7\n\t" \ + "mov %%g6, %%g3\n\t" \ + "or %%o7, %%lo(here - 0x8), %%o7\n\t" \ + "rd %%psr, %%g4\n\t" \ + "std %%sp, [%%g6 + %4]\n\t" \ + "rd %%wim, %%g5\n\t" \ + "wr %%g4, 0x20, %%psr\n\t" \ + "nop\n\t" \ + "std %%g4, [%%g6 + %3]\n\t" \ + "ldd [%2 + %3], %%g4\n\t" \ + "mov %2, %%g6\n\t" \ + ".globl patchme_store_new_current\n" \ +"patchme_store_new_current:\n\t" \ + "st %2, [%1]\n\t" \ + "wr %%g4, 0x20, %%psr\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop\n\t" /* LEON needs all 3 nops: load to %sp depends on CWP. */ \ + "ldd [%%g6 + %4], %%sp\n\t" \ + "wr %%g5, 0x0, %%wim\n\t" \ + "ldd [%%sp + 0x00], %%l0\n\t" \ + "ldd [%%sp + 0x38], %%i6\n\t" \ + "wr %%g4, 0x0, %%psr\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "jmpl %%o7 + 0x8, %%g0\n\t" \ + " ld [%%g3 + %5], %0\n\t" \ + "here:\n" \ + : "=&r" (last) \ + : "r" (&(current_set[hard_smp_processor_id()])), \ + "r" (task_thread_info(next)), \ + "i" (TI_KPSR), \ + "i" (TI_KSP), \ + "i" (TI_TASK) \ + : "g1", "g2", "g3", "g4", "g5", "g7", \ + "l0", "l1", "l3", "l4", "l5", "l6", "l7", \ + "i0", "i1", "i2", "i3", "i4", "i5", \ + "o0", "o1", "o2", "o3", "o7"); \ + } while(0) + +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void synchronize_user_stack(void); + +#endif /* __SPARC_SWITCH_TO_H */ diff --git a/arch/sparc/include/asm/switch_to_64.h b/arch/sparc/include/asm/switch_to_64.h new file mode 100644 index 00000000000..10e76332dc9 --- /dev/null +++ b/arch/sparc/include/asm/switch_to_64.h @@ -0,0 +1,71 @@ +#ifndef __SPARC64_SWITCH_TO_64_H +#define __SPARC64_SWITCH_TO_64_H + +#include <asm/visasm.h> + +#define prepare_arch_switch(next) \ +do { \ + flushw_all(); \ +} while (0) + + /* See what happens when you design the chip correctly? + * + * We tell gcc we clobber all non-fixed-usage registers except + * for l0/l1. It will use one for 'next' and the other to hold + * the output value of 'last'. 'next' is not referenced again + * past the invocation of switch_to in the scheduler, so we need + * not preserve it's value. Hairy, but it lets us remove 2 loads + * and 2 stores in this critical code path. -DaveM + */ +#define switch_to(prev, next, last) \ +do { save_and_clear_fpu(); \ + /* If you are tempted to conditionalize the following */ \ + /* so that ASI is only written if it changes, think again. */ \ + __asm__ __volatile__("wr %%g0, %0, %%asi" \ + : : "r" (task_thread_info(next)->current_ds));\ + trap_block[current_thread_info()->cpu].thread = \ + task_thread_info(next); \ + __asm__ __volatile__( \ + "mov %%g4, %%g7\n\t" \ + "stx %%i6, [%%sp + 2047 + 0x70]\n\t" \ + "stx %%i7, [%%sp + 2047 + 0x78]\n\t" \ + "rdpr %%wstate, %%o5\n\t" \ + "stx %%o6, [%%g6 + %6]\n\t" \ + "stb %%o5, [%%g6 + %5]\n\t" \ + "rdpr %%cwp, %%o5\n\t" \ + "stb %%o5, [%%g6 + %8]\n\t" \ + "wrpr %%g0, 15, %%pil\n\t" \ + "mov %4, %%g6\n\t" \ + "ldub [%4 + %8], %%g1\n\t" \ + "wrpr %%g1, %%cwp\n\t" \ + "ldx [%%g6 + %6], %%o6\n\t" \ + "ldub [%%g6 + %5], %%o5\n\t" \ + "ldub [%%g6 + %7], %%o7\n\t" \ + "wrpr %%o5, 0x0, %%wstate\n\t" \ + "ldx [%%sp + 2047 + 0x70], %%i6\n\t" \ + "ldx [%%sp + 2047 + 0x78], %%i7\n\t" \ + "ldx [%%g6 + %9], %%g4\n\t" \ + "wrpr %%g0, 14, %%pil\n\t" \ + "brz,pt %%o7, switch_to_pc\n\t" \ + " mov %%g7, %0\n\t" \ + "sethi %%hi(ret_from_fork), %%g1\n\t" \ + "jmpl %%g1 + %%lo(ret_from_fork), %%g0\n\t" \ + " nop\n\t" \ + ".globl switch_to_pc\n\t" \ + "switch_to_pc:\n\t" \ + : "=&r" (last), "=r" (current), "=r" (current_thread_info_reg), \ + "=r" (__local_per_cpu_offset) \ + : "0" (task_thread_info(next)), \ + "i" (TI_WSTATE), "i" (TI_KSP), "i" (TI_NEW_CHILD), \ + "i" (TI_CWP), "i" (TI_TASK) \ + : "cc", \ + "g1", "g2", "g3", "g7", \ + "l1", "l2", "l3", "l4", "l5", "l6", "l7", \ + "i0", "i1", "i2", "i3", "i4", "i5", \ + "o0", "o1", "o2", "o3", "o4", "o5", "o7"); \ +} while(0) + +void synchronize_user_stack(void); +void fault_in_user_windows(void); + +#endif /* __SPARC64_SWITCH_TO_64_H */ diff --git a/arch/sparc/include/asm/syscalls.h b/arch/sparc/include/asm/syscalls.h index 45a43f637a1..b0a0db8ea61 100644 --- a/arch/sparc/include/asm/syscalls.h +++ b/arch/sparc/include/asm/syscalls.h @@ -3,11 +3,9 @@ struct pt_regs; -extern asmlinkage long sparc_do_fork(unsigned long clone_flags, - unsigned long stack_start, - struct pt_regs *regs, - unsigned long stack_size); - -extern asmlinkage int sparc_execve(struct pt_regs *regs); +asmlinkage long sparc_do_fork(unsigned long clone_flags, + unsigned long stack_start, + struct pt_regs *regs, + unsigned long stack_size); #endif /* _SPARC64_SYSCALLS_H */ diff --git a/arch/sparc/include/asm/sysen.h b/arch/sparc/include/asm/sysen.h deleted file mode 100644 index 6af34abde6e..00000000000 --- a/arch/sparc/include/asm/sysen.h +++ /dev/null @@ -1,15 +0,0 @@ -/* - * sysen.h: Bit fields within the "System Enable" register accessed via - * the ASI_CONTROL address space at address AC_SYSENABLE. - * - * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) - */ - -#ifndef _SPARC_SYSEN_H -#define _SPARC_SYSEN_H - -#define SENABLE_DVMA 0x20 /* enable dvma transfers */ -#define SENABLE_CACHE 0x10 /* enable VAC cache */ -#define SENABLE_RESET 0x04 /* reset whole machine, danger Will Robinson */ - -#endif /* _SPARC_SYSEN_H */ diff --git a/arch/sparc/include/asm/system.h b/arch/sparc/include/asm/system.h deleted file mode 100644 index 7944a7cfc99..00000000000 --- a/arch/sparc/include/asm/system.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef ___ASM_SPARC_SYSTEM_H -#define ___ASM_SPARC_SYSTEM_H -#if defined(__sparc__) && defined(__arch64__) -#include <asm/system_64.h> -#else -#include <asm/system_32.h> -#endif -#endif diff --git a/arch/sparc/include/asm/system_32.h b/arch/sparc/include/asm/system_32.h deleted file mode 100644 index 890036b3689..00000000000 --- a/arch/sparc/include/asm/system_32.h +++ /dev/null @@ -1,289 +0,0 @@ -#ifndef __SPARC_SYSTEM_H -#define __SPARC_SYSTEM_H - -#include <linux/kernel.h> -#include <linux/threads.h> /* NR_CPUS */ -#include <linux/thread_info.h> - -#include <asm/page.h> -#include <asm/psr.h> -#include <asm/ptrace.h> -#include <asm/btfixup.h> -#include <asm/smp.h> - -#ifndef __ASSEMBLY__ - -#include <linux/irqflags.h> - -static inline unsigned int probe_irq_mask(unsigned long val) -{ - return 0; -} - -/* - * Sparc (general) CPU types - */ -enum sparc_cpu { - sun4 = 0x00, - sun4c = 0x01, - sun4m = 0x02, - sun4d = 0x03, - sun4e = 0x04, - sun4u = 0x05, /* V8 ploos ploos */ - sun_unknown = 0x06, - ap1000 = 0x07, /* almost a sun4m */ - sparc_leon = 0x08, /* Leon SoC */ -}; - -/* Really, userland should not be looking at any of this... */ -#ifdef __KERNEL__ - -extern enum sparc_cpu sparc_cpu_model; - -#define ARCH_SUN4C (sparc_cpu_model==sun4c) - -#define SUN4M_NCPUS 4 /* Architectural limit of sun4m. */ - -extern char reboot_command[]; - -extern struct thread_info *current_set[NR_CPUS]; - -extern unsigned long empty_bad_page; -extern unsigned long empty_bad_page_table; -extern unsigned long empty_zero_page; - -extern void sun_do_break(void); -extern int serial_console; -extern int stop_a_enabled; -extern int scons_pwroff; - -static inline int con_is_present(void) -{ - return serial_console ? 0 : 1; -} - -/* When a context switch happens we must flush all user windows so that - * the windows of the current process are flushed onto its stack. This - * way the windows are all clean for the next process and the stack - * frames are up to date. - */ -extern void flush_user_windows(void); -extern void kill_user_windows(void); -extern void synchronize_user_stack(void); -extern void fpsave(unsigned long *fpregs, unsigned long *fsr, - void *fpqueue, unsigned long *fpqdepth); - -#ifdef CONFIG_SMP -#define SWITCH_ENTER(prv) \ - do { \ - if (test_tsk_thread_flag(prv, TIF_USEDFPU)) { \ - put_psr(get_psr() | PSR_EF); \ - fpsave(&(prv)->thread.float_regs[0], &(prv)->thread.fsr, \ - &(prv)->thread.fpqueue[0], &(prv)->thread.fpqdepth); \ - clear_tsk_thread_flag(prv, TIF_USEDFPU); \ - (prv)->thread.kregs->psr &= ~PSR_EF; \ - } \ - } while(0) - -#define SWITCH_DO_LAZY_FPU(next) /* */ -#else -#define SWITCH_ENTER(prv) /* */ -#define SWITCH_DO_LAZY_FPU(nxt) \ - do { \ - if (last_task_used_math != (nxt)) \ - (nxt)->thread.kregs->psr&=~PSR_EF; \ - } while(0) -#endif - -extern void flushw_all(void); - -/* - * Flush windows so that the VM switch which follows - * would not pull the stack from under us. - * - * SWITCH_ENTER and SWITH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) - * XXX WTF is the above comment? Found in late teen 2.4.x. - */ -#define prepare_arch_switch(next) do { \ - __asm__ __volatile__( \ - ".globl\tflush_patch_switch\nflush_patch_switch:\n\t" \ - "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ - "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ - "save %sp, -0x40, %sp\n\t" \ - "restore; restore; restore; restore; restore; restore; restore"); \ -} while(0) - - /* Much care has gone into this code, do not touch it. - * - * We need to loadup regs l0/l1 for the newly forked child - * case because the trap return path relies on those registers - * holding certain values, gcc is told that they are clobbered. - * Gcc needs registers for 3 values in and 1 value out, so we - * clobber every non-fixed-usage register besides l2/l3/o4/o5. -DaveM - * - * Hey Dave, that do not touch sign is too much of an incentive - * - Anton & Pete - */ -#define switch_to(prev, next, last) do { \ - SWITCH_ENTER(prev); \ - SWITCH_DO_LAZY_FPU(next); \ - cpumask_set_cpu(smp_processor_id(), mm_cpumask(next->active_mm)); \ - __asm__ __volatile__( \ - "sethi %%hi(here - 0x8), %%o7\n\t" \ - "mov %%g6, %%g3\n\t" \ - "or %%o7, %%lo(here - 0x8), %%o7\n\t" \ - "rd %%psr, %%g4\n\t" \ - "std %%sp, [%%g6 + %4]\n\t" \ - "rd %%wim, %%g5\n\t" \ - "wr %%g4, 0x20, %%psr\n\t" \ - "nop\n\t" \ - "std %%g4, [%%g6 + %3]\n\t" \ - "ldd [%2 + %3], %%g4\n\t" \ - "mov %2, %%g6\n\t" \ - ".globl patchme_store_new_current\n" \ -"patchme_store_new_current:\n\t" \ - "st %2, [%1]\n\t" \ - "wr %%g4, 0x20, %%psr\n\t" \ - "nop\n\t" \ - "nop\n\t" \ - "nop\n\t" /* LEON needs all 3 nops: load to %sp depends on CWP. */ \ - "ldd [%%g6 + %4], %%sp\n\t" \ - "wr %%g5, 0x0, %%wim\n\t" \ - "ldd [%%sp + 0x00], %%l0\n\t" \ - "ldd [%%sp + 0x38], %%i6\n\t" \ - "wr %%g4, 0x0, %%psr\n\t" \ - "nop\n\t" \ - "nop\n\t" \ - "jmpl %%o7 + 0x8, %%g0\n\t" \ - " ld [%%g3 + %5], %0\n\t" \ - "here:\n" \ - : "=&r" (last) \ - : "r" (&(current_set[hard_smp_processor_id()])), \ - "r" (task_thread_info(next)), \ - "i" (TI_KPSR), \ - "i" (TI_KSP), \ - "i" (TI_TASK) \ - : "g1", "g2", "g3", "g4", "g5", "g7", \ - "l0", "l1", "l3", "l4", "l5", "l6", "l7", \ - "i0", "i1", "i2", "i3", "i4", "i5", \ - "o0", "o1", "o2", "o3", "o7"); \ - } while(0) - -/* XXX Change this if we ever use a PSO mode kernel. */ -#define mb() __asm__ __volatile__ ("" : : : "memory") -#define rmb() mb() -#define wmb() mb() -#define read_barrier_depends() do { } while(0) -#define set_mb(__var, __value) do { __var = __value; mb(); } while(0) -#define smp_mb() __asm__ __volatile__("":::"memory") -#define smp_rmb() __asm__ __volatile__("":::"memory") -#define smp_wmb() __asm__ __volatile__("":::"memory") -#define smp_read_barrier_depends() do { } while(0) - -#define nop() __asm__ __volatile__ ("nop") - -/* This has special calling conventions */ -#ifndef CONFIG_SMP -BTFIXUPDEF_CALL(void, ___xchg32, void) -#endif - -static inline unsigned long xchg_u32(__volatile__ unsigned long *m, unsigned long val) -{ -#ifdef CONFIG_SMP - __asm__ __volatile__("swap [%2], %0" - : "=&r" (val) - : "0" (val), "r" (m) - : "memory"); - return val; -#else - register unsigned long *ptr asm("g1"); - register unsigned long ret asm("g2"); - - ptr = (unsigned long *) m; - ret = val; - - /* Note: this is magic and the nop there is - really needed. */ - __asm__ __volatile__( - "mov %%o7, %%g4\n\t" - "call ___f____xchg32\n\t" - " nop\n\t" - : "=&r" (ret) - : "0" (ret), "r" (ptr) - : "g3", "g4", "g7", "memory", "cc"); - - return ret; -#endif -} - -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) - -extern void __xchg_called_with_bad_pointer(void); - -static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, int size) -{ - switch (size) { - case 4: - return xchg_u32(ptr, x); - }; - __xchg_called_with_bad_pointer(); - return x; -} - -/* Emulate cmpxchg() the same way we emulate atomics, - * by hashing the object address and indexing into an array - * of spinlocks to get a bit of performance... - * - * See arch/sparc/lib/atomic32.c for implementation. - * - * Cribbed from <asm-parisc/atomic.h> - */ -#define __HAVE_ARCH_CMPXCHG 1 - -/* bug catcher for when unsupported size is used - won't link */ -extern void __cmpxchg_called_with_bad_pointer(void); -/* we only need to support cmpxchg of a u32 on sparc */ -extern unsigned long __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_); - -/* don't worry...optimizer will get rid of most of this */ -static inline unsigned long -__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) -{ - switch (size) { - case 4: - return __cmpxchg_u32((u32 *)ptr, (u32)old, (u32)new_); - default: - __cmpxchg_called_with_bad_pointer(); - break; - } - return old; -} - -#define cmpxchg(ptr, o, n) \ -({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ - (unsigned long)_n_, sizeof(*(ptr))); \ -}) - -#include <asm-generic/cmpxchg-local.h> - -/* - * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make - * them available. - */ -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local_generic((ptr), (unsigned long)(o),\ - (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) - -extern void die_if_kernel(char *str, struct pt_regs *regs) __attribute__ ((noreturn)); - -#endif /* __KERNEL__ */ - -#endif /* __ASSEMBLY__ */ - -#define arch_align_stack(x) (x) - -#endif /* !(__SPARC_SYSTEM_H) */ diff --git a/arch/sparc/include/asm/system_64.h b/arch/sparc/include/asm/system_64.h deleted file mode 100644 index e3b65d8cf41..00000000000 --- a/arch/sparc/include/asm/system_64.h +++ /dev/null @@ -1,335 +0,0 @@ -#ifndef __SPARC64_SYSTEM_H -#define __SPARC64_SYSTEM_H - -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/visasm.h> - -#ifndef __ASSEMBLY__ - -#include <linux/irqflags.h> -#include <asm-generic/cmpxchg-local.h> - -/* - * Sparc (general) CPU types - */ -enum sparc_cpu { - sun4 = 0x00, - sun4c = 0x01, - sun4m = 0x02, - sun4d = 0x03, - sun4e = 0x04, - sun4u = 0x05, /* V8 ploos ploos */ - sun_unknown = 0x06, - ap1000 = 0x07, /* almost a sun4m */ -}; - -#define sparc_cpu_model sun4u - -/* This cannot ever be a sun4c :) That's just history. */ -#define ARCH_SUN4C 0 - -extern const char *sparc_cpu_type; -extern const char *sparc_fpu_type; -extern const char *sparc_pmu_type; - -extern char reboot_command[]; - -/* These are here in an effort to more fully work around Spitfire Errata - * #51. Essentially, if a memory barrier occurs soon after a mispredicted - * branch, the chip can stop executing instructions until a trap occurs. - * Therefore, if interrupts are disabled, the chip can hang forever. - * - * It used to be believed that the memory barrier had to be right in the - * delay slot, but a case has been traced recently wherein the memory barrier - * was one instruction after the branch delay slot and the chip still hung. - * The offending sequence was the following in sym_wakeup_done() of the - * sym53c8xx_2 driver: - * - * call sym_ccb_from_dsa, 0 - * movge %icc, 0, %l0 - * brz,pn %o0, .LL1303 - * mov %o0, %l2 - * membar #LoadLoad - * - * The branch has to be mispredicted for the bug to occur. Therefore, we put - * the memory barrier explicitly into a "branch always, predicted taken" - * delay slot to avoid the problem case. - */ -#define membar_safe(type) \ -do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ - " membar " type "\n" \ - "1:\n" \ - : : : "memory"); \ -} while (0) - -/* The kernel always executes in TSO memory model these days, - * and furthermore most sparc64 chips implement more stringent - * memory ordering than required by the specifications. - */ -#define mb() membar_safe("#StoreLoad") -#define rmb() __asm__ __volatile__("":::"memory") -#define wmb() __asm__ __volatile__("":::"memory") - -#endif - -#define nop() __asm__ __volatile__ ("nop") - -#define read_barrier_depends() do { } while(0) -#define set_mb(__var, __value) \ - do { __var = __value; membar_safe("#StoreLoad"); } while(0) - -#ifdef CONFIG_SMP -#define smp_mb() mb() -#define smp_rmb() rmb() -#define smp_wmb() wmb() -#else -#define smp_mb() __asm__ __volatile__("":::"memory") -#define smp_rmb() __asm__ __volatile__("":::"memory") -#define smp_wmb() __asm__ __volatile__("":::"memory") -#endif - -#define smp_read_barrier_depends() do { } while(0) - -#define flushi(addr) __asm__ __volatile__ ("flush %0" : : "r" (addr) : "memory") - -#define flushw_all() __asm__ __volatile__("flushw") - -/* Performance counter register access. */ -#define read_pcr(__p) __asm__ __volatile__("rd %%pcr, %0" : "=r" (__p)) -#define write_pcr(__p) __asm__ __volatile__("wr %0, 0x0, %%pcr" : : "r" (__p)) -#define read_pic(__p) __asm__ __volatile__("rd %%pic, %0" : "=r" (__p)) - -/* Blackbird errata workaround. See commentary in - * arch/sparc64/kernel/smp.c:smp_percpu_timer_interrupt() - * for more information. - */ -#define write_pic(__p) \ - __asm__ __volatile__("ba,pt %%xcc, 99f\n\t" \ - " nop\n\t" \ - ".align 64\n" \ - "99:wr %0, 0x0, %%pic\n\t" \ - "rd %%pic, %%g0" : : "r" (__p)) -#define reset_pic() write_pic(0) - -#ifndef __ASSEMBLY__ - -extern void sun_do_break(void); -extern int stop_a_enabled; -extern int scons_pwroff; - -extern void fault_in_user_windows(void); -extern void synchronize_user_stack(void); - -extern void __flushw_user(void); -#define flushw_user() __flushw_user() - -#define flush_user_windows flushw_user -#define flush_register_windows flushw_all - -/* Don't hold the runqueue lock over context switch */ -#define __ARCH_WANT_UNLOCKED_CTXSW -#define prepare_arch_switch(next) \ -do { \ - flushw_all(); \ -} while (0) - - /* See what happens when you design the chip correctly? - * - * We tell gcc we clobber all non-fixed-usage registers except - * for l0/l1. It will use one for 'next' and the other to hold - * the output value of 'last'. 'next' is not referenced again - * past the invocation of switch_to in the scheduler, so we need - * not preserve it's value. Hairy, but it lets us remove 2 loads - * and 2 stores in this critical code path. -DaveM - */ -#define switch_to(prev, next, last) \ -do { flush_tlb_pending(); \ - save_and_clear_fpu(); \ - /* If you are tempted to conditionalize the following */ \ - /* so that ASI is only written if it changes, think again. */ \ - __asm__ __volatile__("wr %%g0, %0, %%asi" \ - : : "r" (__thread_flag_byte_ptr(task_thread_info(next))[TI_FLAG_BYTE_CURRENT_DS]));\ - trap_block[current_thread_info()->cpu].thread = \ - task_thread_info(next); \ - __asm__ __volatile__( \ - "mov %%g4, %%g7\n\t" \ - "stx %%i6, [%%sp + 2047 + 0x70]\n\t" \ - "stx %%i7, [%%sp + 2047 + 0x78]\n\t" \ - "rdpr %%wstate, %%o5\n\t" \ - "stx %%o6, [%%g6 + %6]\n\t" \ - "stb %%o5, [%%g6 + %5]\n\t" \ - "rdpr %%cwp, %%o5\n\t" \ - "stb %%o5, [%%g6 + %8]\n\t" \ - "wrpr %%g0, 15, %%pil\n\t" \ - "mov %4, %%g6\n\t" \ - "ldub [%4 + %8], %%g1\n\t" \ - "wrpr %%g1, %%cwp\n\t" \ - "ldx [%%g6 + %6], %%o6\n\t" \ - "ldub [%%g6 + %5], %%o5\n\t" \ - "ldub [%%g6 + %7], %%o7\n\t" \ - "wrpr %%o5, 0x0, %%wstate\n\t" \ - "ldx [%%sp + 2047 + 0x70], %%i6\n\t" \ - "ldx [%%sp + 2047 + 0x78], %%i7\n\t" \ - "ldx [%%g6 + %9], %%g4\n\t" \ - "wrpr %%g0, 14, %%pil\n\t" \ - "brz,pt %%o7, switch_to_pc\n\t" \ - " mov %%g7, %0\n\t" \ - "sethi %%hi(ret_from_syscall), %%g1\n\t" \ - "jmpl %%g1 + %%lo(ret_from_syscall), %%g0\n\t" \ - " nop\n\t" \ - ".globl switch_to_pc\n\t" \ - "switch_to_pc:\n\t" \ - : "=&r" (last), "=r" (current), "=r" (current_thread_info_reg), \ - "=r" (__local_per_cpu_offset) \ - : "0" (task_thread_info(next)), \ - "i" (TI_WSTATE), "i" (TI_KSP), "i" (TI_NEW_CHILD), \ - "i" (TI_CWP), "i" (TI_TASK) \ - : "cc", \ - "g1", "g2", "g3", "g7", \ - "l1", "l2", "l3", "l4", "l5", "l6", "l7", \ - "i0", "i1", "i2", "i3", "i4", "i5", \ - "o0", "o1", "o2", "o3", "o4", "o5", "o7"); \ -} while(0) - -static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val) -{ - unsigned long tmp1, tmp2; - - __asm__ __volatile__( -" mov %0, %1\n" -"1: lduw [%4], %2\n" -" cas [%4], %2, %0\n" -" cmp %2, %0\n" -" bne,a,pn %%icc, 1b\n" -" mov %1, %0\n" - : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) - : "0" (val), "r" (m) - : "cc", "memory"); - return val; -} - -static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long val) -{ - unsigned long tmp1, tmp2; - - __asm__ __volatile__( -" mov %0, %1\n" -"1: ldx [%4], %2\n" -" casx [%4], %2, %0\n" -" cmp %2, %0\n" -" bne,a,pn %%xcc, 1b\n" -" mov %1, %0\n" - : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) - : "0" (val), "r" (m) - : "cc", "memory"); - return val; -} - -#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) - -extern void __xchg_called_with_bad_pointer(void); - -static inline unsigned long __xchg(unsigned long x, __volatile__ void * ptr, - int size) -{ - switch (size) { - case 4: - return xchg32(ptr, x); - case 8: - return xchg64(ptr, x); - }; - __xchg_called_with_bad_pointer(); - return x; -} - -extern void die_if_kernel(char *str, struct pt_regs *regs) __attribute__ ((noreturn)); - -/* - * Atomic compare and exchange. Compare OLD with MEM, if identical, - * store NEW in MEM. Return the initial value in MEM. Success is - * indicated by comparing RETURN with OLD. - */ - -#define __HAVE_ARCH_CMPXCHG 1 - -static inline unsigned long -__cmpxchg_u32(volatile int *m, int old, int new) -{ - __asm__ __volatile__("cas [%2], %3, %0" - : "=&r" (new) - : "0" (new), "r" (m), "r" (old) - : "memory"); - - return new; -} - -static inline unsigned long -__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) -{ - __asm__ __volatile__("casx [%2], %3, %0" - : "=&r" (new) - : "0" (new), "r" (m), "r" (old) - : "memory"); - - return new; -} - -/* This function doesn't exist, so you'll get a linker error - if something tries to do an invalid cmpxchg(). */ -extern void __cmpxchg_called_with_bad_pointer(void); - -static inline unsigned long -__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) -{ - switch (size) { - case 4: - return __cmpxchg_u32(ptr, old, new); - case 8: - return __cmpxchg_u64(ptr, old, new); - } - __cmpxchg_called_with_bad_pointer(); - return old; -} - -#define cmpxchg(ptr,o,n) \ - ({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ - (unsigned long)_n_, sizeof(*(ptr))); \ - }) - -/* - * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make - * them available. - */ - -static inline unsigned long __cmpxchg_local(volatile void *ptr, - unsigned long old, - unsigned long new, int size) -{ - switch (size) { - case 4: - case 8: return __cmpxchg(ptr, old, new, size); - default: - return __cmpxchg_local_generic(ptr, old, new, size); - } - - return old; -} - -#define cmpxchg_local(ptr, o, n) \ - ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ - (unsigned long)(n), sizeof(*(ptr)))) -#define cmpxchg64_local(ptr, o, n) \ - ({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ - }) - -#endif /* !(__ASSEMBLY__) */ - -#define arch_align_stack(x) (x) - -#endif /* !(__SPARC64_SYSTEM_H) */ diff --git a/arch/sparc/include/asm/termbits.h b/arch/sparc/include/asm/termbits.h index 23b10ff08df..948067065ac 100644 --- a/arch/sparc/include/asm/termbits.h +++ b/arch/sparc/include/asm/termbits.h @@ -1,266 +1,8 @@ #ifndef _SPARC_TERMBITS_H #define _SPARC_TERMBITS_H -#include <linux/posix_types.h> +#include <uapi/asm/termbits.h> -typedef unsigned char cc_t; -typedef unsigned int speed_t; - -#if defined(__sparc__) && defined(__arch64__) -typedef unsigned int tcflag_t; -#else -typedef unsigned long tcflag_t; -#endif - -#define NCC 8 -struct termio { - unsigned short c_iflag; /* input mode flags */ - unsigned short c_oflag; /* output mode flags */ - unsigned short c_cflag; /* control mode flags */ - unsigned short c_lflag; /* local mode flags */ - unsigned char c_line; /* line discipline */ - unsigned char c_cc[NCC]; /* control characters */ -}; - -#define NCCS 17 -struct termios { - tcflag_t c_iflag; /* input mode flags */ - tcflag_t c_oflag; /* output mode flags */ - tcflag_t c_cflag; /* control mode flags */ - tcflag_t c_lflag; /* local mode flags */ - cc_t c_line; /* line discipline */ -#ifndef __KERNEL__ - cc_t c_cc[NCCS]; /* control characters */ -#else - cc_t c_cc[NCCS+2]; /* kernel needs 2 more to hold vmin/vtime */ -#define SIZEOF_USER_TERMIOS sizeof (struct termios) - (2*sizeof (cc_t)) -#endif -}; - -struct termios2 { - tcflag_t c_iflag; /* input mode flags */ - tcflag_t c_oflag; /* output mode flags */ - tcflag_t c_cflag; /* control mode flags */ - tcflag_t c_lflag; /* local mode flags */ - cc_t c_line; /* line discipline */ - cc_t c_cc[NCCS+2]; /* control characters */ - speed_t c_ispeed; /* input speed */ - speed_t c_ospeed; /* output speed */ -}; - -struct ktermios { - tcflag_t c_iflag; /* input mode flags */ - tcflag_t c_oflag; /* output mode flags */ - tcflag_t c_cflag; /* control mode flags */ - tcflag_t c_lflag; /* local mode flags */ - cc_t c_line; /* line discipline */ - cc_t c_cc[NCCS+2]; /* control characters */ - speed_t c_ispeed; /* input speed */ - speed_t c_ospeed; /* output speed */ -}; - -/* c_cc characters */ -#define VINTR 0 -#define VQUIT 1 -#define VERASE 2 -#define VKILL 3 -#define VEOF 4 -#define VEOL 5 -#define VEOL2 6 -#define VSWTC 7 -#define VSTART 8 -#define VSTOP 9 - - - -#define VSUSP 10 -#define VDSUSP 11 /* SunOS POSIX nicety I do believe... */ -#define VREPRINT 12 -#define VDISCARD 13 -#define VWERASE 14 -#define VLNEXT 15 - -/* Kernel keeps vmin/vtime separated, user apps assume vmin/vtime is - * shared with eof/eol - */ -#ifdef __KERNEL__ #define VMIN 16 #define VTIME 17 -#else -#define VMIN VEOF -#define VTIME VEOL -#endif - -/* c_iflag bits */ -#define IGNBRK 0x00000001 -#define BRKINT 0x00000002 -#define IGNPAR 0x00000004 -#define PARMRK 0x00000008 -#define INPCK 0x00000010 -#define ISTRIP 0x00000020 -#define INLCR 0x00000040 -#define IGNCR 0x00000080 -#define ICRNL 0x00000100 -#define IUCLC 0x00000200 -#define IXON 0x00000400 -#define IXANY 0x00000800 -#define IXOFF 0x00001000 -#define IMAXBEL 0x00002000 -#define IUTF8 0x00004000 - -/* c_oflag bits */ -#define OPOST 0x00000001 -#define OLCUC 0x00000002 -#define ONLCR 0x00000004 -#define OCRNL 0x00000008 -#define ONOCR 0x00000010 -#define ONLRET 0x00000020 -#define OFILL 0x00000040 -#define OFDEL 0x00000080 -#define NLDLY 0x00000100 -#define NL0 0x00000000 -#define NL1 0x00000100 -#define CRDLY 0x00000600 -#define CR0 0x00000000 -#define CR1 0x00000200 -#define CR2 0x00000400 -#define CR3 0x00000600 -#define TABDLY 0x00001800 -#define TAB0 0x00000000 -#define TAB1 0x00000800 -#define TAB2 0x00001000 -#define TAB3 0x00001800 -#define XTABS 0x00001800 -#define BSDLY 0x00002000 -#define BS0 0x00000000 -#define BS1 0x00002000 -#define VTDLY 0x00004000 -#define VT0 0x00000000 -#define VT1 0x00004000 -#define FFDLY 0x00008000 -#define FF0 0x00000000 -#define FF1 0x00008000 -#define PAGEOUT 0x00010000 /* SUNOS specific */ -#define WRAP 0x00020000 /* SUNOS specific */ - -/* c_cflag bit meaning */ -#define CBAUD 0x0000100f -#define B0 0x00000000 /* hang up */ -#define B50 0x00000001 -#define B75 0x00000002 -#define B110 0x00000003 -#define B134 0x00000004 -#define B150 0x00000005 -#define B200 0x00000006 -#define B300 0x00000007 -#define B600 0x00000008 -#define B1200 0x00000009 -#define B1800 0x0000000a -#define B2400 0x0000000b -#define B4800 0x0000000c -#define B9600 0x0000000d -#define B19200 0x0000000e -#define B38400 0x0000000f -#define EXTA B19200 -#define EXTB B38400 -#define CSIZE 0x00000030 -#define CS5 0x00000000 -#define CS6 0x00000010 -#define CS7 0x00000020 -#define CS8 0x00000030 -#define CSTOPB 0x00000040 -#define CREAD 0x00000080 -#define PARENB 0x00000100 -#define PARODD 0x00000200 -#define HUPCL 0x00000400 -#define CLOCAL 0x00000800 -#define CBAUDEX 0x00001000 -/* We'll never see these speeds with the Zilogs, but for completeness... */ -#define BOTHER 0x00001000 -#define B57600 0x00001001 -#define B115200 0x00001002 -#define B230400 0x00001003 -#define B460800 0x00001004 -/* This is what we can do with the Zilogs. */ -#define B76800 0x00001005 -/* This is what we can do with the SAB82532. */ -#define B153600 0x00001006 -#define B307200 0x00001007 -#define B614400 0x00001008 -#define B921600 0x00001009 -/* And these are the rest... */ -#define B500000 0x0000100a -#define B576000 0x0000100b -#define B1000000 0x0000100c -#define B1152000 0x0000100d -#define B1500000 0x0000100e -#define B2000000 0x0000100f -/* These have totally bogus values and nobody uses them - so far. Later on we'd have to use say 0x10000x and - adjust CBAUD constant and drivers accordingly. -#define B2500000 0x00001010 -#define B3000000 0x00001011 -#define B3500000 0x00001012 -#define B4000000 0x00001013 */ -#define CIBAUD 0x100f0000 /* input baud rate (not used) */ -#define CMSPAR 0x40000000 /* mark or space (stick) parity */ -#define CRTSCTS 0x80000000 /* flow control */ - -#define IBSHIFT 16 /* Shift from CBAUD to CIBAUD */ - -/* c_lflag bits */ -#define ISIG 0x00000001 -#define ICANON 0x00000002 -#define XCASE 0x00000004 -#define ECHO 0x00000008 -#define ECHOE 0x00000010 -#define ECHOK 0x00000020 -#define ECHONL 0x00000040 -#define NOFLSH 0x00000080 -#define TOSTOP 0x00000100 -#define ECHOCTL 0x00000200 -#define ECHOPRT 0x00000400 -#define ECHOKE 0x00000800 -#define DEFECHO 0x00001000 /* SUNOS thing, what is it? */ -#define FLUSHO 0x00002000 -#define PENDIN 0x00004000 -#define IEXTEN 0x00008000 -#define EXTPROC 0x00010000 - -/* modem lines */ -#define TIOCM_LE 0x001 -#define TIOCM_DTR 0x002 -#define TIOCM_RTS 0x004 -#define TIOCM_ST 0x008 -#define TIOCM_SR 0x010 -#define TIOCM_CTS 0x020 -#define TIOCM_CAR 0x040 -#define TIOCM_RNG 0x080 -#define TIOCM_DSR 0x100 -#define TIOCM_CD TIOCM_CAR -#define TIOCM_RI TIOCM_RNG -#define TIOCM_OUT1 0x2000 -#define TIOCM_OUT2 0x4000 -#define TIOCM_LOOP 0x8000 - -/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ -#define TIOCSER_TEMT 0x01 /* Transmitter physically empty */ - - -/* tcflow() and TCXONC use these */ -#define TCOOFF 0 -#define TCOON 1 -#define TCIOFF 2 -#define TCION 3 - -/* tcflush() and TCFLSH use these */ -#define TCIFLUSH 0 -#define TCOFLUSH 1 -#define TCIOFLUSH 2 - -/* tcsetattr uses these */ -#define TCSANOW 0 -#define TCSADRAIN 1 -#define TCSAFLUSH 2 - #endif /* !(_SPARC_TERMBITS_H) */ diff --git a/arch/sparc/include/asm/termios.h b/arch/sparc/include/asm/termios.h index e8ba9539964..0c2414ddd52 100644 --- a/arch/sparc/include/asm/termios.h +++ b/arch/sparc/include/asm/termios.h @@ -1,46 +1,8 @@ #ifndef _SPARC_TERMIOS_H #define _SPARC_TERMIOS_H -#include <asm/ioctls.h> -#include <asm/termbits.h> +#include <uapi/asm/termios.h> -#if defined(__KERNEL__) || defined(__DEFINE_BSD_TERMIOS) -struct sgttyb { - char sg_ispeed; - char sg_ospeed; - char sg_erase; - char sg_kill; - short sg_flags; -}; - -struct tchars { - char t_intrc; - char t_quitc; - char t_startc; - char t_stopc; - char t_eofc; - char t_brkc; -}; - -struct ltchars { - char t_suspc; - char t_dsuspc; - char t_rprntc; - char t_flushc; - char t_werasc; - char t_lnextc; -}; -#endif /* __KERNEL__ */ - -struct winsize { - unsigned short ws_row; - unsigned short ws_col; - unsigned short ws_xpixel; - unsigned short ws_ypixel; -}; - -#ifdef __KERNEL__ -#include <linux/module.h> /* * c_cc characters in the termio structure. Oh, how I love being @@ -181,6 +143,4 @@ struct winsize { err; \ }) -#endif /* __KERNEL__ */ - #endif /* _SPARC_TERMIOS_H */ diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h index 9dd0318d3dd..96efa7adc22 100644 --- a/arch/sparc/include/asm/thread_info_32.h +++ b/arch/sparc/include/asm/thread_info_32.h @@ -15,7 +15,6 @@ #ifndef __ASSEMBLY__ -#include <asm/btfixup.h> #include <asm/ptrace.h> #include <asm/page.h> @@ -78,24 +77,12 @@ register struct thread_info *current_thread_info_reg asm("g6"); /* * thread information allocation */ -#define THREAD_INFO_ORDER 1 - -#define __HAVE_ARCH_THREAD_INFO_ALLOCATOR - -BTFIXUPDEF_CALL(struct thread_info *, alloc_thread_info, void) -#define alloc_thread_info(tsk) BTFIXUP_CALL(alloc_thread_info)() - -BTFIXUPDEF_CALL(void, free_thread_info, struct thread_info *) -#define free_thread_info(ti) BTFIXUP_CALL(free_thread_info)(ti) +#define THREAD_SIZE_ORDER 1 #endif /* __ASSEMBLY__ */ -/* - * Size of kernel stack for each process. - * Observe the order of get_free_pages() in alloc_thread_info(). - * The sun4 has 8K stack too, because it's short on memory, and 16K is a waste. - */ -#define THREAD_SIZE 8192 +/* Size of kernel stack for each process */ +#define THREAD_SIZE (2 * PAGE_SIZE) /* * Offsets in thread_info structure, used in assembly code @@ -118,8 +105,6 @@ BTFIXUPDEF_CALL(void, free_thread_info, struct thread_info *) #define TI_W_SAVED 0x250 /* #define TI_RESTART_BLOCK 0x25n */ /* Nobody cares */ -#define PREEMPT_ACTIVE 0x4000000 - /* * thread information flag bit numbers */ @@ -133,21 +118,17 @@ BTFIXUPDEF_CALL(void, free_thread_info, struct thread_info *) #define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling * TIF_NEED_RESCHED */ #define TIF_MEMDIE 10 /* is terminating due to OOM killer */ -#define TIF_FREEZE 11 /* is freezing for suspend */ /* as above, but as bit values */ #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) #define _TIF_SIGPENDING (1<<TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) -#define _TIF_RESTORE_SIGMASK (1<<TIF_RESTORE_SIGMASK) #define _TIF_USEDFPU (1<<TIF_USEDFPU) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_DO_NOTIFY_RESUME_MASK (_TIF_NOTIFY_RESUME | \ - _TIF_SIGPENDING | \ - _TIF_RESTORE_SIGMASK) -#define _TIF_FREEZE (1<<TIF_FREEZE) + _TIF_SIGPENDING) #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index fb2ea7705a4..a5f01ac6d0f 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h @@ -14,12 +14,12 @@ #define TI_FLAG_FAULT_CODE_SHIFT 56 #define TI_FLAG_BYTE_WSTATE 1 #define TI_FLAG_WSTATE_SHIFT 48 -#define TI_FLAG_BYTE_CWP 2 -#define TI_FLAG_CWP_SHIFT 40 -#define TI_FLAG_BYTE_CURRENT_DS 3 -#define TI_FLAG_CURRENT_DS_SHIFT 32 -#define TI_FLAG_BYTE_FPDEPTH 4 -#define TI_FLAG_FPDEPTH_SHIFT 24 +#define TI_FLAG_BYTE_NOERROR 2 +#define TI_FLAG_BYTE_NOERROR_SHIFT 40 +#define TI_FLAG_BYTE_FPDEPTH 3 +#define TI_FLAG_FPDEPTH_SHIFT 32 +#define TI_FLAG_BYTE_CWP 4 +#define TI_FLAG_CWP_SHIFT 24 #define TI_FLAG_BYTE_WSAVED 5 #define TI_FLAG_WSAVED_SHIFT 16 @@ -47,7 +47,7 @@ struct thread_info { struct exec_domain *exec_domain; int preempt_count; /* 0 => preemptable, <0 => BUG */ __u8 new_child; - __u8 syscall_noerror; + __u8 current_ds; __u16 cpu; unsigned long *utraps; @@ -74,9 +74,9 @@ struct thread_info { #define TI_FAULT_CODE (TI_FLAGS + TI_FLAG_BYTE_FAULT_CODE) #define TI_WSTATE (TI_FLAGS + TI_FLAG_BYTE_WSTATE) #define TI_CWP (TI_FLAGS + TI_FLAG_BYTE_CWP) -#define TI_CURRENT_DS (TI_FLAGS + TI_FLAG_BYTE_CURRENT_DS) #define TI_FPDEPTH (TI_FLAGS + TI_FLAG_BYTE_FPDEPTH) #define TI_WSAVED (TI_FLAGS + TI_FLAG_BYTE_WSAVED) +#define TI_SYS_NOERROR (TI_FLAGS + TI_FLAG_BYTE_NOERROR) #define TI_FPSAVED 0x00000010 #define TI_KSP 0x00000018 #define TI_FAULT_ADDR 0x00000020 @@ -84,7 +84,7 @@ struct thread_info { #define TI_EXEC_DOMAIN 0x00000030 #define TI_PRE_COUNT 0x00000038 #define TI_NEW_CHILD 0x0000003c -#define TI_SYS_NOERROR 0x0000003d +#define TI_CURRENT_DS 0x0000003d #define TI_CPU 0x0000003e #define TI_UTRAPS 0x00000040 #define TI_REG_WINDOW 0x00000048 @@ -111,8 +111,6 @@ struct thread_info { #define THREAD_SHIFT PAGE_SHIFT #endif /* PAGE_SHIFT == 13 */ -#define PREEMPT_ACTIVE 0x10000000 - /* * macros/functions for gaining access to the thread information structure */ @@ -121,7 +119,7 @@ struct thread_info { #define INIT_THREAD_INFO(tsk) \ { \ .task = &tsk, \ - .flags = ((unsigned long)ASI_P) << TI_FLAG_CURRENT_DS_SHIFT, \ + .current_ds = ASI_P, \ .exec_domain = &default_exec_domain, \ .preempt_count = INIT_PREEMPT_COUNT, \ .restart_block = { \ @@ -138,32 +136,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); /* thread information allocation */ #if PAGE_SHIFT == 13 -#define __THREAD_INFO_ORDER 1 +#define THREAD_SIZE_ORDER 1 #else /* PAGE_SHIFT == 13 */ -#define __THREAD_INFO_ORDER 0 +#define THREAD_SIZE_ORDER 0 #endif /* PAGE_SHIFT == 13 */ -#define __HAVE_ARCH_THREAD_INFO_ALLOCATOR - -#ifdef CONFIG_DEBUG_STACK_USAGE -#define alloc_thread_info(tsk) \ -({ \ - struct thread_info *ret; \ - \ - ret = (struct thread_info *) \ - __get_free_pages(GFP_KERNEL, __THREAD_INFO_ORDER); \ - if (ret) \ - memset(ret, 0, PAGE_SIZE<<__THREAD_INFO_ORDER); \ - ret; \ -}) -#else -#define alloc_thread_info(tsk) \ - ((struct thread_info *)__get_free_pages(GFP_KERNEL, __THREAD_INFO_ORDER)) -#endif - -#define free_thread_info(ti) \ - free_pages((unsigned long)(ti),__THREAD_INFO_ORDER) - #define __thread_flag_byte_ptr(ti) \ ((unsigned char *)(&((ti)->flags))) #define __cur_thread_flag_byte_ptr __thread_flag_byte_ptr(current_thread_info()) @@ -174,13 +151,12 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define set_thread_wstate(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSTATE] = (val)) #define get_thread_cwp() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CWP]) #define set_thread_cwp(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CWP] = (val)) -#define get_thread_current_ds() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CURRENT_DS]) -#define set_thread_current_ds(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CURRENT_DS] = (val)) +#define get_thread_noerror() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_NOERROR]) +#define set_thread_noerror(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_NOERROR] = (val)) #define get_thread_fpdepth() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FPDEPTH]) #define set_thread_fpdepth(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FPDEPTH] = (val)) #define get_thread_wsaved() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED]) #define set_thread_wsaved(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED] = (val)) - #endif /* !(__ASSEMBLY__) */ /* @@ -214,7 +190,7 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ /* flag bit 6 is available */ #define TIF_32BIT 7 /* 32-bit binary */ -/* flag bit 8 is available */ +#define TIF_NOHZ 8 /* in adaptive nohz mode */ #define TIF_SECCOMP 9 /* secure computing */ #define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ #define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ @@ -225,7 +201,6 @@ register struct thread_info *current_thread_info_reg asm("g6"); /* flag bit 12 is available */ #define TIF_MEMDIE 13 /* is terminating due to OOM killer */ #define TIF_POLLING_NRFLAG 14 -#define TIF_FREEZE 15 /* is freezing for suspend */ #define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE) #define _TIF_NOTIFY_RESUME (1<<TIF_NOTIFY_RESUME) @@ -233,11 +208,11 @@ register struct thread_info *current_thread_info_reg asm("g6"); #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) #define _TIF_UNALIGNED (1<<TIF_UNALIGNED) #define _TIF_32BIT (1<<TIF_32BIT) +#define _TIF_NOHZ (1<<TIF_NOHZ) #define _TIF_SECCOMP (1<<TIF_SECCOMP) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) -#define _TIF_FREEZE (1<<TIF_FREEZE) #define _TIF_USER_WORK_MASK ((0xff << TI_FLAG_WSAVED_SHIFT) | \ _TIF_DO_NOTIFY_RESUME_MASK | \ @@ -261,8 +236,30 @@ static inline void set_restore_sigmask(void) { struct thread_info *ti = current_thread_info(); ti->status |= TS_RESTORE_SIGMASK; - set_bit(TIF_SIGPENDING, &ti->flags); + WARN_ON(!test_bit(TIF_SIGPENDING, &ti->flags)); +} +static inline void clear_restore_sigmask(void) +{ + current_thread_info()->status &= ~TS_RESTORE_SIGMASK; +} +static inline bool test_restore_sigmask(void) +{ + return current_thread_info()->status & TS_RESTORE_SIGMASK; +} +static inline bool test_and_clear_restore_sigmask(void) +{ + struct thread_info *ti = current_thread_info(); + if (!(ti->status & TS_RESTORE_SIGMASK)) + return false; + ti->status &= ~TS_RESTORE_SIGMASK; + return true; } + +#define thread32_stack_is_64bit(__SP) (((__SP) & 0x1) != 0) +#define test_thread_64bit_stack(__SP) \ + ((test_thread_flag(TIF_32BIT) && !thread32_stack_is_64bit(__SP)) ? \ + false : true) + #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ diff --git a/arch/sparc/include/asm/timer_32.h b/arch/sparc/include/asm/timer_32.h index 2ec030ef381..f8e708a0aa5 100644 --- a/arch/sparc/include/asm/timer_32.h +++ b/arch/sparc/include/asm/timer_32.h @@ -8,13 +8,37 @@ #ifndef _SPARC_TIMER_H #define _SPARC_TIMER_H -#include <asm/system.h> /* For SUN4M_NCPUS */ -#include <asm/btfixup.h> +#include <linux/clocksource.h> +#include <linux/irqreturn.h> -extern __volatile__ unsigned int *master_l10_counter; +#include <asm-generic/percpu.h> -/* FIXME: Make do_[gs]ettimeofday btfixup calls */ -BTFIXUPDEF_CALL(int, bus_do_settimeofday, struct timespec *tv) -#define bus_do_settimeofday(tv) BTFIXUP_CALL(bus_do_settimeofday)(tv) +#include <asm/cpu_type.h> /* For SUN4M_NCPUS */ + +#define SBUS_CLOCK_RATE 2000000 /* 2MHz */ +#define TIMER_VALUE_SHIFT 9 +#define TIMER_VALUE_MASK 0x3fffff +#define TIMER_LIMIT_BIT (1 << 31) /* Bit 31 in Counter-Timer register */ + +/* The counter timer register has the value offset by 9 bits. + * From sun4m manual: + * When a counter reaches the value in the corresponding limit register, + * the Limit bit is set and the counter is set to 500 nS (i.e. 0x00000200). + * + * To compensate for this add one to the value. + */ +static inline unsigned int timer_value(unsigned int value) +{ + return (value + 1) << TIMER_VALUE_SHIFT; +} + +extern volatile u32 __iomem *master_l10_counter; + +irqreturn_t notrace timer_interrupt(int dummy, void *dev_id); + +#ifdef CONFIG_SMP +DECLARE_PER_CPU(struct clock_event_device, sparc32_clockevent); +void register_percpu_ce(int cpu); +#endif #endif /* !(_SPARC_TIMER_H) */ diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h index ef3c3682deb..fce41503400 100644 --- a/arch/sparc/include/asm/timer_64.h +++ b/arch/sparc/include/asm/timer_64.h @@ -23,8 +23,8 @@ struct sparc64_tick_ops { extern struct sparc64_tick_ops *tick_ops; -extern unsigned long sparc64_get_clock_tick(unsigned int cpu); -extern void __devinit setup_sparc64_timer(void); -extern void __init time_init(void); +unsigned long sparc64_get_clock_tick(unsigned int cpu); +void setup_sparc64_timer(void); +void __init time_init(void); #endif /* _SPARC64_TIMER_H */ diff --git a/arch/sparc/include/asm/timex_32.h b/arch/sparc/include/asm/timex_32.h index a254750e4c0..b6ccdb0d6f7 100644 --- a/arch/sparc/include/asm/timex_32.h +++ b/arch/sparc/include/asm/timex_32.h @@ -12,5 +12,4 @@ typedef unsigned long cycles_t; #define get_cycles() (0) -extern u32 (*do_arch_gettimeoffset)(void); #endif diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h index dca406b9b6f..4cb392f75d2 100644 --- a/arch/sparc/include/asm/tlb_64.h +++ b/arch/sparc/include/asm/tlb_64.h @@ -7,105 +7,26 @@ #include <asm/tlbflush.h> #include <asm/mmu_context.h> -#define TLB_BATCH_NR 192 - -/* - * For UP we don't need to worry about TLB flush - * and page free order so much.. - */ #ifdef CONFIG_SMP - #define FREE_PTE_NR 506 - #define tlb_fast_mode(bp) ((bp)->pages_nr == ~0U) -#else - #define FREE_PTE_NR 1 - #define tlb_fast_mode(bp) 1 -#endif - -struct mmu_gather { - struct mm_struct *mm; - unsigned int pages_nr; - unsigned int need_flush; - unsigned int fullmm; - unsigned int tlb_nr; - unsigned long vaddrs[TLB_BATCH_NR]; - struct page *pages[FREE_PTE_NR]; -}; - -DECLARE_PER_CPU(struct mmu_gather, mmu_gathers); - -#ifdef CONFIG_SMP -extern void smp_flush_tlb_pending(struct mm_struct *, +void smp_flush_tlb_pending(struct mm_struct *, unsigned long, unsigned long *); #endif -extern void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); -extern void flush_tlb_pending(void); - -static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush) -{ - struct mmu_gather *mp = &get_cpu_var(mmu_gathers); - - BUG_ON(mp->tlb_nr); - - mp->mm = mm; - mp->pages_nr = num_online_cpus() > 1 ? 0U : ~0U; - mp->fullmm = full_mm_flush; - - return mp; -} - - -static inline void tlb_flush_mmu(struct mmu_gather *mp) -{ - if (!mp->fullmm) - flush_tlb_pending(); - if (mp->need_flush) { - free_pages_and_swap_cache(mp->pages, mp->pages_nr); - mp->pages_nr = 0; - mp->need_flush = 0; - } - -} - #ifdef CONFIG_SMP -extern void smp_flush_tlb_mm(struct mm_struct *mm); +void smp_flush_tlb_mm(struct mm_struct *mm); #define do_flush_tlb_mm(mm) smp_flush_tlb_mm(mm) #else #define do_flush_tlb_mm(mm) __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT) #endif -static inline void tlb_finish_mmu(struct mmu_gather *mp, unsigned long start, unsigned long end) -{ - tlb_flush_mmu(mp); - - if (mp->fullmm) - mp->fullmm = 0; +void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); +void flush_tlb_pending(void); - /* keep the page table cache within bounds */ - check_pgt_cache(); - - put_cpu_var(mmu_gathers); -} - -static inline void tlb_remove_page(struct mmu_gather *mp, struct page *page) -{ - if (tlb_fast_mode(mp)) { - free_page_and_swap_cache(page); - return; - } - mp->need_flush = 1; - mp->pages[mp->pages_nr++] = page; - if (mp->pages_nr >= FREE_PTE_NR) - tlb_flush_mmu(mp); -} - -#define tlb_remove_tlb_entry(mp,ptep,addr) do { } while (0) -#define pte_free_tlb(mp, ptepage, addr) pte_free((mp)->mm, ptepage) -#define pmd_free_tlb(mp, pmdp, addr) pmd_free((mp)->mm, pmdp) -#define pud_free_tlb(tlb,pudp, addr) __pud_free_tlb(tlb,pudp,addr) - -#define tlb_migrate_finish(mm) do { } while (0) #define tlb_start_vma(tlb, vma) do { } while (0) #define tlb_end_vma(tlb, vma) do { } while (0) +#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) +#define tlb_flush(tlb) flush_tlb_pending() + +#include <asm-generic/tlb.h> #endif /* _SPARC64_TLB_H */ diff --git a/arch/sparc/include/asm/tlbflush_32.h b/arch/sparc/include/asm/tlbflush_32.h index fe0a71abc9b..a5c4142130f 100644 --- a/arch/sparc/include/asm/tlbflush_32.h +++ b/arch/sparc/include/asm/tlbflush_32.h @@ -1,52 +1,16 @@ #ifndef _SPARC_TLBFLUSH_H #define _SPARC_TLBFLUSH_H -#include <linux/mm.h> -// #include <asm/processor.h> - -/* - * TLB flushing: - * - * - flush_tlb() flushes the current mm struct TLBs XXX Exists? - * - flush_tlb_all() flushes all processes TLBs - * - flush_tlb_mm(mm) flushes the specified mm context TLB's - * - flush_tlb_page(vma, vmaddr) flushes one page - * - flush_tlb_range(vma, start, end) flushes a range of pages - * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages - */ - -#ifdef CONFIG_SMP - -BTFIXUPDEF_CALL(void, local_flush_tlb_all, void) -BTFIXUPDEF_CALL(void, local_flush_tlb_mm, struct mm_struct *) -BTFIXUPDEF_CALL(void, local_flush_tlb_range, struct vm_area_struct *, unsigned long, unsigned long) -BTFIXUPDEF_CALL(void, local_flush_tlb_page, struct vm_area_struct *, unsigned long) - -#define local_flush_tlb_all() BTFIXUP_CALL(local_flush_tlb_all)() -#define local_flush_tlb_mm(mm) BTFIXUP_CALL(local_flush_tlb_mm)(mm) -#define local_flush_tlb_range(vma,start,end) BTFIXUP_CALL(local_flush_tlb_range)(vma,start,end) -#define local_flush_tlb_page(vma,addr) BTFIXUP_CALL(local_flush_tlb_page)(vma,addr) - -extern void smp_flush_tlb_all(void); -extern void smp_flush_tlb_mm(struct mm_struct *mm); -extern void smp_flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, - unsigned long end); -extern void smp_flush_tlb_page(struct vm_area_struct *mm, unsigned long page); - -#endif /* CONFIG_SMP */ - -BTFIXUPDEF_CALL(void, flush_tlb_all, void) -BTFIXUPDEF_CALL(void, flush_tlb_mm, struct mm_struct *) -BTFIXUPDEF_CALL(void, flush_tlb_range, struct vm_area_struct *, unsigned long, unsigned long) -BTFIXUPDEF_CALL(void, flush_tlb_page, struct vm_area_struct *, unsigned long) - -#define flush_tlb_all() BTFIXUP_CALL(flush_tlb_all)() -#define flush_tlb_mm(mm) BTFIXUP_CALL(flush_tlb_mm)(mm) -#define flush_tlb_range(vma,start,end) BTFIXUP_CALL(flush_tlb_range)(vma,start,end) -#define flush_tlb_page(vma,addr) BTFIXUP_CALL(flush_tlb_page)(vma,addr) - -// #define flush_tlb() flush_tlb_mm(current->active_mm) /* XXX Sure? */ +#include <asm/cachetlb_32.h> + +#define flush_tlb_all() \ + sparc32_cachetlb_ops->tlb_all() +#define flush_tlb_mm(mm) \ + sparc32_cachetlb_ops->tlb_mm(mm) +#define flush_tlb_range(vma, start, end) \ + sparc32_cachetlb_ops->tlb_range(vma, start, end) +#define flush_tlb_page(vma, addr) \ + sparc32_cachetlb_ops->tlb_page(vma, addr) /* * This is a kludge, until I know better. --zaitcev XXX diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h index fbb675dbe0c..816d8202fa0 100644 --- a/arch/sparc/include/asm/tlbflush_64.h +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -1,27 +1,50 @@ #ifndef _SPARC64_TLBFLUSH_H #define _SPARC64_TLBFLUSH_H -#include <linux/mm.h> #include <asm/mmu_context.h> /* TSB flush operations. */ -struct mmu_gather; -extern void flush_tsb_kernel_range(unsigned long start, unsigned long end); -extern void flush_tsb_user(struct mmu_gather *mp); + +#define TLB_BATCH_NR 192 + +struct tlb_batch { + struct mm_struct *mm; + unsigned long tlb_nr; + unsigned long active; + unsigned long vaddrs[TLB_BATCH_NR]; +}; + +void flush_tsb_kernel_range(unsigned long start, unsigned long end); +void flush_tsb_user(struct tlb_batch *tb); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr); /* TLB flush operations. */ -extern void flush_tlb_pending(void); +static inline void flush_tlb_mm(struct mm_struct *mm) +{ +} -#define flush_tlb_range(vma,start,end) \ - do { (void)(start); flush_tlb_pending(); } while (0) -#define flush_tlb_page(vma,addr) flush_tlb_pending() -#define flush_tlb_mm(mm) flush_tlb_pending() +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ +} -/* Local cpu only. */ -extern void __flush_tlb_all(void); +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ +} + +#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE -extern void __flush_tlb_kernel_range(unsigned long start, unsigned long end); +void flush_tlb_pending(void); +void arch_enter_lazy_mmu_mode(void); +void arch_leave_lazy_mmu_mode(void); +#define arch_flush_lazy_mmu_mode() do {} while (0) + +/* Local cpu only. */ +void __flush_tlb_all(void); +void __flush_tlb_page(unsigned long context, unsigned long vaddr); +void __flush_tlb_kernel_range(unsigned long start, unsigned long end); #ifndef CONFIG_SMP @@ -30,15 +53,24 @@ do { flush_tsb_kernel_range(start,end); \ __flush_tlb_kernel_range(start,end); \ } while (0) +static inline void global_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) +{ + __flush_tlb_page(CTX_HWBITS(mm->context), vaddr); +} + #else /* CONFIG_SMP */ -extern void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); +void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); +void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr); #define flush_tlb_kernel_range(start, end) \ do { flush_tsb_kernel_range(start,end); \ smp_flush_tlb_kernel_range(start, end); \ } while (0) +#define global_flush_tlb_page(mm, vaddr) \ + smp_flush_tlb_page(mm, vaddr) + #endif /* ! CONFIG_SMP */ #endif /* _SPARC64_TLBFLUSH_H */ diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h index 1c79f32734a..ed8f071132e 100644 --- a/arch/sparc/include/asm/topology_64.h +++ b/arch/sparc/include/asm/topology_64.h @@ -18,7 +18,7 @@ static inline int cpu_to_node(int cpu) struct pci_bus; #ifdef CONFIG_PCI -extern int pcibus_to_node(struct pci_bus *pbus); +int pcibus_to_node(struct pci_bus *pbus); #else static inline int pcibus_to_node(struct pci_bus *pbus) { @@ -31,25 +31,6 @@ static inline int pcibus_to_node(struct pci_bus *pbus) cpu_all_mask : \ cpumask_of_node(pcibus_to_node(bus))) -#define SD_NODE_INIT (struct sched_domain) { \ - .min_interval = 8, \ - .max_interval = 32, \ - .busy_factor = 32, \ - .imbalance_pct = 125, \ - .cache_nice_tries = 2, \ - .busy_idx = 3, \ - .idle_idx = 2, \ - .newidle_idx = 0, \ - .wake_idx = 0, \ - .forkexec_idx = 0, \ - .flags = SD_LOAD_BALANCE \ - | SD_BALANCE_FORK \ - | SD_BALANCE_EXEC \ - | SD_SERIALIZE, \ - .last_balance = jiffies, \ - .balance_interval = 1, \ -} - #else /* CONFIG_NUMA */ #include <asm-generic/topology.h> @@ -61,10 +42,12 @@ static inline int pcibus_to_node(struct pci_bus *pbus) #define topology_core_id(cpu) (cpu_data(cpu).core_id) #define topology_core_cpumask(cpu) (&cpu_core_map[cpu]) #define topology_thread_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) -#define mc_capable() (sparc64_multi_core) -#define smt_capable() (sparc64_multi_core) #endif /* CONFIG_SMP */ -#define cpu_coregroup_mask(cpu) (&cpu_core_map[cpu]) +extern cpumask_t cpu_core_map[NR_CPUS]; +static inline const struct cpumask *cpu_coregroup_mask(int cpu) +{ + return &cpu_core_map[cpu]; +} #endif /* _ASM_SPARC64_TOPOLOGY_H */ diff --git a/arch/sparc/include/asm/trap_block.h b/arch/sparc/include/asm/trap_block.h index 7e26b2db621..6fd4436d32f 100644 --- a/arch/sparc/include/asm/trap_block.h +++ b/arch/sparc/include/asm/trap_block.h @@ -51,11 +51,11 @@ struct trap_per_cpu { unsigned long __per_cpu_base; } __attribute__((aligned(64))); extern struct trap_per_cpu trap_block[NR_CPUS]; -extern void init_cur_cpu_trap(struct thread_info *); -extern void setup_tba(void); +void init_cur_cpu_trap(struct thread_info *); +void setup_tba(void); extern int ncpus_probed; -extern unsigned long real_hard_smp_processor_id(void); +unsigned long real_hard_smp_processor_id(void); struct cpuid_patch_entry { unsigned int addr; diff --git a/arch/sparc/include/asm/traps.h b/arch/sparc/include/asm/traps.h index 3aa62dde343..51abcb1f9b3 100644 --- a/arch/sparc/include/asm/traps.h +++ b/arch/sparc/include/asm/traps.h @@ -3,14 +3,12 @@ * * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) */ - #ifndef _SPARC_TRAPS_H #define _SPARC_TRAPS_H -#define NUM_SPARC_TRAPS 255 +#include <uapi/asm/traps.h> #ifndef __ASSEMBLY__ -#ifdef __KERNEL__ /* This is for V8 compliant Sparc CPUS */ struct tt_entry { unsigned long inst_one; @@ -22,112 +20,5 @@ struct tt_entry { /* We set this to _start in system setup. */ extern struct tt_entry *sparc_ttable; -#endif /* (__KERNEL__) */ #endif /* !(__ASSEMBLY__) */ - -/* For patching the trap table at boot time, we need to know how to - * form various common Sparc instructions. Thus these macros... - */ - -#define SPARC_MOV_CONST_L3(const) (0xa6102000 | (const&0xfff)) - -/* The following assumes that the branch lies before the place we - * are branching to. This is the case for a trap vector... - * You have been warned. - */ -#define SPARC_BRANCH(dest_addr, inst_addr) \ - (0x10800000 | (((dest_addr-inst_addr)>>2)&0x3fffff)) - -#define SPARC_RD_PSR_L0 (0xa1480000) -#define SPARC_RD_WIM_L3 (0xa7500000) -#define SPARC_NOP (0x01000000) - -/* Various interesting trap levels. */ -/* First, hardware traps. */ -#define SP_TRAP_TFLT 0x1 /* Text fault */ -#define SP_TRAP_II 0x2 /* Illegal Instruction */ -#define SP_TRAP_PI 0x3 /* Privileged Instruction */ -#define SP_TRAP_FPD 0x4 /* Floating Point Disabled */ -#define SP_TRAP_WOVF 0x5 /* Window Overflow */ -#define SP_TRAP_WUNF 0x6 /* Window Underflow */ -#define SP_TRAP_MNA 0x7 /* Memory Address Unaligned */ -#define SP_TRAP_FPE 0x8 /* Floating Point Exception */ -#define SP_TRAP_DFLT 0x9 /* Data Fault */ -#define SP_TRAP_TOF 0xa /* Tag Overflow */ -#define SP_TRAP_WDOG 0xb /* Watchpoint Detected */ -#define SP_TRAP_IRQ1 0x11 /* IRQ level 1 */ -#define SP_TRAP_IRQ2 0x12 /* IRQ level 2 */ -#define SP_TRAP_IRQ3 0x13 /* IRQ level 3 */ -#define SP_TRAP_IRQ4 0x14 /* IRQ level 4 */ -#define SP_TRAP_IRQ5 0x15 /* IRQ level 5 */ -#define SP_TRAP_IRQ6 0x16 /* IRQ level 6 */ -#define SP_TRAP_IRQ7 0x17 /* IRQ level 7 */ -#define SP_TRAP_IRQ8 0x18 /* IRQ level 8 */ -#define SP_TRAP_IRQ9 0x19 /* IRQ level 9 */ -#define SP_TRAP_IRQ10 0x1a /* IRQ level 10 */ -#define SP_TRAP_IRQ11 0x1b /* IRQ level 11 */ -#define SP_TRAP_IRQ12 0x1c /* IRQ level 12 */ -#define SP_TRAP_IRQ13 0x1d /* IRQ level 13 */ -#define SP_TRAP_IRQ14 0x1e /* IRQ level 14 */ -#define SP_TRAP_IRQ15 0x1f /* IRQ level 15 Non-maskable */ -#define SP_TRAP_RACC 0x20 /* Register Access Error ??? */ -#define SP_TRAP_IACC 0x21 /* Instruction Access Error */ -#define SP_TRAP_CPDIS 0x24 /* Co-Processor Disabled */ -#define SP_TRAP_BADFL 0x25 /* Unimplemented Flush Instruction */ -#define SP_TRAP_CPEXP 0x28 /* Co-Processor Exception */ -#define SP_TRAP_DACC 0x29 /* Data Access Error */ -#define SP_TRAP_DIVZ 0x2a /* Divide By Zero */ -#define SP_TRAP_DSTORE 0x2b /* Data Store Error ??? */ -#define SP_TRAP_DMM 0x2c /* Data Access MMU Miss ??? */ -#define SP_TRAP_IMM 0x3c /* Instruction Access MMU Miss ??? */ - -/* Now the Software Traps... */ -#define SP_TRAP_SUNOS 0x80 /* SunOS System Call */ -#define SP_TRAP_SBPT 0x81 /* Software Breakpoint */ -#define SP_TRAP_SDIVZ 0x82 /* Software Divide-by-Zero trap */ -#define SP_TRAP_FWIN 0x83 /* Flush Windows */ -#define SP_TRAP_CWIN 0x84 /* Clean Windows */ -#define SP_TRAP_RCHK 0x85 /* Range Check */ -#define SP_TRAP_FUNA 0x86 /* Fix Unaligned Access */ -#define SP_TRAP_IOWFL 0x87 /* Integer Overflow */ -#define SP_TRAP_SOLARIS 0x88 /* Solaris System Call */ -#define SP_TRAP_NETBSD 0x89 /* NetBSD System Call */ -#define SP_TRAP_LINUX 0x90 /* Linux System Call */ - -/* Names used for compatibility with SunOS */ -#define ST_SYSCALL 0x00 -#define ST_BREAKPOINT 0x01 -#define ST_DIV0 0x02 -#define ST_FLUSH_WINDOWS 0x03 -#define ST_CLEAN_WINDOWS 0x04 -#define ST_RANGE_CHECK 0x05 -#define ST_FIX_ALIGN 0x06 -#define ST_INT_OVERFLOW 0x07 - -/* Special traps... */ -#define SP_TRAP_KBPT1 0xfe /* KADB/PROM Breakpoint one */ -#define SP_TRAP_KBPT2 0xff /* KADB/PROM Breakpoint two */ - -/* Handy Macros */ -/* Is this a trap we never expect to get? */ -#define BAD_TRAP_P(level) \ - ((level > SP_TRAP_WDOG && level < SP_TRAP_IRQ1) || \ - (level > SP_TRAP_IACC && level < SP_TRAP_CPDIS) || \ - (level > SP_TRAP_BADFL && level < SP_TRAP_CPEXP) || \ - (level > SP_TRAP_DMM && level < SP_TRAP_IMM) || \ - (level > SP_TRAP_IMM && level < SP_TRAP_SUNOS) || \ - (level > SP_TRAP_LINUX && level < SP_TRAP_KBPT1)) - -/* Is this a Hardware trap? */ -#define HW_TRAP_P(level) ((level > 0) && (level < SP_TRAP_SUNOS)) - -/* Is this a Software trap? */ -#define SW_TRAP_P(level) ((level >= SP_TRAP_SUNOS) && (level <= SP_TRAP_KBPT2)) - -/* Is this a system call for some OS we know about? */ -#define SCALL_TRAP_P(level) ((level == SP_TRAP_SUNOS) || \ - (level == SP_TRAP_SOLARIS) || \ - (level == SP_TRAP_NETBSD) || \ - (level == SP_TRAP_LINUX)) - #endif /* !(_SPARC_TRAPS_H) */ diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index 83c571d8c8a..90916f955ca 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h @@ -133,29 +133,6 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; sub TSB, 0x8, TSB; \ TSB_STORE(TSB, TAG); -#define KTSB_LOAD_QUAD(TSB, REG) \ - ldda [TSB] ASI_NUCLEUS_QUAD_LDD, REG; - -#define KTSB_STORE(ADDR, VAL) \ - stxa VAL, [ADDR] ASI_N; - -#define KTSB_LOCK_TAG(TSB, REG1, REG2) \ -99: lduwa [TSB] ASI_N, REG1; \ - sethi %hi(TSB_TAG_LOCK_HIGH), REG2;\ - andcc REG1, REG2, %g0; \ - bne,pn %icc, 99b; \ - nop; \ - casa [TSB] ASI_N, REG1, REG2;\ - cmp REG1, REG2; \ - bne,pn %icc, 99b; \ - nop; \ - -#define KTSB_WRITE(TSB, TTE, TAG) \ - add TSB, 0x8, TSB; \ - stxa TTE, [TSB] ASI_N; \ - sub TSB, 0x8, TSB; \ - stxa TAG, [TSB] ASI_N; - /* Do a kernel page table walk. Leaves physical PTE pointer in * REG1. Jumps to FAIL_LABEL on early page table walk termination. * VADDR will not be clobbered, but REG2 will. @@ -165,45 +142,75 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; or REG1, %lo(swapper_pg_dir), REG1; \ sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - andn REG2, 0x3, REG2; \ - lduw [REG1 + REG2], REG1; \ + andn REG2, 0x7, REG2; \ + ldx [REG1 + REG2], REG1; \ brz,pn REG1, FAIL_LABEL; \ sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, 11, REG1; \ - andn REG2, 0x3, REG2; \ - lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ brz,pn REG1, FAIL_LABEL; \ sllx VADDR, 64 - PMD_SHIFT, REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, 11, REG1; \ andn REG2, 0x7, REG2; \ add REG1, REG2, REG1; - /* Do a user page table walk in MMU globals. Leaves physical PTE - * pointer in REG1. Jumps to FAIL_LABEL on early page table walk - * termination. Physical base of page tables is in PHYS_PGD which - * will not be modified. + /* PMD has been loaded into REG1, interpret the value, seeing + * if it is a HUGE PMD or a normal one. If it is not valid + * then jump to FAIL_LABEL. If it is a HUGE PMD, and it + * translates to a valid PTE, branch to PTE_LABEL. + * + * We have to propagate the 4MB bit of the virtual address + * because we are fabricating 8MB pages using 4MB hw pages. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PMD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(4 * 1024 * 1024), REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ +700: +#else +#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + nop; +#endif + + /* Do a user page table walk in MMU globals. Leaves final, + * valid, PTE value in REG1. Jumps to FAIL_LABEL on early + * page table walk termination or if the PTE is not valid. + * + * Physical base of page tables is in PHYS_PGD which will not + * be modified. * * VADDR will not be clobbered, but REG1 and REG2 will. */ #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - andn REG2, 0x3, REG2; \ - lduwa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ + andn REG2, 0x7, REG2; \ + ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ brz,pn REG1, FAIL_LABEL; \ sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, 11, REG1; \ - andn REG2, 0x3, REG2; \ - lduwa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ - brz,pn REG1, FAIL_LABEL; \ - sllx VADDR, 64 - PMD_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ + sllx VADDR, 64 - PMD_SHIFT, REG2; \ srlx REG2, 64 - PAGE_SHIFT, REG2; \ - sllx REG1, 11, REG1; \ andn REG2, 0x7, REG2; \ - add REG1, REG2, REG1; + add REG1, REG2, REG1; \ + ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ + brgez,pn REG1, FAIL_LABEL; \ + nop; \ +800: /* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0. * If no entry is found, FAIL_LABEL will be branched to. On success @@ -239,6 +246,8 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; (KERNEL_TSB_SIZE_BYTES / 16) #define KERNEL_TSB4M_NENTRIES 4096 +#define KTSB_PHYS_SHIFT 15 + /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries * and the found TTE will be left in REG1. REG3 and REG4 must @@ -247,13 +256,22 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * VADDR and TAG will be preserved and not clobbered by this macro. */ #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ - sethi %hi(swapper_tsb), REG1; \ +661: sethi %hi(swapper_tsb), REG1; \ or REG1, %lo(swapper_tsb), REG1; \ + .section .swapper_tsb_phys_patch, "ax"; \ + .word 661b; \ + .previous; \ +661: nop; \ + .section .tsb_ldquad_phys_patch, "ax"; \ + .word 661b; \ + sllx REG1, KTSB_PHYS_SHIFT, REG1; \ + sllx REG1, KTSB_PHYS_SHIFT, REG1; \ + .previous; \ srlx VADDR, PAGE_SHIFT, REG2; \ and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ sllx REG2, 4, REG2; \ add REG1, REG2, REG2; \ - KTSB_LOAD_QUAD(REG2, REG3); \ + TSB_LOAD_QUAD(REG2, REG3); \ cmp REG3, TAG; \ be,a,pt %xcc, OK_LABEL; \ mov REG4, REG1; @@ -263,12 +281,21 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * we can make use of that for the index computation. */ #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ - sethi %hi(swapper_4m_tsb), REG1; \ +661: sethi %hi(swapper_4m_tsb), REG1; \ or REG1, %lo(swapper_4m_tsb), REG1; \ + .section .swapper_4m_tsb_phys_patch, "ax"; \ + .word 661b; \ + .previous; \ +661: nop; \ + .section .tsb_ldquad_phys_patch, "ax"; \ + .word 661b; \ + sllx REG1, KTSB_PHYS_SHIFT, REG1; \ + sllx REG1, KTSB_PHYS_SHIFT, REG1; \ + .previous; \ and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ sllx REG2, 4, REG2; \ add REG1, REG2, REG2; \ - KTSB_LOAD_QUAD(REG2, REG3); \ + TSB_LOAD_QUAD(REG2, REG3); \ cmp REG3, TAG; \ be,a,pt %xcc, OK_LABEL; \ mov REG4, REG1; diff --git a/arch/sparc/include/asm/ttable.h b/arch/sparc/include/asm/ttable.h index 48f2807d326..71b5a67522a 100644 --- a/arch/sparc/include/asm/ttable.h +++ b/arch/sparc/include/asm/ttable.h @@ -372,7 +372,9 @@ etrap_spill_fixup_64bit: \ /* Normal 32bit spill */ #define SPILL_2_GENERIC(ASI) \ - srl %sp, 0, %sp; \ + and %sp, 1, %g3; \ + brnz,pn %g3, (. - (128 + 4)); \ + srl %sp, 0, %sp; \ stwa %l0, [%sp + %g0] ASI; \ mov 0x04, %g3; \ stwa %l1, [%sp + %g3] ASI; \ @@ -398,14 +400,16 @@ etrap_spill_fixup_64bit: \ stwa %i6, [%g1 + %g0] ASI; \ stwa %i7, [%g1 + %g3] ASI; \ saved; \ - retry; nop; nop; \ + retry; \ b,a,pt %xcc, spill_fixup_dax; \ b,a,pt %xcc, spill_fixup_mna; \ b,a,pt %xcc, spill_fixup; #define SPILL_2_GENERIC_ETRAP \ etrap_user_spill_32bit: \ - srl %sp, 0, %sp; \ + and %sp, 1, %g3; \ + brnz,pn %g3, etrap_user_spill_64bit; \ + srl %sp, 0, %sp; \ stwa %l0, [%sp + 0x00] %asi; \ stwa %l1, [%sp + 0x04] %asi; \ stwa %l2, [%sp + 0x08] %asi; \ @@ -427,7 +431,7 @@ etrap_user_spill_32bit: \ ba,pt %xcc, etrap_save; \ wrpr %g1, %cwp; \ nop; nop; nop; nop; \ - nop; nop; nop; nop; \ + nop; nop; \ ba,a,pt %xcc, etrap_spill_fixup_32bit; \ ba,a,pt %xcc, etrap_spill_fixup_32bit; \ ba,a,pt %xcc, etrap_spill_fixup_32bit; @@ -592,7 +596,9 @@ user_rtt_fill_64bit: \ /* Normal 32bit fill */ #define FILL_2_GENERIC(ASI) \ - srl %sp, 0, %sp; \ + and %sp, 1, %g3; \ + brnz,pn %g3, (. - (128 + 4)); \ + srl %sp, 0, %sp; \ lduwa [%sp + %g0] ASI, %l0; \ mov 0x04, %g2; \ mov 0x08, %g3; \ @@ -616,14 +622,16 @@ user_rtt_fill_64bit: \ lduwa [%g1 + %g3] ASI, %i6; \ lduwa [%g1 + %g5] ASI, %i7; \ restored; \ - retry; nop; nop; nop; nop; \ + retry; nop; nop; \ b,a,pt %xcc, fill_fixup_dax; \ b,a,pt %xcc, fill_fixup_mna; \ b,a,pt %xcc, fill_fixup; #define FILL_2_GENERIC_RTRAP \ user_rtt_fill_32bit: \ - srl %sp, 0, %sp; \ + and %sp, 1, %g3; \ + brnz,pn %g3, user_rtt_fill_64bit; \ + srl %sp, 0, %sp; \ lduwa [%sp + 0x00] %asi, %l0; \ lduwa [%sp + 0x04] %asi, %l1; \ lduwa [%sp + 0x08] %asi, %l2; \ @@ -643,7 +651,7 @@ user_rtt_fill_32bit: \ ba,pt %xcc, user_rtt_pre_restore; \ restored; \ nop; nop; nop; nop; nop; \ - nop; nop; nop; nop; nop; \ + nop; nop; nop; \ ba,a,pt %xcc, user_rtt_fill_fixup; \ ba,a,pt %xcc, user_rtt_fill_fixup; \ ba,a,pt %xcc, user_rtt_fill_fixup; diff --git a/arch/sparc/include/asm/types.h b/arch/sparc/include/asm/types.h deleted file mode 100644 index 09c79a9c851..00000000000 --- a/arch/sparc/include/asm/types.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef _SPARC_TYPES_H -#define _SPARC_TYPES_H -/* - * This file is never included by application software unless - * explicitly requested (e.g., via linux/types.h) in which case the - * application is Linux specific so (user-) name space pollution is - * not a major issue. However, for interoperability, libraries still - * need to be careful to avoid a name clashes. - */ - -#if defined(__sparc__) - -#include <asm-generic/int-ll64.h> - -#ifndef __ASSEMBLY__ - -typedef unsigned short umode_t; - -#endif /* __ASSEMBLY__ */ - -#ifdef __KERNEL__ - -#ifndef __ASSEMBLY__ - -/* Dma addresses come in generic and 64-bit flavours. */ - -typedef u32 dma_addr_t; - -#if defined(__arch64__) - -/*** SPARC 64 bit ***/ -typedef u64 dma64_addr_t; -#else -/*** SPARC 32 bit ***/ -typedef u32 dma64_addr_t; - -#endif /* defined(__arch64__) */ - -#endif /* __ASSEMBLY__ */ - -#endif /* __KERNEL__ */ - -#endif /* defined(__sparc__) */ - -#endif /* defined(_SPARC_TYPES_H) */ diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h index e88fbe5c045..bd56c28fff9 100644 --- a/arch/sparc/include/asm/uaccess.h +++ b/arch/sparc/include/asm/uaccess.h @@ -5,4 +5,10 @@ #else #include <asm/uaccess_32.h> #endif + +#define user_addr_max() \ + (segment_eq(get_fs(), USER_DS) ? TASK_SIZE : ~0UL) + +long strncpy_from_user(char *dest, const char __user *src, long count); + #endif diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 8303ac48103..9634d086fc5 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -12,11 +12,12 @@ #include <linux/sched.h> #include <linux/string.h> #include <linux/errno.h> -#include <asm/vac-ops.h> #endif #ifndef __ASSEMBLY__ +#include <asm/processor.h> + #define ARCH_HAS_SORT_EXTABLE #define ARCH_HAS_SEARCH_EXTABLE @@ -77,9 +78,9 @@ struct exception_table_entry }; /* Returns 0 if exception not found and fixup otherwise. */ -extern unsigned long search_extables_range(unsigned long addr, unsigned long *g2); +unsigned long search_extables_range(unsigned long addr, unsigned long *g2); -extern void __ret_efault(void); +void __ret_efault(void); /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right @@ -151,7 +152,7 @@ __asm__ __volatile__( \ : "=&r" (ret) : "r" (x), "m" (*__m(addr)), \ "i" (-EFAULT)) -extern int __put_user_bad(void); +int __put_user_bad(void); #define __get_user_check(x,addr,size,type) ({ \ register int __gu_ret; \ @@ -243,9 +244,9 @@ __asm__ __volatile__( \ ".previous\n\t" \ : "=&r" (x) : "m" (*__m(addr)), "i" (retval)) -extern int __get_user_bad(void); +int __get_user_bad(void); -extern unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); +unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) { @@ -305,34 +306,8 @@ static inline unsigned long clear_user(void __user *addr, unsigned long n) return n; } -extern long __strncpy_from_user(char *dest, const char __user *src, long count); - -static inline long strncpy_from_user(char *dest, const char __user *src, long count) -{ - if (__access_ok((unsigned long) src, count)) - return __strncpy_from_user(dest, src, count); - else - return -EFAULT; -} - -extern long __strlen_user(const char __user *); -extern long __strnlen_user(const char __user *, long len); - -static inline long strlen_user(const char __user *str) -{ - if (!access_ok(VERIFY_READ, str, 0)) - return 0; - else - return __strlen_user(str); -} - -static inline long strnlen_user(const char __user *str, long len) -{ - if (!access_ok(VERIFY_READ, str, 0)) - return 0; - else - return __strnlen_user(str, len); -} +__must_check long strlen_user(const char __user *str); +__must_check long strnlen_user(const char __user *str, long n); #endif /* __ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index 3e1449f0779..c990a5e577f 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -11,13 +11,14 @@ #include <linux/string.h> #include <linux/thread_info.h> #include <asm/asi.h> -#include <asm/system.h> #include <asm/spitfire.h> #include <asm-generic/uaccess-unaligned.h> #endif #ifndef __ASSEMBLY__ +#include <asm/processor.h> + /* * Sparc64 is segmented, though more like the M68K than the I386. * We use the secondary ASI to address user memory, which references a @@ -37,14 +38,14 @@ #define VERIFY_READ 0 #define VERIFY_WRITE 1 -#define get_fs() ((mm_segment_t) { get_thread_current_ds() }) +#define get_fs() ((mm_segment_t){(current_thread_info()->current_ds)}) #define get_ds() (KERNEL_DS) #define segment_eq(a,b) ((a).seg == (b).seg) #define set_fs(val) \ do { \ - set_thread_current_ds((val).seg); \ + current_thread_info()->current_ds =(val).seg; \ __asm__ __volatile__ ("wr %%g0, %0, %%asi" : : "r" ((val).seg)); \ } while(0) @@ -75,8 +76,8 @@ struct exception_table_entry { unsigned int insn, fixup; }; -extern void __ret_efault(void); -extern void __retl_efault(void); +void __ret_efault(void); +void __retl_efault(void); /* Uh, these should become the main single-value transfer routines.. * They automatically use the right size if we just have the right @@ -133,7 +134,7 @@ __asm__ __volatile__( \ : "=r" (ret) : "r" (x), "r" (__m(addr)), \ "i" (-EFAULT)) -extern int __put_user_bad(void); +int __put_user_bad(void); #define __get_user_nocheck(data,addr,size,type) ({ \ register int __gu_ret; \ @@ -203,13 +204,13 @@ __asm__ __volatile__( \ ".previous\n\t" \ : "=r" (x) : "r" (__m(addr)), "i" (retval)) -extern int __get_user_bad(void); +int __get_user_bad(void); -extern unsigned long __must_check ___copy_from_user(void *to, - const void __user *from, - unsigned long size); -extern unsigned long copy_from_user_fixup(void *to, const void __user *from, - unsigned long size); +unsigned long __must_check ___copy_from_user(void *to, + const void __user *from, + unsigned long size); +unsigned long copy_from_user_fixup(void *to, const void __user *from, + unsigned long size); static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long size) { @@ -222,11 +223,11 @@ copy_from_user(void *to, const void __user *from, unsigned long size) } #define __copy_from_user copy_from_user -extern unsigned long __must_check ___copy_to_user(void __user *to, - const void *from, - unsigned long size); -extern unsigned long copy_to_user_fixup(void __user *to, const void *from, - unsigned long size); +unsigned long __must_check ___copy_to_user(void __user *to, + const void *from, + unsigned long size); +unsigned long copy_to_user_fixup(void __user *to, const void *from, + unsigned long size); static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long size) { @@ -238,11 +239,11 @@ copy_to_user(void __user *to, const void *from, unsigned long size) } #define __copy_to_user copy_to_user -extern unsigned long __must_check ___copy_in_user(void __user *to, - const void __user *from, - unsigned long size); -extern unsigned long copy_in_user_fixup(void __user *to, void __user *from, - unsigned long size); +unsigned long __must_check ___copy_in_user(void __user *to, + const void __user *from, + unsigned long size); +unsigned long copy_in_user_fixup(void __user *to, void __user *from, + unsigned long size); static inline unsigned long __must_check copy_in_user(void __user *to, void __user *from, unsigned long size) { @@ -254,21 +255,20 @@ copy_in_user(void __user *to, void __user *from, unsigned long size) } #define __copy_in_user copy_in_user -extern unsigned long __must_check __clear_user(void __user *, unsigned long); +unsigned long __must_check __clear_user(void __user *, unsigned long); #define clear_user __clear_user -extern long __must_check __strncpy_from_user(char *dest, const char __user *src, long count); - -#define strncpy_from_user __strncpy_from_user +__must_check long strlen_user(const char __user *str); +__must_check long strnlen_user(const char __user *str, long n); -extern long __strlen_user(const char __user *); -extern long __strnlen_user(const char __user *, long len); +#define __copy_to_user_inatomic __copy_to_user +#define __copy_from_user_inatomic __copy_from_user -#define strlen_user __strlen_user -#define strnlen_user __strnlen_user -#define __copy_to_user_inatomic ___copy_to_user -#define __copy_from_user_inatomic ___copy_from_user +struct pt_regs; +unsigned long compute_effective_address(struct pt_regs *, + unsigned int insn, + unsigned int rd); #endif /* __ASSEMBLY__ */ diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h index 03eb5a8f6f9..0aac1e8f296 100644 --- a/arch/sparc/include/asm/unistd.h +++ b/arch/sparc/include/asm/unistd.h @@ -1,6 +1,3 @@ -#ifndef _SPARC_UNISTD_H -#define _SPARC_UNISTD_H - /* * System calls under the Sparc. * @@ -14,414 +11,20 @@ * * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) */ -#ifndef __32bit_syscall_numbers__ -#ifndef __arch64__ -#define __32bit_syscall_numbers__ -#endif -#endif +#ifndef _SPARC_UNISTD_H +#define _SPARC_UNISTD_H + +#include <uapi/asm/unistd.h> -#define __NR_restart_syscall 0 /* Linux Specific */ -#define __NR_exit 1 /* Common */ -#define __NR_fork 2 /* Common */ -#define __NR_read 3 /* Common */ -#define __NR_write 4 /* Common */ -#define __NR_open 5 /* Common */ -#define __NR_close 6 /* Common */ -#define __NR_wait4 7 /* Common */ -#define __NR_creat 8 /* Common */ -#define __NR_link 9 /* Common */ -#define __NR_unlink 10 /* Common */ -#define __NR_execv 11 /* SunOS Specific */ -#define __NR_chdir 12 /* Common */ -#define __NR_chown 13 /* Common */ -#define __NR_mknod 14 /* Common */ -#define __NR_chmod 15 /* Common */ -#define __NR_lchown 16 /* Common */ -#define __NR_brk 17 /* Common */ -#define __NR_perfctr 18 /* Performance counter operations */ -#define __NR_lseek 19 /* Common */ -#define __NR_getpid 20 /* Common */ -#define __NR_capget 21 /* Linux Specific */ -#define __NR_capset 22 /* Linux Specific */ -#define __NR_setuid 23 /* Implemented via setreuid in SunOS */ -#define __NR_getuid 24 /* Common */ -#define __NR_vmsplice 25 /* ENOSYS under SunOS */ -#define __NR_ptrace 26 /* Common */ -#define __NR_alarm 27 /* Implemented via setitimer in SunOS */ -#define __NR_sigaltstack 28 /* Common */ -#define __NR_pause 29 /* Is sigblock(0)->sigpause() in SunOS */ -#define __NR_utime 30 /* Implemented via utimes() under SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_lchown32 31 /* Linux sparc32 specific */ -#define __NR_fchown32 32 /* Linux sparc32 specific */ -#endif -#define __NR_access 33 /* Common */ -#define __NR_nice 34 /* Implemented via get/setpriority() in SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_chown32 35 /* Linux sparc32 specific */ -#endif -#define __NR_sync 36 /* Common */ -#define __NR_kill 37 /* Common */ -#define __NR_stat 38 /* Common */ -#define __NR_sendfile 39 /* Linux Specific */ -#define __NR_lstat 40 /* Common */ -#define __NR_dup 41 /* Common */ -#define __NR_pipe 42 /* Common */ -#define __NR_times 43 /* Implemented via getrusage() in SunOS */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getuid32 44 /* Linux sparc32 specific */ -#endif -#define __NR_umount2 45 /* Linux Specific */ -#define __NR_setgid 46 /* Implemented via setregid() in SunOS */ -#define __NR_getgid 47 /* Common */ -#define __NR_signal 48 /* Implemented via sigvec() in SunOS */ -#define __NR_geteuid 49 /* SunOS calls getuid() */ -#define __NR_getegid 50 /* SunOS calls getgid() */ -#define __NR_acct 51 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getgid32 53 /* Linux sparc32 specific */ -#else -#define __NR_memory_ordering 52 /* Linux Specific */ -#endif -#define __NR_ioctl 54 /* Common */ -#define __NR_reboot 55 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_mmap2 56 /* Linux sparc32 Specific */ -#endif -#define __NR_symlink 57 /* Common */ -#define __NR_readlink 58 /* Common */ -#define __NR_execve 59 /* Common */ -#define __NR_umask 60 /* Common */ -#define __NR_chroot 61 /* Common */ -#define __NR_fstat 62 /* Common */ -#define __NR_fstat64 63 /* Linux Specific */ -#define __NR_getpagesize 64 /* Common */ -#define __NR_msync 65 /* Common in newer 1.3.x revs... */ -#define __NR_vfork 66 /* Common */ -#define __NR_pread64 67 /* Linux Specific */ -#define __NR_pwrite64 68 /* Linux Specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_geteuid32 69 /* Linux sparc32, sbrk under SunOS */ -#define __NR_getegid32 70 /* Linux sparc32, sstk under SunOS */ -#endif -#define __NR_mmap 71 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setreuid32 72 /* Linux sparc32, vadvise under SunOS */ -#endif -#define __NR_munmap 73 /* Common */ -#define __NR_mprotect 74 /* Common */ -#define __NR_madvise 75 /* Common */ -#define __NR_vhangup 76 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_truncate64 77 /* Linux sparc32 Specific */ -#endif -#define __NR_mincore 78 /* Common */ -#define __NR_getgroups 79 /* Common */ -#define __NR_setgroups 80 /* Common */ -#define __NR_getpgrp 81 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setgroups32 82 /* Linux sparc32, setpgrp under SunOS */ -#endif -#define __NR_setitimer 83 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_ftruncate64 84 /* Linux sparc32 Specific */ -#endif -#define __NR_swapon 85 /* Common */ -#define __NR_getitimer 86 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setuid32 87 /* Linux sparc32, gethostname under SunOS */ -#endif -#define __NR_sethostname 88 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setgid32 89 /* Linux sparc32, getdtablesize under SunOS */ -#endif -#define __NR_dup2 90 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setfsuid32 91 /* Linux sparc32, getdopt under SunOS */ -#endif -#define __NR_fcntl 92 /* Common */ -#define __NR_select 93 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setfsgid32 94 /* Linux sparc32, setdopt under SunOS */ -#endif -#define __NR_fsync 95 /* Common */ -#define __NR_setpriority 96 /* Common */ -#define __NR_socket 97 /* Common */ -#define __NR_connect 98 /* Common */ -#define __NR_accept 99 /* Common */ -#define __NR_getpriority 100 /* Common */ -#define __NR_rt_sigreturn 101 /* Linux Specific */ -#define __NR_rt_sigaction 102 /* Linux Specific */ -#define __NR_rt_sigprocmask 103 /* Linux Specific */ -#define __NR_rt_sigpending 104 /* Linux Specific */ -#define __NR_rt_sigtimedwait 105 /* Linux Specific */ -#define __NR_rt_sigqueueinfo 106 /* Linux Specific */ -#define __NR_rt_sigsuspend 107 /* Linux Specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_setresuid32 108 /* Linux Specific, sigvec under SunOS */ -#define __NR_getresuid32 109 /* Linux Specific, sigblock under SunOS */ -#define __NR_setresgid32 110 /* Linux Specific, sigsetmask under SunOS */ -#define __NR_getresgid32 111 /* Linux Specific, sigpause under SunOS */ -#define __NR_setregid32 112 /* Linux sparc32, sigstack under SunOS */ -#else -#define __NR_setresuid 108 /* Linux Specific, sigvec under SunOS */ -#define __NR_getresuid 109 /* Linux Specific, sigblock under SunOS */ -#define __NR_setresgid 110 /* Linux Specific, sigsetmask under SunOS */ -#define __NR_getresgid 111 /* Linux Specific, sigpause under SunOS */ -#endif -#define __NR_recvmsg 113 /* Common */ -#define __NR_sendmsg 114 /* Common */ -#ifdef __32bit_syscall_numbers__ -#define __NR_getgroups32 115 /* Linux sparc32, vtrace under SunOS */ -#endif -#define __NR_gettimeofday 116 /* Common */ -#define __NR_getrusage 117 /* Common */ -#define __NR_getsockopt 118 /* Common */ -#define __NR_getcwd 119 /* Linux Specific */ -#define __NR_readv 120 /* Common */ -#define __NR_writev 121 /* Common */ -#define __NR_settimeofday 122 /* Common */ -#define __NR_fchown 123 /* Common */ -#define __NR_fchmod 124 /* Common */ -#define __NR_recvfrom 125 /* Common */ -#define __NR_setreuid 126 /* Common */ -#define __NR_setregid 127 /* Common */ -#define __NR_rename 128 /* Common */ -#define __NR_truncate 129 /* Common */ -#define __NR_ftruncate 130 /* Common */ -#define __NR_flock 131 /* Common */ -#define __NR_lstat64 132 /* Linux Specific */ -#define __NR_sendto 133 /* Common */ -#define __NR_shutdown 134 /* Common */ -#define __NR_socketpair 135 /* Common */ -#define __NR_mkdir 136 /* Common */ -#define __NR_rmdir 137 /* Common */ -#define __NR_utimes 138 /* SunOS Specific */ -#define __NR_stat64 139 /* Linux Specific */ -#define __NR_sendfile64 140 /* adjtime under SunOS */ -#define __NR_getpeername 141 /* Common */ -#define __NR_futex 142 /* gethostid under SunOS */ -#define __NR_gettid 143 /* ENOSYS under SunOS */ -#define __NR_getrlimit 144 /* Common */ -#define __NR_setrlimit 145 /* Common */ -#define __NR_pivot_root 146 /* Linux Specific, killpg under SunOS */ -#define __NR_prctl 147 /* ENOSYS under SunOS */ -#define __NR_pciconfig_read 148 /* ENOSYS under SunOS */ -#define __NR_pciconfig_write 149 /* ENOSYS under SunOS */ -#define __NR_getsockname 150 /* Common */ -#define __NR_inotify_init 151 /* Linux specific */ -#define __NR_inotify_add_watch 152 /* Linux specific */ -#define __NR_poll 153 /* Common */ -#define __NR_getdents64 154 /* Linux specific */ -#ifdef __32bit_syscall_numbers__ -#define __NR_fcntl64 155 /* Linux sparc32 Specific */ -#endif -#define __NR_inotify_rm_watch 156 /* Linux specific */ -#define __NR_statfs 157 /* Common */ -#define __NR_fstatfs 158 /* Common */ -#define __NR_umount 159 /* Common */ -#define __NR_sched_set_affinity 160 /* Linux specific, async_daemon under SunOS */ -#define __NR_sched_get_affinity 161 /* Linux specific, getfh under SunOS */ -#define __NR_getdomainname 162 /* SunOS Specific */ -#define __NR_setdomainname 163 /* Common */ -#ifndef __32bit_syscall_numbers__ -#define __NR_utrap_install 164 /* SYSV ABI/v9 required */ -#endif -#define __NR_quotactl 165 /* Common */ -#define __NR_set_tid_address 166 /* Linux specific, exportfs under SunOS */ -#define __NR_mount 167 /* Common */ -#define __NR_ustat 168 /* Common */ -#define __NR_setxattr 169 /* SunOS: semsys */ -#define __NR_lsetxattr 170 /* SunOS: msgsys */ -#define __NR_fsetxattr 171 /* SunOS: shmsys */ -#define __NR_getxattr 172 /* SunOS: auditsys */ -#define __NR_lgetxattr 173 /* SunOS: rfssys */ -#define __NR_getdents 174 /* Common */ -#define __NR_setsid 175 /* Common */ -#define __NR_fchdir 176 /* Common */ -#define __NR_fgetxattr 177 /* SunOS: fchroot */ -#define __NR_listxattr 178 /* SunOS: vpixsys */ -#define __NR_llistxattr 179 /* SunOS: aioread */ -#define __NR_flistxattr 180 /* SunOS: aiowrite */ -#define __NR_removexattr 181 /* SunOS: aiowait */ -#define __NR_lremovexattr 182 /* SunOS: aiocancel */ -#define __NR_sigpending 183 /* Common */ -#define __NR_query_module 184 /* Linux Specific */ -#define __NR_setpgid 185 /* Common */ -#define __NR_fremovexattr 186 /* SunOS: pathconf */ -#define __NR_tkill 187 /* SunOS: fpathconf */ -#define __NR_exit_group 188 /* Linux specific, sysconf undef SunOS */ -#define __NR_uname 189 /* Linux Specific */ -#define __NR_init_module 190 /* Linux Specific */ -#define __NR_personality 191 /* Linux Specific */ -#define __NR_remap_file_pages 192 /* Linux Specific */ -#define __NR_epoll_create 193 /* Linux Specific */ -#define __NR_epoll_ctl 194 /* Linux Specific */ -#define __NR_epoll_wait 195 /* Linux Specific */ -#define __NR_ioprio_set 196 /* Linux Specific */ -#define __NR_getppid 197 /* Linux Specific */ -#define __NR_sigaction 198 /* Linux Specific */ -#define __NR_sgetmask 199 /* Linux Specific */ -#define __NR_ssetmask 200 /* Linux Specific */ -#define __NR_sigsuspend 201 /* Linux Specific */ -#define __NR_oldlstat 202 /* Linux Specific */ -#define __NR_uselib 203 /* Linux Specific */ -#define __NR_readdir 204 /* Linux Specific */ -#define __NR_readahead 205 /* Linux Specific */ -#define __NR_socketcall 206 /* Linux Specific */ -#define __NR_syslog 207 /* Linux Specific */ -#define __NR_lookup_dcookie 208 /* Linux Specific */ -#define __NR_fadvise64 209 /* Linux Specific */ -#define __NR_fadvise64_64 210 /* Linux Specific */ -#define __NR_tgkill 211 /* Linux Specific */ -#define __NR_waitpid 212 /* Linux Specific */ -#define __NR_swapoff 213 /* Linux Specific */ -#define __NR_sysinfo 214 /* Linux Specific */ -#define __NR_ipc 215 /* Linux Specific */ -#define __NR_sigreturn 216 /* Linux Specific */ -#define __NR_clone 217 /* Linux Specific */ -#define __NR_ioprio_get 218 /* Linux Specific */ -#define __NR_adjtimex 219 /* Linux Specific */ -#define __NR_sigprocmask 220 /* Linux Specific */ -#define __NR_create_module 221 /* Linux Specific */ -#define __NR_delete_module 222 /* Linux Specific */ -#define __NR_get_kernel_syms 223 /* Linux Specific */ -#define __NR_getpgid 224 /* Linux Specific */ -#define __NR_bdflush 225 /* Linux Specific */ -#define __NR_sysfs 226 /* Linux Specific */ -#define __NR_afs_syscall 227 /* Linux Specific */ -#define __NR_setfsuid 228 /* Linux Specific */ -#define __NR_setfsgid 229 /* Linux Specific */ -#define __NR__newselect 230 /* Linux Specific */ #ifdef __32bit_syscall_numbers__ -#define __NR_time 231 /* Linux Specific */ #else -#ifdef __KERNEL__ #define __NR_time 231 /* Linux sparc32 */ #endif -#endif -#define __NR_splice 232 /* Linux Specific */ -#define __NR_stime 233 /* Linux Specific */ -#define __NR_statfs64 234 /* Linux Specific */ -#define __NR_fstatfs64 235 /* Linux Specific */ -#define __NR__llseek 236 /* Linux Specific */ -#define __NR_mlock 237 -#define __NR_munlock 238 -#define __NR_mlockall 239 -#define __NR_munlockall 240 -#define __NR_sched_setparam 241 -#define __NR_sched_getparam 242 -#define __NR_sched_setscheduler 243 -#define __NR_sched_getscheduler 244 -#define __NR_sched_yield 245 -#define __NR_sched_get_priority_max 246 -#define __NR_sched_get_priority_min 247 -#define __NR_sched_rr_get_interval 248 -#define __NR_nanosleep 249 -#define __NR_mremap 250 -#define __NR__sysctl 251 -#define __NR_getsid 252 -#define __NR_fdatasync 253 -#define __NR_nfsservctl 254 -#define __NR_sync_file_range 255 -#define __NR_clock_settime 256 -#define __NR_clock_gettime 257 -#define __NR_clock_getres 258 -#define __NR_clock_nanosleep 259 -#define __NR_sched_getaffinity 260 -#define __NR_sched_setaffinity 261 -#define __NR_timer_settime 262 -#define __NR_timer_gettime 263 -#define __NR_timer_getoverrun 264 -#define __NR_timer_delete 265 -#define __NR_timer_create 266 -/* #define __NR_vserver 267 Reserved for VSERVER */ -#define __NR_io_setup 268 -#define __NR_io_destroy 269 -#define __NR_io_submit 270 -#define __NR_io_cancel 271 -#define __NR_io_getevents 272 -#define __NR_mq_open 273 -#define __NR_mq_unlink 274 -#define __NR_mq_timedsend 275 -#define __NR_mq_timedreceive 276 -#define __NR_mq_notify 277 -#define __NR_mq_getsetattr 278 -#define __NR_waitid 279 -#define __NR_tee 280 -#define __NR_add_key 281 -#define __NR_request_key 282 -#define __NR_keyctl 283 -#define __NR_openat 284 -#define __NR_mkdirat 285 -#define __NR_mknodat 286 -#define __NR_fchownat 287 -#define __NR_futimesat 288 -#define __NR_fstatat64 289 -#define __NR_unlinkat 290 -#define __NR_renameat 291 -#define __NR_linkat 292 -#define __NR_symlinkat 293 -#define __NR_readlinkat 294 -#define __NR_fchmodat 295 -#define __NR_faccessat 296 -#define __NR_pselect6 297 -#define __NR_ppoll 298 -#define __NR_unshare 299 -#define __NR_set_robust_list 300 -#define __NR_get_robust_list 301 -#define __NR_migrate_pages 302 -#define __NR_mbind 303 -#define __NR_get_mempolicy 304 -#define __NR_set_mempolicy 305 -#define __NR_kexec_load 306 -#define __NR_move_pages 307 -#define __NR_getcpu 308 -#define __NR_epoll_pwait 309 -#define __NR_utimensat 310 -#define __NR_signalfd 311 -#define __NR_timerfd_create 312 -#define __NR_eventfd 313 -#define __NR_fallocate 314 -#define __NR_timerfd_settime 315 -#define __NR_timerfd_gettime 316 -#define __NR_signalfd4 317 -#define __NR_eventfd2 318 -#define __NR_epoll_create1 319 -#define __NR_dup3 320 -#define __NR_pipe2 321 -#define __NR_inotify_init1 322 -#define __NR_accept4 323 -#define __NR_preadv 324 -#define __NR_pwritev 325 -#define __NR_rt_tgsigqueueinfo 326 -#define __NR_perf_event_open 327 -#define __NR_recvmmsg 328 -#define __NR_fanotify_init 329 -#define __NR_fanotify_mark 330 -#define __NR_prlimit64 331 - -#define NR_syscalls 332 - -#ifdef __32bit_syscall_numbers__ -/* Sparc 32-bit only has the "setresuid32", "getresuid32" variants, - * it never had the plain ones and there is no value to adding those - * old versions into the syscall table. - */ -#define __IGNORE_setresuid -#define __IGNORE_getresuid -#define __IGNORE_setresgid -#define __IGNORE_getresgid -#endif - -#ifdef __KERNEL__ -#define __ARCH_WANT_IPC_PARSE_VERSION #define __ARCH_WANT_OLD_READDIR #define __ARCH_WANT_STAT64 #define __ARCH_WANT_SYS_ALARM #define __ARCH_WANT_SYS_GETHOSTNAME #define __ARCH_WANT_SYS_PAUSE -#define __ARCH_WANT_SYS_SGETMASK #define __ARCH_WANT_SYS_SIGNAL #define __ARCH_WANT_SYS_TIME #define __ARCH_WANT_SYS_UTIME @@ -434,21 +37,11 @@ #define __ARCH_WANT_SYS_OLDUMOUNT #define __ARCH_WANT_SYS_SIGPENDING #define __ARCH_WANT_SYS_SIGPROCMASK -#define __ARCH_WANT_SYS_RT_SIGSUSPEND #ifdef __32bit_syscall_numbers__ #define __ARCH_WANT_SYS_IPC #else #define __ARCH_WANT_COMPAT_SYS_TIME -#define __ARCH_WANT_COMPAT_SYS_RT_SIGSUSPEND +#define __ARCH_WANT_COMPAT_SYS_SENDFILE #endif -/* - * "Conditional" syscalls - * - * What we want is __attribute__((weak,alias("sys_ni_syscall"))), - * but it doesn't work on all toolchains, so we just do it by hand - */ -#define cond_syscall(x) asm(".weak\t" #x "\n\t.set\t" #x ",sys_ni_syscall") - -#endif /* __KERNEL__ */ #endif /* _SPARC_UNISTD_H */ diff --git a/arch/sparc/include/asm/vac-ops.h b/arch/sparc/include/asm/vac-ops.h deleted file mode 100644 index a63e88ef042..00000000000 --- a/arch/sparc/include/asm/vac-ops.h +++ /dev/null @@ -1,127 +0,0 @@ -#ifndef _SPARC_VAC_OPS_H -#define _SPARC_VAC_OPS_H - -/* vac-ops.h: Inline assembly routines to do operations on the Sparc - * VAC (virtual address cache) for the sun4c. - * - * Copyright (C) 1994, David S. Miller (davem@caip.rutgers.edu) - */ - -#include <asm/sysen.h> -#include <asm/contregs.h> -#include <asm/asi.h> - -/* The SUN4C models have a virtually addressed write-through - * cache. - * - * The cache tags are directly accessible through an ASI and - * each have the form: - * - * ------------------------------------------------------------ - * | MBZ | CONTEXT | WRITE | PRIV | VALID | MBZ | TagID | MBZ | - * ------------------------------------------------------------ - * 31 25 24 22 21 20 19 18 16 15 2 1 0 - * - * MBZ: These bits are either unused and/or reserved and should - * be written as zeroes. - * - * CONTEXT: Records the context to which this cache line belongs. - * - * WRITE: A copy of the writable bit from the mmu pte access bits. - * - * PRIV: A copy of the privileged bit from the pte access bits. - * - * VALID: If set, this line is valid, else invalid. - * - * TagID: Fourteen bits of tag ID. - * - * Every virtual address is seen by the cache like this: - * - * ---------------------------------------- - * | RESV | TagID | LINE | BYTE-in-LINE | - * ---------------------------------------- - * 31 30 29 16 15 4 3 0 - * - * RESV: Unused/reserved. - * - * TagID: Used to match the Tag-ID in that vac tags. - * - * LINE: Which line within the cache - * - * BYTE-in-LINE: Which byte within the cache line. - */ - -/* Sun4c VAC Tags */ -#define S4CVACTAG_CID 0x01c00000 -#define S4CVACTAG_W 0x00200000 -#define S4CVACTAG_P 0x00100000 -#define S4CVACTAG_V 0x00080000 -#define S4CVACTAG_TID 0x0000fffc - -/* Sun4c VAC Virtual Address */ -/* These aren't used, why bother? (Anton) */ -#if 0 -#define S4CVACVA_TID 0x3fff0000 -#define S4CVACVA_LINE 0x0000fff0 -#define S4CVACVA_BIL 0x0000000f -#endif - -/* The indexing of cache lines creates a problem. Because the line - * field of a virtual address extends past the page offset within - * the virtual address it is possible to have what are called - * 'bad aliases' which will create inconsistencies. So we must make - * sure that within a context that if a physical page is mapped - * more than once, that 'extra' line bits are the same. If this is - * not the case, and thus is a 'bad alias' we must turn off the - * cacheable bit in the pte's of all such pages. - */ - -#define S4CVAC_BADBITS 0x0000f000 - -/* The following is true if vaddr1 and vaddr2 would cause - * a 'bad alias'. - */ -#define S4CVAC_BADALIAS(vaddr1, vaddr2) \ - ((((unsigned long) (vaddr1)) ^ ((unsigned long) (vaddr2))) & \ - (S4CVAC_BADBITS)) - -/* The following structure describes the characteristics of a sun4c - * VAC as probed from the prom during boot time. - */ -struct sun4c_vac_props { - unsigned int num_bytes; /* Size of the cache */ - unsigned int do_hwflushes; /* Hardware flushing available? */ - unsigned int linesize; /* Size of each line in bytes */ - unsigned int log2lsize; /* log2(linesize) */ - unsigned int on; /* VAC is enabled */ -}; - -extern struct sun4c_vac_props sun4c_vacinfo; - -/* sun4c_enable_vac() enables the sun4c virtual address cache. */ -static inline void sun4c_enable_vac(void) -{ - __asm__ __volatile__("lduba [%0] %1, %%g1\n\t" - "or %%g1, %2, %%g1\n\t" - "stba %%g1, [%0] %1\n\t" - : /* no outputs */ - : "r" ((unsigned int) AC_SENABLE), - "i" (ASI_CONTROL), "i" (SENABLE_CACHE) - : "g1", "memory"); - sun4c_vacinfo.on = 1; -} - -/* sun4c_disable_vac() disables the virtual address cache. */ -static inline void sun4c_disable_vac(void) -{ - __asm__ __volatile__("lduba [%0] %1, %%g1\n\t" - "andn %%g1, %2, %%g1\n\t" - "stba %%g1, [%0] %1\n\t" - : /* no outputs */ - : "r" ((unsigned int) AC_SENABLE), - "i" (ASI_CONTROL), "i" (SENABLE_CACHE) - : "g1", "memory"); - sun4c_vacinfo.on = 0; -} - -#endif /* !(_SPARC_VAC_OPS_H) */ diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h index 541e13755ce..c3dbcf90203 100644 --- a/arch/sparc/include/asm/vaddrs.h +++ b/arch/sparc/include/asm/vaddrs.h @@ -30,26 +30,32 @@ */ #define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */ +#ifndef __ASSEMBLY__ +#include <asm/kmap_types.h> + +enum fixed_addresses { + FIX_HOLE, +#ifdef CONFIG_HIGHMEM + FIX_KMAP_BEGIN, + FIX_KMAP_END = (KM_TYPE_NR * NR_CPUS), +#endif + __end_of_fixed_addresses +}; +#endif + +/* Leave one empty page between IO pages at 0xfd000000 and + * the top of the fixmap. + */ +#define FIXADDR_TOP (0xfcfff000UL) +#define FIXADDR_SIZE ((FIX_KMAP_END + 1) << PAGE_SHIFT) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) + #define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */ #define IOBASE_VADDR 0xfe000000 #define IOBASE_END 0xfe600000 -/* - * On the sun4/4c we need a place - * to reliably map locked down kernel data. This includes the - * task_struct and kernel stack pages of each process plus the - * scsi buffers during dvma IO transfers, also the floppy buffers - * during pseudo dma which runs with traps off (no faults allowed). - * Some quick calculations yield: - * NR_TASKS <512> * (3 * PAGE_SIZE) == 0x600000 - * Subtract this from 0xc00000 and you get 0x927C0 of vm left - * over to map SCSI dvma + floppy pseudo-dma buffers. So be - * careful if you change NR_TASKS or else there won't be enough - * room for it all. - */ -#define SUN4C_LOCK_VADDR 0xff000000 -#define SUN4C_LOCK_END 0xffc00000 - #define KADB_DEBUGGER_BEGVM 0xffc00000 /* Where kern debugger is in virt-mem */ #define KADB_DEBUGGER_ENDVM 0xffd00000 #define DEBUG_FIRSTVADDR KADB_DEBUGGER_BEGVM diff --git a/arch/sparc/include/asm/vga.h b/arch/sparc/include/asm/vga.h index c69d5b2ba19..ec0e9967d93 100644 --- a/arch/sparc/include/asm/vga.h +++ b/arch/sparc/include/asm/vga.h @@ -7,6 +7,7 @@ #ifndef _LINUX_ASM_VGA_H_ #define _LINUX_ASM_VGA_H_ +#include <linux/bug.h> #include <asm/types.h> #define VT_BUF_HAVE_RW diff --git a/arch/sparc/include/asm/vio.h b/arch/sparc/include/asm/vio.h index 9d83d3bcb49..e0f6c399f1d 100644 --- a/arch/sparc/include/asm/vio.h +++ b/arch/sparc/include/asm/vio.h @@ -284,6 +284,7 @@ struct vio_dev { }; struct vio_driver { + const char *name; struct list_head node; const struct vio_device_id *id_table; int (*probe)(struct vio_dev *dev, const struct vio_device_id *id); @@ -371,8 +372,14 @@ do { if (vio->debug & VIO_DEBUG_##TYPE) \ vio->vdev->channel_id, ## a); \ } while (0) -extern int vio_register_driver(struct vio_driver *drv); -extern void vio_unregister_driver(struct vio_driver *drv); +int __vio_register_driver(struct vio_driver *drv, struct module *owner, + const char *mod_name); +/* + * vio_register_driver must be a macro so that KBUILD_MODNAME can be expanded + */ +#define vio_register_driver(driver) \ + __vio_register_driver(driver, THIS_MODULE, KBUILD_MODNAME) +void vio_unregister_driver(struct vio_driver *drv); static inline struct vio_driver *to_vio_driver(struct device_driver *drv) { @@ -384,21 +391,21 @@ static inline struct vio_dev *to_vio_dev(struct device *dev) return container_of(dev, struct vio_dev, dev); } -extern int vio_ldc_send(struct vio_driver_state *vio, void *data, int len); -extern void vio_link_state_change(struct vio_driver_state *vio, int event); -extern void vio_conn_reset(struct vio_driver_state *vio); -extern int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt); -extern int vio_validate_sid(struct vio_driver_state *vio, - struct vio_msg_tag *tp); -extern u32 vio_send_sid(struct vio_driver_state *vio); -extern int vio_ldc_alloc(struct vio_driver_state *vio, - struct ldc_channel_config *base_cfg, void *event_arg); -extern void vio_ldc_free(struct vio_driver_state *vio); -extern int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, - u8 dev_class, struct vio_version *ver_table, - int ver_table_size, struct vio_driver_ops *ops, - char *name); - -extern void vio_port_up(struct vio_driver_state *vio); +int vio_ldc_send(struct vio_driver_state *vio, void *data, int len); +void vio_link_state_change(struct vio_driver_state *vio, int event); +void vio_conn_reset(struct vio_driver_state *vio); +int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt); +int vio_validate_sid(struct vio_driver_state *vio, + struct vio_msg_tag *tp); +u32 vio_send_sid(struct vio_driver_state *vio); +int vio_ldc_alloc(struct vio_driver_state *vio, + struct ldc_channel_config *base_cfg, void *event_arg); +void vio_ldc_free(struct vio_driver_state *vio); +int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, + u8 dev_class, struct vio_version *ver_table, + int ver_table_size, struct vio_driver_ops *ops, + char *name); + +void vio_port_up(struct vio_driver_state *vio); #endif /* _SPARC64_VIO_H */ diff --git a/arch/sparc/include/asm/visasm.h b/arch/sparc/include/asm/visasm.h index 39ca301920d..b2667375928 100644 --- a/arch/sparc/include/asm/visasm.h +++ b/arch/sparc/include/asm/visasm.h @@ -57,7 +57,8 @@ static inline void save_and_clear_fpu(void) { " " : : "i" (FPRS_FEF|FPRS_DU) : "o5", "g1", "g2", "g3", "g7", "cc"); } -extern int vis_emul(struct pt_regs *, unsigned int); + +int vis_emul(struct pt_regs *, unsigned int); #endif #endif /* _SPARC64_ASI_H */ diff --git a/arch/sparc/include/asm/winmacro.h b/arch/sparc/include/asm/winmacro.h index 5b0a06dc3bc..9b7b21764cd 100644 --- a/arch/sparc/include/asm/winmacro.h +++ b/arch/sparc/include/asm/winmacro.h @@ -103,28 +103,24 @@ st %scratch, [%cur_reg + TI_W_SAVED]; #ifdef CONFIG_SMP -#define LOAD_CURRENT4M(dest_reg, idreg) \ - rd %tbr, %idreg; \ - sethi %hi(current_set), %dest_reg; \ - srl %idreg, 10, %idreg; \ - or %dest_reg, %lo(current_set), %dest_reg; \ - and %idreg, 0xc, %idreg; \ - ld [%idreg + %dest_reg], %dest_reg; - -#define LOAD_CURRENT4D(dest_reg, idreg) \ - lda [%g0] ASI_M_VIKING_TMP1, %idreg; \ - sethi %hi(C_LABEL(current_set)), %dest_reg; \ - sll %idreg, 2, %idreg; \ - or %dest_reg, %lo(C_LABEL(current_set)), %dest_reg; \ - ld [%idreg + %dest_reg], %dest_reg; - -/* Blackbox - take care with this... - check smp4m and smp4d before changing this. */ -#define LOAD_CURRENT(dest_reg, idreg) \ - sethi %hi(___b_load_current), %idreg; \ - sethi %hi(current_set), %dest_reg; \ - sethi %hi(boot_cpu_id4), %idreg; \ - or %dest_reg, %lo(current_set), %dest_reg; \ - ldub [%idreg + %lo(boot_cpu_id4)], %idreg; \ +#define LOAD_CURRENT(dest_reg, idreg) \ +661: rd %tbr, %idreg; \ + srl %idreg, 10, %idreg; \ + and %idreg, 0xc, %idreg; \ + .section .cpuid_patch, "ax"; \ + /* Instruction location. */ \ + .word 661b; \ + /* SUN4D implementation. */ \ + lda [%g0] ASI_M_VIKING_TMP1, %idreg; \ + sll %idreg, 2, %idreg; \ + nop; \ + /* LEON implementation. */ \ + rd %asr17, %idreg; \ + srl %idreg, 0x1c, %idreg; \ + sll %idreg, 0x02, %idreg; \ + .previous; \ + sethi %hi(current_set), %dest_reg; \ + or %dest_reg, %lo(current_set), %dest_reg;\ ld [%idreg + %dest_reg], %dest_reg; #else #define LOAD_CURRENT(dest_reg, idreg) \ diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h index bee4bf4be3a..50c88285603 100644 --- a/arch/sparc/include/asm/xor_64.h +++ b/arch/sparc/include/asm/xor_64.h @@ -20,13 +20,13 @@ #include <asm/spitfire.h> -extern void xor_vis_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_vis_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_vis_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_vis_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_vis_2(unsigned long, unsigned long *, unsigned long *); +void xor_vis_3(unsigned long, unsigned long *, unsigned long *, + unsigned long *); +void xor_vis_4(unsigned long, unsigned long *, unsigned long *, + unsigned long *, unsigned long *); +void xor_vis_5(unsigned long, unsigned long *, unsigned long *, + unsigned long *, unsigned long *, unsigned long *); /* XXX Ugh, write cheetah versions... -DaveM */ @@ -38,13 +38,13 @@ static struct xor_block_template xor_block_VIS = { .do_5 = xor_vis_5, }; -extern void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); +void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); +void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, + unsigned long *); +void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, + unsigned long *, unsigned long *); +void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, + unsigned long *, unsigned long *, unsigned long *); static struct xor_block_template xor_block_niagara = { .name = "Niagara", @@ -65,6 +65,9 @@ static struct xor_block_template xor_block_niagara = { #define XOR_SELECT_TEMPLATE(FASTEST) \ ((tlb_type == hypervisor && \ (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || \ - sun4v_chip_type == SUN4V_CHIP_NIAGARA2)) ? \ + sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || \ + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || \ + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || \ + sun4v_chip_type == SUN4V_CHIP_NIAGARA5)) ? \ &xor_block_niagara : \ &xor_block_VIS) diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild new file mode 100644 index 00000000000..b5843ee09fb --- /dev/null +++ b/arch/sparc/include/uapi/asm/Kbuild @@ -0,0 +1,50 @@ +# UAPI Header export list +# User exported sparc header files + +include include/uapi/asm-generic/Kbuild.asm + +header-y += apc.h +header-y += asi.h +header-y += auxvec.h +header-y += bitsperlong.h +header-y += byteorder.h +header-y += display7seg.h +header-y += envctrl.h +header-y += errno.h +header-y += fbio.h +header-y += fcntl.h +header-y += ioctl.h +header-y += ioctls.h +header-y += ipcbuf.h +header-y += jsflash.h +header-y += kvm_para.h +header-y += mman.h +header-y += msgbuf.h +header-y += openpromio.h +header-y += param.h +header-y += perfctr.h +header-y += poll.h +header-y += posix_types.h +header-y += psr.h +header-y += psrcompat.h +header-y += pstate.h +header-y += ptrace.h +header-y += resource.h +header-y += sembuf.h +header-y += setup.h +header-y += shmbuf.h +header-y += sigcontext.h +header-y += siginfo.h +header-y += signal.h +header-y += socket.h +header-y += sockios.h +header-y += stat.h +header-y += statfs.h +header-y += swab.h +header-y += termbits.h +header-y += termios.h +header-y += traps.h +header-y += uctx.h +header-y += unistd.h +header-y += utrap.h +header-y += watchdog.h diff --git a/arch/sparc/include/asm/apc.h b/arch/sparc/include/uapi/asm/apc.h index 24e9a7d4d97..24e9a7d4d97 100644 --- a/arch/sparc/include/asm/apc.h +++ b/arch/sparc/include/uapi/asm/apc.h diff --git a/arch/sparc/include/asm/asi.h b/arch/sparc/include/uapi/asm/asi.h index b2e3db63a64..aace6f31371 100644 --- a/arch/sparc/include/asm/asi.h +++ b/arch/sparc/include/uapi/asm/asi.h @@ -40,11 +40,7 @@ #define ASI_M_UNA01 0x01 /* Same here... */ #define ASI_M_MXCC 0x02 /* Access to TI VIKING MXCC registers */ #define ASI_M_FLUSH_PROBE 0x03 /* Reference MMU Flush/Probe; rw, ss */ -#ifndef CONFIG_SPARC_LEON #define ASI_M_MMUREGS 0x04 /* MMU Registers; rw, ss */ -#else -#define ASI_M_MMUREGS 0x19 -#endif /* CONFIG_SPARC_LEON */ #define ASI_M_TLBDIAG 0x05 /* MMU TLB only Diagnostics */ #define ASI_M_DIAGS 0x06 /* Reference MMU Diagnostics */ #define ASI_M_IODIAG 0x07 /* MMU I/O TLB only Diagnostics */ @@ -112,6 +108,20 @@ #define ASI_M_ACTION 0x4c /* Breakpoint Action Register (GNU/Viking) */ +/* LEON ASI */ +#define ASI_LEON_NOCACHE 0x01 + +#define ASI_LEON_DCACHE_MISS 0x01 + +#define ASI_LEON_CACHEREGS 0x02 +#define ASI_LEON_IFLUSH 0x10 +#define ASI_LEON_DFLUSH 0x11 + +#define ASI_LEON_MMUFLUSH 0x18 +#define ASI_LEON_MMUREGS 0x19 +#define ASI_LEON_BYPASS 0x1c +#define ASI_LEON_FLUSH_PAGE 0x10 + /* V9 Architecture mandary ASIs. */ #define ASI_N 0x04 /* Nucleus */ #define ASI_NL 0x0c /* Nucleus, little endian */ @@ -131,7 +141,8 @@ /* SpitFire and later extended ASIs. The "(III)" marker designates * UltraSparc-III and later specific ASIs. The "(CMT)" marker designates * Chip Multi Threading specific ASIs. "(NG)" designates Niagara specific - * ASIs, "(4V)" designates SUN4V specific ASIs. + * ASIs, "(4V)" designates SUN4V specific ASIs. "(NG4)" designates SPARC-T4 + * and later ASIs. */ #define ASI_PHYS_USE_EC 0x14 /* PADDR, E-cachable */ #define ASI_PHYS_BYPASS_EC_E 0x15 /* PADDR, E-bit */ @@ -233,6 +244,7 @@ #define ASI_UDBL_CONTROL_R 0x7f /* External UDB control regs rd low*/ #define ASI_INTR_R 0x7f /* IRQ vector dispatch read */ #define ASI_INTR_DATAN_R 0x7f /* (III) In irq vector data reg N */ +#define ASI_PIC 0xb0 /* (NG4) PIC registers */ #define ASI_PST8_P 0xc0 /* Primary, 8 8-bit, partial */ #define ASI_PST8_S 0xc1 /* Secondary, 8 8-bit, partial */ #define ASI_PST16_P 0xc2 /* Primary, 4 16-bit, partial */ @@ -258,9 +270,28 @@ #define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load, * primary, implicit */ +#define ASI_BLK_INIT_QUAD_LDD_S 0xe3 /* (NG) init-store, twin load, + * secondary, implicit + */ #define ASI_BLK_P 0xf0 /* Primary, blk ld/st */ #define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */ +#define ASI_ST_BLKINIT_MRU_P 0xf2 /* (NG4) init-store, twin load, + * Most-Recently-Used, primary, + * implicit + */ +#define ASI_ST_BLKINIT_MRU_S 0xf2 /* (NG4) init-store, twin load, + * Most-Recently-Used, secondary, + * implicit + */ #define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */ #define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */ +#define ASI_ST_BLKINIT_MRU_PL 0xfa /* (NG4) init-store, twin load, + * Most-Recently-Used, primary, + * implicit, little-endian + */ +#define ASI_ST_BLKINIT_MRU_SL 0xfb /* (NG4) init-store, twin load, + * Most-Recently-Used, secondary, + * implicit, little-endian + */ #endif /* _SPARC_ASI_H */ diff --git a/arch/sparc/include/asm/auxvec.h b/arch/sparc/include/uapi/asm/auxvec.h index ad6f360261f..ad6f360261f 100644 --- a/arch/sparc/include/asm/auxvec.h +++ b/arch/sparc/include/uapi/asm/auxvec.h diff --git a/arch/sparc/include/asm/bitsperlong.h b/arch/sparc/include/uapi/asm/bitsperlong.h index 40dcaa3aaa5..40dcaa3aaa5 100644 --- a/arch/sparc/include/asm/bitsperlong.h +++ b/arch/sparc/include/uapi/asm/bitsperlong.h diff --git a/arch/sparc/include/asm/byteorder.h b/arch/sparc/include/uapi/asm/byteorder.h index ccc1b6b7de6..ccc1b6b7de6 100644 --- a/arch/sparc/include/asm/byteorder.h +++ b/arch/sparc/include/uapi/asm/byteorder.h diff --git a/arch/sparc/include/asm/display7seg.h b/arch/sparc/include/uapi/asm/display7seg.h index 86d4a901df2..86d4a901df2 100644 --- a/arch/sparc/include/asm/display7seg.h +++ b/arch/sparc/include/uapi/asm/display7seg.h diff --git a/arch/sparc/include/asm/envctrl.h b/arch/sparc/include/uapi/asm/envctrl.h index 624fa7e2da8..624fa7e2da8 100644 --- a/arch/sparc/include/asm/envctrl.h +++ b/arch/sparc/include/uapi/asm/envctrl.h diff --git a/arch/sparc/include/asm/errno.h b/arch/sparc/include/uapi/asm/errno.h index 4e2bc490d71..20423e17285 100644 --- a/arch/sparc/include/asm/errno.h +++ b/arch/sparc/include/uapi/asm/errno.h @@ -40,7 +40,7 @@ #define EPROCLIM 67 /* SUNOS: Too many processes */ #define EUSERS 68 /* Too many users */ #define EDQUOT 69 /* Quota exceeded */ -#define ESTALE 70 /* Stale NFS file handle */ +#define ESTALE 70 /* Stale file handle */ #define EREMOTE 71 /* Object is remote */ #define ENOSTR 72 /* Device not a stream */ #define ETIME 73 /* Timer expired */ @@ -112,4 +112,6 @@ #define ERFKILL 134 /* Operation not possible due to RF-kill */ +#define EHWPOISON 135 /* Memory page has hardware error */ + #endif diff --git a/arch/sparc/include/uapi/asm/fbio.h b/arch/sparc/include/uapi/asm/fbio.h new file mode 100644 index 00000000000..d6cea07afb6 --- /dev/null +++ b/arch/sparc/include/uapi/asm/fbio.h @@ -0,0 +1,259 @@ +#ifndef _UAPI__LINUX_FBIO_H +#define _UAPI__LINUX_FBIO_H + +#include <linux/compiler.h> +#include <linux/types.h> + +/* Constants used for fbio SunOS compatibility */ +/* (C) 1996 Miguel de Icaza */ + +/* Frame buffer types */ +#define FBTYPE_NOTYPE -1 +#define FBTYPE_SUN1BW 0 /* mono */ +#define FBTYPE_SUN1COLOR 1 +#define FBTYPE_SUN2BW 2 +#define FBTYPE_SUN2COLOR 3 +#define FBTYPE_SUN2GP 4 +#define FBTYPE_SUN5COLOR 5 +#define FBTYPE_SUN3COLOR 6 +#define FBTYPE_MEMCOLOR 7 +#define FBTYPE_SUN4COLOR 8 + +#define FBTYPE_NOTSUN1 9 +#define FBTYPE_NOTSUN2 10 +#define FBTYPE_NOTSUN3 11 + +#define FBTYPE_SUNFAST_COLOR 12 /* cg6 */ +#define FBTYPE_SUNROP_COLOR 13 +#define FBTYPE_SUNFB_VIDEO 14 +#define FBTYPE_SUNGIFB 15 +#define FBTYPE_SUNGPLAS 16 +#define FBTYPE_SUNGP3 17 +#define FBTYPE_SUNGT 18 +#define FBTYPE_SUNLEO 19 /* zx Leo card */ +#define FBTYPE_MDICOLOR 20 /* cg14 */ +#define FBTYPE_TCXCOLOR 21 /* SUNW,tcx card */ + +#define FBTYPE_LASTPLUSONE 21 /* This is not last + 1 in fact... */ + +/* Does not seem to be listed in the Sun file either */ +#define FBTYPE_CREATOR 22 +#define FBTYPE_PCI_IGA1682 23 +#define FBTYPE_P9100COLOR 24 + +#define FBTYPE_PCI_GENERIC 1000 +#define FBTYPE_PCI_MACH64 1001 + +/* fbio ioctls */ +/* Returned by FBIOGTYPE */ +struct fbtype { + int fb_type; /* fb type, see above */ + int fb_height; /* pixels */ + int fb_width; /* pixels */ + int fb_depth; + int fb_cmsize; /* color map entries */ + int fb_size; /* fb size in bytes */ +}; +#define FBIOGTYPE _IOR('F', 0, struct fbtype) + +struct fbcmap { + int index; /* first element (0 origin) */ + int count; + unsigned char __user *red; + unsigned char __user *green; + unsigned char __user *blue; +}; + +#ifndef __KERNEL__ +#define FBIOPUTCMAP _IOW('F', 3, struct fbcmap) +#define FBIOGETCMAP _IOW('F', 4, struct fbcmap) +#endif + +/* # of device specific values */ +#define FB_ATTR_NDEVSPECIFIC 8 +/* # of possible emulations */ +#define FB_ATTR_NEMUTYPES 4 + +struct fbsattr { + int flags; + int emu_type; /* -1 if none */ + int dev_specific[FB_ATTR_NDEVSPECIFIC]; +}; + +struct fbgattr { + int real_type; /* real frame buffer type */ + int owner; /* unknown */ + struct fbtype fbtype; /* real frame buffer fbtype */ + struct fbsattr sattr; + int emu_types[FB_ATTR_NEMUTYPES]; /* supported emulations */ +}; +#define FBIOSATTR _IOW('F', 5, struct fbgattr) /* Unsupported: */ +#define FBIOGATTR _IOR('F', 6, struct fbgattr) /* supported */ + +#define FBIOSVIDEO _IOW('F', 7, int) +#define FBIOGVIDEO _IOR('F', 8, int) + +struct fbcursor { + short set; /* what to set, choose from the list above */ + short enable; /* cursor on/off */ + struct fbcurpos pos; /* cursor position */ + struct fbcurpos hot; /* cursor hot spot */ + struct fbcmap cmap; /* color map info */ + struct fbcurpos size; /* cursor bit map size */ + char __user *image; /* cursor image bits */ + char __user *mask; /* cursor mask bits */ +}; + +/* set/get cursor attributes/shape */ +#define FBIOSCURSOR _IOW('F', 24, struct fbcursor) +#define FBIOGCURSOR _IOWR('F', 25, struct fbcursor) + +/* set/get cursor position */ +#define FBIOSCURPOS _IOW('F', 26, struct fbcurpos) +#define FBIOGCURPOS _IOW('F', 27, struct fbcurpos) + +/* get max cursor size */ +#define FBIOGCURMAX _IOR('F', 28, struct fbcurpos) + +/* wid manipulation */ +struct fb_wid_alloc { +#define FB_WID_SHARED_8 0 +#define FB_WID_SHARED_24 1 +#define FB_WID_DBL_8 2 +#define FB_WID_DBL_24 3 + __u32 wa_type; + __s32 wa_index; /* Set on return */ + __u32 wa_count; +}; +struct fb_wid_item { + __u32 wi_type; + __s32 wi_index; + __u32 wi_attrs; + __u32 wi_values[32]; +}; +struct fb_wid_list { + __u32 wl_flags; + __u32 wl_count; + struct fb_wid_item *wl_list; +}; + +#define FBIO_WID_ALLOC _IOWR('F', 30, struct fb_wid_alloc) +#define FBIO_WID_FREE _IOW('F', 31, struct fb_wid_alloc) +#define FBIO_WID_PUT _IOW('F', 32, struct fb_wid_list) +#define FBIO_WID_GET _IOWR('F', 33, struct fb_wid_list) + +/* Creator ioctls */ +#define FFB_IOCTL ('F'<<8) +#define FFB_SYS_INFO (FFB_IOCTL|80) +#define FFB_CLUTREAD (FFB_IOCTL|81) +#define FFB_CLUTPOST (FFB_IOCTL|82) +#define FFB_SETDIAGMODE (FFB_IOCTL|83) +#define FFB_GETMONITORID (FFB_IOCTL|84) +#define FFB_GETVIDEOMODE (FFB_IOCTL|85) +#define FFB_SETVIDEOMODE (FFB_IOCTL|86) +#define FFB_SETSERVER (FFB_IOCTL|87) +#define FFB_SETOVCTL (FFB_IOCTL|88) +#define FFB_GETOVCTL (FFB_IOCTL|89) +#define FFB_GETSAXNUM (FFB_IOCTL|90) +#define FFB_FBDEBUG (FFB_IOCTL|91) + +/* Cg14 ioctls */ +#define MDI_IOCTL ('M'<<8) +#define MDI_RESET (MDI_IOCTL|1) +#define MDI_GET_CFGINFO (MDI_IOCTL|2) +#define MDI_SET_PIXELMODE (MDI_IOCTL|3) +# define MDI_32_PIX 32 +# define MDI_16_PIX 16 +# define MDI_8_PIX 8 + +struct mdi_cfginfo { + int mdi_ncluts; /* Number of implemented CLUTs in this MDI */ + int mdi_type; /* FBTYPE name */ + int mdi_height; /* height */ + int mdi_width; /* width */ + int mdi_size; /* available ram */ + int mdi_mode; /* 8bpp, 16bpp or 32bpp */ + int mdi_pixfreq; /* pixel clock (from PROM) */ +}; + +/* SparcLinux specific ioctl for the MDI, should be replaced for + * the SET_XLUT/SET_CLUTn ioctls instead + */ +#define MDI_CLEAR_XLUT (MDI_IOCTL|9) + +/* leo & ffb ioctls */ +struct fb_clut_alloc { + __u32 clutid; /* Set on return */ + __u32 flag; + __u32 index; +}; + +struct fb_clut { +#define FB_CLUT_WAIT 0x00000001 /* Not yet implemented */ + __u32 flag; + __u32 clutid; + __u32 offset; + __u32 count; + char * red; + char * green; + char * blue; +}; + +struct fb_clut32 { + __u32 flag; + __u32 clutid; + __u32 offset; + __u32 count; + __u32 red; + __u32 green; + __u32 blue; +}; + +#define LEO_CLUTALLOC _IOWR('L', 53, struct fb_clut_alloc) +#define LEO_CLUTFREE _IOW('L', 54, struct fb_clut_alloc) +#define LEO_CLUTREAD _IOW('L', 55, struct fb_clut) +#define LEO_CLUTPOST _IOW('L', 56, struct fb_clut) +#define LEO_SETGAMMA _IOW('L', 68, int) /* Not yet implemented */ +#define LEO_GETGAMMA _IOR('L', 69, int) /* Not yet implemented */ + + +/* These are exported to userland for applications to use */ +/* Mappable offsets for the cg14: control registers */ +#define MDI_DIRECT_MAP 0x10000000 +#define MDI_CTLREG_MAP 0x20000000 +#define MDI_CURSOR_MAP 0x30000000 +#define MDI_SHDW_VRT_MAP 0x40000000 + +/* Mappable offsets for the cg14: frame buffer resolutions */ +/* 32 bits */ +#define MDI_CHUNKY_XBGR_MAP 0x50000000 +#define MDI_CHUNKY_BGR_MAP 0x60000000 + +/* 16 bits */ +#define MDI_PLANAR_X16_MAP 0x70000000 +#define MDI_PLANAR_C16_MAP 0x80000000 + +/* 8 bit is done as CG3 MMAP offset */ +/* 32 bits, planar */ +#define MDI_PLANAR_X32_MAP 0x90000000 +#define MDI_PLANAR_B32_MAP 0xa0000000 +#define MDI_PLANAR_G32_MAP 0xb0000000 +#define MDI_PLANAR_R32_MAP 0xc0000000 + +/* Mappable offsets on leo */ +#define LEO_SS0_MAP 0x00000000 +#define LEO_LC_SS0_USR_MAP 0x00800000 +#define LEO_LD_SS0_MAP 0x00801000 +#define LEO_LX_CURSOR_MAP 0x00802000 +#define LEO_SS1_MAP 0x00803000 +#define LEO_LC_SS1_USR_MAP 0x01003000 +#define LEO_LD_SS1_MAP 0x01004000 +#define LEO_UNK_MAP 0x01005000 +#define LEO_LX_KRN_MAP 0x01006000 +#define LEO_LC_SS0_KRN_MAP 0x01007000 +#define LEO_LC_SS1_KRN_MAP 0x01008000 +#define LEO_LD_GBL_MAP 0x01009000 +#define LEO_UNK2_MAP 0x0100a000 + + +#endif /* _UAPI__LINUX_FBIO_H */ diff --git a/arch/sparc/include/asm/fcntl.h b/arch/sparc/include/uapi/asm/fcntl.h index 38f37b333cc..7e8ace5bf76 100644 --- a/arch/sparc/include/asm/fcntl.h +++ b/arch/sparc/include/uapi/asm/fcntl.h @@ -34,6 +34,9 @@ #define __O_SYNC 0x800000 #define O_SYNC (__O_SYNC|O_DSYNC) +#define O_PATH 0x1000000 +#define __O_TMPFILE 0x2000000 + #define F_GETOWN 5 /* for sockets. */ #define F_SETOWN 6 /* for sockets. */ #define F_GETLK 7 diff --git a/arch/sparc/include/asm/ioctl.h b/arch/sparc/include/uapi/asm/ioctl.h index 7d6bd51321b..7d6bd51321b 100644 --- a/arch/sparc/include/asm/ioctl.h +++ b/arch/sparc/include/uapi/asm/ioctl.h diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h new file mode 100644 index 00000000000..897d1723fa1 --- /dev/null +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -0,0 +1,134 @@ +#ifndef _UAPI_ASM_SPARC_IOCTLS_H +#define _UAPI_ASM_SPARC_IOCTLS_H + +#include <asm/ioctl.h> + +/* Big T */ +#define TCGETA _IOR('T', 1, struct termio) +#define TCSETA _IOW('T', 2, struct termio) +#define TCSETAW _IOW('T', 3, struct termio) +#define TCSETAF _IOW('T', 4, struct termio) +#define TCSBRK _IO('T', 5) +#define TCXONC _IO('T', 6) +#define TCFLSH _IO('T', 7) +#define TCGETS _IOR('T', 8, struct termios) +#define TCSETS _IOW('T', 9, struct termios) +#define TCSETSW _IOW('T', 10, struct termios) +#define TCSETSF _IOW('T', 11, struct termios) +#define TCGETS2 _IOR('T', 12, struct termios2) +#define TCSETS2 _IOW('T', 13, struct termios2) +#define TCSETSW2 _IOW('T', 14, struct termios2) +#define TCSETSF2 _IOW('T', 15, struct termios2) +#define TIOCGDEV _IOR('T',0x32, unsigned int) /* Get primary device node of /dev/console */ +#define TIOCVHANGUP _IO('T', 0x37) +#define TIOCGPKT _IOR('T', 0x38, int) /* Get packet mode state */ +#define TIOCGPTLCK _IOR('T', 0x39, int) /* Get Pty lock state */ +#define TIOCGEXCL _IOR('T', 0x40, int) /* Get exclusive mode state */ + +/* Note that all the ioctls that are not available in Linux have a + * double underscore on the front to: a) avoid some programs to + * think we support some ioctls under Linux (autoconfiguration stuff) + */ +/* Little t */ +#define TIOCGETD _IOR('t', 0, int) +#define TIOCSETD _IOW('t', 1, int) +#define __TIOCHPCL _IO('t', 2) /* SunOS Specific */ +#define __TIOCMODG _IOR('t', 3, int) /* SunOS Specific */ +#define __TIOCMODS _IOW('t', 4, int) /* SunOS Specific */ +#define __TIOCGETP _IOR('t', 8, struct sgttyb) /* SunOS Specific */ +#define __TIOCSETP _IOW('t', 9, struct sgttyb) /* SunOS Specific */ +#define __TIOCSETN _IOW('t', 10, struct sgttyb) /* SunOS Specific */ +#define TIOCEXCL _IO('t', 13) +#define TIOCNXCL _IO('t', 14) +#define __TIOCFLUSH _IOW('t', 16, int) /* SunOS Specific */ +#define __TIOCSETC _IOW('t', 17, struct tchars) /* SunOS Specific */ +#define __TIOCGETC _IOR('t', 18, struct tchars) /* SunOS Specific */ +#define __TIOCTCNTL _IOW('t', 32, int) /* SunOS Specific */ +#define __TIOCSIGNAL _IOW('t', 33, int) /* SunOS Specific */ +#define __TIOCSETX _IOW('t', 34, int) /* SunOS Specific */ +#define __TIOCGETX _IOR('t', 35, int) /* SunOS Specific */ +#define TIOCCONS _IO('t', 36) +#define TIOCGSOFTCAR _IOR('t', 100, int) +#define TIOCSSOFTCAR _IOW('t', 101, int) +#define __TIOCUCNTL _IOW('t', 102, int) /* SunOS Specific */ +#define TIOCSWINSZ _IOW('t', 103, struct winsize) +#define TIOCGWINSZ _IOR('t', 104, struct winsize) +#define __TIOCREMOTE _IOW('t', 105, int) /* SunOS Specific */ +#define TIOCMGET _IOR('t', 106, int) +#define TIOCMBIC _IOW('t', 107, int) +#define TIOCMBIS _IOW('t', 108, int) +#define TIOCMSET _IOW('t', 109, int) +#define TIOCSTART _IO('t', 110) +#define TIOCSTOP _IO('t', 111) +#define TIOCPKT _IOW('t', 112, int) +#define TIOCNOTTY _IO('t', 113) +#define TIOCSTI _IOW('t', 114, char) +#define TIOCOUTQ _IOR('t', 115, int) +#define __TIOCGLTC _IOR('t', 116, struct ltchars) /* SunOS Specific */ +#define __TIOCSLTC _IOW('t', 117, struct ltchars) /* SunOS Specific */ +/* 118 is the non-posix setpgrp tty ioctl */ +/* 119 is the non-posix getpgrp tty ioctl */ +#define __TIOCCDTR _IO('t', 120) /* SunOS Specific */ +#define __TIOCSDTR _IO('t', 121) /* SunOS Specific */ +#define TIOCCBRK _IO('t', 122) +#define TIOCSBRK _IO('t', 123) +#define __TIOCLGET _IOW('t', 124, int) /* SunOS Specific */ +#define __TIOCLSET _IOW('t', 125, int) /* SunOS Specific */ +#define __TIOCLBIC _IOW('t', 126, int) /* SunOS Specific */ +#define __TIOCLBIS _IOW('t', 127, int) /* SunOS Specific */ +#define __TIOCISPACE _IOR('t', 128, int) /* SunOS Specific */ +#define __TIOCISIZE _IOR('t', 129, int) /* SunOS Specific */ +#define TIOCSPGRP _IOW('t', 130, int) +#define TIOCGPGRP _IOR('t', 131, int) +#define TIOCSCTTY _IO('t', 132) +#define TIOCGSID _IOR('t', 133, int) +/* Get minor device of a pty master's FD -- Solaris equiv is ISPTM */ +#define TIOCGPTN _IOR('t', 134, unsigned int) /* Get Pty Number */ +#define TIOCSPTLCK _IOW('t', 135, int) /* Lock/unlock PTY */ +#define TIOCSIG _IOW('t', 136, int) /* Generate signal on Pty slave */ + +/* Little f */ +#define FIOCLEX _IO('f', 1) +#define FIONCLEX _IO('f', 2) +#define FIOASYNC _IOW('f', 125, int) +#define FIONBIO _IOW('f', 126, int) +#define FIONREAD _IOR('f', 127, int) +#define TIOCINQ FIONREAD +#define FIOQSIZE _IOR('f', 128, loff_t) + +/* SCARY Rutgers local SunOS kernel hackery, perhaps I will support it + * someday. This is completely bogus, I know... + */ +#define __TCGETSTAT _IO('T', 200) /* Rutgers specific */ +#define __TCSETSTAT _IO('T', 201) /* Rutgers specific */ + +/* Linux specific, no SunOS equivalent. */ +#define TIOCLINUX 0x541C +#define TIOCGSERIAL 0x541E +#define TIOCSSERIAL 0x541F +#define TCSBRKP 0x5425 +#define TIOCSERCONFIG 0x5453 +#define TIOCSERGWILD 0x5454 +#define TIOCSERSWILD 0x5455 +#define TIOCGLCKTRMIOS 0x5456 +#define TIOCSLCKTRMIOS 0x5457 +#define TIOCSERGSTRUCT 0x5458 /* For debugging only */ +#define TIOCSERGETLSR 0x5459 /* Get line status register */ +#define TIOCSERGETMULTI 0x545A /* Get multiport config */ +#define TIOCSERSETMULTI 0x545B /* Set multiport config */ +#define TIOCMIWAIT 0x545C /* Wait for change on serial input line(s) */ +#define TIOCGICOUNT 0x545D /* Read serial port inline interrupt counts */ + +/* Kernel definitions */ + +/* Used for packet mode */ +#define TIOCPKT_DATA 0 +#define TIOCPKT_FLUSHREAD 1 +#define TIOCPKT_FLUSHWRITE 2 +#define TIOCPKT_STOP 4 +#define TIOCPKT_START 8 +#define TIOCPKT_NOSTOP 16 +#define TIOCPKT_DOSTOP 32 +#define TIOCPKT_IOCTL 64 + +#endif /* _UAPI_ASM_SPARC_IOCTLS_H */ diff --git a/arch/sparc/include/asm/ipcbuf.h b/arch/sparc/include/uapi/asm/ipcbuf.h index 66013b4fe10..66013b4fe10 100644 --- a/arch/sparc/include/asm/ipcbuf.h +++ b/arch/sparc/include/uapi/asm/ipcbuf.h diff --git a/arch/sparc/include/asm/jsflash.h b/arch/sparc/include/uapi/asm/jsflash.h index 0717d9e39d2..0717d9e39d2 100644 --- a/arch/sparc/include/asm/jsflash.h +++ b/arch/sparc/include/uapi/asm/jsflash.h diff --git a/arch/sparc/include/uapi/asm/kvm_para.h b/arch/sparc/include/uapi/asm/kvm_para.h new file mode 100644 index 00000000000..14fab8f0b95 --- /dev/null +++ b/arch/sparc/include/uapi/asm/kvm_para.h @@ -0,0 +1 @@ +#include <asm-generic/kvm_para.h> diff --git a/arch/sparc/include/uapi/asm/mman.h b/arch/sparc/include/uapi/asm/mman.h new file mode 100644 index 00000000000..0b14df33cff --- /dev/null +++ b/arch/sparc/include/uapi/asm/mman.h @@ -0,0 +1,27 @@ +#ifndef _UAPI__SPARC_MMAN_H__ +#define _UAPI__SPARC_MMAN_H__ + +#include <asm-generic/mman-common.h> + +/* SunOS'ified... */ + +#define MAP_RENAME MAP_ANONYMOUS /* In SunOS terminology */ +#define MAP_NORESERVE 0x40 /* don't reserve swap pages */ +#define MAP_INHERIT 0x80 /* SunOS doesn't do this, but... */ +#define MAP_LOCKED 0x100 /* lock the mapping */ +#define _MAP_NEW 0x80000000 /* Binary compatibility is fun... */ + +#define MAP_GROWSDOWN 0x0200 /* stack-like segment */ +#define MAP_DENYWRITE 0x0800 /* ETXTBSY */ +#define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ + +#define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ +#define MCL_FUTURE 0x4000 /* lock all additions to address space */ + +#define MAP_POPULATE 0x8000 /* populate (prefault) pagetables */ +#define MAP_NONBLOCK 0x10000 /* do not block on IO */ +#define MAP_STACK 0x20000 /* give out an address that is best suited for process/thread stacks */ +#define MAP_HUGETLB 0x40000 /* create a huge page mapping */ + + +#endif /* _UAPI__SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/asm/msgbuf.h b/arch/sparc/include/uapi/asm/msgbuf.h index efc7cbe9788..efc7cbe9788 100644 --- a/arch/sparc/include/asm/msgbuf.h +++ b/arch/sparc/include/uapi/asm/msgbuf.h diff --git a/arch/sparc/include/asm/openpromio.h b/arch/sparc/include/uapi/asm/openpromio.h index 917fb8e9c63..917fb8e9c63 100644 --- a/arch/sparc/include/asm/openpromio.h +++ b/arch/sparc/include/uapi/asm/openpromio.h diff --git a/arch/sparc/include/asm/param.h b/arch/sparc/include/uapi/asm/param.h index 0bc356bf8c5..0bc356bf8c5 100644 --- a/arch/sparc/include/asm/param.h +++ b/arch/sparc/include/uapi/asm/param.h diff --git a/arch/sparc/include/asm/perfctr.h b/arch/sparc/include/uapi/asm/perfctr.h index 8d8720a8770..214feefa577 100644 --- a/arch/sparc/include/asm/perfctr.h +++ b/arch/sparc/include/uapi/asm/perfctr.h @@ -54,11 +54,6 @@ enum perfctr_opcode { PERFCTR_GETPCR }; -/* I don't want the kernel's namespace to be polluted with this - * stuff when this file is included. --DaveM - */ -#ifndef __KERNEL__ - #define PRIV 0x00000001 #define SYS 0x00000002 #define USR 0x00000004 @@ -168,6 +163,4 @@ struct vcounter_struct { unsigned long long vcnt1; }; -#endif /* !(__KERNEL__) */ - #endif /* !(PERF_COUNTER_API) */ diff --git a/arch/sparc/include/asm/poll.h b/arch/sparc/include/uapi/asm/poll.h index 091d3ad2e83..091d3ad2e83 100644 --- a/arch/sparc/include/asm/poll.h +++ b/arch/sparc/include/uapi/asm/poll.h diff --git a/arch/sparc/include/uapi/asm/posix_types.h b/arch/sparc/include/uapi/asm/posix_types.h new file mode 100644 index 00000000000..156220ed99e --- /dev/null +++ b/arch/sparc/include/uapi/asm/posix_types.h @@ -0,0 +1,49 @@ +/* + * This file is generally used by user-level software, so you need to + * be a little careful about namespace pollution etc. Also, we cannot + * assume GCC is being used. + */ + +#ifndef __SPARC_POSIX_TYPES_H +#define __SPARC_POSIX_TYPES_H + +#if defined(__sparc__) && defined(__arch64__) +/* sparc 64 bit */ + +typedef unsigned short __kernel_old_uid_t; +typedef unsigned short __kernel_old_gid_t; +#define __kernel_old_uid_t __kernel_old_uid_t + +/* Note this piece of asymmetry from the v9 ABI. */ +typedef int __kernel_suseconds_t; +#define __kernel_suseconds_t __kernel_suseconds_t + +#else +/* sparc 32 bit */ + +typedef unsigned int __kernel_size_t; +typedef int __kernel_ssize_t; +typedef long int __kernel_ptrdiff_t; +#define __kernel_size_t __kernel_size_t + +typedef unsigned short __kernel_ipc_pid_t; +#define __kernel_ipc_pid_t __kernel_ipc_pid_t + +typedef unsigned short __kernel_uid_t; +typedef unsigned short __kernel_gid_t; +#define __kernel_uid_t __kernel_uid_t + +typedef unsigned short __kernel_mode_t; +#define __kernel_mode_t __kernel_mode_t + +typedef long __kernel_daddr_t; +#define __kernel_daddr_t __kernel_daddr_t + +typedef unsigned short __kernel_old_dev_t; +#define __kernel_old_dev_t __kernel_old_dev_t + +#endif /* defined(__sparc__) && defined(__arch64__) */ + +#include <asm-generic/posix_types.h> + +#endif /* __SPARC_POSIX_TYPES_H */ diff --git a/arch/sparc/include/uapi/asm/psr.h b/arch/sparc/include/uapi/asm/psr.h new file mode 100644 index 00000000000..2f0ed856530 --- /dev/null +++ b/arch/sparc/include/uapi/asm/psr.h @@ -0,0 +1,47 @@ +/* + * psr.h: This file holds the macros for masking off various parts of + * the processor status register on the Sparc. This is valid + * for Version 8. On the V9 this is renamed to the PSTATE + * register and its members are accessed as fields like + * PSTATE.PRIV for the current CPU privilege level. + * + * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _UAPI__LINUX_SPARC_PSR_H +#define _UAPI__LINUX_SPARC_PSR_H + +/* The Sparc PSR fields are laid out as the following: + * + * ------------------------------------------------------------------------ + * | impl | vers | icc | resv | EC | EF | PIL | S | PS | ET | CWP | + * | 31-28 | 27-24 | 23-20 | 19-14 | 13 | 12 | 11-8 | 7 | 6 | 5 | 4-0 | + * ------------------------------------------------------------------------ + */ +#define PSR_CWP 0x0000001f /* current window pointer */ +#define PSR_ET 0x00000020 /* enable traps field */ +#define PSR_PS 0x00000040 /* previous privilege level */ +#define PSR_S 0x00000080 /* current privilege level */ +#define PSR_PIL 0x00000f00 /* processor interrupt level */ +#define PSR_EF 0x00001000 /* enable floating point */ +#define PSR_EC 0x00002000 /* enable co-processor */ +#define PSR_SYSCALL 0x00004000 /* inside of a syscall */ +#define PSR_LE 0x00008000 /* SuperSparcII little-endian */ +#define PSR_ICC 0x00f00000 /* integer condition codes */ +#define PSR_C 0x00100000 /* carry bit */ +#define PSR_V 0x00200000 /* overflow bit */ +#define PSR_Z 0x00400000 /* zero bit */ +#define PSR_N 0x00800000 /* negative bit */ +#define PSR_VERS 0x0f000000 /* cpu-version field */ +#define PSR_IMPL 0xf0000000 /* cpu-implementation field */ + +#define PSR_VERS_SHIFT 24 +#define PSR_IMPL_SHIFT 28 +#define PSR_VERS_SHIFTED_MASK 0xf +#define PSR_IMPL_SHIFTED_MASK 0xf + +#define PSR_IMPL_TI 0x4 +#define PSR_IMPL_LEON 0xf + + +#endif /* _UAPI__LINUX_SPARC_PSR_H */ diff --git a/arch/sparc/include/asm/psrcompat.h b/arch/sparc/include/uapi/asm/psrcompat.h index 44b6327dbbf..44b6327dbbf 100644 --- a/arch/sparc/include/asm/psrcompat.h +++ b/arch/sparc/include/uapi/asm/psrcompat.h diff --git a/arch/sparc/include/asm/pstate.h b/arch/sparc/include/uapi/asm/pstate.h index a26a53777bb..4b6b998afd9 100644 --- a/arch/sparc/include/asm/pstate.h +++ b/arch/sparc/include/uapi/asm/pstate.h @@ -88,4 +88,18 @@ #define VERS_MAXTL _AC(0x000000000000ff00,UL) /* Max Trap Level. */ #define VERS_MAXWIN _AC(0x000000000000001f,UL) /* Max RegWindow Idx.*/ +/* Compatability Feature Register (%asr26), SPARC-T4 and later */ +#define CFR_AES _AC(0x0000000000000001,UL) /* Supports AES opcodes */ +#define CFR_DES _AC(0x0000000000000002,UL) /* Supports DES opcodes */ +#define CFR_KASUMI _AC(0x0000000000000004,UL) /* Supports KASUMI opcodes */ +#define CFR_CAMELLIA _AC(0x0000000000000008,UL) /* Supports CAMELLIA opcodes*/ +#define CFR_MD5 _AC(0x0000000000000010,UL) /* Supports MD5 opcodes */ +#define CFR_SHA1 _AC(0x0000000000000020,UL) /* Supports SHA1 opcodes */ +#define CFR_SHA256 _AC(0x0000000000000040,UL) /* Supports SHA256 opcodes */ +#define CFR_SHA512 _AC(0x0000000000000080,UL) /* Supports SHA512 opcodes */ +#define CFR_MPMUL _AC(0x0000000000000100,UL) /* Supports MPMUL opcodes */ +#define CFR_MONTMUL _AC(0x0000000000000200,UL) /* Supports MONTMUL opcodes */ +#define CFR_MONTSQR _AC(0x0000000000000400,UL) /* Supports MONTSQR opcodes */ +#define CFR_CRC32C _AC(0x0000000000000800,UL) /* Supports CRC32C opcodes */ + #endif /* !(_SPARC64_PSTATE_H) */ diff --git a/arch/sparc/include/uapi/asm/ptrace.h b/arch/sparc/include/uapi/asm/ptrace.h new file mode 100644 index 00000000000..56fe4ea73fe --- /dev/null +++ b/arch/sparc/include/uapi/asm/ptrace.h @@ -0,0 +1,352 @@ +#ifndef _UAPI__SPARC_PTRACE_H +#define _UAPI__SPARC_PTRACE_H + +#if defined(__sparc__) && defined(__arch64__) +/* 64 bit sparc */ +#include <asm/pstate.h> + +/* This struct defines the way the registers are stored on the + * stack during a system call and basically all traps. + */ + +/* This magic value must have the low 9 bits clear, + * as that is where we encode the %tt value, see below. + */ +#define PT_REGS_MAGIC 0x57ac6c00 + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> + +struct pt_regs { + unsigned long u_regs[16]; /* globals and ins */ + unsigned long tstate; + unsigned long tpc; + unsigned long tnpc; + unsigned int y; + + /* We encode a magic number, PT_REGS_MAGIC, along + * with the %tt (trap type) register value at trap + * entry time. The magic number allows us to identify + * accurately a trap stack frame in the stack + * unwinder, and the %tt value allows us to test + * things like "in a system call" etc. for an arbitray + * process. + * + * The PT_REGS_MAGIC is chosen such that it can be + * loaded completely using just a sethi instruction. + */ + unsigned int magic; +}; + +struct pt_regs32 { + unsigned int psr; + unsigned int pc; + unsigned int npc; + unsigned int y; + unsigned int u_regs[16]; /* globals and ins */ +}; + +/* A V9 register window */ +struct reg_window { + unsigned long locals[8]; + unsigned long ins[8]; +}; + +/* A 32-bit register window. */ +struct reg_window32 { + unsigned int locals[8]; + unsigned int ins[8]; +}; + +/* A V9 Sparc stack frame */ +struct sparc_stackf { + unsigned long locals[8]; + unsigned long ins[6]; + struct sparc_stackf *fp; + unsigned long callers_pc; + char *structptr; + unsigned long xargs[6]; + unsigned long xxargs[1]; +}; + +/* A 32-bit Sparc stack frame */ +struct sparc_stackf32 { + unsigned int locals[8]; + unsigned int ins[6]; + unsigned int fp; + unsigned int callers_pc; + unsigned int structptr; + unsigned int xargs[6]; + unsigned int xxargs[1]; +}; + +struct sparc_trapf { + unsigned long locals[8]; + unsigned long ins[8]; + unsigned long _unused; + struct pt_regs *regs; +}; +#endif /* (!__ASSEMBLY__) */ +#else +/* 32 bit sparc */ + +#include <asm/psr.h> + +/* This struct defines the way the registers are stored on the + * stack during a system call and basically all traps. + */ +#ifndef __ASSEMBLY__ + +#include <linux/types.h> + +struct pt_regs { + unsigned long psr; + unsigned long pc; + unsigned long npc; + unsigned long y; + unsigned long u_regs[16]; /* globals and ins */ +}; + +/* A 32-bit register window. */ +struct reg_window32 { + unsigned long locals[8]; + unsigned long ins[8]; +}; + +/* A Sparc stack frame */ +struct sparc_stackf { + unsigned long locals[8]; + unsigned long ins[6]; + struct sparc_stackf *fp; + unsigned long callers_pc; + char *structptr; + unsigned long xargs[6]; + unsigned long xxargs[1]; +}; +#endif /* (!__ASSEMBLY__) */ + +#endif /* (defined(__sparc__) && defined(__arch64__))*/ + +#ifndef __ASSEMBLY__ + +#define TRACEREG_SZ sizeof(struct pt_regs) +#define STACKFRAME_SZ sizeof(struct sparc_stackf) + +#define TRACEREG32_SZ sizeof(struct pt_regs32) +#define STACKFRAME32_SZ sizeof(struct sparc_stackf32) + +#endif /* (!__ASSEMBLY__) */ + +#define UREG_G0 0 +#define UREG_G1 1 +#define UREG_G2 2 +#define UREG_G3 3 +#define UREG_G4 4 +#define UREG_G5 5 +#define UREG_G6 6 +#define UREG_G7 7 +#define UREG_I0 8 +#define UREG_I1 9 +#define UREG_I2 10 +#define UREG_I3 11 +#define UREG_I4 12 +#define UREG_I5 13 +#define UREG_I6 14 +#define UREG_I7 15 +#define UREG_FP UREG_I6 +#define UREG_RETPC UREG_I7 + +#if defined(__sparc__) && defined(__arch64__) +/* 64 bit sparc */ + +#ifndef __ASSEMBLY__ + + +#else /* __ASSEMBLY__ */ +/* For assembly code. */ +#define TRACEREG_SZ 0xa0 +#define STACKFRAME_SZ 0xc0 + +#define TRACEREG32_SZ 0x50 +#define STACKFRAME32_SZ 0x60 +#endif /* __ASSEMBLY__ */ + +#else /* (defined(__sparc__) && defined(__arch64__)) */ + +/* 32 bit sparc */ + +#ifndef __ASSEMBLY__ + + +#else /* (!__ASSEMBLY__) */ +/* For assembly code. */ +#define TRACEREG_SZ 0x50 +#define STACKFRAME_SZ 0x60 +#endif /* (!__ASSEMBLY__) */ + +#endif /* (defined(__sparc__) && defined(__arch64__)) */ + + +/* These are for pt_regs. */ +#define PT_V9_G0 0x00 +#define PT_V9_G1 0x08 +#define PT_V9_G2 0x10 +#define PT_V9_G3 0x18 +#define PT_V9_G4 0x20 +#define PT_V9_G5 0x28 +#define PT_V9_G6 0x30 +#define PT_V9_G7 0x38 +#define PT_V9_I0 0x40 +#define PT_V9_I1 0x48 +#define PT_V9_I2 0x50 +#define PT_V9_I3 0x58 +#define PT_V9_I4 0x60 +#define PT_V9_I5 0x68 +#define PT_V9_I6 0x70 +#define PT_V9_FP PT_V9_I6 +#define PT_V9_I7 0x78 +#define PT_V9_TSTATE 0x80 +#define PT_V9_TPC 0x88 +#define PT_V9_TNPC 0x90 +#define PT_V9_Y 0x98 +#define PT_V9_MAGIC 0x9c +#define PT_TSTATE PT_V9_TSTATE +#define PT_TPC PT_V9_TPC +#define PT_TNPC PT_V9_TNPC + +/* These for pt_regs32. */ +#define PT_PSR 0x0 +#define PT_PC 0x4 +#define PT_NPC 0x8 +#define PT_Y 0xc +#define PT_G0 0x10 +#define PT_WIM PT_G0 +#define PT_G1 0x14 +#define PT_G2 0x18 +#define PT_G3 0x1c +#define PT_G4 0x20 +#define PT_G5 0x24 +#define PT_G6 0x28 +#define PT_G7 0x2c +#define PT_I0 0x30 +#define PT_I1 0x34 +#define PT_I2 0x38 +#define PT_I3 0x3c +#define PT_I4 0x40 +#define PT_I5 0x44 +#define PT_I6 0x48 +#define PT_FP PT_I6 +#define PT_I7 0x4c + +/* Reg_window offsets */ +#define RW_V9_L0 0x00 +#define RW_V9_L1 0x08 +#define RW_V9_L2 0x10 +#define RW_V9_L3 0x18 +#define RW_V9_L4 0x20 +#define RW_V9_L5 0x28 +#define RW_V9_L6 0x30 +#define RW_V9_L7 0x38 +#define RW_V9_I0 0x40 +#define RW_V9_I1 0x48 +#define RW_V9_I2 0x50 +#define RW_V9_I3 0x58 +#define RW_V9_I4 0x60 +#define RW_V9_I5 0x68 +#define RW_V9_I6 0x70 +#define RW_V9_I7 0x78 + +#define RW_L0 0x00 +#define RW_L1 0x04 +#define RW_L2 0x08 +#define RW_L3 0x0c +#define RW_L4 0x10 +#define RW_L5 0x14 +#define RW_L6 0x18 +#define RW_L7 0x1c +#define RW_I0 0x20 +#define RW_I1 0x24 +#define RW_I2 0x28 +#define RW_I3 0x2c +#define RW_I4 0x30 +#define RW_I5 0x34 +#define RW_I6 0x38 +#define RW_I7 0x3c + +/* Stack_frame offsets */ +#define SF_V9_L0 0x00 +#define SF_V9_L1 0x08 +#define SF_V9_L2 0x10 +#define SF_V9_L3 0x18 +#define SF_V9_L4 0x20 +#define SF_V9_L5 0x28 +#define SF_V9_L6 0x30 +#define SF_V9_L7 0x38 +#define SF_V9_I0 0x40 +#define SF_V9_I1 0x48 +#define SF_V9_I2 0x50 +#define SF_V9_I3 0x58 +#define SF_V9_I4 0x60 +#define SF_V9_I5 0x68 +#define SF_V9_FP 0x70 +#define SF_V9_PC 0x78 +#define SF_V9_RETP 0x80 +#define SF_V9_XARG0 0x88 +#define SF_V9_XARG1 0x90 +#define SF_V9_XARG2 0x98 +#define SF_V9_XARG3 0xa0 +#define SF_V9_XARG4 0xa8 +#define SF_V9_XARG5 0xb0 +#define SF_V9_XXARG 0xb8 + +#define SF_L0 0x00 +#define SF_L1 0x04 +#define SF_L2 0x08 +#define SF_L3 0x0c +#define SF_L4 0x10 +#define SF_L5 0x14 +#define SF_L6 0x18 +#define SF_L7 0x1c +#define SF_I0 0x20 +#define SF_I1 0x24 +#define SF_I2 0x28 +#define SF_I3 0x2c +#define SF_I4 0x30 +#define SF_I5 0x34 +#define SF_FP 0x38 +#define SF_PC 0x3c +#define SF_RETP 0x40 +#define SF_XARG0 0x44 +#define SF_XARG1 0x48 +#define SF_XARG2 0x4c +#define SF_XARG3 0x50 +#define SF_XARG4 0x54 +#define SF_XARG5 0x58 +#define SF_XXARG 0x5c + + +/* Stuff for the ptrace system call */ +#define PTRACE_SPARC_DETACH 11 +#define PTRACE_GETREGS 12 +#define PTRACE_SETREGS 13 +#define PTRACE_GETFPREGS 14 +#define PTRACE_SETFPREGS 15 +#define PTRACE_READDATA 16 +#define PTRACE_WRITEDATA 17 +#define PTRACE_READTEXT 18 +#define PTRACE_WRITETEXT 19 +#define PTRACE_GETFPAREGS 20 +#define PTRACE_SETFPAREGS 21 + +/* There are for debugging 64-bit processes, either from a 32 or 64 bit + * parent. Thus their complements are for debugging 32-bit processes only. + */ + +#define PTRACE_GETREGS64 22 +#define PTRACE_SETREGS64 23 +/* PTRACE_SYSCALL is 24 */ +#define PTRACE_GETFPREGS64 25 +#define PTRACE_SETFPREGS64 26 + +#endif /* _UAPI__SPARC_PTRACE_H */ diff --git a/arch/sparc/include/asm/resource.h b/arch/sparc/include/uapi/asm/resource.h index fe163cafb4c..fe163cafb4c 100644 --- a/arch/sparc/include/asm/resource.h +++ b/arch/sparc/include/uapi/asm/resource.h diff --git a/arch/sparc/include/asm/sembuf.h b/arch/sparc/include/uapi/asm/sembuf.h index faee1be08d6..faee1be08d6 100644 --- a/arch/sparc/include/asm/sembuf.h +++ b/arch/sparc/include/uapi/asm/sembuf.h diff --git a/arch/sparc/include/uapi/asm/setup.h b/arch/sparc/include/uapi/asm/setup.h new file mode 100644 index 00000000000..53376845087 --- /dev/null +++ b/arch/sparc/include/uapi/asm/setup.h @@ -0,0 +1,15 @@ +/* + * Just a place holder. + */ + +#ifndef _UAPI_SPARC_SETUP_H +#define _UAPI_SPARC_SETUP_H + +#if defined(__sparc__) && defined(__arch64__) +# define COMMAND_LINE_SIZE 2048 +#else +# define COMMAND_LINE_SIZE 256 +#endif + + +#endif /* _UAPI_SPARC_SETUP_H */ diff --git a/arch/sparc/include/asm/shmbuf.h b/arch/sparc/include/uapi/asm/shmbuf.h index 83a16055363..83a16055363 100644 --- a/arch/sparc/include/asm/shmbuf.h +++ b/arch/sparc/include/uapi/asm/shmbuf.h diff --git a/arch/sparc/include/uapi/asm/sigcontext.h b/arch/sparc/include/uapi/asm/sigcontext.h new file mode 100644 index 00000000000..ae5704fa77a --- /dev/null +++ b/arch/sparc/include/uapi/asm/sigcontext.h @@ -0,0 +1,4 @@ +/* + * There isn't anything here anymore, but the file must not be empty or patch + * will delete it. + */ diff --git a/arch/sparc/include/uapi/asm/siginfo.h b/arch/sparc/include/uapi/asm/siginfo.h new file mode 100644 index 00000000000..2d9b79ccaa5 --- /dev/null +++ b/arch/sparc/include/uapi/asm/siginfo.h @@ -0,0 +1,25 @@ +#ifndef _UAPI__SPARC_SIGINFO_H +#define _UAPI__SPARC_SIGINFO_H + +#if defined(__sparc__) && defined(__arch64__) + +#define __ARCH_SI_PREAMBLE_SIZE (4 * sizeof(int)) +#define __ARCH_SI_BAND_T int + +#endif /* defined(__sparc__) && defined(__arch64__) */ + + +#define __ARCH_SI_TRAPNO + +#include <asm-generic/siginfo.h> + + +#define SI_NOINFO 32767 /* no information in siginfo_t */ + +/* + * SIGEMT si_codes + */ +#define EMT_TAGOVF (__SI_FAULT|1) /* tag overflow */ +#define NSIGEMT 1 + +#endif /* _UAPI__SPARC_SIGINFO_H */ diff --git a/arch/sparc/include/uapi/asm/signal.h b/arch/sparc/include/uapi/asm/signal.h new file mode 100644 index 00000000000..f387400fcfd --- /dev/null +++ b/arch/sparc/include/uapi/asm/signal.h @@ -0,0 +1,181 @@ +#ifndef _UAPI__SPARC_SIGNAL_H +#define _UAPI__SPARC_SIGNAL_H + +#include <asm/sigcontext.h> +#include <linux/compiler.h> + + +/* On the Sparc the signal handlers get passed a 'sub-signal' code + * for certain signal types, which we document here. + */ +#define SIGHUP 1 +#define SIGINT 2 +#define SIGQUIT 3 +#define SIGILL 4 +#define SUBSIG_STACK 0 +#define SUBSIG_ILLINST 2 +#define SUBSIG_PRIVINST 3 +#define SUBSIG_BADTRAP(t) (0x80 + (t)) + +#define SIGTRAP 5 +#define SIGABRT 6 +#define SIGIOT 6 + +#define SIGEMT 7 +#define SUBSIG_TAG 10 + +#define SIGFPE 8 +#define SUBSIG_FPDISABLED 0x400 +#define SUBSIG_FPERROR 0x404 +#define SUBSIG_FPINTOVFL 0x001 +#define SUBSIG_FPSTSIG 0x002 +#define SUBSIG_IDIVZERO 0x014 +#define SUBSIG_FPINEXACT 0x0c4 +#define SUBSIG_FPDIVZERO 0x0c8 +#define SUBSIG_FPUNFLOW 0x0cc +#define SUBSIG_FPOPERROR 0x0d0 +#define SUBSIG_FPOVFLOW 0x0d4 + +#define SIGKILL 9 +#define SIGBUS 10 +#define SUBSIG_BUSTIMEOUT 1 +#define SUBSIG_ALIGNMENT 2 +#define SUBSIG_MISCERROR 5 + +#define SIGSEGV 11 +#define SUBSIG_NOMAPPING 3 +#define SUBSIG_PROTECTION 4 +#define SUBSIG_SEGERROR 5 + +#define SIGSYS 12 + +#define SIGPIPE 13 +#define SIGALRM 14 +#define SIGTERM 15 +#define SIGURG 16 + +/* SunOS values which deviate from the Linux/i386 ones */ +#define SIGSTOP 17 +#define SIGTSTP 18 +#define SIGCONT 19 +#define SIGCHLD 20 +#define SIGTTIN 21 +#define SIGTTOU 22 +#define SIGIO 23 +#define SIGPOLL SIGIO /* SysV name for SIGIO */ +#define SIGXCPU 24 +#define SIGXFSZ 25 +#define SIGVTALRM 26 +#define SIGPROF 27 +#define SIGWINCH 28 +#define SIGLOST 29 +#define SIGPWR SIGLOST +#define SIGUSR1 30 +#define SIGUSR2 31 + +/* Most things should be clean enough to redefine this at will, if care + is taken to make libc match. */ + +#define __OLD_NSIG 32 +#define __NEW_NSIG 64 +#ifdef __arch64__ +#define _NSIG_BPW 64 +#else +#define _NSIG_BPW 32 +#endif +#define _NSIG_WORDS (__NEW_NSIG / _NSIG_BPW) + +#define SIGRTMIN 32 +#define SIGRTMAX __NEW_NSIG + +#if defined(__KERNEL__) || defined(__WANT_POSIX1B_SIGNALS__) +#define _NSIG __NEW_NSIG +#define __new_sigset_t sigset_t +#define __new_sigaction sigaction +#define __new_sigaction32 sigaction32 +#define __old_sigset_t old_sigset_t +#define __old_sigaction old_sigaction +#define __old_sigaction32 old_sigaction32 +#else +#define _NSIG __OLD_NSIG +#define NSIG _NSIG +#define __old_sigset_t sigset_t +#define __old_sigaction sigaction +#define __old_sigaction32 sigaction32 +#endif + +#ifndef __ASSEMBLY__ + +typedef unsigned long __old_sigset_t; /* at least 32 bits */ + +typedef struct { + unsigned long sig[_NSIG_WORDS]; +} __new_sigset_t; + +/* A SunOS sigstack */ +struct sigstack { + /* XXX 32-bit pointers pinhead XXX */ + char *the_stack; + int cur_status; +}; + +/* Sigvec flags */ +#define _SV_SSTACK 1u /* This signal handler should use sig-stack */ +#define _SV_INTR 2u /* Sig return should not restart system call */ +#define _SV_RESET 4u /* Set handler to SIG_DFL upon taken signal */ +#define _SV_IGNCHILD 8u /* Do not send SIGCHLD */ + +/* + * sa_flags values: SA_STACK is not currently supported, but will allow the + * usage of signal stacks by using the (now obsolete) sa_restorer field in + * the sigaction structure as a stack pointer. This is now possible due to + * the changes in signal handling. LBT 010493. + * SA_RESTART flag to get restarting signals (which were the default long ago) + */ +#define SA_NOCLDSTOP _SV_IGNCHILD +#define SA_STACK _SV_SSTACK +#define SA_ONSTACK _SV_SSTACK +#define SA_RESTART _SV_INTR +#define SA_ONESHOT _SV_RESET +#define SA_NODEFER 0x20u +#define SA_NOCLDWAIT 0x100u +#define SA_SIGINFO 0x200u + +#define SA_NOMASK SA_NODEFER + +#define SIG_BLOCK 0x01 /* for blocking signals */ +#define SIG_UNBLOCK 0x02 /* for unblocking signals */ +#define SIG_SETMASK 0x04 /* for setting the signal mask */ + +#define MINSIGSTKSZ 4096 +#define SIGSTKSZ 16384 + + +#include <asm-generic/signal-defs.h> + +#ifndef __KERNEL__ +struct __new_sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; + __sigrestore_t sa_restorer; /* not used by Linux/SPARC yet */ + __new_sigset_t sa_mask; +}; + +struct __old_sigaction { + __sighandler_t sa_handler; + __old_sigset_t sa_mask; + unsigned long sa_flags; + void (*sa_restorer)(void); /* not used by Linux/SPARC yet */ +}; +#endif + +typedef struct sigaltstack { + void __user *ss_sp; + int ss_flags; + size_t ss_size; +} stack_t; + + +#endif /* !(__ASSEMBLY__) */ + +#endif /* _UAPI__SPARC_SIGNAL_H */ diff --git a/arch/sparc/include/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h index 9d3fefcff2f..54d9608681b 100644 --- a/arch/sparc/include/asm/socket.h +++ b/arch/sparc/include/uapi/asm/socket.h @@ -15,7 +15,7 @@ #define SO_PEERCRED 0x0040 #define SO_LINGER 0x0080 #define SO_OOBINLINE 0x0100 -/* To add :#define SO_REUSEPORT 0x0200 */ +#define SO_REUSEPORT 0x0200 #define SO_BSDCOMPAT 0x0400 #define SO_RCVLOWAT 0x0800 #define SO_SNDLOWAT 0x1000 @@ -41,6 +41,7 @@ #define SO_ATTACH_FILTER 0x001a #define SO_DETACH_FILTER 0x001b +#define SO_GET_FILTER SO_ATTACH_FILTER #define SO_PEERNAME 0x001c #define SO_TIMESTAMP 0x001d @@ -58,6 +59,23 @@ #define SO_RXQ_OVFL 0x0024 +#define SO_WIFI_STATUS 0x0025 +#define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 0x0026 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 0x0027 + +#define SO_LOCK_FILTER 0x0028 + +#define SO_SELECT_ERR_QUEUE 0x0029 + +#define SO_BUSY_POLL 0x0030 + +#define SO_MAX_PACING_RATE 0x0031 + +#define SO_BPF_EXTENSIONS 0x0032 + /* Security levels - as per NRL IPv6 - don't actually do anything */ #define SO_SECURITY_AUTHENTICATION 0x5001 #define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 diff --git a/arch/sparc/include/asm/sockios.h b/arch/sparc/include/uapi/asm/sockios.h index 990ea746486..990ea746486 100644 --- a/arch/sparc/include/asm/sockios.h +++ b/arch/sparc/include/uapi/asm/sockios.h diff --git a/arch/sparc/include/asm/stat.h b/arch/sparc/include/uapi/asm/stat.h index a232e9e1f4e..a232e9e1f4e 100644 --- a/arch/sparc/include/asm/stat.h +++ b/arch/sparc/include/uapi/asm/stat.h diff --git a/arch/sparc/include/asm/statfs.h b/arch/sparc/include/uapi/asm/statfs.h index 55e607ad461..55e607ad461 100644 --- a/arch/sparc/include/asm/statfs.h +++ b/arch/sparc/include/uapi/asm/statfs.h diff --git a/arch/sparc/include/asm/swab.h b/arch/sparc/include/uapi/asm/swab.h index a34ad079487..a34ad079487 100644 --- a/arch/sparc/include/asm/swab.h +++ b/arch/sparc/include/uapi/asm/swab.h diff --git a/arch/sparc/include/uapi/asm/termbits.h b/arch/sparc/include/uapi/asm/termbits.h new file mode 100644 index 00000000000..dd91642fcca --- /dev/null +++ b/arch/sparc/include/uapi/asm/termbits.h @@ -0,0 +1,263 @@ +#ifndef _UAPI_SPARC_TERMBITS_H +#define _UAPI_SPARC_TERMBITS_H + +#include <linux/posix_types.h> + +typedef unsigned char cc_t; +typedef unsigned int speed_t; + +#if defined(__sparc__) && defined(__arch64__) +typedef unsigned int tcflag_t; +#else +typedef unsigned long tcflag_t; +#endif + +#define NCC 8 +struct termio { + unsigned short c_iflag; /* input mode flags */ + unsigned short c_oflag; /* output mode flags */ + unsigned short c_cflag; /* control mode flags */ + unsigned short c_lflag; /* local mode flags */ + unsigned char c_line; /* line discipline */ + unsigned char c_cc[NCC]; /* control characters */ +}; + +#define NCCS 17 +struct termios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ +#ifndef __KERNEL__ + cc_t c_cc[NCCS]; /* control characters */ +#else + cc_t c_cc[NCCS+2]; /* kernel needs 2 more to hold vmin/vtime */ +#define SIZEOF_USER_TERMIOS sizeof (struct termios) - (2*sizeof (cc_t)) +#endif +}; + +struct termios2 { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ + cc_t c_cc[NCCS+2]; /* control characters */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +struct ktermios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ + cc_t c_cc[NCCS+2]; /* control characters */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +/* c_cc characters */ +#define VINTR 0 +#define VQUIT 1 +#define VERASE 2 +#define VKILL 3 +#define VEOF 4 +#define VEOL 5 +#define VEOL2 6 +#define VSWTC 7 +#define VSTART 8 +#define VSTOP 9 + + + +#define VSUSP 10 +#define VDSUSP 11 /* SunOS POSIX nicety I do believe... */ +#define VREPRINT 12 +#define VDISCARD 13 +#define VWERASE 14 +#define VLNEXT 15 + +/* Kernel keeps vmin/vtime separated, user apps assume vmin/vtime is + * shared with eof/eol + */ +#ifndef __KERNEL__ +#define VMIN VEOF +#define VTIME VEOL +#endif + +/* c_iflag bits */ +#define IGNBRK 0x00000001 +#define BRKINT 0x00000002 +#define IGNPAR 0x00000004 +#define PARMRK 0x00000008 +#define INPCK 0x00000010 +#define ISTRIP 0x00000020 +#define INLCR 0x00000040 +#define IGNCR 0x00000080 +#define ICRNL 0x00000100 +#define IUCLC 0x00000200 +#define IXON 0x00000400 +#define IXANY 0x00000800 +#define IXOFF 0x00001000 +#define IMAXBEL 0x00002000 +#define IUTF8 0x00004000 + +/* c_oflag bits */ +#define OPOST 0x00000001 +#define OLCUC 0x00000002 +#define ONLCR 0x00000004 +#define OCRNL 0x00000008 +#define ONOCR 0x00000010 +#define ONLRET 0x00000020 +#define OFILL 0x00000040 +#define OFDEL 0x00000080 +#define NLDLY 0x00000100 +#define NL0 0x00000000 +#define NL1 0x00000100 +#define CRDLY 0x00000600 +#define CR0 0x00000000 +#define CR1 0x00000200 +#define CR2 0x00000400 +#define CR3 0x00000600 +#define TABDLY 0x00001800 +#define TAB0 0x00000000 +#define TAB1 0x00000800 +#define TAB2 0x00001000 +#define TAB3 0x00001800 +#define XTABS 0x00001800 +#define BSDLY 0x00002000 +#define BS0 0x00000000 +#define BS1 0x00002000 +#define VTDLY 0x00004000 +#define VT0 0x00000000 +#define VT1 0x00004000 +#define FFDLY 0x00008000 +#define FF0 0x00000000 +#define FF1 0x00008000 +#define PAGEOUT 0x00010000 /* SUNOS specific */ +#define WRAP 0x00020000 /* SUNOS specific */ + +/* c_cflag bit meaning */ +#define CBAUD 0x0000100f +#define B0 0x00000000 /* hang up */ +#define B50 0x00000001 +#define B75 0x00000002 +#define B110 0x00000003 +#define B134 0x00000004 +#define B150 0x00000005 +#define B200 0x00000006 +#define B300 0x00000007 +#define B600 0x00000008 +#define B1200 0x00000009 +#define B1800 0x0000000a +#define B2400 0x0000000b +#define B4800 0x0000000c +#define B9600 0x0000000d +#define B19200 0x0000000e +#define B38400 0x0000000f +#define EXTA B19200 +#define EXTB B38400 +#define CSIZE 0x00000030 +#define CS5 0x00000000 +#define CS6 0x00000010 +#define CS7 0x00000020 +#define CS8 0x00000030 +#define CSTOPB 0x00000040 +#define CREAD 0x00000080 +#define PARENB 0x00000100 +#define PARODD 0x00000200 +#define HUPCL 0x00000400 +#define CLOCAL 0x00000800 +#define CBAUDEX 0x00001000 +/* We'll never see these speeds with the Zilogs, but for completeness... */ +#define BOTHER 0x00001000 +#define B57600 0x00001001 +#define B115200 0x00001002 +#define B230400 0x00001003 +#define B460800 0x00001004 +/* This is what we can do with the Zilogs. */ +#define B76800 0x00001005 +/* This is what we can do with the SAB82532. */ +#define B153600 0x00001006 +#define B307200 0x00001007 +#define B614400 0x00001008 +#define B921600 0x00001009 +/* And these are the rest... */ +#define B500000 0x0000100a +#define B576000 0x0000100b +#define B1000000 0x0000100c +#define B1152000 0x0000100d +#define B1500000 0x0000100e +#define B2000000 0x0000100f +/* These have totally bogus values and nobody uses them + so far. Later on we'd have to use say 0x10000x and + adjust CBAUD constant and drivers accordingly. +#define B2500000 0x00001010 +#define B3000000 0x00001011 +#define B3500000 0x00001012 +#define B4000000 0x00001013 */ +#define CIBAUD 0x100f0000 /* input baud rate (not used) */ +#define CMSPAR 0x40000000 /* mark or space (stick) parity */ +#define CRTSCTS 0x80000000 /* flow control */ + +#define IBSHIFT 16 /* Shift from CBAUD to CIBAUD */ + +/* c_lflag bits */ +#define ISIG 0x00000001 +#define ICANON 0x00000002 +#define XCASE 0x00000004 +#define ECHO 0x00000008 +#define ECHOE 0x00000010 +#define ECHOK 0x00000020 +#define ECHONL 0x00000040 +#define NOFLSH 0x00000080 +#define TOSTOP 0x00000100 +#define ECHOCTL 0x00000200 +#define ECHOPRT 0x00000400 +#define ECHOKE 0x00000800 +#define DEFECHO 0x00001000 /* SUNOS thing, what is it? */ +#define FLUSHO 0x00002000 +#define PENDIN 0x00004000 +#define IEXTEN 0x00008000 +#define EXTPROC 0x00010000 + +/* modem lines */ +#define TIOCM_LE 0x001 +#define TIOCM_DTR 0x002 +#define TIOCM_RTS 0x004 +#define TIOCM_ST 0x008 +#define TIOCM_SR 0x010 +#define TIOCM_CTS 0x020 +#define TIOCM_CAR 0x040 +#define TIOCM_RNG 0x080 +#define TIOCM_DSR 0x100 +#define TIOCM_CD TIOCM_CAR +#define TIOCM_RI TIOCM_RNG +#define TIOCM_OUT1 0x2000 +#define TIOCM_OUT2 0x4000 +#define TIOCM_LOOP 0x8000 + +/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ +#define TIOCSER_TEMT 0x01 /* Transmitter physically empty */ + + +/* tcflow() and TCXONC use these */ +#define TCOOFF 0 +#define TCOON 1 +#define TCIOFF 2 +#define TCION 3 + +/* tcflush() and TCFLSH use these */ +#define TCIFLUSH 0 +#define TCOFLUSH 1 +#define TCIOFLUSH 2 + +/* tcsetattr uses these */ +#define TCSANOW 0 +#define TCSADRAIN 1 +#define TCSAFLUSH 2 + +#endif /* _UAPI_SPARC_TERMBITS_H */ diff --git a/arch/sparc/include/uapi/asm/termios.h b/arch/sparc/include/uapi/asm/termios.h new file mode 100644 index 00000000000..ea6f09e51e5 --- /dev/null +++ b/arch/sparc/include/uapi/asm/termios.h @@ -0,0 +1,43 @@ +#ifndef _UAPI_SPARC_TERMIOS_H +#define _UAPI_SPARC_TERMIOS_H + +#include <asm/ioctls.h> +#include <asm/termbits.h> + +#if defined(__KERNEL__) || defined(__DEFINE_BSD_TERMIOS) +struct sgttyb { + char sg_ispeed; + char sg_ospeed; + char sg_erase; + char sg_kill; + short sg_flags; +}; + +struct tchars { + char t_intrc; + char t_quitc; + char t_startc; + char t_stopc; + char t_eofc; + char t_brkc; +}; + +struct ltchars { + char t_suspc; + char t_dsuspc; + char t_rprntc; + char t_flushc; + char t_werasc; + char t_lnextc; +}; +#endif /* __KERNEL__ */ + +struct winsize { + unsigned short ws_row; + unsigned short ws_col; + unsigned short ws_xpixel; + unsigned short ws_ypixel; +}; + + +#endif /* _UAPI_SPARC_TERMIOS_H */ diff --git a/arch/sparc/include/uapi/asm/traps.h b/arch/sparc/include/uapi/asm/traps.h new file mode 100644 index 00000000000..a4eceace6cc --- /dev/null +++ b/arch/sparc/include/uapi/asm/traps.h @@ -0,0 +1,120 @@ +/* + * traps.h: Format of entries for the Sparc trap table. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _UAPI_SPARC_TRAPS_H +#define _UAPI_SPARC_TRAPS_H + +#define NUM_SPARC_TRAPS 255 + +#ifndef __ASSEMBLY__ +#endif /* !(__ASSEMBLY__) */ + +/* For patching the trap table at boot time, we need to know how to + * form various common Sparc instructions. Thus these macros... + */ + +#define SPARC_MOV_CONST_L3(const) (0xa6102000 | (const&0xfff)) + +/* The following assumes that the branch lies before the place we + * are branching to. This is the case for a trap vector... + * You have been warned. + */ +#define SPARC_BRANCH(dest_addr, inst_addr) \ + (0x10800000 | (((dest_addr-inst_addr)>>2)&0x3fffff)) + +#define SPARC_RD_PSR_L0 (0xa1480000) +#define SPARC_RD_WIM_L3 (0xa7500000) +#define SPARC_NOP (0x01000000) + +/* Various interesting trap levels. */ +/* First, hardware traps. */ +#define SP_TRAP_TFLT 0x1 /* Text fault */ +#define SP_TRAP_II 0x2 /* Illegal Instruction */ +#define SP_TRAP_PI 0x3 /* Privileged Instruction */ +#define SP_TRAP_FPD 0x4 /* Floating Point Disabled */ +#define SP_TRAP_WOVF 0x5 /* Window Overflow */ +#define SP_TRAP_WUNF 0x6 /* Window Underflow */ +#define SP_TRAP_MNA 0x7 /* Memory Address Unaligned */ +#define SP_TRAP_FPE 0x8 /* Floating Point Exception */ +#define SP_TRAP_DFLT 0x9 /* Data Fault */ +#define SP_TRAP_TOF 0xa /* Tag Overflow */ +#define SP_TRAP_WDOG 0xb /* Watchpoint Detected */ +#define SP_TRAP_IRQ1 0x11 /* IRQ level 1 */ +#define SP_TRAP_IRQ2 0x12 /* IRQ level 2 */ +#define SP_TRAP_IRQ3 0x13 /* IRQ level 3 */ +#define SP_TRAP_IRQ4 0x14 /* IRQ level 4 */ +#define SP_TRAP_IRQ5 0x15 /* IRQ level 5 */ +#define SP_TRAP_IRQ6 0x16 /* IRQ level 6 */ +#define SP_TRAP_IRQ7 0x17 /* IRQ level 7 */ +#define SP_TRAP_IRQ8 0x18 /* IRQ level 8 */ +#define SP_TRAP_IRQ9 0x19 /* IRQ level 9 */ +#define SP_TRAP_IRQ10 0x1a /* IRQ level 10 */ +#define SP_TRAP_IRQ11 0x1b /* IRQ level 11 */ +#define SP_TRAP_IRQ12 0x1c /* IRQ level 12 */ +#define SP_TRAP_IRQ13 0x1d /* IRQ level 13 */ +#define SP_TRAP_IRQ14 0x1e /* IRQ level 14 */ +#define SP_TRAP_IRQ15 0x1f /* IRQ level 15 Non-maskable */ +#define SP_TRAP_RACC 0x20 /* Register Access Error ??? */ +#define SP_TRAP_IACC 0x21 /* Instruction Access Error */ +#define SP_TRAP_CPDIS 0x24 /* Co-Processor Disabled */ +#define SP_TRAP_BADFL 0x25 /* Unimplemented Flush Instruction */ +#define SP_TRAP_CPEXP 0x28 /* Co-Processor Exception */ +#define SP_TRAP_DACC 0x29 /* Data Access Error */ +#define SP_TRAP_DIVZ 0x2a /* Divide By Zero */ +#define SP_TRAP_DSTORE 0x2b /* Data Store Error ??? */ +#define SP_TRAP_DMM 0x2c /* Data Access MMU Miss ??? */ +#define SP_TRAP_IMM 0x3c /* Instruction Access MMU Miss ??? */ + +/* Now the Software Traps... */ +#define SP_TRAP_SUNOS 0x80 /* SunOS System Call */ +#define SP_TRAP_SBPT 0x81 /* Software Breakpoint */ +#define SP_TRAP_SDIVZ 0x82 /* Software Divide-by-Zero trap */ +#define SP_TRAP_FWIN 0x83 /* Flush Windows */ +#define SP_TRAP_CWIN 0x84 /* Clean Windows */ +#define SP_TRAP_RCHK 0x85 /* Range Check */ +#define SP_TRAP_FUNA 0x86 /* Fix Unaligned Access */ +#define SP_TRAP_IOWFL 0x87 /* Integer Overflow */ +#define SP_TRAP_SOLARIS 0x88 /* Solaris System Call */ +#define SP_TRAP_NETBSD 0x89 /* NetBSD System Call */ +#define SP_TRAP_LINUX 0x90 /* Linux System Call */ + +/* Names used for compatibility with SunOS */ +#define ST_SYSCALL 0x00 +#define ST_BREAKPOINT 0x01 +#define ST_DIV0 0x02 +#define ST_FLUSH_WINDOWS 0x03 +#define ST_CLEAN_WINDOWS 0x04 +#define ST_RANGE_CHECK 0x05 +#define ST_FIX_ALIGN 0x06 +#define ST_INT_OVERFLOW 0x07 + +/* Special traps... */ +#define SP_TRAP_KBPT1 0xfe /* KADB/PROM Breakpoint one */ +#define SP_TRAP_KBPT2 0xff /* KADB/PROM Breakpoint two */ + +/* Handy Macros */ +/* Is this a trap we never expect to get? */ +#define BAD_TRAP_P(level) \ + ((level > SP_TRAP_WDOG && level < SP_TRAP_IRQ1) || \ + (level > SP_TRAP_IACC && level < SP_TRAP_CPDIS) || \ + (level > SP_TRAP_BADFL && level < SP_TRAP_CPEXP) || \ + (level > SP_TRAP_DMM && level < SP_TRAP_IMM) || \ + (level > SP_TRAP_IMM && level < SP_TRAP_SUNOS) || \ + (level > SP_TRAP_LINUX && level < SP_TRAP_KBPT1)) + +/* Is this a Hardware trap? */ +#define HW_TRAP_P(level) ((level > 0) && (level < SP_TRAP_SUNOS)) + +/* Is this a Software trap? */ +#define SW_TRAP_P(level) ((level >= SP_TRAP_SUNOS) && (level <= SP_TRAP_KBPT2)) + +/* Is this a system call for some OS we know about? */ +#define SCALL_TRAP_P(level) ((level == SP_TRAP_SUNOS) || \ + (level == SP_TRAP_SOLARIS) || \ + (level == SP_TRAP_NETBSD) || \ + (level == SP_TRAP_LINUX)) + +#endif /* _UAPI_SPARC_TRAPS_H */ diff --git a/arch/sparc/include/asm/uctx.h b/arch/sparc/include/uapi/asm/uctx.h index dc937c75ffd..dc937c75ffd 100644 --- a/arch/sparc/include/asm/uctx.h +++ b/arch/sparc/include/uapi/asm/uctx.h diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h new file mode 100644 index 00000000000..42f2bca1d33 --- /dev/null +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -0,0 +1,431 @@ +/* + * System calls under the Sparc. + * + * Don't be scared by the ugly clobbers, it is the only way I can + * think of right now to force the arguments into fixed registers + * before the trap into the system call with gcc 'asm' statements. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * + * SunOS compatibility based upon preliminary work which is: + * + * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) + */ +#ifndef _UAPI_SPARC_UNISTD_H +#define _UAPI_SPARC_UNISTD_H + +#ifndef __32bit_syscall_numbers__ +#ifndef __arch64__ +#define __32bit_syscall_numbers__ +#endif +#endif + +#define __NR_restart_syscall 0 /* Linux Specific */ +#define __NR_exit 1 /* Common */ +#define __NR_fork 2 /* Common */ +#define __NR_read 3 /* Common */ +#define __NR_write 4 /* Common */ +#define __NR_open 5 /* Common */ +#define __NR_close 6 /* Common */ +#define __NR_wait4 7 /* Common */ +#define __NR_creat 8 /* Common */ +#define __NR_link 9 /* Common */ +#define __NR_unlink 10 /* Common */ +#define __NR_execv 11 /* SunOS Specific */ +#define __NR_chdir 12 /* Common */ +#define __NR_chown 13 /* Common */ +#define __NR_mknod 14 /* Common */ +#define __NR_chmod 15 /* Common */ +#define __NR_lchown 16 /* Common */ +#define __NR_brk 17 /* Common */ +#define __NR_perfctr 18 /* Performance counter operations */ +#define __NR_lseek 19 /* Common */ +#define __NR_getpid 20 /* Common */ +#define __NR_capget 21 /* Linux Specific */ +#define __NR_capset 22 /* Linux Specific */ +#define __NR_setuid 23 /* Implemented via setreuid in SunOS */ +#define __NR_getuid 24 /* Common */ +#define __NR_vmsplice 25 /* ENOSYS under SunOS */ +#define __NR_ptrace 26 /* Common */ +#define __NR_alarm 27 /* Implemented via setitimer in SunOS */ +#define __NR_sigaltstack 28 /* Common */ +#define __NR_pause 29 /* Is sigblock(0)->sigpause() in SunOS */ +#define __NR_utime 30 /* Implemented via utimes() under SunOS */ +#ifdef __32bit_syscall_numbers__ +#define __NR_lchown32 31 /* Linux sparc32 specific */ +#define __NR_fchown32 32 /* Linux sparc32 specific */ +#endif +#define __NR_access 33 /* Common */ +#define __NR_nice 34 /* Implemented via get/setpriority() in SunOS */ +#ifdef __32bit_syscall_numbers__ +#define __NR_chown32 35 /* Linux sparc32 specific */ +#endif +#define __NR_sync 36 /* Common */ +#define __NR_kill 37 /* Common */ +#define __NR_stat 38 /* Common */ +#define __NR_sendfile 39 /* Linux Specific */ +#define __NR_lstat 40 /* Common */ +#define __NR_dup 41 /* Common */ +#define __NR_pipe 42 /* Common */ +#define __NR_times 43 /* Implemented via getrusage() in SunOS */ +#ifdef __32bit_syscall_numbers__ +#define __NR_getuid32 44 /* Linux sparc32 specific */ +#endif +#define __NR_umount2 45 /* Linux Specific */ +#define __NR_setgid 46 /* Implemented via setregid() in SunOS */ +#define __NR_getgid 47 /* Common */ +#define __NR_signal 48 /* Implemented via sigvec() in SunOS */ +#define __NR_geteuid 49 /* SunOS calls getuid() */ +#define __NR_getegid 50 /* SunOS calls getgid() */ +#define __NR_acct 51 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_getgid32 53 /* Linux sparc32 specific */ +#else +#define __NR_memory_ordering 52 /* Linux Specific */ +#endif +#define __NR_ioctl 54 /* Common */ +#define __NR_reboot 55 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_mmap2 56 /* Linux sparc32 Specific */ +#endif +#define __NR_symlink 57 /* Common */ +#define __NR_readlink 58 /* Common */ +#define __NR_execve 59 /* Common */ +#define __NR_umask 60 /* Common */ +#define __NR_chroot 61 /* Common */ +#define __NR_fstat 62 /* Common */ +#define __NR_fstat64 63 /* Linux Specific */ +#define __NR_getpagesize 64 /* Common */ +#define __NR_msync 65 /* Common in newer 1.3.x revs... */ +#define __NR_vfork 66 /* Common */ +#define __NR_pread64 67 /* Linux Specific */ +#define __NR_pwrite64 68 /* Linux Specific */ +#ifdef __32bit_syscall_numbers__ +#define __NR_geteuid32 69 /* Linux sparc32, sbrk under SunOS */ +#define __NR_getegid32 70 /* Linux sparc32, sstk under SunOS */ +#endif +#define __NR_mmap 71 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setreuid32 72 /* Linux sparc32, vadvise under SunOS */ +#endif +#define __NR_munmap 73 /* Common */ +#define __NR_mprotect 74 /* Common */ +#define __NR_madvise 75 /* Common */ +#define __NR_vhangup 76 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_truncate64 77 /* Linux sparc32 Specific */ +#endif +#define __NR_mincore 78 /* Common */ +#define __NR_getgroups 79 /* Common */ +#define __NR_setgroups 80 /* Common */ +#define __NR_getpgrp 81 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setgroups32 82 /* Linux sparc32, setpgrp under SunOS */ +#endif +#define __NR_setitimer 83 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_ftruncate64 84 /* Linux sparc32 Specific */ +#endif +#define __NR_swapon 85 /* Common */ +#define __NR_getitimer 86 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setuid32 87 /* Linux sparc32, gethostname under SunOS */ +#endif +#define __NR_sethostname 88 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setgid32 89 /* Linux sparc32, getdtablesize under SunOS */ +#endif +#define __NR_dup2 90 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setfsuid32 91 /* Linux sparc32, getdopt under SunOS */ +#endif +#define __NR_fcntl 92 /* Common */ +#define __NR_select 93 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setfsgid32 94 /* Linux sparc32, setdopt under SunOS */ +#endif +#define __NR_fsync 95 /* Common */ +#define __NR_setpriority 96 /* Common */ +#define __NR_socket 97 /* Common */ +#define __NR_connect 98 /* Common */ +#define __NR_accept 99 /* Common */ +#define __NR_getpriority 100 /* Common */ +#define __NR_rt_sigreturn 101 /* Linux Specific */ +#define __NR_rt_sigaction 102 /* Linux Specific */ +#define __NR_rt_sigprocmask 103 /* Linux Specific */ +#define __NR_rt_sigpending 104 /* Linux Specific */ +#define __NR_rt_sigtimedwait 105 /* Linux Specific */ +#define __NR_rt_sigqueueinfo 106 /* Linux Specific */ +#define __NR_rt_sigsuspend 107 /* Linux Specific */ +#ifdef __32bit_syscall_numbers__ +#define __NR_setresuid32 108 /* Linux Specific, sigvec under SunOS */ +#define __NR_getresuid32 109 /* Linux Specific, sigblock under SunOS */ +#define __NR_setresgid32 110 /* Linux Specific, sigsetmask under SunOS */ +#define __NR_getresgid32 111 /* Linux Specific, sigpause under SunOS */ +#define __NR_setregid32 112 /* Linux sparc32, sigstack under SunOS */ +#else +#define __NR_setresuid 108 /* Linux Specific, sigvec under SunOS */ +#define __NR_getresuid 109 /* Linux Specific, sigblock under SunOS */ +#define __NR_setresgid 110 /* Linux Specific, sigsetmask under SunOS */ +#define __NR_getresgid 111 /* Linux Specific, sigpause under SunOS */ +#endif +#define __NR_recvmsg 113 /* Common */ +#define __NR_sendmsg 114 /* Common */ +#ifdef __32bit_syscall_numbers__ +#define __NR_getgroups32 115 /* Linux sparc32, vtrace under SunOS */ +#endif +#define __NR_gettimeofday 116 /* Common */ +#define __NR_getrusage 117 /* Common */ +#define __NR_getsockopt 118 /* Common */ +#define __NR_getcwd 119 /* Linux Specific */ +#define __NR_readv 120 /* Common */ +#define __NR_writev 121 /* Common */ +#define __NR_settimeofday 122 /* Common */ +#define __NR_fchown 123 /* Common */ +#define __NR_fchmod 124 /* Common */ +#define __NR_recvfrom 125 /* Common */ +#define __NR_setreuid 126 /* Common */ +#define __NR_setregid 127 /* Common */ +#define __NR_rename 128 /* Common */ +#define __NR_truncate 129 /* Common */ +#define __NR_ftruncate 130 /* Common */ +#define __NR_flock 131 /* Common */ +#define __NR_lstat64 132 /* Linux Specific */ +#define __NR_sendto 133 /* Common */ +#define __NR_shutdown 134 /* Common */ +#define __NR_socketpair 135 /* Common */ +#define __NR_mkdir 136 /* Common */ +#define __NR_rmdir 137 /* Common */ +#define __NR_utimes 138 /* SunOS Specific */ +#define __NR_stat64 139 /* Linux Specific */ +#define __NR_sendfile64 140 /* adjtime under SunOS */ +#define __NR_getpeername 141 /* Common */ +#define __NR_futex 142 /* gethostid under SunOS */ +#define __NR_gettid 143 /* ENOSYS under SunOS */ +#define __NR_getrlimit 144 /* Common */ +#define __NR_setrlimit 145 /* Common */ +#define __NR_pivot_root 146 /* Linux Specific, killpg under SunOS */ +#define __NR_prctl 147 /* ENOSYS under SunOS */ +#define __NR_pciconfig_read 148 /* ENOSYS under SunOS */ +#define __NR_pciconfig_write 149 /* ENOSYS under SunOS */ +#define __NR_getsockname 150 /* Common */ +#define __NR_inotify_init 151 /* Linux specific */ +#define __NR_inotify_add_watch 152 /* Linux specific */ +#define __NR_poll 153 /* Common */ +#define __NR_getdents64 154 /* Linux specific */ +#ifdef __32bit_syscall_numbers__ +#define __NR_fcntl64 155 /* Linux sparc32 Specific */ +#endif +#define __NR_inotify_rm_watch 156 /* Linux specific */ +#define __NR_statfs 157 /* Common */ +#define __NR_fstatfs 158 /* Common */ +#define __NR_umount 159 /* Common */ +#define __NR_sched_set_affinity 160 /* Linux specific, async_daemon under SunOS */ +#define __NR_sched_get_affinity 161 /* Linux specific, getfh under SunOS */ +#define __NR_getdomainname 162 /* SunOS Specific */ +#define __NR_setdomainname 163 /* Common */ +#ifndef __32bit_syscall_numbers__ +#define __NR_utrap_install 164 /* SYSV ABI/v9 required */ +#endif +#define __NR_quotactl 165 /* Common */ +#define __NR_set_tid_address 166 /* Linux specific, exportfs under SunOS */ +#define __NR_mount 167 /* Common */ +#define __NR_ustat 168 /* Common */ +#define __NR_setxattr 169 /* SunOS: semsys */ +#define __NR_lsetxattr 170 /* SunOS: msgsys */ +#define __NR_fsetxattr 171 /* SunOS: shmsys */ +#define __NR_getxattr 172 /* SunOS: auditsys */ +#define __NR_lgetxattr 173 /* SunOS: rfssys */ +#define __NR_getdents 174 /* Common */ +#define __NR_setsid 175 /* Common */ +#define __NR_fchdir 176 /* Common */ +#define __NR_fgetxattr 177 /* SunOS: fchroot */ +#define __NR_listxattr 178 /* SunOS: vpixsys */ +#define __NR_llistxattr 179 /* SunOS: aioread */ +#define __NR_flistxattr 180 /* SunOS: aiowrite */ +#define __NR_removexattr 181 /* SunOS: aiowait */ +#define __NR_lremovexattr 182 /* SunOS: aiocancel */ +#define __NR_sigpending 183 /* Common */ +#define __NR_query_module 184 /* Linux Specific */ +#define __NR_setpgid 185 /* Common */ +#define __NR_fremovexattr 186 /* SunOS: pathconf */ +#define __NR_tkill 187 /* SunOS: fpathconf */ +#define __NR_exit_group 188 /* Linux specific, sysconf undef SunOS */ +#define __NR_uname 189 /* Linux Specific */ +#define __NR_init_module 190 /* Linux Specific */ +#define __NR_personality 191 /* Linux Specific */ +#define __NR_remap_file_pages 192 /* Linux Specific */ +#define __NR_epoll_create 193 /* Linux Specific */ +#define __NR_epoll_ctl 194 /* Linux Specific */ +#define __NR_epoll_wait 195 /* Linux Specific */ +#define __NR_ioprio_set 196 /* Linux Specific */ +#define __NR_getppid 197 /* Linux Specific */ +#define __NR_sigaction 198 /* Linux Specific */ +#define __NR_sgetmask 199 /* Linux Specific */ +#define __NR_ssetmask 200 /* Linux Specific */ +#define __NR_sigsuspend 201 /* Linux Specific */ +#define __NR_oldlstat 202 /* Linux Specific */ +#define __NR_uselib 203 /* Linux Specific */ +#define __NR_readdir 204 /* Linux Specific */ +#define __NR_readahead 205 /* Linux Specific */ +#define __NR_socketcall 206 /* Linux Specific */ +#define __NR_syslog 207 /* Linux Specific */ +#define __NR_lookup_dcookie 208 /* Linux Specific */ +#define __NR_fadvise64 209 /* Linux Specific */ +#define __NR_fadvise64_64 210 /* Linux Specific */ +#define __NR_tgkill 211 /* Linux Specific */ +#define __NR_waitpid 212 /* Linux Specific */ +#define __NR_swapoff 213 /* Linux Specific */ +#define __NR_sysinfo 214 /* Linux Specific */ +#define __NR_ipc 215 /* Linux Specific */ +#define __NR_sigreturn 216 /* Linux Specific */ +#define __NR_clone 217 /* Linux Specific */ +#define __NR_ioprio_get 218 /* Linux Specific */ +#define __NR_adjtimex 219 /* Linux Specific */ +#define __NR_sigprocmask 220 /* Linux Specific */ +#define __NR_create_module 221 /* Linux Specific */ +#define __NR_delete_module 222 /* Linux Specific */ +#define __NR_get_kernel_syms 223 /* Linux Specific */ +#define __NR_getpgid 224 /* Linux Specific */ +#define __NR_bdflush 225 /* Linux Specific */ +#define __NR_sysfs 226 /* Linux Specific */ +#define __NR_afs_syscall 227 /* Linux Specific */ +#define __NR_setfsuid 228 /* Linux Specific */ +#define __NR_setfsgid 229 /* Linux Specific */ +#define __NR__newselect 230 /* Linux Specific */ +#ifdef __32bit_syscall_numbers__ +#define __NR_time 231 /* Linux Specific */ +#else +#endif +#define __NR_splice 232 /* Linux Specific */ +#define __NR_stime 233 /* Linux Specific */ +#define __NR_statfs64 234 /* Linux Specific */ +#define __NR_fstatfs64 235 /* Linux Specific */ +#define __NR__llseek 236 /* Linux Specific */ +#define __NR_mlock 237 +#define __NR_munlock 238 +#define __NR_mlockall 239 +#define __NR_munlockall 240 +#define __NR_sched_setparam 241 +#define __NR_sched_getparam 242 +#define __NR_sched_setscheduler 243 +#define __NR_sched_getscheduler 244 +#define __NR_sched_yield 245 +#define __NR_sched_get_priority_max 246 +#define __NR_sched_get_priority_min 247 +#define __NR_sched_rr_get_interval 248 +#define __NR_nanosleep 249 +#define __NR_mremap 250 +#define __NR__sysctl 251 +#define __NR_getsid 252 +#define __NR_fdatasync 253 +#define __NR_nfsservctl 254 +#define __NR_sync_file_range 255 +#define __NR_clock_settime 256 +#define __NR_clock_gettime 257 +#define __NR_clock_getres 258 +#define __NR_clock_nanosleep 259 +#define __NR_sched_getaffinity 260 +#define __NR_sched_setaffinity 261 +#define __NR_timer_settime 262 +#define __NR_timer_gettime 263 +#define __NR_timer_getoverrun 264 +#define __NR_timer_delete 265 +#define __NR_timer_create 266 +/* #define __NR_vserver 267 Reserved for VSERVER */ +#define __NR_io_setup 268 +#define __NR_io_destroy 269 +#define __NR_io_submit 270 +#define __NR_io_cancel 271 +#define __NR_io_getevents 272 +#define __NR_mq_open 273 +#define __NR_mq_unlink 274 +#define __NR_mq_timedsend 275 +#define __NR_mq_timedreceive 276 +#define __NR_mq_notify 277 +#define __NR_mq_getsetattr 278 +#define __NR_waitid 279 +#define __NR_tee 280 +#define __NR_add_key 281 +#define __NR_request_key 282 +#define __NR_keyctl 283 +#define __NR_openat 284 +#define __NR_mkdirat 285 +#define __NR_mknodat 286 +#define __NR_fchownat 287 +#define __NR_futimesat 288 +#define __NR_fstatat64 289 +#define __NR_unlinkat 290 +#define __NR_renameat 291 +#define __NR_linkat 292 +#define __NR_symlinkat 293 +#define __NR_readlinkat 294 +#define __NR_fchmodat 295 +#define __NR_faccessat 296 +#define __NR_pselect6 297 +#define __NR_ppoll 298 +#define __NR_unshare 299 +#define __NR_set_robust_list 300 +#define __NR_get_robust_list 301 +#define __NR_migrate_pages 302 +#define __NR_mbind 303 +#define __NR_get_mempolicy 304 +#define __NR_set_mempolicy 305 +#define __NR_kexec_load 306 +#define __NR_move_pages 307 +#define __NR_getcpu 308 +#define __NR_epoll_pwait 309 +#define __NR_utimensat 310 +#define __NR_signalfd 311 +#define __NR_timerfd_create 312 +#define __NR_eventfd 313 +#define __NR_fallocate 314 +#define __NR_timerfd_settime 315 +#define __NR_timerfd_gettime 316 +#define __NR_signalfd4 317 +#define __NR_eventfd2 318 +#define __NR_epoll_create1 319 +#define __NR_dup3 320 +#define __NR_pipe2 321 +#define __NR_inotify_init1 322 +#define __NR_accept4 323 +#define __NR_preadv 324 +#define __NR_pwritev 325 +#define __NR_rt_tgsigqueueinfo 326 +#define __NR_perf_event_open 327 +#define __NR_recvmmsg 328 +#define __NR_fanotify_init 329 +#define __NR_fanotify_mark 330 +#define __NR_prlimit64 331 +#define __NR_name_to_handle_at 332 +#define __NR_open_by_handle_at 333 +#define __NR_clock_adjtime 334 +#define __NR_syncfs 335 +#define __NR_sendmmsg 336 +#define __NR_setns 337 +#define __NR_process_vm_readv 338 +#define __NR_process_vm_writev 339 +#define __NR_kern_features 340 +#define __NR_kcmp 341 +#define __NR_finit_module 342 +#define __NR_sched_setattr 343 +#define __NR_sched_getattr 344 +#define __NR_renameat2 345 + +#define NR_syscalls 346 + +/* Bitmask values returned from kern_features system call. */ +#define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 + +#ifdef __32bit_syscall_numbers__ +/* Sparc 32-bit only has the "setresuid32", "getresuid32" variants, + * it never had the plain ones and there is no value to adding those + * old versions into the syscall table. + */ +#define __IGNORE_setresuid +#define __IGNORE_getresuid +#define __IGNORE_setresgid +#define __IGNORE_getresgid +#endif + +#endif /* _UAPI_SPARC_UNISTD_H */ diff --git a/arch/sparc/include/asm/utrap.h b/arch/sparc/include/uapi/asm/utrap.h index b10e527c22d..b10e527c22d 100644 --- a/arch/sparc/include/asm/utrap.h +++ b/arch/sparc/include/uapi/asm/utrap.h diff --git a/arch/sparc/include/asm/watchdog.h b/arch/sparc/include/uapi/asm/watchdog.h index 5baf2d3919c..5baf2d3919c 100644 --- a/arch/sparc/include/asm/watchdog.h +++ b/arch/sparc/include/uapi/asm/watchdog.h diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index 599398fbbc7..7cf9c6ea3f1 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -1,3 +1,4 @@ + # # Makefile for the linux kernel. # @@ -6,7 +7,6 @@ asflags-y := -ansi ccflags-y := -Werror extra-y := head_$(BITS).o -extra-y += init_task.o # Undefine sparc when processing vmlinux.lds - it is used # And teach CPP we are doing $(BITS) builds (for this case) @@ -28,10 +28,11 @@ obj-y += traps_$(BITS).o # IRQ obj-y += irq_$(BITS).o -obj-$(CONFIG_SPARC32) += sun4m_irq.o sun4c_irq.o sun4d_irq.o +obj-$(CONFIG_SPARC32) += sun4m_irq.o sun4d_irq.o obj-y += process_$(BITS).o obj-y += signal_$(BITS).o +obj-y += sigutil_$(BITS).o obj-$(CONFIG_SPARC32) += ioport.o obj-y += setup_$(BITS).o obj-y += idprom.o @@ -41,19 +42,17 @@ obj-y += time_$(BITS).o obj-$(CONFIG_SPARC32) += windows.o obj-y += cpu.o obj-$(CONFIG_SPARC32) += devices.o -obj-$(CONFIG_SPARC32) += tadpole.o -obj-$(CONFIG_SPARC32) += tick14.o obj-y += ptrace_$(BITS).o obj-y += unaligned_$(BITS).o obj-y += una_asm_$(BITS).o -obj-$(CONFIG_SPARC32) += muldiv.o obj-y += prom_common.o obj-y += prom_$(BITS).o obj-y += of_device_common.o obj-y += of_device_$(BITS).o obj-$(CONFIG_SPARC64) += prom_irqtrans.o -obj-$(CONFIG_SPARC_LEON)+= leon_kernel.o +obj-$(CONFIG_SPARC32) += leon_kernel.o +obj-$(CONFIG_SPARC32) += leon_pmc.o obj-$(CONFIG_SPARC64) += reboot.o obj-$(CONFIG_SPARC64) += sysfs.o @@ -71,13 +70,12 @@ obj-$(CONFIG_SPARC64) += pcr.o obj-$(CONFIG_SPARC64) += nmi.o obj-$(CONFIG_SPARC64_SMP) += cpumap.o -# sparc32 do not use GENERIC_HARDIRQS but uses the generic devres implementation -obj-$(CONFIG_SPARC32) += devres.o -devres-y := ../../../kernel/irq/devres.o - obj-y += dma.o -obj-$(CONFIG_SPARC32_PCI) += pcic.o +obj-$(CONFIG_PCIC_PCI) += pcic.o +obj-$(CONFIG_LEON_PCI) += leon_pci.o +obj-$(CONFIG_SPARC_GRPCI2)+= leon_pci_grpci2.o +obj-$(CONFIG_SPARC_GRPCI1)+= leon_pci_grpci1.o obj-$(CONFIG_SMP) += trampoline_$(BITS).o smp_$(BITS).o obj-$(CONFIG_SPARC32_SMP) += sun4m_smp.o sun4d_smp.o leon_smp.o @@ -101,13 +99,10 @@ obj-$(CONFIG_STACKTRACE) += stacktrace.o obj-$(CONFIG_SPARC64_PCI) += pci.o pci_common.o psycho_common.o obj-$(CONFIG_SPARC64_PCI) += pci_psycho.o pci_sabre.o pci_schizo.o obj-$(CONFIG_SPARC64_PCI) += pci_sun4v.o pci_sun4v_asm.o pci_fire.o -obj-$(CONFIG_PCI_MSI) += pci_msi.o +obj-$(CONFIG_SPARC64_PCI_MSI) += pci_msi.o obj-$(CONFIG_COMPAT) += sys32.o sys_sparc32.o signal32.o -# sparc64 cpufreq -obj-$(CONFIG_US3_FREQ) += us3_cpufreq.o -obj-$(CONFIG_US2E_FREQ) += us2e_cpufreq.o obj-$(CONFIG_US3_MC) += chmc.o obj-$(CONFIG_KPROBES) += kprobes.o diff --git a/arch/sparc/kernel/apc.c b/arch/sparc/kernel/apc.c index 52de4a9424e..eefda32b595 100644 --- a/arch/sparc/kernel/apc.c +++ b/arch/sparc/kernel/apc.c @@ -13,12 +13,14 @@ #include <linux/pm.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/module.h> #include <asm/io.h> #include <asm/oplib.h> #include <asm/uaccess.h> #include <asm/auxio.h> #include <asm/apc.h> +#include <asm/processor.h> /* Debugging * @@ -30,7 +32,7 @@ #define APC_DEVNAME "apc" static u8 __iomem *regs; -static int apc_no_idle __devinitdata = 0; +static int apc_no_idle = 0; #define apc_readb(offs) (sbus_readb(regs+offs)) #define apc_writeb(val, offs) (sbus_writeb(val, regs+offs)) @@ -123,7 +125,7 @@ static long apc_ioctl(struct file *f, unsigned int cmd, unsigned long __arg) default: return -EINVAL; - }; + } return 0; } @@ -137,8 +139,7 @@ static const struct file_operations apc_fops = { static struct miscdevice apc_miscdev = { APC_MINOR, APC_DEVNAME, &apc_fops }; -static int __devinit apc_probe(struct platform_device *op, - const struct of_device_id *match) +static int apc_probe(struct platform_device *op) { int err; @@ -158,7 +159,7 @@ static int __devinit apc_probe(struct platform_device *op, /* Assign power management IDLE handler */ if (!apc_no_idle) - pm_idle = apc_swift_idle; + sparc_idle = apc_swift_idle; printk(KERN_INFO "%s: power management initialized%s\n", APC_DEVNAME, apc_no_idle ? " (CPU idle disabled)" : ""); @@ -166,7 +167,7 @@ static int __devinit apc_probe(struct platform_device *op, return 0; } -static struct of_device_id __initdata apc_match[] = { +static struct of_device_id apc_match[] = { { .name = APC_OBPNAME, }, @@ -174,7 +175,7 @@ static struct of_device_id __initdata apc_match[] = { }; MODULE_DEVICE_TABLE(of, apc_match); -static struct of_platform_driver apc_driver = { +static struct platform_driver apc_driver = { .driver = { .name = "apc", .owner = THIS_MODULE, @@ -185,7 +186,7 @@ static struct of_platform_driver apc_driver = { static int __init apc_init(void) { - return of_register_platform_driver(&apc_driver); + return platform_driver_register(&apc_driver); } /* This driver is not critical to the boot process diff --git a/arch/sparc/kernel/asm-offsets.c b/arch/sparc/kernel/asm-offsets.c index 68f7e1118e9..f76389a3234 100644 --- a/arch/sparc/kernel/asm-offsets.c +++ b/arch/sparc/kernel/asm-offsets.c @@ -14,6 +14,8 @@ // #include <linux/mm.h> #include <linux/kbuild.h> +#include <asm/hibernate.h> + #ifdef CONFIG_SPARC32 int sparc32_foo(void) { @@ -24,6 +26,19 @@ int sparc32_foo(void) #else int sparc64_foo(void) { +#ifdef CONFIG_HIBERNATION + BLANK(); + OFFSET(SC_REG_FP, saved_context, fp); + OFFSET(SC_REG_CWP, saved_context, cwp); + OFFSET(SC_REG_WSTATE, saved_context, wstate); + + OFFSET(SC_REG_TICK, saved_context, tick); + OFFSET(SC_REG_PSTATE, saved_context, pstate); + + OFFSET(SC_REG_G4, saved_context, g4); + OFFSET(SC_REG_G5, saved_context, g5); + OFFSET(SC_REG_G6, saved_context, g6); +#endif return 0; } #endif @@ -34,6 +49,8 @@ int foo(void) DEFINE(AOFF_task_thread, offsetof(struct task_struct, thread)); BLANK(); DEFINE(AOFF_mm_context, offsetof(struct mm_struct, context)); + BLANK(); + DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm)); /* DEFINE(NUM_USER_SEGMENTS, TASK_SIZE>>28); */ return 0; diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c index 8fff0ac63d5..24361b494a9 100644 --- a/arch/sparc/kernel/audit.c +++ b/arch/sparc/kernel/audit.c @@ -3,6 +3,8 @@ #include <linux/audit.h> #include <asm/unistd.h> +#include "kernel.h" + static unsigned dir_class[] = { #include <asm-generic/audit_dir_write.h> ~0U @@ -40,7 +42,6 @@ int audit_classify_arch(int arch) int audit_classify_syscall(int abi, unsigned syscall) { #ifdef CONFIG_COMPAT - extern int sparc32_classify_syscall(unsigned); if (abi == AUDIT_ARCH_SPARC) return sparc32_classify_syscall(syscall); #endif @@ -61,11 +62,6 @@ int audit_classify_syscall(int abi, unsigned syscall) static int __init audit_classes_init(void) { #ifdef CONFIG_COMPAT - extern __u32 sparc32_dir_class[]; - extern __u32 sparc32_write_class[]; - extern __u32 sparc32_read_class[]; - extern __u32 sparc32_chattr_class[]; - extern __u32 sparc32_signal_class[]; audit_register_class(AUDIT_CLASS_WRITE_32, sparc32_write_class); audit_register_class(AUDIT_CLASS_READ_32, sparc32_read_class); audit_register_class(AUDIT_CLASS_DIR_WRITE_32, sparc32_dir_class); diff --git a/arch/sparc/kernel/auxio_32.c b/arch/sparc/kernel/auxio_32.c index 8505e0ac78b..ae88c223e4d 100644 --- a/arch/sparc/kernel/auxio_32.c +++ b/arch/sparc/kernel/auxio_32.c @@ -8,10 +8,15 @@ #include <linux/spinlock.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/export.h> + #include <asm/oplib.h> #include <asm/io.h> #include <asm/auxio.h> #include <asm/string.h> /* memset(), Linux has no bzero() */ +#include <asm/cpu_type.h> + +#include "kernel.h" /* Probe and map in the Auxiliary I/O register */ @@ -30,7 +35,6 @@ void __init auxio_probe(void) switch (sparc_cpu_model) { case sparc_leon: case sun4d: - case sun4: return; default: break; @@ -63,9 +67,8 @@ void __init auxio_probe(void) r.start = auxregs[0].phys_addr; r.end = auxregs[0].phys_addr + auxregs[0].reg_size - 1; auxio_register = of_ioremap(&r, 0, auxregs[0].reg_size, "auxio"); - /* Fix the address on sun4m and sun4c. */ - if((((unsigned long) auxregs[0].phys_addr) & 3) == 3 || - sparc_cpu_model == sun4c) + /* Fix the address on sun4m. */ + if ((((unsigned long) auxregs[0].phys_addr) & 3) == 3) auxio_register += (3 - ((unsigned long)auxio_register & 3)); set_auxio(AUXIO_LED, 0); @@ -84,12 +87,7 @@ void set_auxio(unsigned char bits_on, unsigned char bits_off) unsigned char regval; unsigned long flags; spin_lock_irqsave(&auxio_lock, flags); - switch(sparc_cpu_model) { - case sun4c: - regval = sbus_readb(auxio_register); - sbus_writeb(((regval | bits_on) & ~bits_off) | AUXIO_ORMEIN, - auxio_register); - break; + switch (sparc_cpu_model) { case sun4m: if(!auxio_register) break; /* VME chassis sun4m, no auxio. */ @@ -101,14 +99,14 @@ void set_auxio(unsigned char bits_on, unsigned char bits_off) break; default: panic("Can't set AUXIO register on this machine."); - }; + } spin_unlock_irqrestore(&auxio_lock, flags); } EXPORT_SYMBOL(set_auxio); /* sun4m power control register (AUXIO2) */ -volatile unsigned char * auxio_power_register = NULL; +volatile u8 __iomem *auxio_power_register = NULL; void __init auxio_power_probe(void) { @@ -132,8 +130,8 @@ void __init auxio_power_probe(void) r.flags = regs.which_io & 0xF; r.start = regs.phys_addr; r.end = regs.phys_addr + regs.reg_size - 1; - auxio_power_register = (unsigned char *) of_ioremap(&r, 0, - regs.reg_size, "auxpower"); + auxio_power_register = + (u8 __iomem *)of_ioremap(&r, 0, regs.reg_size, "auxpower"); /* Display a quick message on the console. */ if (auxio_power_register) diff --git a/arch/sparc/kernel/auxio_64.c b/arch/sparc/kernel/auxio_64.c index 3efd3c5af6a..86e55778e4a 100644 --- a/arch/sparc/kernel/auxio_64.c +++ b/arch/sparc/kernel/auxio_64.c @@ -93,7 +93,7 @@ void auxio_set_lte(int on) } EXPORT_SYMBOL(auxio_set_lte); -static struct of_device_id __initdata auxio_match[] = { +static const struct of_device_id auxio_match[] = { { .name = "auxio", }, @@ -102,8 +102,7 @@ static struct of_device_id __initdata auxio_match[] = { MODULE_DEVICE_TABLE(of, auxio_match); -static int __devinit auxio_probe(struct platform_device *dev, - const struct of_device_id *match) +static int auxio_probe(struct platform_device *dev) { struct device_node *dp = dev->dev.of_node; unsigned long size; @@ -132,7 +131,7 @@ static int __devinit auxio_probe(struct platform_device *dev, return 0; } -static struct of_platform_driver auxio_driver = { +static struct platform_driver auxio_driver = { .probe = auxio_probe, .driver = { .name = "auxio", @@ -143,7 +142,7 @@ static struct of_platform_driver auxio_driver = { static int __init auxio_init(void) { - return of_register_platform_driver(&auxio_driver); + return platform_driver_register(&auxio_driver); } /* Must be after subsys_initcall() so that busses are probed. Must diff --git a/arch/sparc/kernel/btext.c b/arch/sparc/kernel/btext.c index 89aa4eb20cf..987f7ec497c 100644 --- a/arch/sparc/kernel/btext.c +++ b/arch/sparc/kernel/btext.c @@ -6,7 +6,6 @@ #include <linux/kernel.h> #include <linux/string.h> #include <linux/init.h> -#include <linux/module.h> #include <linux/console.h> #include <asm/btext.h> @@ -138,7 +137,7 @@ static void scrollscreen(void) } #endif /* ndef NO_SCROLL */ -void btext_drawchar(char c) +static void btext_drawchar(char c) { int cline = 0; #ifdef NO_SCROLL diff --git a/arch/sparc/kernel/central.c b/arch/sparc/kernel/central.c index cfa2624c533..052b5a44318 100644 --- a/arch/sparc/kernel/central.c +++ b/arch/sparc/kernel/central.c @@ -6,6 +6,7 @@ #include <linux/kernel.h> #include <linux/types.h> #include <linux/slab.h> +#include <linux/export.h> #include <linux/string.h> #include <linux/init.h> #include <linux/of_device.h> @@ -32,7 +33,7 @@ struct fhc { struct platform_device leds_pdev; }; -static int __devinit clock_board_calc_nslots(struct clock_board *p) +static int clock_board_calc_nslots(struct clock_board *p) { u8 reg = upa_readb(p->clock_regs + CLOCK_STAT1) & 0xc0; @@ -59,8 +60,7 @@ static int __devinit clock_board_calc_nslots(struct clock_board *p) } } -static int __devinit clock_board_probe(struct platform_device *op, - const struct of_device_id *match) +static int clock_board_probe(struct platform_device *op) { struct clock_board *p = kzalloc(sizeof(*p), GFP_KERNEL); int err = -ENOMEM; @@ -141,14 +141,14 @@ out_free: goto out; } -static struct of_device_id __initdata clock_board_match[] = { +static const struct of_device_id clock_board_match[] = { { .name = "clock-board", }, {}, }; -static struct of_platform_driver clock_board_driver = { +static struct platform_driver clock_board_driver = { .probe = clock_board_probe, .driver = { .name = "clock_board", @@ -157,8 +157,7 @@ static struct of_platform_driver clock_board_driver = { }, }; -static int __devinit fhc_probe(struct platform_device *op, - const struct of_device_id *match) +static int fhc_probe(struct platform_device *op) { struct fhc *p = kzalloc(sizeof(*p), GFP_KERNEL); int err = -ENOMEM; @@ -247,14 +246,14 @@ out_free: goto out; } -static struct of_device_id __initdata fhc_match[] = { +static const struct of_device_id fhc_match[] = { { .name = "fhc", }, {}, }; -static struct of_platform_driver fhc_driver = { +static struct platform_driver fhc_driver = { .probe = fhc_probe, .driver = { .name = "fhc", @@ -265,9 +264,9 @@ static struct of_platform_driver fhc_driver = { static int __init sunfire_init(void) { - (void) of_register_platform_driver(&fhc_driver); - (void) of_register_platform_driver(&clock_board_driver); + (void) platform_driver_register(&fhc_driver); + (void) platform_driver_register(&clock_board_driver); return 0; } -subsys_initcall(sunfire_init); +fs_initcall(sunfire_init); diff --git a/arch/sparc/kernel/chmc.c b/arch/sparc/kernel/chmc.c index 08c466ebb32..dbb210d74e2 100644 --- a/arch/sparc/kernel/chmc.c +++ b/arch/sparc/kernel/chmc.c @@ -336,9 +336,9 @@ static int jbusmc_print_dimm(int syndrome_code, return 0; } -static u64 __devinit jbusmc_dimm_group_size(u64 base, - const struct linux_prom64_registers *mem_regs, - int num_mem_regs) +static u64 jbusmc_dimm_group_size(u64 base, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) { u64 max = base + (8UL * 1024 * 1024 * 1024); u64 max_seen = base; @@ -363,10 +363,10 @@ static u64 __devinit jbusmc_dimm_group_size(u64 base, return max_seen - base; } -static void __devinit jbusmc_construct_one_dimm_group(struct jbusmc *p, - unsigned long index, - const struct linux_prom64_registers *mem_regs, - int num_mem_regs) +static void jbusmc_construct_one_dimm_group(struct jbusmc *p, + unsigned long index, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) { struct jbusmc_dimm_group *dp = &p->dimm_groups[index]; @@ -378,9 +378,9 @@ static void __devinit jbusmc_construct_one_dimm_group(struct jbusmc *p, dp->size = jbusmc_dimm_group_size(dp->base_addr, mem_regs, num_mem_regs); } -static void __devinit jbusmc_construct_dimm_groups(struct jbusmc *p, - const struct linux_prom64_registers *mem_regs, - int num_mem_regs) +static void jbusmc_construct_dimm_groups(struct jbusmc *p, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) { if (p->mc_reg_1 & JB_MC_REG1_DIMM1_BANK0) { jbusmc_construct_one_dimm_group(p, 0, mem_regs, num_mem_regs); @@ -392,8 +392,7 @@ static void __devinit jbusmc_construct_dimm_groups(struct jbusmc *p, } } -static int __devinit jbusmc_probe(struct platform_device *op, - const struct of_device_id *match) +static int jbusmc_probe(struct platform_device *op) { const struct linux_prom64_registers *mem_regs; struct device_node *mem_node; @@ -665,7 +664,7 @@ static void chmc_interpret_one_decode_reg(struct chmc *p, int which_bank, u64 va case 0x0: bp->interleave = 16; break; - }; + } /* UK[10] is reserved, and UK[11] is not set for the SDRAM * bank size definition. @@ -690,8 +689,7 @@ static void chmc_fetch_decode_regs(struct chmc *p) chmc_read_mcreg(p, CHMCTRL_DECODE4)); } -static int __devinit chmc_probe(struct platform_device *op, - const struct of_device_id *match) +static int chmc_probe(struct platform_device *op) { struct device_node *dp = op->dev.of_node; unsigned long ver; @@ -765,31 +763,30 @@ out_free: goto out; } -static int __devinit us3mc_probe(struct platform_device *op, - const struct of_device_id *match) +static int us3mc_probe(struct platform_device *op) { if (mc_type == MC_TYPE_SAFARI) - return chmc_probe(op, match); + return chmc_probe(op); else if (mc_type == MC_TYPE_JBUS) - return jbusmc_probe(op, match); + return jbusmc_probe(op); return -ENODEV; } -static void __devexit chmc_destroy(struct platform_device *op, struct chmc *p) +static void chmc_destroy(struct platform_device *op, struct chmc *p) { list_del(&p->list); of_iounmap(&op->resource[0], p->regs, 0x48); kfree(p); } -static void __devexit jbusmc_destroy(struct platform_device *op, struct jbusmc *p) +static void jbusmc_destroy(struct platform_device *op, struct jbusmc *p) { mc_list_del(&p->list); of_iounmap(&op->resource[0], p->regs, JBUSMC_REGS_SIZE); kfree(p); } -static int __devexit us3mc_remove(struct platform_device *op) +static int us3mc_remove(struct platform_device *op) { void *p = dev_get_drvdata(&op->dev); @@ -810,14 +807,14 @@ static const struct of_device_id us3mc_match[] = { }; MODULE_DEVICE_TABLE(of, us3mc_match); -static struct of_platform_driver us3mc_driver = { +static struct platform_driver us3mc_driver = { .driver = { .name = "us3mc", .owner = THIS_MODULE, .of_match_table = us3mc_match, }, .probe = us3mc_probe, - .remove = __devexit_p(us3mc_remove), + .remove = us3mc_remove, }; static inline bool us3mc_platform(void) @@ -848,7 +845,7 @@ static int __init us3mc_init(void) ret = register_dimm_printer(us3mc_dimm_printer); if (!ret) { - ret = of_register_platform_driver(&us3mc_driver); + ret = platform_driver_register(&us3mc_driver); if (ret) unregister_dimm_printer(us3mc_dimm_printer); } @@ -859,7 +856,7 @@ static void __exit us3mc_cleanup(void) { if (us3mc_platform()) { unregister_dimm_printer(us3mc_dimm_printer); - of_unregister_platform_driver(&us3mc_driver); + platform_driver_unregister(&us3mc_driver); } } diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c index d865575b25b..7062263d09c 100644 --- a/arch/sparc/kernel/compat_audit.c +++ b/arch/sparc/kernel/compat_audit.c @@ -1,5 +1,6 @@ #define __32bit_syscall_numbers__ #include <asm/unistd.h> +#include "kernel.h" unsigned sparc32_dir_class[] = { #include <asm-generic/audit_dir_write.h> diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c index 0dc714fa23d..82a3a71c451 100644 --- a/arch/sparc/kernel/cpu.c +++ b/arch/sparc/kernel/cpu.c @@ -4,14 +4,17 @@ * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/seq_file.h> #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/smp.h> #include <linux/threads.h> #include <asm/spitfire.h> +#include <asm/pgtable.h> #include <asm/oplib.h> +#include <asm/setup.h> #include <asm/page.h> #include <asm/head.h> #include <asm/psr.h> @@ -19,10 +22,14 @@ #include <asm/cpudata.h> #include "kernel.h" +#include "entry.h" DEFINE_PER_CPU(cpuinfo_sparc, __cpu_data) = { 0 }; EXPORT_PER_CPU_SYMBOL(__cpu_data); +int ncpus_probed; +unsigned int fsr_storage; + struct cpu_info { int psr_vers; const char *name; @@ -115,7 +122,7 @@ static const struct manufacturer_info __initconst manufacturer_info[] = { FPU(-1, NULL) } },{ - 4, + PSR_IMPL_TI, .cpu_info = { CPU(0, "Texas Instruments, Inc. - SuperSparc-(II)"), /* SparcClassic -- borned STP1010TAB-50*/ @@ -185,7 +192,7 @@ static const struct manufacturer_info __initconst manufacturer_info[] = { FPU(-1, NULL) } },{ - 0xF, /* Aeroflex Gaisler */ + PSR_IMPL_LEON, /* Aeroflex Gaisler */ .cpu_info = { CPU(3, "LEON"), CPU(-1, NULL) @@ -247,13 +254,12 @@ static const struct manufacturer_info __initconst manufacturer_info[] = { * machine type value into consideration too. I will fix this. */ -const char *sparc_cpu_type; -const char *sparc_fpu_type; +static const char *sparc_cpu_type; +static const char *sparc_fpu_type; const char *sparc_pmu_type; -unsigned int fsr_storage; -static void set_cpu_and_fpu(int psr_impl, int psr_vers, int fpu_vers) +static void __init set_cpu_and_fpu(int psr_impl, int psr_vers, int fpu_vers) { const struct manufacturer_info *manuf; int i; @@ -313,27 +319,148 @@ static void set_cpu_and_fpu(int psr_impl, int psr_vers, int fpu_vers) } #ifdef CONFIG_SPARC32 -void __cpuinit cpu_probe(void) +static int show_cpuinfo(struct seq_file *m, void *__unused) +{ + seq_printf(m, + "cpu\t\t: %s\n" + "fpu\t\t: %s\n" + "promlib\t\t: Version %d Revision %d\n" + "prom\t\t: %d.%d\n" + "type\t\t: %s\n" + "ncpus probed\t: %d\n" + "ncpus active\t: %d\n" +#ifndef CONFIG_SMP + "CPU0Bogo\t: %lu.%02lu\n" + "CPU0ClkTck\t: %ld\n" +#endif + , + sparc_cpu_type, + sparc_fpu_type , + romvec->pv_romvers, + prom_rev, + romvec->pv_printrev >> 16, + romvec->pv_printrev & 0xffff, + &cputypval[0], + ncpus_probed, + num_online_cpus() +#ifndef CONFIG_SMP + , cpu_data(0).udelay_val/(500000/HZ), + (cpu_data(0).udelay_val/(5000/HZ)) % 100, + cpu_data(0).clock_tick +#endif + ); + +#ifdef CONFIG_SMP + smp_bogo(m); +#endif + mmu_info(m); +#ifdef CONFIG_SMP + smp_info(m); +#endif + return 0; +} +#endif /* CONFIG_SPARC32 */ + +#ifdef CONFIG_SPARC64 +unsigned int dcache_parity_tl1_occurred; +unsigned int icache_parity_tl1_occurred; + + +static int show_cpuinfo(struct seq_file *m, void *__unused) +{ + seq_printf(m, + "cpu\t\t: %s\n" + "fpu\t\t: %s\n" + "pmu\t\t: %s\n" + "prom\t\t: %s\n" + "type\t\t: %s\n" + "ncpus probed\t: %d\n" + "ncpus active\t: %d\n" + "D$ parity tl1\t: %u\n" + "I$ parity tl1\t: %u\n" +#ifndef CONFIG_SMP + "Cpu0ClkTck\t: %016lx\n" +#endif + , + sparc_cpu_type, + sparc_fpu_type, + sparc_pmu_type, + prom_version, + ((tlb_type == hypervisor) ? + "sun4v" : + "sun4u"), + ncpus_probed, + num_online_cpus(), + dcache_parity_tl1_occurred, + icache_parity_tl1_occurred +#ifndef CONFIG_SMP + , cpu_data(0).clock_tick +#endif + ); + cpucap_info(m); +#ifdef CONFIG_SMP + smp_bogo(m); +#endif + mmu_info(m); +#ifdef CONFIG_SMP + smp_info(m); +#endif + return 0; +} +#endif /* CONFIG_SPARC64 */ + +static void *c_start(struct seq_file *m, loff_t *pos) +{ + /* The pointer we are returning is arbitrary, + * it just has to be non-NULL and not IS_ERR + * in the success case. + */ + return *pos == 0 ? &c_start : NULL; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + ++*pos; + return c_start(m, pos); +} + +static void c_stop(struct seq_file *m, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start =c_start, + .next = c_next, + .stop = c_stop, + .show = show_cpuinfo, +}; + +#ifdef CONFIG_SPARC32 +static int __init cpu_type_probe(void) { int psr_impl, psr_vers, fpu_vers; int psr; - psr_impl = ((get_psr() >> 28) & 0xf); - psr_vers = ((get_psr() >> 24) & 0xf); + psr_impl = ((get_psr() >> PSR_IMPL_SHIFT) & PSR_IMPL_SHIFTED_MASK); + psr_vers = ((get_psr() >> PSR_VERS_SHIFT) & PSR_VERS_SHIFTED_MASK); psr = get_psr(); put_psr(psr | PSR_EF); -#ifdef CONFIG_SPARC_LEON - fpu_vers = 7; -#else - fpu_vers = ((get_fsr() >> 17) & 0x7); -#endif + + if (psr_impl == PSR_IMPL_LEON) + fpu_vers = get_psr() & PSR_EF ? ((get_fsr() >> 17) & 0x7) : 7; + else + fpu_vers = ((get_fsr() >> 17) & 0x7); put_psr(psr); set_cpu_and_fpu(psr_impl, psr_vers, fpu_vers); + + return 0; } -#else +#endif /* CONFIG_SPARC32 */ + +#ifdef CONFIG_SPARC64 static void __init sun4v_cpu_probe(void) { switch (sun4v_chip_type) { @@ -349,11 +476,36 @@ static void __init sun4v_cpu_probe(void) sparc_pmu_type = "niagara2"; break; + case SUN4V_CHIP_NIAGARA3: + sparc_cpu_type = "UltraSparc T3 (Niagara3)"; + sparc_fpu_type = "UltraSparc T3 integrated FPU"; + sparc_pmu_type = "niagara3"; + break; + + case SUN4V_CHIP_NIAGARA4: + sparc_cpu_type = "UltraSparc T4 (Niagara4)"; + sparc_fpu_type = "UltraSparc T4 integrated FPU"; + sparc_pmu_type = "niagara4"; + break; + + case SUN4V_CHIP_NIAGARA5: + sparc_cpu_type = "UltraSparc T5 (Niagara5)"; + sparc_fpu_type = "UltraSparc T5 integrated FPU"; + sparc_pmu_type = "niagara5"; + break; + + case SUN4V_CHIP_SPARC64X: + sparc_cpu_type = "SPARC64-X"; + sparc_fpu_type = "SPARC64-X integrated FPU"; + sparc_pmu_type = "sparc64-x"; + break; + default: printk(KERN_WARNING "CPU: Unknown sun4v cpu type [%s]\n", prom_cpu_compatible); sparc_cpu_type = "Unknown SUN4V CPU"; sparc_fpu_type = "Unknown SUN4V FPU"; + sparc_pmu_type = "Unknown SUN4V PMU"; break; } } @@ -374,6 +526,6 @@ static int __init cpu_type_probe(void) } return 0; } +#endif /* CONFIG_SPARC64 */ early_initcall(cpu_type_probe); -#endif diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c index 8de64c8126b..de1c844dfab 100644 --- a/arch/sparc/kernel/cpumap.c +++ b/arch/sparc/kernel/cpumap.c @@ -3,10 +3,9 @@ * Copyright (C) 2009 Hong H. Pham <hong.pham@windriver.com> */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/kernel.h> -#include <linux/init.h> #include <linux/cpumask.h> #include <linux/spinlock.h> #include <asm/cpudata.h> @@ -202,7 +201,7 @@ static struct cpuinfo_tree *build_cpuinfo_tree(void) new_tree->total_nodes = n; memcpy(&new_tree->level, tmp_level, sizeof(tmp_level)); - prev_cpu = cpu = first_cpu(cpu_online_map); + prev_cpu = cpu = cpumask_first(cpu_online_mask); /* Initialize all levels in the tree with the first CPU */ for (level = CPUINFO_LVL_PROC; level >= CPUINFO_LVL_ROOT; level--) { @@ -324,6 +323,10 @@ static int iterate_cpu(struct cpuinfo_tree *t, unsigned int root_index) switch (sun4v_chip_type) { case SUN4V_CHIP_NIAGARA1: case SUN4V_CHIP_NIAGARA2: + case SUN4V_CHIP_NIAGARA3: + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC64X: rover_inc_table = niagara_iterate_method; break; default: @@ -381,7 +384,7 @@ static int simple_map_to_cpu(unsigned int index) } /* Impossible, since num_online_cpus() <= num_possible_cpus() */ - return first_cpu(cpu_online_map); + return cpumask_first(cpu_online_mask); } static int _map_to_cpu(unsigned int index) diff --git a/arch/sparc/kernel/cpumap.h b/arch/sparc/kernel/cpumap.h index e639880ab86..9dac398c434 100644 --- a/arch/sparc/kernel/cpumap.h +++ b/arch/sparc/kernel/cpumap.h @@ -2,8 +2,8 @@ #define _CPUMAP_H #ifdef CONFIG_SMP -extern void cpu_map_rebuild(void); -extern int map_to_cpu(unsigned int index); +void cpu_map_rebuild(void); +int map_to_cpu(unsigned int index); #define cpu_map_init() cpu_map_rebuild() #else #define cpu_map_init() do {} while (0) diff --git a/arch/sparc/kernel/devices.c b/arch/sparc/kernel/devices.c index d2eddd6647c..8d5d09f09ca 100644 --- a/arch/sparc/kernel/devices.c +++ b/arch/sparc/kernel/devices.c @@ -17,12 +17,11 @@ #include <asm/oplib.h> #include <asm/prom.h> #include <asm/smp.h> -#include <asm/system.h> #include <asm/cpudata.h> +#include <asm/cpu_type.h> +#include <asm/setup.h> -extern void cpu_probe(void); -extern void clock_stop_probe(void); /* tadpole.c */ -extern void sun4c_probe_memerr_reg(void); +#include "kernel.h" static char *cpu_mid_prop(void) { @@ -115,7 +114,7 @@ int cpu_get_hwmid(phandle prom_node) void __init device_scan(void) { - prom_printf("Booting Linux...\n"); + printk(KERN_NOTICE "Booting Linux...\n"); #ifndef CONFIG_SMP { @@ -133,15 +132,6 @@ void __init device_scan(void) } #endif /* !CONFIG_SMP */ - cpu_probe(); - { - extern void auxio_probe(void); - extern void auxio_power_probe(void); - auxio_probe(); - auxio_power_probe(); - } - clock_stop_probe(); - - if (ARCH_SUN4C) - sun4c_probe_memerr_reg(); + auxio_probe(); + auxio_power_probe(); } diff --git a/arch/sparc/kernel/dma.c b/arch/sparc/kernel/dma.c index e1ba8ee21b9..b667aa6f28f 100644 --- a/arch/sparc/kernel/dma.c +++ b/arch/sparc/kernel/dma.c @@ -1,5 +1,4 @@ #include <linux/kernel.h> -#include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/dma-debug.h> diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c index 4a700f4b79c..dff60abbea0 100644 --- a/arch/sparc/kernel/ds.c +++ b/arch/sparc/kernel/ds.c @@ -15,18 +15,21 @@ #include <linux/reboot.h> #include <linux/cpu.h> +#include <asm/hypervisor.h> #include <asm/ldc.h> #include <asm/vio.h> #include <asm/mdesc.h> #include <asm/head.h> #include <asm/irq.h> +#include "kernel.h" + #define DRV_MODULE_NAME "ds" #define PFX DRV_MODULE_NAME ": " #define DRV_MODULE_VERSION "1.0" #define DRV_MODULE_RELDATE "Jul 11, 2007" -static char version[] __devinitdata = +static char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); MODULE_DESCRIPTION("Sun LDOM domain services driver"); @@ -497,7 +500,7 @@ static void dr_cpu_init_response(struct ds_data *resp, u64 req_num, tag->num_records = ncpus; i = 0; - for_each_cpu_mask(cpu, *mask) { + for_each_cpu(cpu, mask) { ent[i].cpu = cpu; ent[i].result = DR_CPU_RES_OK; ent[i].stat = default_stat; @@ -525,16 +528,14 @@ static void dr_cpu_mark(struct ds_data *resp, int cpu, int ncpus, } } -static int __cpuinit dr_cpu_configure(struct ds_info *dp, - struct ds_cap_state *cp, - u64 req_num, - cpumask_t *mask) +static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, + u64 req_num, cpumask_t *mask) { struct ds_data *resp; int resp_len, ncpus, cpu; unsigned long flags; - ncpus = cpus_weight(*mask); + ncpus = cpumask_weight(mask); resp_len = dr_cpu_size_response(ncpus); resp = kzalloc(resp_len, GFP_KERNEL); if (!resp) @@ -547,7 +548,7 @@ static int __cpuinit dr_cpu_configure(struct ds_info *dp, mdesc_populate_present_mask(mask); mdesc_fill_in_cpu_data(mask); - for_each_cpu_mask(cpu, *mask) { + for_each_cpu(cpu, mask) { int err; printk(KERN_INFO "ds-%llu: Starting cpu %d...\n", @@ -593,7 +594,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, int resp_len, ncpus, cpu; unsigned long flags; - ncpus = cpus_weight(*mask); + ncpus = cpumask_weight(mask); resp_len = dr_cpu_size_response(ncpus); resp = kzalloc(resp_len, GFP_KERNEL); if (!resp) @@ -603,7 +604,7 @@ static int dr_cpu_unconfigure(struct ds_info *dp, resp_len, ncpus, mask, DR_CPU_STAT_UNCONFIGURED); - for_each_cpu_mask(cpu, *mask) { + for_each_cpu(cpu, mask) { int err; printk(KERN_INFO "ds-%llu: Shutting down cpu %d...\n", @@ -624,9 +625,8 @@ static int dr_cpu_unconfigure(struct ds_info *dp, return 0; } -static void __cpuinit dr_cpu_data(struct ds_info *dp, - struct ds_cap_state *cp, - void *buf, int len) +static void dr_cpu_data(struct ds_info *dp, struct ds_cap_state *cp, void *buf, + int len) { struct ds_data *data = buf; struct dr_cpu_tag *tag = (struct dr_cpu_tag *) (data + 1); @@ -649,13 +649,13 @@ static void __cpuinit dr_cpu_data(struct ds_info *dp, purge_dups(cpu_list, tag->num_records); - cpus_clear(mask); + cpumask_clear(&mask); for (i = 0; i < tag->num_records; i++) { if (cpu_list[i] == CPU_SENTINEL) continue; if (cpu_list[i] < nr_cpu_ids) - cpu_set(cpu_list[i], mask); + cpumask_set_cpu(cpu_list[i], &mask); } if (tag->type == DR_CPU_CONFIGURE) @@ -780,6 +780,16 @@ void ldom_set_var(const char *var, const char *value) char *base, *p; int msg_len, loops; + if (strlen(var) + strlen(value) + 2 > + sizeof(pkt) - sizeof(pkt.header)) { + printk(KERN_ERR PFX + "contents length: %zu, which more than max: %lu," + "so could not set (%s) variable to (%s).\n", + strlen(var) + strlen(value) + 2, + sizeof(pkt) - sizeof(pkt.header), var, value); + return; + } + memset(&pkt, 0, sizeof(pkt)); pkt.header.data.tag.type = DS_DATA; pkt.header.data.handle = cp->handle; @@ -828,18 +838,32 @@ void ldom_set_var(const char *var, const char *value) } } +static char full_boot_str[256] __attribute__((aligned(32))); +static int reboot_data_supported; + void ldom_reboot(const char *boot_command) { /* Don't bother with any of this if the boot_command * is empty. */ if (boot_command && strlen(boot_command)) { - char full_boot_str[256]; + unsigned long len; + + snprintf(full_boot_str, sizeof(full_boot_str), "boot %s", + boot_command); + len = strlen(full_boot_str); - strcpy(full_boot_str, "boot "); - strcpy(full_boot_str + strlen("boot "), boot_command); + if (reboot_data_supported) { + unsigned long ra = kimage_addr_to_ra(full_boot_str); + unsigned long hv_ret; - ldom_set_var("reboot-command", full_boot_str); + hv_ret = sun4v_reboot_data_set(ra, len); + if (hv_ret != HV_EOK) + pr_err("SUN4V: Unable to set reboot data " + "hv_ret=%lu\n", hv_ret); + } else { + ldom_set_var("reboot-command", full_boot_str); + } } sun4v_mach_sir(); } @@ -851,7 +875,7 @@ void ldom_power_off(void) static void ds_conn_reset(struct ds_info *dp) { - printk(KERN_ERR "ds-%llu: ds_conn_reset() from %p\n", + printk(KERN_ERR "ds-%llu: ds_conn_reset() from %pf\n", dp->id, __builtin_return_address(0)); } @@ -1129,8 +1153,7 @@ static void ds_event(void *arg, int event) spin_unlock_irqrestore(&ds_lock, flags); } -static int __devinit ds_probe(struct vio_dev *vdev, - const struct vio_device_id *id) +static int ds_probe(struct vio_dev *vdev, const struct vio_device_id *id) { static int ds_version_printed; struct ldc_channel_config ds_cfg = { @@ -1164,13 +1187,11 @@ static int __devinit ds_probe(struct vio_dev *vdev, dp->rcv_buf_len = 4096; - dp->ds_states = kzalloc(sizeof(ds_states_template), - GFP_KERNEL); + dp->ds_states = kmemdup(ds_states_template, + sizeof(ds_states_template), GFP_KERNEL); if (!dp->ds_states) goto out_free_rcv_buf; - memcpy(dp->ds_states, ds_states_template, - sizeof(ds_states_template)); dp->num_ds_states = ARRAY_SIZE(ds_states_template); for (i = 0; i < dp->num_ds_states; i++) @@ -1218,7 +1239,7 @@ static int ds_remove(struct vio_dev *vdev) return 0; } -static struct vio_device_id __initdata ds_match[] = { +static const struct vio_device_id ds_match[] = { { .type = "domain-services-port", }, @@ -1229,17 +1250,24 @@ static struct vio_driver ds_driver = { .id_table = ds_match, .probe = ds_probe, .remove = ds_remove, - .driver = { - .name = "ds", - .owner = THIS_MODULE, - } + .name = "ds", }; static int __init ds_init(void) { + unsigned long hv_ret, major, minor; + + if (tlb_type == hypervisor) { + hv_ret = sun4v_get_version(HV_GRP_REBOOT_DATA, &major, &minor); + if (hv_ret == HV_EOK) { + pr_info("SUN4V: Reboot data supported (maj=%lu,min=%lu).\n", + major, minor); + reboot_data_supported = 1; + } + } kthread_run(ds_thread, NULL, "kldomd"); return vio_register_driver(&ds_driver); } -subsys_initcall(ds_init); +fs_initcall(ds_init); diff --git a/arch/sparc/kernel/ebus.c b/arch/sparc/kernel/ebus.c index 77dbf6d45fa..acf8314cec4 100644 --- a/arch/sparc/kernel/ebus.c +++ b/arch/sparc/kernel/ebus.c @@ -4,10 +4,9 @@ * Copyright (C) 1999 David S. Miller (davem@redhat.com) */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/types.h> -#include <linux/init.h> #include <linux/interrupt.h> #include <linux/delay.h> diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S index 1504df8ddf7..33c02b15f47 100644 --- a/arch/sparc/kernel/entry.S +++ b/arch/sparc/kernel/entry.S @@ -7,6 +7,7 @@ * Copyright (C) 1997 Anton Blanchard (anton@progsoc.uts.edu.au) */ +#include <linux/linkage.h> #include <linux/errno.h> #include <asm/head.h> @@ -17,10 +18,8 @@ #include <asm/asm-offsets.h> #include <asm/psr.h> #include <asm/vaddrs.h> -#include <asm/memreg.h> #include <asm/page.h> #include <asm/pgtable.h> -#include <asm/pgtsun4c.h> #include <asm/winmacro.h> #include <asm/signal.h> #include <asm/obio.h> @@ -125,22 +124,11 @@ floppy_tdone: set auxio_register, %l7 ld [%l7], %l7 - set sparc_cpu_model, %l5 - ld [%l5], %l5 - subcc %l5, 1, %g0 /* enum { sun4c = 1 }; */ - be 1f - ldub [%l7], %l5 + ldub [%l7], %l5 or %l5, 0xc2, %l5 stb %l5, [%l7] andn %l5, 0x02, %l5 - b 2f - nop - -1: - or %l5, 0xf4, %l5 - stb %l5, [%l7] - andn %l5, 0x04, %l5 2: /* Kill some time so the bits set */ @@ -229,7 +217,7 @@ real_irq_entry: #ifdef CONFIG_SMP .globl patchme_maybe_smp_msg - cmp %l7, 12 + cmp %l7, 11 patchme_maybe_smp_msg: bgu maybe_smp4m_msg nop @@ -266,22 +254,30 @@ smp4m_ticker: WRITE_PAUSE RESTORE_ALL +#define GET_PROCESSOR4M_ID(reg) \ + rd %tbr, %reg; \ + srl %reg, 12, %reg; \ + and %reg, 3, %reg; + /* Here is where we check for possible SMP IPI passed to us * on some level other than 15 which is the NMI and only used * for cross calls. That has a separate entry point below. + * + * IPIs are sent on Level 12, 13 and 14. See IRQ_IPI_*. */ maybe_smp4m_msg: GET_PROCESSOR4M_ID(o3) sethi %hi(sun4m_irq_percpu), %l5 sll %o3, 2, %o3 or %l5, %lo(sun4m_irq_percpu), %o5 - sethi %hi(0x40000000), %o2 + sethi %hi(0x70000000), %o2 ! Check all soft-IRQs ld [%o5 + %o3], %o1 ld [%o1 + 0x00], %o3 ! sun4m_irq_percpu[cpu]->pending andcc %o3, %o2, %g0 be,a smp4m_ticker cmp %l7, 14 - st %o2, [%o1 + 0x04] ! sun4m_irq_percpu[cpu]->clear=0x40000000 + /* Soft-IRQ IPI */ + st %o2, [%o1 + 0x04] ! sun4m_irq_percpu[cpu]->clear=0x70000000 WRITE_PAUSE ld [%o1 + 0x00], %g0 ! sun4m_irq_percpu[cpu]->pending WRITE_PAUSE @@ -290,9 +286,27 @@ maybe_smp4m_msg: WRITE_PAUSE wr %l4, PSR_ET, %psr WRITE_PAUSE - call smp_reschedule_irq + srl %o3, 28, %o2 ! shift for simpler checks below +maybe_smp4m_msg_check_single: + andcc %o2, 0x1, %g0 + beq,a maybe_smp4m_msg_check_mask + andcc %o2, 0x2, %g0 + call smp_call_function_single_interrupt nop - + andcc %o2, 0x2, %g0 +maybe_smp4m_msg_check_mask: + beq,a maybe_smp4m_msg_check_resched + andcc %o2, 0x4, %g0 + call smp_call_function_interrupt + nop + andcc %o2, 0x4, %g0 +maybe_smp4m_msg_check_resched: + /* rescheduling is done in RESTORE_ALL regardless, but incr stats */ + beq,a maybe_smp4m_msg_out + nop + call smp_resched_interrupt + nop +maybe_smp4m_msg_out: RESTORE_ALL .align 4 @@ -307,7 +321,7 @@ linux_trap_ipi15_sun4m: ld [%o5 + %o0], %o5 ld [%o5 + 0x00], %o3 ! sun4m_irq_percpu[cpu]->pending andcc %o3, %o2, %g0 - be 1f ! Must be an NMI async memory error + be sun4m_nmi_error ! Must be an NMI async memory error st %o2, [%o5 + 0x04] ! sun4m_irq_percpu[cpu]->clear=0x80000000 WRITE_PAUSE ld [%o5 + 0x00], %g0 ! sun4m_irq_percpu[cpu]->pending @@ -321,27 +335,6 @@ linux_trap_ipi15_sun4m: nop b ret_trap_lockless_ipi clr %l6 -1: - /* NMI async memory error handling. */ - sethi %hi(0x80000000), %l4 - sethi %hi(sun4m_irq_global), %o5 - ld [%o5 + %lo(sun4m_irq_global)], %l5 - st %l4, [%l5 + 0x0c] ! sun4m_irq_global->mask_set=0x80000000 - WRITE_PAUSE - ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending - WRITE_PAUSE - or %l0, PSR_PIL, %l4 - wr %l4, 0x0, %psr - WRITE_PAUSE - wr %l4, PSR_ET, %psr - WRITE_PAUSE - call sun4m_nmi - nop - st %l4, [%l5 + 0x08] ! sun4m_irq_global->mask_clear=0x80000000 - WRITE_PAUSE - ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending - WRITE_PAUSE - RESTORE_ALL .globl smp4d_ticker /* SMP per-cpu ticker interrupts are handled specially. */ @@ -400,19 +393,18 @@ linux_trap_ipi15_sun4d: /* FIXME */ 1: b,a 1b -#ifdef CONFIG_SPARC_LEON - - .globl smpleon_ticker - /* SMP per-cpu ticker interrupts are handled specially. */ -smpleon_ticker: + .globl smpleon_ipi + .extern leon_ipi_interrupt + /* SMP per-cpu IPI interrupts are handled specially. */ +smpleon_ipi: SAVE_ALL or %l0, PSR_PIL, %g2 wr %g2, 0x0, %psr WRITE_PAUSE wr %g2, PSR_ET, %psr WRITE_PAUSE - call leon_percpu_timer_interrupt - add %sp, STACKFRAME_SZ, %o0 + call leonsmp_ipi_interrupt + add %sp, STACKFRAME_SZ, %o1 ! pt_regs wr %l0, PSR_ET, %psr WRITE_PAUSE RESTORE_ALL @@ -431,8 +423,6 @@ linux_trap_ipi15_leon: b ret_trap_lockless_ipi clr %l6 -#endif /* CONFIG_SPARC_LEON */ - #endif /* CONFIG_SMP */ /* This routine handles illegal instructions and privileged @@ -739,326 +729,37 @@ setcc_trap_handler: jmp %l2 ! advance over trap instruction rett %l2 + 0x4 ! like this... - .align 4 - .globl linux_trap_nmi_sun4c -linux_trap_nmi_sun4c: - SAVE_ALL - - /* Ugh, we need to clear the IRQ line. This is now - * a very sun4c specific trap handler... - */ - sethi %hi(interrupt_enable), %l5 - ld [%l5 + %lo(interrupt_enable)], %l5 - ldub [%l5], %l6 - andn %l6, INTS_ENAB, %l6 - stb %l6, [%l5] - - /* Now it is safe to re-enable traps without recursion. */ - or %l0, PSR_PIL, %l0 - wr %l0, PSR_ET, %psr +sun4m_nmi_error: + /* NMI async memory error handling. */ + sethi %hi(0x80000000), %l4 + sethi %hi(sun4m_irq_global), %o5 + ld [%o5 + %lo(sun4m_irq_global)], %l5 + st %l4, [%l5 + 0x0c] ! sun4m_irq_global->mask_set=0x80000000 WRITE_PAUSE - - /* Now call the c-code with the pt_regs frame ptr and the - * memory error registers as arguments. The ordering chosen - * here is due to unlatching semantics. - */ - sethi %hi(AC_SYNC_ERR), %o0 - add %o0, 0x4, %o0 - lda [%o0] ASI_CONTROL, %o2 ! sync vaddr - sub %o0, 0x4, %o0 - lda [%o0] ASI_CONTROL, %o1 ! sync error - add %o0, 0xc, %o0 - lda [%o0] ASI_CONTROL, %o4 ! async vaddr - sub %o0, 0x4, %o0 - lda [%o0] ASI_CONTROL, %o3 ! async error - call sparc_lvl15_nmi - add %sp, STACKFRAME_SZ, %o0 - - RESTORE_ALL - - .align 4 - .globl invalid_segment_patch1_ff - .globl invalid_segment_patch2_ff -invalid_segment_patch1_ff: cmp %l4, 0xff -invalid_segment_patch2_ff: mov 0xff, %l3 - - .align 4 - .globl invalid_segment_patch1_1ff - .globl invalid_segment_patch2_1ff -invalid_segment_patch1_1ff: cmp %l4, 0x1ff -invalid_segment_patch2_1ff: mov 0x1ff, %l3 - - .align 4 - .globl num_context_patch1_16, num_context_patch2_16 -num_context_patch1_16: mov 0x10, %l7 -num_context_patch2_16: mov 0x10, %l7 - - .align 4 - .globl vac_linesize_patch_32 -vac_linesize_patch_32: subcc %l7, 32, %l7 - - .align 4 - .globl vac_hwflush_patch1_on, vac_hwflush_patch2_on - -/* - * Ugly, but we cant use hardware flushing on the sun4 and we'd require - * two instructions (Anton) - */ -vac_hwflush_patch1_on: addcc %l7, -PAGE_SIZE, %l7 - -vac_hwflush_patch2_on: sta %g0, [%l3 + %l7] ASI_HWFLUSHSEG - - .globl invalid_segment_patch1, invalid_segment_patch2 - .globl num_context_patch1 - .globl vac_linesize_patch, vac_hwflush_patch1 - .globl vac_hwflush_patch2 - - .align 4 - .globl sun4c_fault - -! %l0 = %psr -! %l1 = %pc -! %l2 = %npc -! %l3 = %wim -! %l7 = 1 for textfault -! We want error in %l5, vaddr in %l6 -sun4c_fault: - sethi %hi(AC_SYNC_ERR), %l4 - add %l4, 0x4, %l6 ! AC_SYNC_VA in %l6 - lda [%l6] ASI_CONTROL, %l5 ! Address - lda [%l4] ASI_CONTROL, %l6 ! Error, retained for a bit - - andn %l5, 0xfff, %l5 ! Encode all info into l7 - srl %l6, 14, %l4 - - and %l4, 2, %l4 - or %l5, %l4, %l4 - - or %l4, %l7, %l7 ! l7 = [addr,write,txtfault] - - andcc %l0, PSR_PS, %g0 - be sun4c_fault_fromuser - andcc %l7, 1, %g0 ! Text fault? - - be 1f - sethi %hi(KERNBASE), %l4 - - mov %l1, %l5 ! PC - -1: - cmp %l5, %l4 - blu sun4c_fault_fromuser - sethi %hi(~((1 << SUN4C_REAL_PGDIR_SHIFT) - 1)), %l4 - - /* If the kernel references a bum kernel pointer, or a pte which - * points to a non existant page in ram, we will run this code - * _forever_ and lock up the machine!!!!! So we must check for - * this condition, the AC_SYNC_ERR bits are what we must examine. - * Also a parity error would make this happen as well. So we just - * check that we are in fact servicing a tlb miss and not some - * other type of fault for the kernel. - */ - andcc %l6, 0x80, %g0 - be sun4c_fault_fromuser - and %l5, %l4, %l5 - - /* Test for NULL pte_t * in vmalloc area. */ - sethi %hi(VMALLOC_START), %l4 - cmp %l5, %l4 - blu,a invalid_segment_patch1 - lduXa [%l5] ASI_SEGMAP, %l4 - - sethi %hi(swapper_pg_dir), %l4 - srl %l5, SUN4C_PGDIR_SHIFT, %l6 - or %l4, %lo(swapper_pg_dir), %l4 - sll %l6, 2, %l6 - ld [%l4 + %l6], %l4 - andcc %l4, PAGE_MASK, %g0 - be sun4c_fault_fromuser - lduXa [%l5] ASI_SEGMAP, %l4 - -invalid_segment_patch1: - cmp %l4, 0x7f - bne 1f - sethi %hi(sun4c_kfree_ring), %l4 - or %l4, %lo(sun4c_kfree_ring), %l4 - ld [%l4 + 0x18], %l3 - deccc %l3 ! do we have a free entry? - bcs,a 2f ! no, unmap one. - sethi %hi(sun4c_kernel_ring), %l4 - - st %l3, [%l4 + 0x18] ! sun4c_kfree_ring.num_entries-- - - ld [%l4 + 0x00], %l6 ! entry = sun4c_kfree_ring.ringhd.next - st %l5, [%l6 + 0x08] ! entry->vaddr = address - - ld [%l6 + 0x00], %l3 ! next = entry->next - ld [%l6 + 0x04], %l7 ! entry->prev - - st %l7, [%l3 + 0x04] ! next->prev = entry->prev - st %l3, [%l7 + 0x00] ! entry->prev->next = next - - sethi %hi(sun4c_kernel_ring), %l4 - or %l4, %lo(sun4c_kernel_ring), %l4 - ! head = &sun4c_kernel_ring.ringhd - - ld [%l4 + 0x00], %l7 ! head->next - - st %l4, [%l6 + 0x04] ! entry->prev = head - st %l7, [%l6 + 0x00] ! entry->next = head->next - st %l6, [%l7 + 0x04] ! head->next->prev = entry - - st %l6, [%l4 + 0x00] ! head->next = entry - - ld [%l4 + 0x18], %l3 - inc %l3 ! sun4c_kernel_ring.num_entries++ - st %l3, [%l4 + 0x18] - b 4f - ld [%l6 + 0x08], %l5 - -2: - or %l4, %lo(sun4c_kernel_ring), %l4 - ! head = &sun4c_kernel_ring.ringhd - - ld [%l4 + 0x04], %l6 ! entry = head->prev - - ld [%l6 + 0x08], %l3 ! tmp = entry->vaddr - - ! Flush segment from the cache. - sethi %hi((64 * 1024)), %l7 -9: -vac_hwflush_patch1: -vac_linesize_patch: - subcc %l7, 16, %l7 - bne 9b -vac_hwflush_patch2: - sta %g0, [%l3 + %l7] ASI_FLUSHSEG - - st %l5, [%l6 + 0x08] ! entry->vaddr = address - - ld [%l6 + 0x00], %l5 ! next = entry->next - ld [%l6 + 0x04], %l7 ! entry->prev - - st %l7, [%l5 + 0x04] ! next->prev = entry->prev - st %l5, [%l7 + 0x00] ! entry->prev->next = next - st %l4, [%l6 + 0x04] ! entry->prev = head - - ld [%l4 + 0x00], %l7 ! head->next - - st %l7, [%l6 + 0x00] ! entry->next = head->next - st %l6, [%l7 + 0x04] ! head->next->prev = entry - st %l6, [%l4 + 0x00] ! head->next = entry - - mov %l3, %l5 ! address = tmp - -4: -num_context_patch1: - mov 0x08, %l7 - - ld [%l6 + 0x08], %l4 - ldub [%l6 + 0x0c], %l3 - or %l4, %l3, %l4 ! encode new vaddr/pseg into l4 - - sethi %hi(AC_CONTEXT), %l3 - lduba [%l3] ASI_CONTROL, %l6 - - /* Invalidate old mapping, instantiate new mapping, - * for each context. Registers l6/l7 are live across - * this loop. - */ -3: deccc %l7 - sethi %hi(AC_CONTEXT), %l3 - stba %l7, [%l3] ASI_CONTROL -invalid_segment_patch2: - mov 0x7f, %l3 - stXa %l3, [%l5] ASI_SEGMAP - andn %l4, 0x1ff, %l3 - bne 3b - stXa %l4, [%l3] ASI_SEGMAP - - sethi %hi(AC_CONTEXT), %l3 - stba %l6, [%l3] ASI_CONTROL - - andn %l4, 0x1ff, %l5 - -1: - sethi %hi(VMALLOC_START), %l4 - cmp %l5, %l4 - - bgeu 1f - mov 1 << (SUN4C_REAL_PGDIR_SHIFT - PAGE_SHIFT), %l7 - - sethi %hi(KERNBASE), %l6 - - sub %l5, %l6, %l4 - srl %l4, PAGE_SHIFT, %l4 - sethi %hi((SUN4C_PAGE_KERNEL & 0xf4000000)), %l3 - or %l3, %l4, %l3 - - sethi %hi(PAGE_SIZE), %l4 - -2: - sta %l3, [%l5] ASI_PTE - deccc %l7 - inc %l3 - bne 2b - add %l5, %l4, %l5 - - b 7f - sethi %hi(sun4c_kernel_faults), %l4 - -1: - srl %l5, SUN4C_PGDIR_SHIFT, %l3 - sethi %hi(swapper_pg_dir), %l4 - or %l4, %lo(swapper_pg_dir), %l4 - sll %l3, 2, %l3 - ld [%l4 + %l3], %l4 - and %l4, PAGE_MASK, %l4 - - srl %l5, (PAGE_SHIFT - 2), %l6 - and %l6, ((SUN4C_PTRS_PER_PTE - 1) << 2), %l6 - add %l6, %l4, %l6 - - sethi %hi(PAGE_SIZE), %l4 - -2: - ld [%l6], %l3 - deccc %l7 - sta %l3, [%l5] ASI_PTE - add %l6, 0x4, %l6 - bne 2b - add %l5, %l4, %l5 - - sethi %hi(sun4c_kernel_faults), %l4 -7: - ld [%l4 + %lo(sun4c_kernel_faults)], %l3 - inc %l3 - st %l3, [%l4 + %lo(sun4c_kernel_faults)] - - /* Restore condition codes */ - wr %l0, 0x0, %psr + ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending WRITE_PAUSE - jmp %l1 - rett %l2 - -sun4c_fault_fromuser: - SAVE_ALL + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + call sun4m_nmi nop - - mov %l7, %o1 ! Decode the info from %l7 - mov %l7, %o2 - and %o1, 1, %o1 ! arg2 = text_faultp - mov %l7, %o3 - and %o2, 2, %o2 ! arg3 = writep - andn %o3, 0xfff, %o3 ! arg4 = faulting address - - wr %l0, PSR_ET, %psr + st %l4, [%l5 + 0x08] ! sun4m_irq_global->mask_clear=0x80000000 + WRITE_PAUSE + ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending WRITE_PAUSE + RESTORE_ALL - call do_sun4c_fault - add %sp, STACKFRAME_SZ, %o0 ! arg1 = pt_regs ptr +#ifndef CONFIG_SMP + .align 4 + .globl linux_trap_ipi15_sun4m +linux_trap_ipi15_sun4m: + SAVE_ALL - RESTORE_ALL + ba sun4m_nmi_error + nop +#endif /* CONFIG_SMP */ .align 4 .globl srmmu_fault @@ -1066,8 +767,11 @@ srmmu_fault: mov 0x400, %l5 mov 0x300, %l4 - lda [%l5] ASI_M_MMUREGS, %l6 ! read sfar first - lda [%l4] ASI_M_MMUREGS, %l5 ! read sfsr last +LEON_PI(lda [%l5] ASI_LEON_MMUREGS, %l6) ! read sfar first +SUN_PI_(lda [%l5] ASI_M_MMUREGS, %l6) ! read sfar first + +LEON_PI(lda [%l4] ASI_LEON_MMUREGS, %l5) ! read sfsr last +SUN_PI_(lda [%l4] ASI_M_MMUREGS, %l5) ! read sfsr last andn %l6, 0xfff, %l6 srl %l5, 6, %l5 ! and encode all info into l7 @@ -1102,23 +806,10 @@ sys_nis_syscall: call c_sys_nis_syscall mov %l5, %o7 - .align 4 - .globl sys_execve -sys_execve: - mov %o7, %l5 - add %sp, STACKFRAME_SZ, %o0 ! pt_regs *regs arg - call sparc_execve - mov %l5, %o7 - - .globl sunos_execv sunos_execv: - st %g0, [%sp + STACKFRAME_SZ + PT_I2] - - call sparc_execve - add %sp, STACKFRAME_SZ, %o0 - - b ret_sys_call - ld [%sp + STACKFRAME_SZ + PT_I0], %o0 + .globl sunos_execv + b sys_execve + clr %i2 .align 4 .globl sys_sparc_pipe @@ -1129,14 +820,6 @@ sys_sparc_pipe: mov %l5, %o7 .align 4 - .globl sys_sigaltstack -sys_sigaltstack: - mov %o7, %l5 - mov %fp, %o2 - call do_sigaltstack - mov %l5, %o7 - - .align 4 .globl sys_sigstack sys_sigstack: mov %o7, %l5 @@ -1156,7 +839,7 @@ sys_sigreturn: nop call syscall_trace - nop + mov 1, %o1 1: /* We don't want to muck with user registers like a @@ -1255,17 +938,9 @@ flush_patch_four: .align 4 linux_sparc_ni_syscall: sethi %hi(sys_ni_syscall), %l7 - b syscall_is_too_hard + b do_syscall or %l7, %lo(sys_ni_syscall), %l7 -linux_fast_syscall: - andn %l7, 3, %l7 - mov %i0, %o0 - mov %i1, %o1 - mov %i2, %o2 - jmpl %l7 + %g0, %g0 - mov %i3, %o3 - linux_syscall_trace: add %sp, STACKFRAME_SZ, %o0 call syscall_trace @@ -1283,10 +958,27 @@ linux_syscall_trace: .globl ret_from_fork ret_from_fork: call schedule_tail - mov %g3, %o0 + ld [%g3 + TI_TASK], %o0 b ret_sys_call ld [%sp + STACKFRAME_SZ + PT_I0], %o0 + .globl ret_from_kernel_thread +ret_from_kernel_thread: + call schedule_tail + ld [%g3 + TI_TASK], %o0 + ld [%sp + STACKFRAME_SZ + PT_G1], %l0 + call %l0 + ld [%sp + STACKFRAME_SZ + PT_G2], %o0 + rd %psr, %l1 + ld [%sp + STACKFRAME_SZ + PT_PSR], %l0 + andn %l0, PSR_CWP, %l0 + nop + and %l1, PSR_CWP, %l1 + or %l0, %l1, %l0 + st %l0, [%sp + STACKFRAME_SZ + PT_PSR] + b ret_sys_call + mov 0, %o0 + /* Linux native system calls enter here... */ .align 4 .globl linux_sparc_syscall @@ -1298,11 +990,8 @@ linux_sparc_syscall: bgeu linux_sparc_ni_syscall sll %g1, 2, %l4 ld [%l7 + %l4], %l7 - andcc %l7, 1, %g0 - bne linux_fast_syscall - /* Just do first insn from SAVE_ALL in the delay slot */ -syscall_is_too_hard: +do_syscall: SAVE_ALL_HEAD rd %wim, %l3 @@ -1462,11 +1151,13 @@ fpload: .globl __ndelay __ndelay: save %sp, -STACKFRAME_SZ, %sp - mov %i0, %o0 - call .umul ! round multiplier up so large ns ok - mov 0x1ae, %o1 ! 2**32 / (1 000 000 000 / HZ) - call .umul - mov %i1, %o1 ! udelay_val + mov %i0, %o0 ! round multiplier up so large ns ok + mov 0x1ae, %o1 ! 2**32 / (1 000 000 000 / HZ) + umul %o0, %o1, %o0 + rd %y, %o1 + mov %i1, %o1 ! udelay_val + umul %o0, %o1, %o0 + rd %y, %o1 ba delay_continue mov %o1, %o0 ! >>32 later for better resolution @@ -1475,18 +1166,21 @@ __udelay: save %sp, -STACKFRAME_SZ, %sp mov %i0, %o0 sethi %hi(0x10c7), %o1 ! round multiplier up so large us ok - call .umul - or %o1, %lo(0x10c7), %o1 ! 2**32 / 1 000 000 - call .umul - mov %i1, %o1 ! udelay_val + or %o1, %lo(0x10c7), %o1 ! 2**32 / 1 000 000 + umul %o0, %o1, %o0 + rd %y, %o1 + mov %i1, %o1 ! udelay_val + umul %o0, %o1, %o0 + rd %y, %o1 sethi %hi(0x028f4b62), %l0 ! Add in rounding constant * 2**32, or %g0, %lo(0x028f4b62), %l0 addcc %o0, %l0, %o0 ! 2**32 * 0.009 999 bcs,a 3f add %o1, 0x01, %o1 3: - call .umul - mov HZ, %o0 ! >>32 earlier for wider range + mov HZ, %o0 ! >>32 earlier for wider range + umul %o0, %o1, %o0 + rd %y, %o1 delay_continue: cmp %o0, 0x0 @@ -1583,7 +1277,7 @@ restore_current: retl nop -#ifdef CONFIG_PCI +#ifdef CONFIG_PCIC_PCI #include <asm/pcic.h> .align 4 @@ -1629,7 +1323,7 @@ pcic_nmi_trap_patch: rd %psr, %l0 .word 0 -#endif /* CONFIG_PCI */ +#endif /* CONFIG_PCIC_PCI */ .globl flushw_all flushw_all: @@ -1649,4 +1343,26 @@ flushw_all: ret restore +#ifdef CONFIG_SMP +ENTRY(hard_smp_processor_id) +661: rd %tbr, %g1 + srl %g1, 12, %o0 + and %o0, 3, %o0 + .section .cpuid_patch, "ax" + /* Instruction location. */ + .word 661b + /* SUN4D implementation. */ + lda [%g0] ASI_M_VIKING_TMP1, %o0 + nop + nop + /* LEON implementation. */ + rd %asr17, %o0 + srl %o0, 0x1c, %o0 + nop + .previous + retl + nop +ENDPROC(hard_smp_processor_id) +#endif + /* End of entry.S */ diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h index c011b932bb1..ebaba6167dd 100644 --- a/arch/sparc/kernel/entry.h +++ b/arch/sparc/kernel/entry.h @@ -6,156 +6,182 @@ #include <linux/init.h> /* irq */ -extern void handler_irq(int irq, struct pt_regs *regs); +void handler_irq(int irq, struct pt_regs *regs); #ifdef CONFIG_SPARC32 /* traps */ -extern void do_hw_interrupt(struct pt_regs *regs, unsigned long type); -extern void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); - -extern void do_priv_instruction(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, - unsigned long npc, - unsigned long psr); -extern void do_fpd_trap(struct pt_regs *regs, unsigned long pc, +void do_hw_interrupt(struct pt_regs *regs, unsigned long type); +void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); + +void do_priv_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpd_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpe_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_watchpoint(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_reg_access(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr); -extern void do_fpe_trap(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_watchpoint(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_reg_access(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -extern void handle_cp_exception(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); +void handle_cp_exception(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); /* entry.S */ -extern void fpsave(unsigned long *fpregs, unsigned long *fsr, - void *fpqueue, unsigned long *fpqdepth); -extern void fpload(unsigned long *fpregs, unsigned long *fsr); +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void fpload(unsigned long *fpregs, unsigned long *fsr); #else /* CONFIG_SPARC32 */ -extern void __init per_cpu_patch(void); -extern void __init sun4v_patch(void); -extern void __init boot_cpu_id_too_large(int cpu); + +#include <asm/trap_block.h> + +struct popc_3insn_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct popc_3insn_patch_entry __popc_3insn_patch, + __popc_3insn_patch_end; + +struct popc_6insn_patch_entry { + unsigned int addr; + unsigned int insns[6]; +}; +extern struct popc_6insn_patch_entry __popc_6insn_patch, + __popc_6insn_patch_end; + +struct pause_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct pause_patch_entry __pause_3insn_patch, + __pause_3insn_patch_end; + +void __init per_cpu_patch(void); +void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, + struct sun4v_1insn_patch_entry *); +void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *, + struct sun4v_2insn_patch_entry *); +void __init sun4v_patch(void); +void __init boot_cpu_id_too_large(int cpu); extern unsigned int dcache_parity_tl1_occurred; extern unsigned int icache_parity_tl1_occurred; -extern asmlinkage void sparc_breakpoint(struct pt_regs *regs); -extern void timer_interrupt(int irq, struct pt_regs *regs); - -extern void do_notify_resume(struct pt_regs *regs, - unsigned long orig_i0, - unsigned long thread_info_flags); - -extern asmlinkage int syscall_trace_enter(struct pt_regs *regs); -extern asmlinkage void syscall_trace_leave(struct pt_regs *regs); - -extern void bad_trap_tl1(struct pt_regs *regs, long lvl); - -extern void do_fpe_common(struct pt_regs *regs); -extern void do_fpieee(struct pt_regs *regs); -extern void do_fpother(struct pt_regs *regs); -extern void do_tof(struct pt_regs *regs); -extern void do_div0(struct pt_regs *regs); -extern void do_illegal_instruction(struct pt_regs *regs); -extern void mem_address_unaligned(struct pt_regs *regs, - unsigned long sfar, - unsigned long sfsr); -extern void sun4v_do_mna(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void do_privop(struct pt_regs *regs); -extern void do_privact(struct pt_regs *regs); -extern void do_cee(struct pt_regs *regs); -extern void do_cee_tl1(struct pt_regs *regs); -extern void do_dae_tl1(struct pt_regs *regs); -extern void do_iae_tl1(struct pt_regs *regs); -extern void do_div0_tl1(struct pt_regs *regs); -extern void do_fpdis_tl1(struct pt_regs *regs); -extern void do_fpieee_tl1(struct pt_regs *regs); -extern void do_fpother_tl1(struct pt_regs *regs); -extern void do_ill_tl1(struct pt_regs *regs); -extern void do_irq_tl1(struct pt_regs *regs); -extern void do_lddfmna_tl1(struct pt_regs *regs); -extern void do_stdfmna_tl1(struct pt_regs *regs); -extern void do_paw(struct pt_regs *regs); -extern void do_paw_tl1(struct pt_regs *regs); -extern void do_vaw(struct pt_regs *regs); -extern void do_vaw_tl1(struct pt_regs *regs); -extern void do_tof_tl1(struct pt_regs *regs); -extern void do_getpsr(struct pt_regs *regs); - -extern void spitfire_insn_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_insn_access_exception_tl1(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_data_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_data_access_exception_tl1(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); -extern void spitfire_access_error(struct pt_regs *regs, - unsigned long status_encoded, - unsigned long afar); - -extern void cheetah_fecc_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_cee_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_deferred_handler(struct pt_regs *regs, - unsigned long afsr, - unsigned long afar); -extern void cheetah_plus_parity_error(int type, struct pt_regs *regs); - -extern void sun4v_insn_access_exception(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_insn_access_exception_tl1(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_data_access_exception(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_data_access_exception_tl1(struct pt_regs *regs, - unsigned long addr, - unsigned long type_ctx); -extern void sun4v_resum_error(struct pt_regs *regs, - unsigned long offset); -extern void sun4v_resum_overflow(struct pt_regs *regs); -extern void sun4v_nonresum_error(struct pt_regs *regs, - unsigned long offset); -extern void sun4v_nonresum_overflow(struct pt_regs *regs); +asmlinkage void sparc_breakpoint(struct pt_regs *regs); +void timer_interrupt(int irq, struct pt_regs *regs); + +void do_notify_resume(struct pt_regs *regs, + unsigned long orig_i0, + unsigned long thread_info_flags); + +asmlinkage int syscall_trace_enter(struct pt_regs *regs); +asmlinkage void syscall_trace_leave(struct pt_regs *regs); + +void bad_trap_tl1(struct pt_regs *regs, long lvl); + +void do_fpieee(struct pt_regs *regs); +void do_fpother(struct pt_regs *regs); +void do_tof(struct pt_regs *regs); +void do_div0(struct pt_regs *regs); +void do_illegal_instruction(struct pt_regs *regs); +void mem_address_unaligned(struct pt_regs *regs, + unsigned long sfar, + unsigned long sfsr); +void sun4v_do_mna(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void do_privop(struct pt_regs *regs); +void do_privact(struct pt_regs *regs); +void do_cee(struct pt_regs *regs); +void do_cee_tl1(struct pt_regs *regs); +void do_dae_tl1(struct pt_regs *regs); +void do_iae_tl1(struct pt_regs *regs); +void do_div0_tl1(struct pt_regs *regs); +void do_fpdis_tl1(struct pt_regs *regs); +void do_fpieee_tl1(struct pt_regs *regs); +void do_fpother_tl1(struct pt_regs *regs); +void do_ill_tl1(struct pt_regs *regs); +void do_irq_tl1(struct pt_regs *regs); +void do_lddfmna_tl1(struct pt_regs *regs); +void do_stdfmna_tl1(struct pt_regs *regs); +void do_paw(struct pt_regs *regs); +void do_paw_tl1(struct pt_regs *regs); +void do_vaw(struct pt_regs *regs); +void do_vaw_tl1(struct pt_regs *regs); +void do_tof_tl1(struct pt_regs *regs); +void do_getpsr(struct pt_regs *regs); + +void spitfire_insn_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_access_error(struct pt_regs *regs, + unsigned long status_encoded, + unsigned long afar); + +void cheetah_fecc_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_cee_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_deferred_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_plus_parity_error(int type, struct pt_regs *regs); + +void sun4v_insn_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_resum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_resum_overflow(struct pt_regs *regs); +void sun4v_nonresum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_nonresum_overflow(struct pt_regs *regs); extern unsigned long sun4v_err_itlb_vaddr; extern unsigned long sun4v_err_itlb_ctx; extern unsigned long sun4v_err_itlb_pte; extern unsigned long sun4v_err_itlb_error; -extern void sun4v_itlb_error_report(struct pt_regs *regs, int tl); +void sun4v_itlb_error_report(struct pt_regs *regs, int tl); extern unsigned long sun4v_err_dtlb_vaddr; extern unsigned long sun4v_err_dtlb_ctx; extern unsigned long sun4v_err_dtlb_pte; extern unsigned long sun4v_err_dtlb_error; -extern void sun4v_dtlb_error_report(struct pt_regs *regs, int tl); -extern void hypervisor_tlbop_error(unsigned long err, - unsigned long op); -extern void hypervisor_tlbop_error_xcall(unsigned long err, - unsigned long op); +void sun4v_dtlb_error_report(struct pt_regs *regs, int tl); +void hypervisor_tlbop_error(unsigned long err, + unsigned long op); +void hypervisor_tlbop_error_xcall(unsigned long err, + unsigned long op); /* WARNING: The error trap handlers in assembly know the precise * layout of the following structure. @@ -213,16 +239,16 @@ extern struct cheetah_err_info *cheetah_error_log; struct ino_bucket { /*0x00*/unsigned long __irq_chain_pa; - /* Virtual interrupt number assigned to this INO. */ -/*0x08*/unsigned int __virt_irq; + /* Interrupt number assigned to this INO. */ +/*0x08*/unsigned int __irq; /*0x0c*/unsigned int __pad; }; extern struct ino_bucket *ivector_table; extern unsigned long ivector_table_pa; -extern void init_irqwork_curcpu(void); -extern void __cpuinit sun4v_register_mondo_queues(int this_cpu); +void init_irqwork_curcpu(void); +void sun4v_register_mondo_queues(int this_cpu); #endif /* CONFIG_SPARC32 */ #endif /* _ENTRY_H */ diff --git a/arch/sparc/kernel/etrap_32.S b/arch/sparc/kernel/etrap_32.S index e806fcdc46d..e3e80d65e39 100644 --- a/arch/sparc/kernel/etrap_32.S +++ b/arch/sparc/kernel/etrap_32.S @@ -216,9 +216,7 @@ tsetup_patch6: /* Call MMU-architecture dependent stack checking * routine. */ - .globl tsetup_mmu_patchme -tsetup_mmu_patchme: - b tsetup_sun4c_stackchk + b tsetup_srmmu_stackchk andcc %sp, 0x7, %g0 /* Architecture specific stack checking routines. When either @@ -228,52 +226,6 @@ tsetup_mmu_patchme: */ #define glob_tmp g1 -tsetup_sun4c_stackchk: - /* Done by caller: andcc %sp, 0x7, %g0 */ - bne trap_setup_user_stack_is_bolixed - sra %sp, 29, %glob_tmp - - add %glob_tmp, 0x1, %glob_tmp - andncc %glob_tmp, 0x1, %g0 - bne trap_setup_user_stack_is_bolixed - and %sp, 0xfff, %glob_tmp ! delay slot - - /* See if our dump area will be on more than one - * page. - */ - add %glob_tmp, 0x38, %glob_tmp - andncc %glob_tmp, 0xff8, %g0 - be tsetup_sun4c_onepage ! only one page to check - lda [%sp] ASI_PTE, %glob_tmp ! have to check first page anyways - -tsetup_sun4c_twopages: - /* Is first page ok permission wise? */ - srl %glob_tmp, 29, %glob_tmp - cmp %glob_tmp, 0x6 - bne trap_setup_user_stack_is_bolixed - add %sp, 0x38, %glob_tmp /* Is second page in vma hole? */ - - sra %glob_tmp, 29, %glob_tmp - add %glob_tmp, 0x1, %glob_tmp - andncc %glob_tmp, 0x1, %g0 - bne trap_setup_user_stack_is_bolixed - add %sp, 0x38, %glob_tmp - - lda [%glob_tmp] ASI_PTE, %glob_tmp - -tsetup_sun4c_onepage: - srl %glob_tmp, 29, %glob_tmp - cmp %glob_tmp, 0x6 ! can user write to it? - bne trap_setup_user_stack_is_bolixed ! failure - nop - - STORE_WINDOW(sp) - - restore %g0, %g0, %g0 - - jmpl %t_retpc + 0x8, %g0 - mov %t_kstack, %sp - .globl tsetup_srmmu_stackchk tsetup_srmmu_stackchk: /* Check results of callers andcc %sp, 0x7, %g0 */ @@ -282,7 +234,8 @@ tsetup_srmmu_stackchk: cmp %glob_tmp, %sp bleu,a 1f - lda [%g0] ASI_M_MMUREGS, %glob_tmp ! read MMU control +LEON_PI( lda [%g0] ASI_LEON_MMUREGS, %glob_tmp) ! read MMU control +SUN_PI_( lda [%g0] ASI_M_MMUREGS, %glob_tmp) ! read MMU control trap_setup_user_stack_is_bolixed: /* From user/kernel into invalid window w/bad user @@ -297,18 +250,25 @@ trap_setup_user_stack_is_bolixed: 1: /* Clear the fault status and turn on the no_fault bit. */ or %glob_tmp, 0x2, %glob_tmp ! or in no_fault bit - sta %glob_tmp, [%g0] ASI_M_MMUREGS ! set it +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) ! set it +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) ! set it /* Dump the registers and cross fingers. */ STORE_WINDOW(sp) /* Clear the no_fault bit and check the status. */ andn %glob_tmp, 0x2, %glob_tmp - sta %glob_tmp, [%g0] ASI_M_MMUREGS +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) + mov AC_M_SFAR, %glob_tmp - lda [%glob_tmp] ASI_M_MMUREGS, %g0 +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) + mov AC_M_SFSR, %glob_tmp - lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp ! save away status of winstore +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %glob_tmp)! save away status of winstore +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp) ! save away status of winstore + andcc %glob_tmp, 0x2, %g0 ! did we fault? bne trap_setup_user_stack_is_bolixed ! failure nop diff --git a/arch/sparc/kernel/etrap_64.S b/arch/sparc/kernel/etrap_64.S index 786b185e6e3..1276ca2567b 100644 --- a/arch/sparc/kernel/etrap_64.S +++ b/arch/sparc/kernel/etrap_64.S @@ -92,8 +92,10 @@ etrap_save: save %g2, -STACK_BIAS, %sp rdpr %wstate, %g2 wrpr %g0, 0, %canrestore sll %g2, 3, %g2 + + /* Set TI_SYS_FPDEPTH to 1 and clear TI_SYS_NOERROR. */ mov 1, %l5 - stb %l5, [%l6 + TI_FPDEPTH] + sth %l5, [%l6 + TI_SYS_NOERROR] wrpr %g3, 0, %otherwin wrpr %g2, 0, %wstate @@ -152,7 +154,9 @@ etrap_save: save %g2, -STACK_BIAS, %sp add %l6, TI_FPSAVED + 1, %l4 srl %l5, 1, %l3 add %l5, 2, %l5 - stb %l5, [%l6 + TI_FPDEPTH] + + /* Set TI_SYS_FPDEPTH to %l5 and clear TI_SYS_NOERROR. */ + sth %l5, [%l6 + TI_SYS_NOERROR] ba,pt %xcc, 2b stb %g0, [%l4 + %l3] nop diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c index 03ab022e51c..0a2d2ddff54 100644 --- a/arch/sparc/kernel/ftrace.c +++ b/arch/sparc/kernel/ftrace.c @@ -82,12 +82,8 @@ int ftrace_update_ftrace_func(ftrace_func_t func) return ftrace_modify_code(ip, old, new); } -int __init ftrace_dyn_arch_init(void *data) +int __init ftrace_dyn_arch_init(void) { - unsigned long *p = data; - - *p = 0; - return 0; } #endif diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index 59423491cef..3d92c0a8f6c 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -26,396 +26,39 @@ #include <asm/pgtsrmmu.h> /* SRMMU_PGDIR_SHIFT */ .data -/* - * The following are used with the prom_vector node-ops to figure out - * the cpu-type +/* The following are used with the prom_vector node-ops to figure out + * the cpu-type */ - - .align 4 -cputyp: - .word 1 - .align 4 .globl cputypval cputypval: - .asciz "sun4c" + .asciz "sun4m" .ascii " " -cputypvalend: -cputypvallen = cputypvar - cputypval - +/* Tested on SS-5, SS-10 */ .align 4 -/* - * Sun people can't spell worth damn. "compatability" indeed. - * At least we *know* we can't spell, and use a spell-checker. - */ - -/* Uh, actually Linus it is I who cannot spell. Too much murky - * Sparc assembly will do this to ya. - */ cputypvar: - .asciz "compatability" - -/* Tested on SS-5, SS-10. Probably someone at Sun applied a spell-checker. */ - .align 4 -cputypvar_sun4m: .asciz "compatible" .align 4 -sun4_notsup: - .asciz "Sparc-Linux sun4 support does no longer exist.\n\n" +notsup: + .asciz "Sparc-Linux sun4/sun4c or MMU-less not supported\n\n" .align 4 sun4e_notsup: .asciz "Sparc-Linux sun4e support does not exist\n\n" .align 4 - /* The Sparc trap table, bootloader gives us control at _start. */ - __HEAD - .globl _stext, _start, __stext - .globl trapbase -_start: /* danger danger */ -__stext: -_stext: -trapbase: -#ifdef CONFIG_SMP -trapbase_cpu0: -#endif -/* We get control passed to us here at t_zero. */ -t_zero: b gokernel; nop; nop; nop; -t_tflt: SPARC_TFAULT /* Inst. Access Exception */ -t_bins: TRAP_ENTRY(0x2, bad_instruction) /* Illegal Instruction */ -t_pins: TRAP_ENTRY(0x3, priv_instruction) /* Privileged Instruction */ -t_fpd: TRAP_ENTRY(0x4, fpd_trap_handler) /* Floating Point Disabled */ -t_wovf: WINDOW_SPILL /* Window Overflow */ -t_wunf: WINDOW_FILL /* Window Underflow */ -t_mna: TRAP_ENTRY(0x7, mna_handler) /* Memory Address Not Aligned */ -t_fpe: TRAP_ENTRY(0x8, fpe_trap_handler) /* Floating Point Exception */ -t_dflt: SPARC_DFAULT /* Data Miss Exception */ -t_tio: TRAP_ENTRY(0xa, do_tag_overflow) /* Tagged Instruction Ovrflw */ -t_wpt: TRAP_ENTRY(0xb, do_watchpoint) /* Watchpoint Detected */ -t_badc: BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) -t_irq1: TRAP_ENTRY_INTERRUPT(1) /* IRQ Software/SBUS Level 1 */ -t_irq2: TRAP_ENTRY_INTERRUPT(2) /* IRQ SBUS Level 2 */ -t_irq3: TRAP_ENTRY_INTERRUPT(3) /* IRQ SCSI/DMA/SBUS Level 3 */ -t_irq4: TRAP_ENTRY_INTERRUPT(4) /* IRQ Software Level 4 */ -t_irq5: TRAP_ENTRY_INTERRUPT(5) /* IRQ SBUS/Ethernet Level 5 */ -t_irq6: TRAP_ENTRY_INTERRUPT(6) /* IRQ Software Level 6 */ -t_irq7: TRAP_ENTRY_INTERRUPT(7) /* IRQ Video/SBUS Level 5 */ -t_irq8: TRAP_ENTRY_INTERRUPT(8) /* IRQ SBUS Level 6 */ -t_irq9: TRAP_ENTRY_INTERRUPT(9) /* IRQ SBUS Level 7 */ -t_irq10:TRAP_ENTRY_INTERRUPT(10) /* IRQ Timer #1 (one we use) */ -t_irq11:TRAP_ENTRY_INTERRUPT(11) /* IRQ Floppy Intr. */ -t_irq12:TRAP_ENTRY_INTERRUPT(12) /* IRQ Zilog serial chip */ -t_irq13:TRAP_ENTRY_INTERRUPT(13) /* IRQ Audio Intr. */ -t_irq14:TRAP_ENTRY_INTERRUPT(14) /* IRQ Timer #2 */ - .globl t_nmi -#ifndef CONFIG_SMP -t_nmi: NMI_TRAP /* Level 15 (NMI) */ -#else -t_nmi: TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) -#endif -t_racc: TRAP_ENTRY(0x20, do_reg_access) /* General Register Access Error */ -t_iacce:BAD_TRAP(0x21) /* Instr Access Error */ -t_bad22:BAD_TRAP(0x22) BAD_TRAP(0x23) -t_cpdis:TRAP_ENTRY(0x24, do_cp_disabled) /* Co-Processor Disabled */ -t_uflsh:SKIP_TRAP(0x25, unimp_flush) /* Unimplemented FLUSH inst. */ -t_bad26:BAD_TRAP(0x26) BAD_TRAP(0x27) -t_cpexc:TRAP_ENTRY(0x28, do_cp_exception) /* Co-Processor Exception */ -t_dacce:SPARC_DFAULT /* Data Access Error */ -t_hwdz: TRAP_ENTRY(0x2a, do_hw_divzero) /* Division by zero, you lose... */ -t_dserr:BAD_TRAP(0x2b) /* Data Store Error */ -t_daccm:BAD_TRAP(0x2c) /* Data Access MMU-Miss */ -t_bad2d:BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) -t_bad32:BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) -t_bad37:BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) -t_iaccm:BAD_TRAP(0x3c) /* Instr Access MMU-Miss */ -t_bad3d:BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) BAD_TRAP(0x41) -t_bad42:BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) BAD_TRAP(0x46) -t_bad47:BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) BAD_TRAP(0x4b) -t_bad4c:BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) BAD_TRAP(0x50) -t_bad51:BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) -t_bad56:BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) -t_bad5b:BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) -t_bad60:BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) -t_bad65:BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) -t_bad6a:BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) -t_bad6f:BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) -t_bad74:BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) -t_bad79:BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) -t_bad7e:BAD_TRAP(0x7e) BAD_TRAP(0x7f) -t_bad80:BAD_TRAP(0x80) /* SunOS System Call */ -t_sbkpt:BREAKPOINT_TRAP /* Software Breakpoint/KGDB */ -t_divz: TRAP_ENTRY(0x82, do_hw_divzero) /* Divide by zero trap */ -t_flwin:TRAP_ENTRY(0x83, do_flush_windows) /* Flush Windows Trap */ -t_clwin:BAD_TRAP(0x84) /* Clean Windows Trap */ -t_rchk: BAD_TRAP(0x85) /* Range Check */ -t_funal:BAD_TRAP(0x86) /* Fix Unaligned Access Trap */ -t_iovf: BAD_TRAP(0x87) /* Integer Overflow Trap */ -t_bad88:BAD_TRAP(0x88) /* Slowaris System Call */ -t_bad89:BAD_TRAP(0x89) /* Net-B.S. System Call */ -t_bad8a:BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) BAD_TRAP(0x8d) BAD_TRAP(0x8e) -t_bad8f:BAD_TRAP(0x8f) -t_linux:LINUX_SYSCALL_TRAP /* Linux System Call */ -t_bad91:BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) BAD_TRAP(0x95) -t_bad96:BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) BAD_TRAP(0x9a) -t_bad9b:BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) BAD_TRAP(0x9f) -t_getcc:GETCC_TRAP /* Get Condition Codes */ -t_setcc:SETCC_TRAP /* Set Condition Codes */ -t_getpsr:GETPSR_TRAP /* Get PSR Register */ -t_bada3:BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) -t_bada7:BAD_TRAP(0xa7) -t_bada8:BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) -t_badac:BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) -t_badb1:BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) -t_badb6:BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) -t_badbb:BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) -t_badc0:BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) -t_badc5:BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) -t_badca:BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) -t_badcf:BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) -t_badd4:BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) -t_badd9:BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) -t_badde:BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) -t_bade3:BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) -t_bade8:BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) -t_baded:BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) -t_badf2:BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) -t_badf7:BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) -t_badfc:BAD_TRAP(0xfc) -t_kgdb: KGDB_TRAP(0xfd) -dbtrap: BAD_TRAP(0xfe) /* Debugger/PROM breakpoint #1 */ -dbtrap2:BAD_TRAP(0xff) /* Debugger/PROM breakpoint #2 */ - - .globl end_traptable -end_traptable: - -#ifdef CONFIG_SMP - /* Trap tables for the other cpus. */ - .globl trapbase_cpu1, trapbase_cpu2, trapbase_cpu3 -trapbase_cpu1: - BAD_TRAP(0x0) SRMMU_TFAULT TRAP_ENTRY(0x2, bad_instruction) - TRAP_ENTRY(0x3, priv_instruction) TRAP_ENTRY(0x4, fpd_trap_handler) - WINDOW_SPILL WINDOW_FILL TRAP_ENTRY(0x7, mna_handler) - TRAP_ENTRY(0x8, fpe_trap_handler) SRMMU_DFAULT - TRAP_ENTRY(0xa, do_tag_overflow) TRAP_ENTRY(0xb, do_watchpoint) - BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) - TRAP_ENTRY_INTERRUPT(1) TRAP_ENTRY_INTERRUPT(2) - TRAP_ENTRY_INTERRUPT(3) TRAP_ENTRY_INTERRUPT(4) - TRAP_ENTRY_INTERRUPT(5) TRAP_ENTRY_INTERRUPT(6) - TRAP_ENTRY_INTERRUPT(7) TRAP_ENTRY_INTERRUPT(8) - TRAP_ENTRY_INTERRUPT(9) TRAP_ENTRY_INTERRUPT(10) - TRAP_ENTRY_INTERRUPT(11) TRAP_ENTRY_INTERRUPT(12) - TRAP_ENTRY_INTERRUPT(13) TRAP_ENTRY_INTERRUPT(14) - TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) - TRAP_ENTRY(0x20, do_reg_access) BAD_TRAP(0x21) BAD_TRAP(0x22) - BAD_TRAP(0x23) TRAP_ENTRY(0x24, do_cp_disabled) SKIP_TRAP(0x25, unimp_flush) - BAD_TRAP(0x26) BAD_TRAP(0x27) TRAP_ENTRY(0x28, do_cp_exception) - SRMMU_DFAULT TRAP_ENTRY(0x2a, do_hw_divzero) BAD_TRAP(0x2b) BAD_TRAP(0x2c) - BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) - BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) - BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) - BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) - BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) - BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) - BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) - BAD_TRAP(0x50) - BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) - BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) - BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) - BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) - BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) - BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) - BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) - BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) - BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) - BAD_TRAP(0x7e) BAD_TRAP(0x7f) - BAD_TRAP(0x80) - BREAKPOINT_TRAP - TRAP_ENTRY(0x82, do_hw_divzero) - TRAP_ENTRY(0x83, do_flush_windows) BAD_TRAP(0x84) BAD_TRAP(0x85) - BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) - BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) - BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) - LINUX_SYSCALL_TRAP BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) - BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) - BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) - BAD_TRAP(0x9f) GETCC_TRAP SETCC_TRAP GETPSR_TRAP - BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) - BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) - BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) - BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) - BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) - BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) - BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) - BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) - BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) - BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) - BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) - BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) - BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) - BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) - BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) - BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) - BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) - BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) - BAD_TRAP(0xfc) KGDB_TRAP(0xfd) BAD_TRAP(0xfe) BAD_TRAP(0xff) - -trapbase_cpu2: - BAD_TRAP(0x0) SRMMU_TFAULT TRAP_ENTRY(0x2, bad_instruction) - TRAP_ENTRY(0x3, priv_instruction) TRAP_ENTRY(0x4, fpd_trap_handler) - WINDOW_SPILL WINDOW_FILL TRAP_ENTRY(0x7, mna_handler) - TRAP_ENTRY(0x8, fpe_trap_handler) SRMMU_DFAULT - TRAP_ENTRY(0xa, do_tag_overflow) TRAP_ENTRY(0xb, do_watchpoint) - BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) - TRAP_ENTRY_INTERRUPT(1) TRAP_ENTRY_INTERRUPT(2) - TRAP_ENTRY_INTERRUPT(3) TRAP_ENTRY_INTERRUPT(4) - TRAP_ENTRY_INTERRUPT(5) TRAP_ENTRY_INTERRUPT(6) - TRAP_ENTRY_INTERRUPT(7) TRAP_ENTRY_INTERRUPT(8) - TRAP_ENTRY_INTERRUPT(9) TRAP_ENTRY_INTERRUPT(10) - TRAP_ENTRY_INTERRUPT(11) TRAP_ENTRY_INTERRUPT(12) - TRAP_ENTRY_INTERRUPT(13) TRAP_ENTRY_INTERRUPT(14) - TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) - TRAP_ENTRY(0x20, do_reg_access) BAD_TRAP(0x21) BAD_TRAP(0x22) - BAD_TRAP(0x23) TRAP_ENTRY(0x24, do_cp_disabled) SKIP_TRAP(0x25, unimp_flush) - BAD_TRAP(0x26) BAD_TRAP(0x27) TRAP_ENTRY(0x28, do_cp_exception) - SRMMU_DFAULT TRAP_ENTRY(0x2a, do_hw_divzero) BAD_TRAP(0x2b) BAD_TRAP(0x2c) - BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) - BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) - BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) - BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) - BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) - BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) - BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) - BAD_TRAP(0x50) - BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) - BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) - BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) - BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) - BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) - BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) - BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) - BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) - BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) - BAD_TRAP(0x7e) BAD_TRAP(0x7f) - BAD_TRAP(0x80) - BREAKPOINT_TRAP - TRAP_ENTRY(0x82, do_hw_divzero) - TRAP_ENTRY(0x83, do_flush_windows) BAD_TRAP(0x84) BAD_TRAP(0x85) - BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) - BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) - BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) - LINUX_SYSCALL_TRAP BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) - BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) - BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) - BAD_TRAP(0x9f) GETCC_TRAP SETCC_TRAP GETPSR_TRAP - BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) - BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) - BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) - BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) - BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) - BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) - BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) - BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) - BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) - BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) - BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) - BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) - BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) - BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) - BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) - BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) - BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) - BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) - BAD_TRAP(0xfc) KGDB_TRAP(0xfd) BAD_TRAP(0xfe) BAD_TRAP(0xff) - -trapbase_cpu3: - BAD_TRAP(0x0) SRMMU_TFAULT TRAP_ENTRY(0x2, bad_instruction) - TRAP_ENTRY(0x3, priv_instruction) TRAP_ENTRY(0x4, fpd_trap_handler) - WINDOW_SPILL WINDOW_FILL TRAP_ENTRY(0x7, mna_handler) - TRAP_ENTRY(0x8, fpe_trap_handler) SRMMU_DFAULT - TRAP_ENTRY(0xa, do_tag_overflow) TRAP_ENTRY(0xb, do_watchpoint) - BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) - TRAP_ENTRY_INTERRUPT(1) TRAP_ENTRY_INTERRUPT(2) - TRAP_ENTRY_INTERRUPT(3) TRAP_ENTRY_INTERRUPT(4) - TRAP_ENTRY_INTERRUPT(5) TRAP_ENTRY_INTERRUPT(6) - TRAP_ENTRY_INTERRUPT(7) TRAP_ENTRY_INTERRUPT(8) - TRAP_ENTRY_INTERRUPT(9) TRAP_ENTRY_INTERRUPT(10) - TRAP_ENTRY_INTERRUPT(11) TRAP_ENTRY_INTERRUPT(12) - TRAP_ENTRY_INTERRUPT(13) TRAP_ENTRY_INTERRUPT(14) - TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) - TRAP_ENTRY(0x20, do_reg_access) BAD_TRAP(0x21) BAD_TRAP(0x22) - BAD_TRAP(0x23) TRAP_ENTRY(0x24, do_cp_disabled) SKIP_TRAP(0x25, unimp_flush) - BAD_TRAP(0x26) BAD_TRAP(0x27) TRAP_ENTRY(0x28, do_cp_exception) - SRMMU_DFAULT TRAP_ENTRY(0x2a, do_hw_divzero) BAD_TRAP(0x2b) BAD_TRAP(0x2c) - BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) - BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) - BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) - BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) - BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) - BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) - BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) - BAD_TRAP(0x50) - BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) - BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) - BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) - BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) - BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) - BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) - BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) - BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) - BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) - BAD_TRAP(0x7e) BAD_TRAP(0x7f) - BAD_TRAP(0x80) - BREAKPOINT_TRAP - TRAP_ENTRY(0x82, do_hw_divzero) - TRAP_ENTRY(0x83, do_flush_windows) BAD_TRAP(0x84) BAD_TRAP(0x85) - BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) - BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) - BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) - LINUX_SYSCALL_TRAP BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) - BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) - BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) - BAD_TRAP(0x9f) GETCC_TRAP SETCC_TRAP GETPSR_TRAP - BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) - BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) - BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) - BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) - BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) - BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) - BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) - BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) - BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) - BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) - BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) - BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) - BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) - BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) - BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) - BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) - BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) - BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) - BAD_TRAP(0xfc) KGDB_TRAP(0xfd) BAD_TRAP(0xfe) BAD_TRAP(0xff) +/* The trap-table - located in the __HEAD section */ +#include "ttable_32.S" -#endif .align PAGE_SIZE /* This was the only reasonable way I could think of to properly align * these page-table data structures. */ - .globl pg0, pg1, pg2, pg3 - .globl empty_bad_page - .globl empty_bad_page_table .globl empty_zero_page - .globl swapper_pg_dir -swapper_pg_dir: .skip PAGE_SIZE -pg0: .skip PAGE_SIZE -pg1: .skip PAGE_SIZE -pg2: .skip PAGE_SIZE -pg3: .skip PAGE_SIZE -empty_bad_page: .skip PAGE_SIZE -empty_bad_page_table: .skip PAGE_SIZE empty_zero_page: .skip PAGE_SIZE .global root_flags @@ -474,7 +117,7 @@ current_pc: tst %o0 be no_sun4u_here mov %g4, %o7 /* Previous %o7. */ - + mov %o0, %l0 ! stash away romvec mov %o0, %g7 ! put it here too mov %o1, %l1 ! stash away debug_vec too @@ -483,7 +126,7 @@ current_pc: set current_pc, %g5 cmp %g3, %g5 be already_mapped - nop + nop /* %l6 will hold the offset we have to subtract * from absolute symbols in order to access areas @@ -523,10 +166,10 @@ copy_prom_lvl14: ldd [%g2 + 0x8], %g4 std %g4, [%g3 + 0x8] ! Copy proms handler -/* Must determine whether we are on a sun4c MMU, SRMMU, or SUN4/400 MUTANT - * MMU so we can remap ourselves properly. DON'T TOUCH %l0 thru %l5 in these - * remapping routines, we need their values afterwards! +/* DON'T TOUCH %l0 thru %l5 in these remapping routines, + * we need their values afterwards! */ + /* Now check whether we are already mapped, if we * are we can skip all this garbage coming up. */ @@ -535,33 +178,49 @@ copy_prom_done: be go_to_highmem ! this will be a nop then nop - set LOAD_ADDR, %g6 + /* Validate that we are in fact running on an + * SRMMU based cpu. + */ + set 0x4000, %g6 cmp %g7, %g6 - bne remap_not_a_sun4 ! This is not a Sun4 + bne not_a_sun4 nop - or %g0, 0x1, %g1 - lduba [%g1] ASI_CONTROL, %g1 ! Only safe to try on Sun4. - subcc %g1, 0x24, %g0 ! Is this a mutant Sun4/400??? - be sun4_mutant_remap ! Ugh, it is... +halt_notsup: + ld [%g7 + 0x68], %o1 + set notsup, %o0 + sub %o0, %l6, %o0 + call %o1 nop + ba halt_me + nop + +not_a_sun4: + /* It looks like this is a machine we support. + * Now find out what MMU we are dealing with + * LEON - identified by the psr.impl field + * Viking - identified by the psr.impl field + * In all other cases a sun4m srmmu. + * We check that the MMU is enabled in all cases. + */ - b sun4_normal_remap ! regular sun4, 2 level mmu + /* Check if this is a LEON CPU */ + rd %psr, %g3 + srl %g3, PSR_IMPL_SHIFT, %g3 + and %g3, PSR_IMPL_SHIFTED_MASK, %g3 + cmp %g3, PSR_IMPL_LEON + be leon_remap /* It is a LEON - jump */ nop -remap_not_a_sun4: - lda [%g0] ASI_M_MMUREGS, %g1 ! same as ASI_PTE on sun4c - and %g1, 0x1, %g1 ! Test SRMMU Enable bit ;-) - cmp %g1, 0x0 - be sun4c_remap ! A sun4c MMU or normal Sun4 + /* Sanity-check, is MMU enabled */ + lda [%g0] ASI_M_MMUREGS, %g1 + andcc %g1, 1, %g0 + be halt_notsup nop -srmmu_remap: - /* First, check for a viking (TI) module. */ - set 0x40000000, %g2 - rd %psr, %g3 - and %g2, %g3, %g3 - subcc %g3, 0x0, %g0 - bz srmmu_nviking + + /* Check for a viking (TI) module. */ + cmp %g3, PSR_IMPL_TI + bne srmmu_not_viking nop /* Figure out what kind of viking we are on. @@ -576,14 +235,14 @@ srmmu_remap: lda [%g0] ASI_M_MMUREGS, %g3 ! peek in the control reg and %g2, %g3, %g3 subcc %g3, 0x0, %g0 - bnz srmmu_nviking ! is in mbus mode + bnz srmmu_not_viking ! is in mbus mode nop - + rd %psr, %g3 ! DO NOT TOUCH %g3 andn %g3, PSR_ET, %g2 wr %g2, 0x0, %psr WRITE_PAUSE - + /* Get context table pointer, then convert to * a physical address, which is 36 bits. */ @@ -606,12 +265,12 @@ srmmu_remap: lda [%g4] ASI_M_BYPASS, %o1 ! This is a level 1 ptr srl %o1, 0x4, %o1 ! Clear low 4 bits sll %o1, 0x8, %o1 ! Make physical - + /* Ok, pull in the PTD. */ lda [%o1] ASI_M_BYPASS, %o2 ! This is the 0x0 16MB pgd /* Calculate to KERNBASE entry. */ - add %o1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %o3 + add %o1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %o3 /* Poke the entry into the calculated address. */ sta %o2, [%o3] ASI_M_BYPASS @@ -641,12 +300,12 @@ srmmu_remap: b go_to_highmem nop +srmmu_not_viking: /* This works on viking's in Mbus mode and all * other MBUS modules. It is virtually the same as * the above madness sans turning traps off and flipping * the AC bit. */ -srmmu_nviking: set AC_M_CTPR, %g1 lda [%g1] ASI_M_MMUREGS, %g1 ! get ctx table ptr sll %g1, 0x4, %g1 ! make physical addr @@ -660,72 +319,29 @@ srmmu_nviking: b go_to_highmem nop ! wheee.... - /* This remaps the kernel on Sun4/4xx machines - * that have the Sun Mutant Three Level MMU. - * It's like a platypus, Sun didn't have the - * SRMMU in conception so they kludged the three - * level logic in the regular Sun4 MMU probably. - * - * Basically, you take each entry in the top level - * directory that maps the low 3MB starting at - * address zero and put the mapping in the KERNBASE - * slots. These top level pgd's are called regmaps. - */ -sun4_mutant_remap: - or %g0, %g0, %g3 ! source base - sethi %hi(KERNBASE), %g4 ! destination base - or %g4, %lo(KERNBASE), %g4 - sethi %hi(0x300000), %g5 - or %g5, %lo(0x300000), %g5 ! upper bound 3MB - or %g0, 0x1, %l6 - sll %l6, 24, %l6 ! Regmap mapping size - add %g3, 0x2, %g3 ! Base magic - add %g4, 0x2, %g4 ! Base magic - - /* Main remapping loop on Sun4-Mutant-MMU. - * "I am not an animal..." -Famous Mutant Person - */ -sun4_mutant_loop: - lduha [%g3] ASI_REGMAP, %g2 ! Get lower entry - stha %g2, [%g4] ASI_REGMAP ! Store in high entry - add %g4, %l6, %g4 ! Move up high memory ptr - subcc %g3, %g5, %g0 ! Reached our limit? - blu sun4_mutant_loop ! Nope, loop again - add %g3, %l6, %g3 ! delay, Move up low ptr - b go_to_highmem ! Jump to high memory. - nop - /* The following is for non-4/4xx sun4 MMU's. */ -sun4_normal_remap: - mov 0, %g3 ! source base - set KERNBASE, %g4 ! destination base - set 0x300000, %g5 ! upper bound 3MB - mov 1, %l6 - sll %l6, 18, %l6 ! sun4 mmu segmap size -sun4_normal_loop: - lduha [%g3] ASI_SEGMAP, %g6 ! load phys_seg - stha %g6, [%g4] ASI_SEGMAP ! stort new virt mapping - add %g3, %l6, %g3 ! increment source pointer - subcc %g3, %g5, %g0 ! reached limit? - blu sun4_normal_loop ! nope, loop again - add %g4, %l6, %g4 ! delay, increment dest ptr - b go_to_highmem +leon_remap: + /* Sanity-check, is MMU enabled */ + lda [%g0] ASI_LEON_MMUREGS, %g1 + andcc %g1, 1, %g0 + be halt_notsup nop - /* The following works for Sun4c MMU's */ -sun4c_remap: - mov 0, %g3 ! source base - set KERNBASE, %g4 ! destination base - set 0x300000, %g5 ! upper bound 3MB - mov 1, %l6 - sll %l6, 18, %l6 ! sun4c mmu segmap size -sun4c_remap_loop: - lda [%g3] ASI_SEGMAP, %g6 ! load phys_seg - sta %g6, [%g4] ASI_SEGMAP ! store new virt mapping - add %g3, %l6, %g3 ! Increment source ptr - subcc %g3, %g5, %g0 ! Reached limit? - bl sun4c_remap_loop ! Nope, loop again - add %g4, %l6, %g4 ! delay, Increment dest ptr + /* Same code as in the srmmu_not_viking case, + * with the LEON ASI for mmuregs + */ + set AC_M_CTPR, %g1 + lda [%g1] ASI_LEON_MMUREGS, %g1 ! get ctx table ptr + sll %g1, 0x4, %g1 ! make physical addr + lda [%g1] ASI_M_BYPASS, %g1 ! ptr to level 1 pg_table + srl %g1, 0x4, %g1 + sll %g1, 0x8, %g1 ! make phys addr for l1 tbl + + lda [%g1] ASI_M_BYPASS, %g2 ! get level1 entry for 0x0 + add %g1, KERNBASE >> (SRMMU_PGDIR_SHIFT - 2), %g3 + sta %g2, [%g3] ASI_M_BYPASS ! place at KERNBASE entry + b go_to_highmem + nop ! wheee.... /* Now do a non-relative jump so that PC is in high-memory */ go_to_highmem: @@ -750,35 +366,13 @@ execute_in_high_mem: sethi %hi(linux_dbvec), %g1 st %o1, [%g1 + %lo(linux_dbvec)] - ld [%o0 + 0x4], %o3 - and %o3, 0x3, %o5 ! get the version - - cmp %o3, 0x2 ! a v2 prom? - be found_version - nop - - /* paul@sfe.com.au */ - cmp %o3, 0x3 ! a v3 prom? - be found_version - nop - -/* Old sun4's pass our load address into %o0 instead of the prom - * pointer. On sun4's you have to hard code the romvec pointer into - * your code. Sun probably still does that because they don't even - * trust their own "OpenBoot" specifications. - */ - set LOAD_ADDR, %g6 - cmp %o0, %g6 ! an old sun4? - be sun4_init - nop - -found_version: -/* Get the machine type via the mysterious romvec node operations. */ - - add %g7, 0x1c, %l1 + /* Get the machine type via the romvec + * getprops node operation + */ + add %g7, 0x1c, %l1 ld [%l1], %l0 ld [%l0], %l0 - call %l0 + call %l0 or %g0, %g0, %o0 ! next_node(0) = first_node or %o0, %g0, %g6 @@ -786,89 +380,65 @@ found_version: or %o1, %lo(cputypvar), %o1 sethi %hi(cputypval), %o2 ! information, the string or %o2, %lo(cputypval), %o2 - ld [%l1], %l0 ! 'compatibility' tells + ld [%l1], %l0 ! 'compatible' tells ld [%l0 + 0xc], %l0 ! that we want 'sun4x' where - call %l0 ! x is one of '', 'c', 'm', - nop ! 'd' or 'e'. %o2 holds pointer + call %l0 ! x is one of 'm', 'd' or 'e'. + nop ! %o2 holds pointer ! to a buf where above string ! will get stored by the prom. - subcc %o0, %g0, %g0 - bpos got_prop ! Got the property - nop - or %g6, %g0, %o0 - sethi %hi(cputypvar_sun4m), %o1 - or %o1, %lo(cputypvar_sun4m), %o1 - sethi %hi(cputypval), %o2 - or %o2, %lo(cputypval), %o2 - ld [%l1], %l0 - ld [%l0 + 0xc], %l0 - call %l0 - nop + /* Check value of "compatible" property. + * "value" => "model" + * leon => sparc_leon + * sun4m => sun4m + * sun4s => sun4m + * sun4d => sun4d + * sun4e => "no_sun4e_here" + * '*' => "no_sun4u_here" + * Check single letters only + */ -got_prop: -#ifdef CONFIG_SPARC_LEON - /* no cpu-type check is needed, it is a SPARC-LEON */ -#ifdef CONFIG_SMP - ba leon_smp_init + set cputypval, %o2 + /* If cputypval[0] == 'l' (lower case letter L) this is leon */ + ldub [%o2], %l1 + cmp %l1, 'l' + be leon_init nop - .global leon_smp_init -leon_smp_init: - sethi %hi(boot_cpu_id), %g1 ! master always 0 - stb %g0, [%g1 + %lo(boot_cpu_id)] - sethi %hi(boot_cpu_id4), %g1 ! master always 0 - stb %g0, [%g1 + %lo(boot_cpu_id4)] - - rd %asr17,%g1 - srl %g1,28,%g1 - - cmp %g0,%g1 - beq sun4c_continue_boot !continue with master - nop - - ba leon_smp_cpu_startup - nop -#else - ba sun4c_continue_boot - nop -#endif -#endif - set cputypval, %o2 + /* Check cputypval[4] to find the sun model */ ldub [%o2 + 0x4], %l1 - cmp %l1, ' ' - be 1f - cmp %l1, 'c' - be 1f - cmp %l1, 'm' - be 1f + cmp %l1, 'm' + be sun4m_init cmp %l1, 's' - be 1f + be sun4m_init cmp %l1, 'd' - be 1f + be sun4d_init cmp %l1, 'e' be no_sun4e_here ! Could be a sun4e. nop b no_sun4u_here ! AIEEE, a V9 sun4u... Get our BIG BROTHER kernel :)) nop -1: set cputypval, %l1 - ldub [%l1 + 0x4], %l1 - cmp %l1, 'm' ! Test for sun4d, sun4e ? - be sun4m_init - cmp %l1, 's' ! Treat sun4s as sun4m - be sun4m_init - cmp %l1, 'd' ! Let us see how the beast will die - be sun4d_init +leon_init: + /* LEON CPU - set boot_cpu_id */ + sethi %hi(boot_cpu_id), %g2 ! boot-cpu index + +#ifdef CONFIG_SMP + ldub [%g2 + %lo(boot_cpu_id)], %g1 + cmp %g1, 0xff ! unset means first CPU + bne leon_smp_cpu_startup ! continue only with master nop +#endif + /* Get CPU-ID from most significant 4-bit of ASR17 */ + rd %asr17, %g1 + srl %g1, 28, %g1 - /* Jump into mmu context zero. */ - set AC_CONTEXT, %g1 - stba %g0, [%g1] ASI_CONTROL + /* Update boot_cpu_id only on boot cpu */ + stub %g1, [%g2 + %lo(boot_cpu_id)] - b sun4c_continue_boot + ba continue_boot nop /* CPUID in bootbus can be found at PA 0xff0140000 */ @@ -893,74 +463,11 @@ sun4d_init: sta %g4, [%g0] ASI_M_VIKING_TMP1 sethi %hi(boot_cpu_id), %g5 stb %g4, [%g5 + %lo(boot_cpu_id)] - sll %g4, 2, %g4 - sethi %hi(boot_cpu_id4), %g5 - stb %g4, [%g5 + %lo(boot_cpu_id4)] #endif /* Fall through to sun4m_init */ sun4m_init: - /* XXX Fucking Cypress... */ - lda [%g0] ASI_M_MMUREGS, %g5 - srl %g5, 28, %g4 - - cmp %g4, 1 - bne 1f - srl %g5, 24, %g4 - - and %g4, 0xf, %g4 - cmp %g4, 7 /* This would be a HyperSparc. */ - - bne 2f - nop - -1: - -#define PATCH_IT(dst, src) \ - set (dst), %g5; \ - set (src), %g4; \ - ld [%g4], %g3; \ - st %g3, [%g5]; \ - ld [%g4+0x4], %g3; \ - st %g3, [%g5+0x4]; - - /* Signed multiply. */ - PATCH_IT(.mul, .mul_patch) - PATCH_IT(.mul+0x08, .mul_patch+0x08) - - /* Signed remainder. */ - PATCH_IT(.rem, .rem_patch) - PATCH_IT(.rem+0x08, .rem_patch+0x08) - PATCH_IT(.rem+0x10, .rem_patch+0x10) - PATCH_IT(.rem+0x18, .rem_patch+0x18) - PATCH_IT(.rem+0x20, .rem_patch+0x20) - PATCH_IT(.rem+0x28, .rem_patch+0x28) - - /* Signed division. */ - PATCH_IT(.div, .div_patch) - PATCH_IT(.div+0x08, .div_patch+0x08) - PATCH_IT(.div+0x10, .div_patch+0x10) - PATCH_IT(.div+0x18, .div_patch+0x18) - PATCH_IT(.div+0x20, .div_patch+0x20) - - /* Unsigned multiply. */ - PATCH_IT(.umul, .umul_patch) - PATCH_IT(.umul+0x08, .umul_patch+0x08) - - /* Unsigned remainder. */ - PATCH_IT(.urem, .urem_patch) - PATCH_IT(.urem+0x08, .urem_patch+0x08) - PATCH_IT(.urem+0x10, .urem_patch+0x10) - PATCH_IT(.urem+0x18, .urem_patch+0x18) - - /* Unsigned division. */ - PATCH_IT(.udiv, .udiv_patch) - PATCH_IT(.udiv+0x08, .udiv_patch+0x08) - PATCH_IT(.udiv+0x10, .udiv_patch+0x10) - -#undef PATCH_IT - /* Ok, the PROM could have done funny things and apple cider could still * be sitting in the fault status/address registers. Read them all to * clear them so we don't get magic faults later on. @@ -968,10 +475,10 @@ sun4m_init: /* This sucks, apparently this makes Vikings call prom panic, will fix later */ 2: rd %psr, %o1 - srl %o1, 28, %o1 ! Get a type of the CPU + srl %o1, PSR_IMPL_SHIFT, %o1 ! Get a type of the CPU - subcc %o1, 4, %g0 ! TI: Viking or MicroSPARC - be sun4c_continue_boot + subcc %o1, PSR_IMPL_TI, %g0 ! TI: Viking or MicroSPARC + be continue_boot nop set AC_M_SFSR, %o0 @@ -981,7 +488,7 @@ sun4m_init: /* Fujitsu MicroSPARC-II has no asynchronous flavors of FARs */ subcc %o1, 0, %g0 - be sun4c_continue_boot + be continue_boot nop set AC_M_AFSR, %o0 @@ -991,16 +498,11 @@ sun4m_init: nop -sun4c_continue_boot: - +continue_boot: /* Aieee, now set PC and nPC, enable traps, give ourselves a stack and it's * show-time! */ - - sethi %hi(cputyp), %o0 - st %g4, [%o0 + %lo(cputyp)] - /* Turn on Supervisor, EnableFloating, and all the PIL bits. * Also puts us in register window zero with traps off. */ @@ -1018,20 +520,31 @@ sun4c_continue_boot: set __bss_start , %o0 ! First address of BSS set _end , %o1 ! Last address of BSS add %o0, 0x1, %o0 -1: +1: stb %g0, [%o0] subcc %o0, %o1, %g0 bl 1b add %o0, 0x1, %o0 + /* If boot_cpu_id has not been setup by machine specific + * init-code above we default it to zero. + */ + sethi %hi(boot_cpu_id), %g2 + ldub [%g2 + %lo(boot_cpu_id)], %g3 + cmp %g3, 0xff + bne 1f + nop + mov %g0, %g3 + stub %g3, [%g2 + %lo(boot_cpu_id)] + +1: sll %g3, 2, %g3 + /* Initialize the uwinmask value for init task just in case. * But first make current_set[boot_cpu_id] point to something useful. */ set init_thread_union, %g6 set current_set, %g2 #ifdef CONFIG_SMP - sethi %hi(boot_cpu_id4), %g3 - ldub [%g3 + %lo(boot_cpu_id4)], %g3 st %g6, [%g2] add %g2, %g3, %g2 #endif @@ -1073,7 +586,7 @@ sun4c_continue_boot: set dest, %g2; \ ld [%g5], %g4; \ st %g4, [%g2]; - + /* Patch for window spills... */ PATCH_INSN(spnwin_patch1_7win, spnwin_patch1) PATCH_INSN(spnwin_patch2_7win, spnwin_patch2) @@ -1124,7 +637,7 @@ sun4c_continue_boot: st %g4, [%g5 + 0x18] st %g4, [%g5 + 0x1c] -2: +2: sethi %hi(nwindows), %g4 st %g3, [%g4 + %lo(nwindows)] ! store final value sub %g3, 0x1, %g3 @@ -1144,35 +657,16 @@ sun4c_continue_boot: wr %g3, PSR_ET, %psr WRITE_PAUSE - /* First we call prom_init() to set up PROMLIB, then - * off to start_kernel(). - */ - + /* Call sparc32_start_kernel(struct linux_romvec *rp) */ sethi %hi(prom_vector_p), %g5 ld [%g5 + %lo(prom_vector_p)], %o0 - call prom_init + call sparc32_start_kernel nop - call start_kernel - nop - /* We should not get here. */ call halt_me nop -sun4_init: - sethi %hi(SUN4_PROM_VECTOR+0x84), %o1 - ld [%o1 + %lo(SUN4_PROM_VECTOR+0x84)], %o1 - set sun4_notsup, %o0 - call %o1 /* printf */ - nop - sethi %hi(SUN4_PROM_VECTOR+0xc4), %o1 - ld [%o1 + %lo(SUN4_PROM_VECTOR+0xc4)], %o1 - call %o1 /* exittomon */ - nop -1: ba 1b ! Cannot exit into KMON - nop - no_sun4e_here: ld [%g7 + 0x68], %o1 set sun4e_notsup, %o0 @@ -1199,7 +693,7 @@ sun4u_5: .asciz "write" .align 4 sun4u_6: - .asciz "\n\rOn sun4u you have to use UltraLinux (64bit) kernel\n\rand not a 32bit sun4[cdem] version\n\r\n\r" + .asciz "\n\rOn sun4u you have to use sparc64 kernel\n\rand not a sparc32 version\n\r\n\r" sun4u_6e: .align 4 sun4u_7: diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index f8f21050448..452f04fe8da 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -85,7 +85,7 @@ sparc_ramdisk_image64: sparc64_boot: mov %o4, %l7 - /* We need to remap the kernel. Use position independant + /* We need to remap the kernel. Use position independent * code to remap us to KERNBASE. * * SILO can invoke us with 32-bit address masking enabled, @@ -132,6 +132,10 @@ prom_sun4v_name: .asciz "sun4v" prom_niagara_prefix: .asciz "SUNW,UltraSPARC-T" +prom_sparc_prefix: + .asciz "SPARC-" +prom_sparc64x_prefix: + .asciz "SPARC64-X" .align 4 prom_root_compatible: .skip 64 @@ -278,8 +282,8 @@ sun4v_chip_type: stx %l2, [%l4 + 0x0] ldx [%sp + 2047 + 128 + 0x50], %l3 ! physaddr low /* 4MB align */ - srlx %l3, 22, %l3 - sllx %l3, 22, %l3 + srlx %l3, ILOG2_4MB, %l3 + sllx %l3, ILOG2_4MB, %l3 stx %l3, [%l4 + 0x8] /* Leave service as-is, "call-method" */ @@ -382,6 +386,22 @@ sun4v_chip_type: 90: ldub [%g7], %g2 ldub [%g1], %g4 cmp %g2, %g4 + bne,pn %icc, 89f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 90b + add %g1, 1, %g1 + ba,pt %xcc, 91f + nop + +89: sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + sethi %hi(prom_sparc_prefix), %g7 + or %g7, %lo(prom_sparc_prefix), %g7 + mov 6, %g3 +90: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 bne,pn %icc, 4f add %g7, 1, %g7 subcc %g3, 1, %g3 @@ -390,6 +410,28 @@ sun4v_chip_type: sethi %hi(prom_cpu_compatible), %g1 or %g1, %lo(prom_cpu_compatible), %g1 + ldub [%g1 + 6], %g2 + cmp %g2, 'T' + be,pt %xcc, 70f + cmp %g2, 'M' + bne,pn %xcc, 49f + nop + +70: ldub [%g1 + 7], %g2 + cmp %g2, '3' + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA3, %g4 + cmp %g2, '4' + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA4, %g4 + cmp %g2, '5' + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA5, %g4 + ba,pt %xcc, 49f + nop + +91: sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 ldub [%g1 + 17], %g2 cmp %g2, '1' be,pt %xcc, 5f @@ -397,7 +439,27 @@ sun4v_chip_type: cmp %g2, '2' be,pt %xcc, 5f mov SUN4V_CHIP_NIAGARA2, %g4 + 4: + /* Athena */ + sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + sethi %hi(prom_sparc64x_prefix), %g7 + or %g7, %lo(prom_sparc64x_prefix), %g7 + mov 9, %g3 +41: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 + bne,pn %icc, 49f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 41b + add %g1, 1, %g1 + mov SUN4V_CHIP_SPARC64X, %g4 + ba,pt %xcc, 5f + nop + +49: mov SUN4V_CHIP_UNKNOWN, %g4 5: sethi %hi(sun4v_chip_type), %g2 or %g2, %lo(sun4v_chip_type), %g2 @@ -514,6 +576,15 @@ niagara_tlb_fixup: cmp %g1, SUN4V_CHIP_NIAGARA2 be,pt %xcc, niagara2_patch nop + cmp %g1, SUN4V_CHIP_NIAGARA3 + be,pt %xcc, niagara2_patch + nop + cmp %g1, SUN4V_CHIP_NIAGARA4 + be,pt %xcc, niagara4_patch + nop + cmp %g1, SUN4V_CHIP_NIAGARA5 + be,pt %xcc, niagara4_patch + nop call generic_patch_copyops nop @@ -523,12 +594,22 @@ niagara_tlb_fixup: nop ba,a,pt %xcc, 80f +niagara4_patch: + call niagara4_patch_copyops + nop + call niagara4_patch_bzero + nop + call niagara4_patch_pageops + nop + + ba,a,pt %xcc, 80f + niagara2_patch: call niagara2_patch_copyops nop call niagara_patch_bzero nop - call niagara2_patch_pageops + call niagara_patch_pageops nop ba,a,pt %xcc, 80f @@ -856,7 +937,7 @@ swapper_4m_tsb: * error and will instead write junk into the relocation and * you'll have an unbootable kernel. */ -#include "ttable.S" +#include "ttable_64.S" ! 0x0000000000428000 diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c index 7c60afb835b..c0a2de0fd62 100644 --- a/arch/sparc/kernel/hvapi.c +++ b/arch/sparc/kernel/hvapi.c @@ -3,7 +3,7 @@ * Copyright (C) 2007 David S. Miller <davem@davemloft.net> */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <asm/hypervisor.h> @@ -28,16 +28,24 @@ static struct api_info api_table[] = { { .group = HV_GRP_CORE, .flags = FLAG_PRE_API }, { .group = HV_GRP_INTR, }, { .group = HV_GRP_SOFT_STATE, }, + { .group = HV_GRP_TM, }, { .group = HV_GRP_PCI, .flags = FLAG_PRE_API }, { .group = HV_GRP_LDOM, }, { .group = HV_GRP_SVC_CHAN, .flags = FLAG_PRE_API }, { .group = HV_GRP_NCS, .flags = FLAG_PRE_API }, { .group = HV_GRP_RNG, }, + { .group = HV_GRP_PBOOT, }, + { .group = HV_GRP_TPM, }, + { .group = HV_GRP_SDIO, }, + { .group = HV_GRP_SDIO_ERR, }, + { .group = HV_GRP_REBOOT_DATA, }, { .group = HV_GRP_NIAG_PERF, .flags = FLAG_PRE_API }, { .group = HV_GRP_FIRE_PERF, }, { .group = HV_GRP_N2_CPU, }, { .group = HV_GRP_NIU, }, { .group = HV_GRP_VF_CPU, }, + { .group = HV_GRP_KT_CPU, }, + { .group = HV_GRP_VT_CPU, }, { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API }, }; @@ -186,7 +194,7 @@ void __init sun4v_hvapi_init(void) bad: prom_printf("HVAPI: Cannot register API group " - "%lx with major(%u) minor(%u)\n", + "%lx with major(%lu) minor(%lu)\n", group, major, minor); prom_halt(); } diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S index 8a5f35ffb15..f3ab509b76a 100644 --- a/arch/sparc/kernel/hvcalls.S +++ b/arch/sparc/kernel/hvcalls.S @@ -798,3 +798,26 @@ ENTRY(sun4v_niagara2_setperf) retl nop ENDPROC(sun4v_niagara2_setperf) + +ENTRY(sun4v_reboot_data_set) + mov HV_FAST_REBOOT_DATA_SET, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_reboot_data_set) + +ENTRY(sun4v_vt_get_perfreg) + mov %o1, %o4 + mov HV_FAST_VT_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_vt_get_perfreg) + +ENTRY(sun4v_vt_set_perfreg) + mov HV_FAST_VT_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vt_set_perfreg) diff --git a/arch/sparc/kernel/hvtramp.S b/arch/sparc/kernel/hvtramp.S index 9365432904d..b7ddcdd1dea 100644 --- a/arch/sparc/kernel/hvtramp.S +++ b/arch/sparc/kernel/hvtramp.S @@ -3,7 +3,6 @@ * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> */ -#include <linux/init.h> #include <asm/thread_info.h> #include <asm/hypervisor.h> @@ -16,7 +15,6 @@ #include <asm/asi.h> #include <asm/pil.h> - __CPUINIT .align 8 .globl hv_cpu_startup, hv_cpu_startup_end @@ -128,8 +126,7 @@ hv_cpu_startup: call smp_callin nop - call cpu_idle - mov 0, %o0 + call cpu_panic nop diff --git a/arch/sparc/kernel/idprom.c b/arch/sparc/kernel/idprom.c index 52a15fe2db1..6bd75012109 100644 --- a/arch/sparc/kernel/idprom.c +++ b/arch/sparc/kernel/idprom.c @@ -8,7 +8,7 @@ #include <linux/kernel.h> #include <linux/types.h> #include <linux/init.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/oplib.h> #include <asm/idprom.h> @@ -25,22 +25,9 @@ static struct idprom idprom_buffer; * of the Sparc CPU and have a meaningful IDPROM machtype value that we * know about. See asm-sparc/machines.h for empirical constants. */ -static struct Sun_Machine_Models Sun_Machines[NUM_SUN_MACHINES] = { -/* First, Sun4's */ -{ .name = "Sun 4/100 Series", .id_machtype = (SM_SUN4 | SM_4_110) }, -{ .name = "Sun 4/200 Series", .id_machtype = (SM_SUN4 | SM_4_260) }, -{ .name = "Sun 4/300 Series", .id_machtype = (SM_SUN4 | SM_4_330) }, -{ .name = "Sun 4/400 Series", .id_machtype = (SM_SUN4 | SM_4_470) }, -/* Now Leon */ +static struct Sun_Machine_Models Sun_Machines[] = { +/* First, Leon */ { .name = "Leon3 System-on-a-Chip", .id_machtype = (M_LEON | M_LEON3_SOC) }, -/* Now, Sun4c's */ -{ .name = "Sun4c SparcStation 1", .id_machtype = (SM_SUN4C | SM_4C_SS1) }, -{ .name = "Sun4c SparcStation IPC", .id_machtype = (SM_SUN4C | SM_4C_IPC) }, -{ .name = "Sun4c SparcStation 1+", .id_machtype = (SM_SUN4C | SM_4C_SS1PLUS) }, -{ .name = "Sun4c SparcStation SLC", .id_machtype = (SM_SUN4C | SM_4C_SLC) }, -{ .name = "Sun4c SparcStation 2", .id_machtype = (SM_SUN4C | SM_4C_SS2) }, -{ .name = "Sun4c SparcStation ELC", .id_machtype = (SM_SUN4C | SM_4C_ELC) }, -{ .name = "Sun4c SparcStation IPX", .id_machtype = (SM_SUN4C | SM_4C_IPX) }, /* Finally, early Sun4m's */ { .name = "Sun4m SparcSystem600", .id_machtype = (SM_SUN4M | SM_4M_SS60) }, { .name = "Sun4m SparcStation10/20", .id_machtype = (SM_SUN4M | SM_4M_SS50) }, @@ -53,7 +40,7 @@ static void __init display_system_type(unsigned char machtype) char sysname[128]; register int i; - for (i = 0; i < NUM_SUN_MACHINES; i++) { + for (i = 0; i < ARRAY_SIZE(Sun_Machines); i++) { if (Sun_Machines[i].id_machtype == machtype) { if (machtype != (SM_SUN4M_OBP | 0x00) || prom_getproperty(prom_root_node, "banner-name", diff --git a/arch/sparc/kernel/init_task.c b/arch/sparc/kernel/init_task.c deleted file mode 100644 index 5fe3d65581f..00000000000 --- a/arch/sparc/kernel/init_task.c +++ /dev/null @@ -1,22 +0,0 @@ -#include <linux/mm.h> -#include <linux/fs.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/init_task.h> -#include <linux/mqueue.h> - -#include <asm/pgtable.h> -#include <asm/uaccess.h> - -static struct signal_struct init_signals = INIT_SIGNALS(init_signals); -static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand); -struct task_struct init_task = INIT_TASK(init_task); -EXPORT_SYMBOL(init_task); - -/* .text section in head.S is aligned at 8k boundary and this gets linked - * right after that so that the init_thread_union is aligned properly as well. - * If this is not aligned on a 8k boundry, then you should change code - * in etrap.S which assumes it. - */ -union thread_union init_thread_union __init_task_data = - { INIT_THREAD_INFO(init_task) }; diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 72509d0e34b..bfa4d0c2df4 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -5,7 +5,7 @@ */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/delay.h> #include <linux/device.h> @@ -21,6 +21,7 @@ #include <asm/iommu.h> #include "iommu_common.h" +#include "kernel.h" #define STC_CTXMATCH_ADDR(STC, CTX) \ ((STC)->strbuf_ctxmatch_base + ((CTX) << 3)) @@ -280,7 +281,8 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) } static void *dma_4u_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp) + dma_addr_t *dma_addrp, gfp_t gfp, + struct dma_attrs *attrs) { unsigned long flags, order, first_page; struct iommu *iommu; @@ -330,16 +332,14 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, } static void dma_4u_free_coherent(struct device *dev, size_t size, - void *cpu, dma_addr_t dvma) + void *cpu, dma_addr_t dvma, + struct dma_attrs *attrs) { struct iommu *iommu; - iopte_t *iopte; unsigned long flags, order, npages; npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; iommu = dev->archdata.iommu; - iopte = iommu->page_table + - ((dvma - iommu->page_table_map_base) >> IO_PAGE_SHIFT); spin_lock_irqsave(&iommu->lock, flags); @@ -828,8 +828,8 @@ static void dma_4u_sync_sg_for_cpu(struct device *dev, } static struct dma_map_ops sun4u_dma_ops = { - .alloc_coherent = dma_4u_alloc_coherent, - .free_coherent = dma_4u_free_coherent, + .alloc = dma_4u_alloc_coherent, + .free = dma_4u_free_coherent, .map_page = dma_4u_map_page, .unmap_page = dma_4u_unmap_page, .map_sg = dma_4u_map_sg, @@ -841,8 +841,6 @@ static struct dma_map_ops sun4u_dma_ops = { struct dma_map_ops *dma_ops = &sun4u_dma_ops; EXPORT_SYMBOL(dma_ops); -extern int pci64_dma_supported(struct pci_dev *pdev, u64 device_mask); - int dma_supported(struct device *dev, u64 device_mask) { struct iommu *iommu = dev->archdata.iommu; @@ -855,7 +853,7 @@ int dma_supported(struct device *dev, u64 device_mask) return 1; #ifdef CONFIG_PCI - if (dev->bus == &pci_bus_type) + if (dev_is_pci(dev)) return pci64_dma_supported(to_pci_dev(dev), device_mask); #endif diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h index 591f5879039..1ec0de4156e 100644 --- a/arch/sparc/kernel/iommu_common.h +++ b/arch/sparc/kernel/iommu_common.h @@ -48,12 +48,12 @@ static inline int is_span_boundary(unsigned long entry, return iommu_is_span_boundary(entry, nr, shift, boundary_size); } -extern unsigned long iommu_range_alloc(struct device *dev, - struct iommu *iommu, - unsigned long npages, - unsigned long *handle); -extern void iommu_range_free(struct iommu *iommu, - dma_addr_t dma_addr, - unsigned long npages); +unsigned long iommu_range_alloc(struct device *dev, + struct iommu *iommu, + unsigned long npages, + unsigned long *handle); +void iommu_range_free(struct iommu *iommu, + dma_addr_t dma_addr, + unsigned long npages); #endif /* _IOMMU_COMMON_H */ diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index 41f7e4e0f72..7f08ec8a7c6 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -50,14 +50,18 @@ #include <asm/io-unit.h> #include <asm/leon.h> -#ifdef CONFIG_SPARC_LEON -#define mmu_inval_dma_area(p, l) leon_flush_dcache_all() -#else -#define mmu_inval_dma_area(p, l) /* Anton pulled it out for 2.4.0-xx */ -#endif +const struct sparc32_dma_ops *sparc32_dma_ops; -static struct resource *_sparc_find_resource(struct resource *r, - unsigned long); +/* This function must make sure that caches and memory are coherent after DMA + * On LEON systems without cache snooping it flushes the entire D-CACHE. + */ +static inline void dma_make_coherent(unsigned long pa, unsigned long len) +{ + if (sparc_cpu_model == sparc_leon) { + if (!sparc_leon3_snooping_enabled()) + leon_flush_dcache_all(); + } +} static void __iomem *_sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz); static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, @@ -134,7 +138,11 @@ void iounmap(volatile void __iomem *virtual) unsigned long vaddr = (unsigned long) virtual & PAGE_MASK; struct resource *res; - if ((res = _sparc_find_resource(&sparc_iomap, vaddr)) == NULL) { + /* + * XXX Too slow. Can have 8192 DVMA pages on sun4m in the worst case. + * This probably warrants some sort of hashing. + */ + if ((res = lookup_resource(&sparc_iomap, vaddr)) == NULL) { printk("free_io/iounmap: cannot free %lx\n", vaddr); return; } @@ -178,7 +186,7 @@ static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, if (name == NULL) name = "???"; - if ((xres = xres_alloc()) != 0) { + if ((xres = xres_alloc()) != NULL) { tack = xres->xname; res = &xres->xres; } else { @@ -219,7 +227,7 @@ _sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz) } pa &= PAGE_MASK; - sparc_mapiorange(bus, pa, res->start, res->end - res->start + 1); + srmmu_mapiorange(bus, pa, res->start, resource_size(res)); return (void __iomem *)(unsigned long)(res->start + offset); } @@ -231,9 +239,9 @@ static void _sparc_free_io(struct resource *res) { unsigned long plen; - plen = res->end - res->start + 1; + plen = resource_size(res); BUG_ON((plen & (PAGE_SIZE-1)) != 0); - sparc_unmapiorange(res->start, plen); + srmmu_unmapiorange(res->start, plen); release_resource(res); } @@ -251,10 +259,11 @@ EXPORT_SYMBOL(sbus_set_sbus64); * CPU may access them without any explicit flushing. */ static void *sbus_alloc_coherent(struct device *dev, size_t len, - dma_addr_t *dma_addrp, gfp_t gfp) + dma_addr_t *dma_addrp, gfp_t gfp, + struct dma_attrs *attrs) { struct platform_device *op = to_platform_device(dev); - unsigned long len_total = (len + PAGE_SIZE-1) & PAGE_MASK; + unsigned long len_total = PAGE_ALIGN(len); unsigned long va; struct resource *res; int order; @@ -280,14 +289,14 @@ static void *sbus_alloc_coherent(struct device *dev, size_t len, printk("sbus_alloc_consistent: cannot occupy 0x%lx", len_total); goto err_nova; } - mmu_inval_dma_area(va, len_total); - // XXX The mmu_map_dma_area does this for us below, see comments. - // sparc_mapiorange(0, virt_to_phys(va), res->start, len_total); + + // XXX The sbus_map_dma_area does this for us below, see comments. + // srmmu_mapiorange(0, virt_to_phys(va), res->start, len_total); /* * XXX That's where sdev would be used. Currently we load * all iommu tables with the same translations. */ - if (mmu_map_dma_area(dev, dma_addrp, va, res->start, len_total) != 0) + if (sbus_map_dma_area(dev, dma_addrp, va, res->start, len_total) != 0) goto err_noiommu; res->name = op->dev.of_node->name; @@ -297,20 +306,20 @@ static void *sbus_alloc_coherent(struct device *dev, size_t len, err_noiommu: release_resource(res); err_nova: - free_pages(va, order); -err_nomem: kfree(res); +err_nomem: + free_pages(va, order); err_nopages: return NULL; } static void sbus_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba) + dma_addr_t ba, struct dma_attrs *attrs) { struct resource *res; struct page *pgv; - if ((res = _sparc_find_resource(&_sparc_dvma, + if ((res = lookup_resource(&_sparc_dvma, (unsigned long)p)) == NULL) { printk("sbus_free_consistent: cannot free %p\n", p); return; @@ -321,19 +330,18 @@ static void sbus_free_coherent(struct device *dev, size_t n, void *p, return; } - n = (n + PAGE_SIZE-1) & PAGE_MASK; - if ((res->end-res->start)+1 != n) { + n = PAGE_ALIGN(n); + if (resource_size(res) != n) { printk("sbus_free_consistent: region 0x%lx asked 0x%zx\n", - (long)((res->end-res->start)+1), n); + (long)resource_size(res), n); return; } release_resource(res); kfree(res); - /* mmu_inval_dma_area(va, n); */ /* it's consistent, isn't it */ pgv = virt_to_page(p); - mmu_unmap_dma_area(dev, ba, n); + sbus_unmap_dma_area(dev, ba, n); __free_pages(pgv, get_order(n)); } @@ -371,11 +379,6 @@ static int sbus_map_sg(struct device *dev, struct scatterlist *sg, int n, enum dma_data_direction dir, struct dma_attrs *attrs) { mmu_get_scsi_sgl(dev, sg, n); - - /* - * XXX sparc64 can return a partial length here. sun4c should do this - * but it currently panics if it can't fulfill the request - Anton - */ return n; } @@ -397,9 +400,9 @@ static void sbus_sync_sg_for_device(struct device *dev, struct scatterlist *sg, BUG(); } -struct dma_map_ops sbus_dma_ops = { - .alloc_coherent = sbus_alloc_coherent, - .free_coherent = sbus_free_coherent, +static struct dma_map_ops sbus_dma_ops = { + .alloc = sbus_alloc_coherent, + .free = sbus_free_coherent, .map_page = sbus_map_page, .unmap_page = sbus_unmap_page, .map_sg = sbus_map_sg, @@ -408,9 +411,6 @@ struct dma_map_ops sbus_dma_ops = { .sync_sg_for_device = sbus_sync_sg_for_device, }; -struct dma_map_ops *dma_ops = &sbus_dma_ops; -EXPORT_SYMBOL(dma_ops); - static int __init sparc_register_ioport(void) { register_proc_sparc_ioport(); @@ -422,16 +422,16 @@ arch_initcall(sparc_register_ioport); #endif /* CONFIG_SBUS */ -#ifdef CONFIG_PCI /* Allocate and map kernel buffer using consistent mode DMA for a device. * hwdev should be valid struct pci_dev pointer for PCI devices. */ static void *pci32_alloc_coherent(struct device *dev, size_t len, - dma_addr_t *pba, gfp_t gfp) + dma_addr_t *pba, gfp_t gfp, + struct dma_attrs *attrs) { - unsigned long len_total = (len + PAGE_SIZE-1) & PAGE_MASK; - unsigned long va; + unsigned long len_total = PAGE_ALIGN(len); + void *va; struct resource *res; int order; @@ -443,34 +443,33 @@ static void *pci32_alloc_coherent(struct device *dev, size_t len, } order = get_order(len_total); - va = __get_free_pages(GFP_KERNEL, order); - if (va == 0) { + va = (void *) __get_free_pages(GFP_KERNEL, order); + if (va == NULL) { printk("pci_alloc_consistent: no %ld pages\n", len_total>>PAGE_SHIFT); - return NULL; + goto err_nopages; } if ((res = kzalloc(sizeof(struct resource), GFP_KERNEL)) == NULL) { - free_pages(va, order); printk("pci_alloc_consistent: no core\n"); - return NULL; + goto err_nomem; } if (allocate_resource(&_sparc_dvma, res, len_total, _sparc_dvma.start, _sparc_dvma.end, PAGE_SIZE, NULL, NULL) != 0) { printk("pci_alloc_consistent: cannot occupy 0x%lx", len_total); - free_pages(va, order); - kfree(res); - return NULL; + goto err_nova; } - mmu_inval_dma_area(va, len_total); -#if 0 -/* P3 */ printk("pci_alloc_consistent: kva %lx uncva %lx phys %lx size %lx\n", - (long)va, (long)res->start, (long)virt_to_phys(va), len_total); -#endif - sparc_mapiorange(0, virt_to_phys(va), res->start, len_total); + srmmu_mapiorange(0, virt_to_phys(va), res->start, len_total); *pba = virt_to_phys(va); /* equals virt_to_bus (R.I.P.) for us. */ return (void *) res->start; + +err_nova: + kfree(res); +err_nomem: + free_pages((unsigned long)va, order); +err_nopages: + return NULL; } /* Free and unmap a consistent DMA buffer. @@ -482,12 +481,11 @@ static void *pci32_alloc_coherent(struct device *dev, size_t len, * past this call are illegal. */ static void pci32_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba) + dma_addr_t ba, struct dma_attrs *attrs) { struct resource *res; - unsigned long pgp; - if ((res = _sparc_find_resource(&_sparc_dvma, + if ((res = lookup_resource(&_sparc_dvma, (unsigned long)p)) == NULL) { printk("pci_free_consistent: cannot free %p\n", p); return; @@ -498,21 +496,19 @@ static void pci32_free_coherent(struct device *dev, size_t n, void *p, return; } - n = (n + PAGE_SIZE-1) & PAGE_MASK; - if ((res->end-res->start)+1 != n) { + n = PAGE_ALIGN(n); + if (resource_size(res) != n) { printk("pci_free_consistent: region 0x%lx asked 0x%lx\n", - (long)((res->end-res->start)+1), (long)n); + (long)resource_size(res), (long)n); return; } - pgp = (unsigned long) phys_to_virt(ba); /* bus_to_virt actually */ - mmu_inval_dma_area(pgp, n); - sparc_unmapiorange((unsigned long)p, n); + dma_make_coherent(ba, n); + srmmu_unmapiorange((unsigned long)p, n); release_resource(res); kfree(res); - - free_pages(pgp, get_order(n)); + free_pages((unsigned long)phys_to_virt(ba), get_order(n)); } /* @@ -527,6 +523,13 @@ static dma_addr_t pci32_map_page(struct device *dev, struct page *page, return page_to_phys(page) + offset; } +static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, + enum dma_data_direction dir, struct dma_attrs *attrs) +{ + if (dir != PCI_DMA_TODEVICE) + dma_make_coherent(ba, PAGE_ALIGN(size)); +} + /* Map a set of buffers described by scatterlist in streaming * mode for DMA. This is the scather-gather version of the * above pci_map_single interface. Here the scatter gather list @@ -551,8 +554,7 @@ static int pci32_map_sg(struct device *device, struct scatterlist *sgl, /* IIep is write-through, not flushing. */ for_each_sg(sgl, sg, nents, n) { - BUG_ON(page_address(sg_page(sg)) == NULL); - sg->dma_address = virt_to_phys(sg_virt(sg)); + sg->dma_address = sg_phys(sg); sg->dma_length = sg->length; } return nents; @@ -571,10 +573,7 @@ static void pci32_unmap_sg(struct device *dev, struct scatterlist *sgl, if (dir != PCI_DMA_TODEVICE) { for_each_sg(sgl, sg, nents, n) { - BUG_ON(page_address(sg_page(sg)) == NULL); - mmu_inval_dma_area( - (unsigned long) page_address(sg_page(sg)), - (sg->length + PAGE_SIZE-1) & PAGE_MASK); + dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); } } } @@ -593,8 +592,7 @@ static void pci32_sync_single_for_cpu(struct device *dev, dma_addr_t ba, size_t size, enum dma_data_direction dir) { if (dir != PCI_DMA_TODEVICE) { - mmu_inval_dma_area((unsigned long)phys_to_virt(ba), - (size + PAGE_SIZE-1) & PAGE_MASK); + dma_make_coherent(ba, PAGE_ALIGN(size)); } } @@ -602,8 +600,7 @@ static void pci32_sync_single_for_device(struct device *dev, dma_addr_t ba, size_t size, enum dma_data_direction dir) { if (dir != PCI_DMA_TODEVICE) { - mmu_inval_dma_area((unsigned long)phys_to_virt(ba), - (size + PAGE_SIZE-1) & PAGE_MASK); + dma_make_coherent(ba, PAGE_ALIGN(size)); } } @@ -621,10 +618,7 @@ static void pci32_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl, if (dir != PCI_DMA_TODEVICE) { for_each_sg(sgl, sg, nents, n) { - BUG_ON(page_address(sg_page(sg)) == NULL); - mmu_inval_dma_area( - (unsigned long) page_address(sg_page(sg)), - (sg->length + PAGE_SIZE-1) & PAGE_MASK); + dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); } } } @@ -637,18 +631,16 @@ static void pci32_sync_sg_for_device(struct device *device, struct scatterlist * if (dir != PCI_DMA_TODEVICE) { for_each_sg(sgl, sg, nents, n) { - BUG_ON(page_address(sg_page(sg)) == NULL); - mmu_inval_dma_area( - (unsigned long) page_address(sg_page(sg)), - (sg->length + PAGE_SIZE-1) & PAGE_MASK); + dma_make_coherent(sg_phys(sg), PAGE_ALIGN(sg->length)); } } } struct dma_map_ops pci32_dma_ops = { - .alloc_coherent = pci32_alloc_coherent, - .free_coherent = pci32_free_coherent, + .alloc = pci32_alloc_coherent, + .free = pci32_free_coherent, .map_page = pci32_map_page, + .unmap_page = pci32_unmap_page, .map_sg = pci32_map_sg, .unmap_sg = pci32_unmap_sg, .sync_single_for_cpu = pci32_sync_single_for_cpu, @@ -658,7 +650,13 @@ struct dma_map_ops pci32_dma_ops = { }; EXPORT_SYMBOL(pci32_dma_ops); -#endif /* CONFIG_PCI */ +/* leon re-uses pci32_dma_ops */ +struct dma_map_ops *leon_dma_ops = &pci32_dma_ops; +EXPORT_SYMBOL(leon_dma_ops); + +struct dma_map_ops *dma_ops = &sbus_dma_ops; +EXPORT_SYMBOL(dma_ops); + /* * Return whether the given PCI device DMA address mask can be @@ -668,10 +666,9 @@ EXPORT_SYMBOL(pci32_dma_ops); */ int dma_supported(struct device *dev, u64 mask) { -#ifdef CONFIG_PCI - if (dev->bus == &pci_bus_type) + if (dev_is_pci(dev)) return 1; -#endif + return 0; } EXPORT_SYMBOL(dma_supported); @@ -684,7 +681,7 @@ static int sparc_io_proc_show(struct seq_file *m, void *v) const char *nm; for (r = root->child; r != NULL; r = r->sibling) { - if ((nm = r->name) == 0) nm = "???"; + if ((nm = r->name) == NULL) nm = "???"; seq_printf(m, "%016llx-%016llx: %s\n", (unsigned long long)r->start, (unsigned long long)r->end, nm); @@ -695,7 +692,7 @@ static int sparc_io_proc_show(struct seq_file *m, void *v) static int sparc_io_proc_open(struct inode *inode, struct file *file) { - return single_open(file, sparc_io_proc_show, PDE(inode)->data); + return single_open(file, sparc_io_proc_show, PDE_DATA(inode)); } static const struct file_operations sparc_io_proc_fops = { @@ -707,25 +704,6 @@ static const struct file_operations sparc_io_proc_fops = { }; #endif /* CONFIG_PROC_FS */ -/* - * This is a version of find_resource and it belongs to kernel/resource.c. - * Until we have agreement with Linus and Martin, it lingers here. - * - * XXX Too slow. Can have 8192 DVMA pages on sun4m in the worst case. - * This probably warrants some sort of hashing. - */ -static struct resource *_sparc_find_resource(struct resource *root, - unsigned long hit) -{ - struct resource *tmp; - - for (tmp = root->child; tmp != 0; tmp = tmp->sibling) { - if (tmp->start <= hit && tmp->end >= hit) - return tmp; - } - return NULL; -} - static void register_proc_sparc_ioport(void) { #ifdef CONFIG_PROC_FS diff --git a/arch/sparc/kernel/irq.h b/arch/sparc/kernel/irq.h index db751388153..70a0b8ddd0b 100644 --- a/arch/sparc/kernel/irq.h +++ b/arch/sparc/kernel/irq.h @@ -1,62 +1,101 @@ -#include <asm/btfixup.h> - -/* Dave Redman (djhr@tadpole.co.uk) - * changed these to function pointers.. it saves cycles and will allow - * the irq dependencies to be split into different files at a later date - * sun4c_irq.c, sun4m_irq.c etc so we could reduce the kernel size. - * Jakub Jelinek (jj@sunsite.mff.cuni.cz) - * Changed these to btfixup entities... It saves cycles :) +#include <linux/platform_device.h> + +#include <asm/cpu_type.h> + +struct irq_bucket { + struct irq_bucket *next; + unsigned int real_irq; + unsigned int irq; + unsigned int pil; +}; + +#define SUN4M_HARD_INT(x) (0x000000001 << (x)) +#define SUN4M_SOFT_INT(x) (0x000010000 << (x)) + +#define SUN4D_MAX_BOARD 10 +#define SUN4D_MAX_IRQ ((SUN4D_MAX_BOARD + 2) << 5) + +/* Map between the irq identifier used in hw to the + * irq_bucket. The map is sufficient large to hold + * the sun4d hw identifiers. + */ +extern struct irq_bucket *irq_map[SUN4D_MAX_IRQ]; + + +/* sun4m specific type definitions */ + +/* This maps direct to CPU specific interrupt registers */ +struct sun4m_irq_percpu { + u32 pending; + u32 clear; + u32 set; +}; + +/* This maps direct to global interrupt registers */ +struct sun4m_irq_global { + u32 pending; + u32 mask; + u32 mask_clear; + u32 mask_set; + u32 interrupt_target; +}; + +extern struct sun4m_irq_percpu __iomem *sun4m_irq_percpu[SUN4M_NCPUS]; +extern struct sun4m_irq_global __iomem *sun4m_irq_global; + +/* The following definitions describe the individual platform features: */ +#define FEAT_L10_CLOCKSOURCE (1 << 0) /* L10 timer is used as a clocksource */ +#define FEAT_L10_CLOCKEVENT (1 << 1) /* L10 timer is used as a clockevent */ +#define FEAT_L14_ONESHOT (1 << 2) /* L14 timer clockevent can oneshot */ + +/* + * Platform specific configuration + * The individual platforms assign their platform + * specifics in their init functions. */ +struct sparc_config { + void (*init_timers)(void); + unsigned int (*build_device_irq)(struct platform_device *op, + unsigned int real_irq); + + /* generic clockevent features - see FEAT_* above */ + int features; + + /* clock rate used for clock event timer */ + int clock_rate; + + /* one period for clock source timer */ + unsigned int cs_period; + + /* function to obtain offsett for cs period */ + unsigned int (*get_cycles_offset)(void); + + void (*clear_clock_irq)(void); + void (*load_profile_irq)(int cpu, unsigned int limit); +}; +extern struct sparc_config sparc_config; -BTFIXUPDEF_CALL(void, disable_irq, unsigned int) -BTFIXUPDEF_CALL(void, enable_irq, unsigned int) -BTFIXUPDEF_CALL(void, disable_pil_irq, unsigned int) -BTFIXUPDEF_CALL(void, enable_pil_irq, unsigned int) -BTFIXUPDEF_CALL(void, clear_clock_irq, void) -BTFIXUPDEF_CALL(void, load_profile_irq, int, unsigned int) - -static inline void __disable_irq(unsigned int irq) -{ - BTFIXUP_CALL(disable_irq)(irq); -} - -static inline void __enable_irq(unsigned int irq) -{ - BTFIXUP_CALL(enable_irq)(irq); -} - -static inline void disable_pil_irq(unsigned int irq) -{ - BTFIXUP_CALL(disable_pil_irq)(irq); -} - -static inline void enable_pil_irq(unsigned int irq) -{ - BTFIXUP_CALL(enable_pil_irq)(irq); -} - -static inline void clear_clock_irq(void) -{ - BTFIXUP_CALL(clear_clock_irq)(); -} - -static inline void load_profile_irq(int cpu, int limit) -{ - BTFIXUP_CALL(load_profile_irq)(cpu, limit); -} - -extern void (*sparc_init_timers)(irq_handler_t lvl10_irq); - -extern void claim_ticker14(irq_handler_t irq_handler, - int irq, - unsigned int timeout); +unsigned int irq_alloc(unsigned int real_irq, unsigned int pil); +void irq_link(unsigned int irq); +void irq_unlink(unsigned int irq); +void handler_irq(unsigned int pil, struct pt_regs *regs); + +unsigned long leon_get_irqmask(unsigned int irq); + +/* irq_32.c */ +void sparc_floppy_irq(int irq, void *dev_id, struct pt_regs *regs); + +/* sun4m_irq.c */ +void sun4m_nmi(struct pt_regs *regs); + +/* sun4d_irq.c */ +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs); #ifdef CONFIG_SMP -BTFIXUPDEF_CALL(void, set_cpu_int, int, int) -BTFIXUPDEF_CALL(void, clear_cpu_int, int, int) -BTFIXUPDEF_CALL(void, set_irq_udt, int) -#define set_cpu_int(cpu,level) BTFIXUP_CALL(set_cpu_int)(cpu,level) -#define clear_cpu_int(cpu,level) BTFIXUP_CALL(clear_cpu_int)(cpu,level) -#define set_irq_udt(cpu) BTFIXUP_CALL(set_irq_udt)(cpu) +/* All SUN4D IPIs are sent on this IRQ, may be shared with hard IRQs */ +#define SUN4D_IPI_IRQ 13 + +void sun4d_ipi_interrupt(void); + #endif diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c index 5ad6e5c5dbb..a979e99f875 100644 --- a/arch/sparc/kernel/irq_32.c +++ b/arch/sparc/kernel/irq_32.c @@ -1,8 +1,8 @@ /* - * arch/sparc/kernel/irq.c: Interrupt request handling routines. On the - * Sparc the IRQs are basically 'cast in stone' - * and you are supposed to probe the prom's device - * node trees to find out who's got which IRQ. + * Interrupt request handling routines. On the + * Sparc the IRQs are basically 'cast in stone' + * and you are supposed to probe the prom's device + * node trees to find out who's got which IRQ. * * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) * Copyright (C) 1995 Miguel de Icaza (miguel@nuclecu.unam.mx) @@ -11,52 +11,22 @@ * Copyright (C) 1998-2000 Anton Blanchard (anton@samba.org) */ -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/ptrace.h> -#include <linux/errno.h> -#include <linux/linkage.h> #include <linux/kernel_stat.h> -#include <linux/signal.h> -#include <linux/interrupt.h> -#include <linux/slab.h> -#include <linux/random.h> -#include <linux/init.h> -#include <linux/smp.h> -#include <linux/delay.h> -#include <linux/threads.h> -#include <linux/spinlock.h> #include <linux/seq_file.h> +#include <linux/export.h> -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/system.h> -#include <asm/psr.h> -#include <asm/smp.h> -#include <asm/vaddrs.h> -#include <asm/timer.h> -#include <asm/openprom.h> -#include <asm/oplib.h> -#include <asm/traps.h> -#include <asm/irq.h> -#include <asm/io.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/pcic.h> #include <asm/cacheflush.h> -#include <asm/irq_regs.h> +#include <asm/cpudata.h> +#include <asm/setup.h> +#include <asm/pcic.h> #include <asm/leon.h> #include "kernel.h" #include "irq.h" -#ifdef CONFIG_SMP -#define SMP_NOP2 "nop; nop;\n\t" -#define SMP_NOP3 "nop; nop; nop;\n\t" -#else -#define SMP_NOP2 -#define SMP_NOP3 -#endif /* SMP */ +/* platform specific irq setup */ +struct sparc_config sparc_config; + unsigned long arch_local_irq_save(void) { unsigned long retval; @@ -64,7 +34,6 @@ unsigned long arch_local_irq_save(void) __asm__ __volatile__( "rd %%psr, %0\n\t" - SMP_NOP3 /* Sun4m + Cypress + SMP bug */ "or %0, %2, %1\n\t" "wr %1, 0, %%psr\n\t" "nop; nop; nop\n" @@ -82,7 +51,6 @@ void arch_local_irq_enable(void) __asm__ __volatile__( "rd %%psr, %0\n\t" - SMP_NOP3 /* Sun4m + Cypress + SMP bug */ "andn %0, %1, %0\n\t" "wr %0, 0, %%psr\n\t" "nop; nop; nop\n" @@ -99,7 +67,6 @@ void arch_local_irq_restore(unsigned long old_psr) __asm__ __volatile__( "rd %%psr, %0\n\t" "and %2, %1, %2\n\t" - SMP_NOP2 /* Sun4m + Cypress + SMP bug */ "andn %0, %1, %0\n\t" "wr %0, %2, %%psr\n\t" "nop; nop; nop\n" @@ -126,309 +93,187 @@ EXPORT_SYMBOL(arch_local_irq_restore); * directed CPU interrupts using the existing enable/disable irq code * with tweaks. * + * Sun4d complicates things even further. IRQ numbers are arbitrary + * 32-bit values in that case. Since this is similar to sparc64, + * we adopt a virtual IRQ numbering scheme as is done there. + * Virutal interrupt numbers are allocated by build_irq(). So NR_IRQS + * just becomes a limit of how many interrupt sources we can handle in + * a single system. Even fully loaded SS2000 machines top off at + * about 32 interrupt sources or so, therefore a NR_IRQS value of 64 + * is more than enough. + * + * We keep a map of per-PIL enable interrupts. These get wired + * up via the irq_chip->startup() method which gets invoked by + * the generic IRQ layer during request_irq(). */ -static void irq_panic(void) -{ - extern char *cputypval; - prom_printf("machine: %s doesn't have irq handlers defined!\n",cputypval); - prom_halt(); -} -void (*sparc_init_timers)(irq_handler_t ) = - (void (*)(irq_handler_t )) irq_panic; +/* Table of allocated irqs. Unused entries has irq == 0 */ +static struct irq_bucket irq_table[NR_IRQS]; +/* Protect access to irq_table */ +static DEFINE_SPINLOCK(irq_table_lock); -/* - * Dave Redman (djhr@tadpole.co.uk) - * - * There used to be extern calls and hard coded values here.. very sucky! - * instead, because some of the devices attach very early, I do something - * equally sucky but at least we'll never try to free statically allocated - * space or call kmalloc before kmalloc_init :(. - * - * In fact it's the timer10 that attaches first.. then timer14 - * then kmalloc_init is called.. then the tty interrupts attach. - * hmmm.... - * - */ -#define MAX_STATIC_ALLOC 4 -struct irqaction static_irqaction[MAX_STATIC_ALLOC]; -int static_irq_count; - -static struct { - struct irqaction *action; - int flags; -} sparc_irq[NR_IRQS]; -#define SPARC_IRQ_INPROGRESS 1 - -/* Used to protect the IRQ action lists */ -DEFINE_SPINLOCK(irq_action_lock); +/* Map between the irq identifier used in hw to the irq_bucket. */ +struct irq_bucket *irq_map[SUN4D_MAX_IRQ]; +/* Protect access to irq_map */ +static DEFINE_SPINLOCK(irq_map_lock); -int show_interrupts(struct seq_file *p, void *v) +/* Allocate a new irq from the irq_table */ +unsigned int irq_alloc(unsigned int real_irq, unsigned int pil) { - int i = *(loff_t *) v; - struct irqaction * action; unsigned long flags; -#ifdef CONFIG_SMP - int j; -#endif + unsigned int i; - if (sparc_cpu_model == sun4d) { - extern int show_sun4d_interrupts(struct seq_file *, void *); - - return show_sun4d_interrupts(p, v); + spin_lock_irqsave(&irq_table_lock, flags); + for (i = 1; i < NR_IRQS; i++) { + if (irq_table[i].real_irq == real_irq && irq_table[i].pil == pil) + goto found; } - spin_lock_irqsave(&irq_action_lock, flags); + + for (i = 1; i < NR_IRQS; i++) { + if (!irq_table[i].irq) + break; + } + if (i < NR_IRQS) { - action = sparc_irq[i].action; - if (!action) - goto out_unlock; - seq_printf(p, "%3d: ", i); -#ifndef CONFIG_SMP - seq_printf(p, "%10u ", kstat_irqs(i)); -#else - for_each_online_cpu(j) { - seq_printf(p, "%10u ", - kstat_cpu(j).irqs[i]); - } -#endif - seq_printf(p, " %c %s", - (action->flags & IRQF_DISABLED) ? '+' : ' ', - action->name); - for (action=action->next; action; action = action->next) { - seq_printf(p, ",%s %s", - (action->flags & IRQF_DISABLED) ? " +" : "", - action->name); - } - seq_putc(p, '\n'); + irq_table[i].real_irq = real_irq; + irq_table[i].irq = i; + irq_table[i].pil = pil; + } else { + printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); + i = 0; } -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); - return 0; +found: + spin_unlock_irqrestore(&irq_table_lock, flags); + + return i; } -void free_irq(unsigned int irq, void *dev_id) +/* Based on a single pil handler_irq may need to call several + * interrupt handlers. Use irq_map as entry to irq_table, + * and let each entry in irq_table point to the next entry. + */ +void irq_link(unsigned int irq) { - struct irqaction * action; - struct irqaction **actionp; - unsigned long flags; - unsigned int cpu_irq; - - if (sparc_cpu_model == sun4d) { - extern void sun4d_free_irq(unsigned int, void *); - - sun4d_free_irq(irq, dev_id); - return; - } - cpu_irq = irq & (NR_IRQS - 1); - if (cpu_irq > 14) { /* 14 irq levels on the sparc */ - printk("Trying to free bogus IRQ %d\n", irq); - return; - } - - spin_lock_irqsave(&irq_action_lock, flags); + struct irq_bucket *p; + unsigned long flags; + unsigned int pil; - actionp = &sparc_irq[cpu_irq].action; - action = *actionp; + BUG_ON(irq >= NR_IRQS); - if (!action->handler) { - printk("Trying to free free IRQ%d\n",irq); - goto out_unlock; - } - if (dev_id) { - for (; action; action = action->next) { - if (action->dev_id == dev_id) - break; - actionp = &action->next; - } - if (!action) { - printk("Trying to free free shared IRQ%d\n",irq); - goto out_unlock; - } - } else if (action->flags & IRQF_SHARED) { - printk("Trying to free shared IRQ%d with NULL device ID\n", irq); - goto out_unlock; - } - if (action->flags & SA_STATIC_ALLOC) - { - /* This interrupt is marked as specially allocated - * so it is a bad idea to free it. - */ - printk("Attempt to free statically allocated IRQ%d (%s)\n", - irq, action->name); - goto out_unlock; - } + spin_lock_irqsave(&irq_map_lock, flags); - *actionp = action->next; + p = &irq_table[irq]; + pil = p->pil; + BUG_ON(pil > SUN4D_MAX_IRQ); + p->next = irq_map[pil]; + irq_map[pil] = p; - spin_unlock_irqrestore(&irq_action_lock, flags); + spin_unlock_irqrestore(&irq_map_lock, flags); +} - synchronize_irq(irq); +void irq_unlink(unsigned int irq) +{ + struct irq_bucket *p, **pnext; + unsigned long flags; - spin_lock_irqsave(&irq_action_lock, flags); + BUG_ON(irq >= NR_IRQS); - kfree(action); + spin_lock_irqsave(&irq_map_lock, flags); - if (!sparc_irq[cpu_irq].action) - __disable_irq(irq); + p = &irq_table[irq]; + BUG_ON(p->pil > SUN4D_MAX_IRQ); + pnext = &irq_map[p->pil]; + while (*pnext != p) + pnext = &(*pnext)->next; + *pnext = p->next; -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); + spin_unlock_irqrestore(&irq_map_lock, flags); } -EXPORT_SYMBOL(free_irq); -/* - * This is called when we want to synchronize with - * interrupts. We may for example tell a device to - * stop sending interrupts: but to make sure there - * are no interrupts that are executing on another - * CPU we need to call this function. - */ -#ifdef CONFIG_SMP -void synchronize_irq(unsigned int irq) +/* /proc/interrupts printing */ +int arch_show_interrupts(struct seq_file *p, int prec) { - unsigned int cpu_irq; - - cpu_irq = irq & (NR_IRQS - 1); - while (sparc_irq[cpu_irq].flags & SPARC_IRQ_INPROGRESS) - cpu_relax(); -} -EXPORT_SYMBOL(synchronize_irq); -#endif /* SMP */ + int j; -void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs) -{ - int i; - struct irqaction * action; - unsigned int cpu_irq; - - cpu_irq = irq & (NR_IRQS - 1); - action = sparc_irq[cpu_irq].action; - - printk("IO device interrupt, irq = %d\n", irq); - printk("PC = %08lx NPC = %08lx FP=%08lx\n", regs->pc, - regs->npc, regs->u_regs[14]); - if (action) { - printk("Expecting: "); - for (i = 0; i < 16; i++) - if (action->handler) - printk("[%s:%d:0x%x] ", action->name, - (int) i, (unsigned int) action->handler); - } - printk("AIEEE\n"); - panic("bogus interrupt received"); +#ifdef CONFIG_SMP + seq_printf(p, "RES: "); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_data(j).irq_resched_count); + seq_printf(p, " IPI rescheduling interrupts\n"); + seq_printf(p, "CAL: "); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_data(j).irq_call_count); + seq_printf(p, " IPI function call interrupts\n"); +#endif + seq_printf(p, "NMI: "); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_data(j).counter); + seq_printf(p, " Non-maskable interrupts\n"); + return 0; } -void handler_irq(int irq, struct pt_regs * regs) +void handler_irq(unsigned int pil, struct pt_regs *regs) { struct pt_regs *old_regs; - struct irqaction * action; - int cpu = smp_processor_id(); -#ifdef CONFIG_SMP - extern void smp4m_irq_rotate(int cpu); -#endif + struct irq_bucket *p; + BUG_ON(pil > 15); old_regs = set_irq_regs(regs); irq_enter(); - disable_pil_irq(irq); -#ifdef CONFIG_SMP - /* Only rotate on lower priority IRQs (scsi, ethernet, etc.). */ - if((sparc_cpu_model==sun4m) && (irq < 10)) - smp4m_irq_rotate(cpu); -#endif - action = sparc_irq[irq].action; - sparc_irq[irq].flags |= SPARC_IRQ_INPROGRESS; - kstat_cpu(cpu).irqs[irq]++; - do { - if (!action || !action->handler) - unexpected_irq(irq, NULL, regs); - action->handler(irq, action->dev_id); - action = action->next; - } while (action); - sparc_irq[irq].flags &= ~SPARC_IRQ_INPROGRESS; - enable_pil_irq(irq); + + p = irq_map[pil]; + while (p) { + struct irq_bucket *next = p->next; + + generic_handle_irq(p->irq); + p = next; + } irq_exit(); set_irq_regs(old_regs); } #if defined(CONFIG_BLK_DEV_FD) || defined(CONFIG_BLK_DEV_FD_MODULE) +static unsigned int floppy_irq; -/* Fast IRQs on the Sparc can only have one routine attached to them, - * thus no sharing possible. - */ -static int request_fast_irq(unsigned int irq, - void (*handler)(void), - unsigned long irqflags, const char *devname) +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler) { - struct irqaction *action; - unsigned long flags; unsigned int cpu_irq; - int ret; -#if defined CONFIG_SMP && !defined CONFIG_SPARC_LEON - struct tt_entry *trap_table; - extern struct tt_entry trapbase_cpu1, trapbase_cpu2, trapbase_cpu3; -#endif - - cpu_irq = irq & (NR_IRQS - 1); - if(cpu_irq > 14) { - ret = -EINVAL; - goto out; - } - if(!handler) { - ret = -EINVAL; - goto out; - } + int err; - spin_lock_irqsave(&irq_action_lock, flags); - action = sparc_irq[cpu_irq].action; - if(action) { - if(action->flags & IRQF_SHARED) - panic("Trying to register fast irq when already shared.\n"); - if(irqflags & IRQF_SHARED) - panic("Trying to register fast irq as shared.\n"); + err = request_irq(irq, irq_handler, 0, "floppy", NULL); + if (err) + return -1; - /* Anyway, someone already owns it so cannot be made fast. */ - printk("request_fast_irq: Trying to register yet already owned.\n"); - ret = -EBUSY; - goto out_unlock; - } + /* Save for later use in floppy interrupt handler */ + floppy_irq = irq; - /* If this is flagged as statically allocated then we use our - * private struct which is never freed. - */ - if (irqflags & SA_STATIC_ALLOC) { - if (static_irq_count < MAX_STATIC_ALLOC) - action = &static_irqaction[static_irq_count++]; - else - printk("Fast IRQ%d (%s) SA_STATIC_ALLOC failed using kmalloc\n", - irq, devname); - } - - if (action == NULL) - action = kmalloc(sizeof(struct irqaction), - GFP_ATOMIC); - - if (!action) { - ret = -ENOMEM; - goto out_unlock; - } + cpu_irq = (irq & (NR_IRQS - 1)); /* Dork with trap table if we get this far. */ #define INSTANTIATE(table) \ table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_one = SPARC_RD_PSR_L0; \ table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_two = \ - SPARC_BRANCH((unsigned long) handler, \ + SPARC_BRANCH((unsigned long) floppy_hardint, \ (unsigned long) &table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_two);\ table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_three = SPARC_RD_WIM_L3; \ table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_four = SPARC_NOP; INSTANTIATE(sparc_ttable) -#if defined CONFIG_SMP && !defined CONFIG_SPARC_LEON - trap_table = &trapbase_cpu1; INSTANTIATE(trap_table) - trap_table = &trapbase_cpu2; INSTANTIATE(trap_table) - trap_table = &trapbase_cpu3; INSTANTIATE(trap_table) + +#if defined CONFIG_SMP + if (sparc_cpu_model != sparc_leon) { + struct tt_entry *trap_table; + + trap_table = &trapbase_cpu1; + INSTANTIATE(trap_table) + trap_table = &trapbase_cpu2; + INSTANTIATE(trap_table) + trap_table = &trapbase_cpu3; + INSTANTIATE(trap_table) + } #endif #undef INSTANTIATE /* @@ -437,24 +282,12 @@ static int request_fast_irq(unsigned int irq, * writing we have no CPU-neutral interface to fine-grained flushes. */ flush_cache_all(); - - action->flags = irqflags; - action->name = devname; - action->dev_id = NULL; - action->next = NULL; - - sparc_irq[cpu_irq].action = action; - - __enable_irq(irq); - - ret = 0; -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); -out: - return ret; + return 0; } +EXPORT_SYMBOL(sparc_floppy_request_irq); -/* These variables are used to access state from the assembler +/* + * These variables are used to access state from the assembler * interrupt handler, floppy_hardint, so we cannot put these in * the floppy driver image because that would not work in the * modular case. @@ -477,155 +310,23 @@ EXPORT_SYMBOL(pdma_base); unsigned long pdma_areasize; EXPORT_SYMBOL(pdma_areasize); -extern void floppy_hardint(void); - -static irq_handler_t floppy_irq_handler; - +/* Use the generic irq support to call floppy_interrupt + * which was setup using request_irq() in sparc_floppy_request_irq(). + * We only have one floppy interrupt so we do not need to check + * for additional handlers being wired up by irq_link() + */ void sparc_floppy_irq(int irq, void *dev_id, struct pt_regs *regs) { struct pt_regs *old_regs; - int cpu = smp_processor_id(); old_regs = set_irq_regs(regs); - disable_pil_irq(irq); irq_enter(); - kstat_cpu(cpu).irqs[irq]++; - floppy_irq_handler(irq, dev_id); + generic_handle_irq(floppy_irq); irq_exit(); - enable_pil_irq(irq); set_irq_regs(old_regs); - // XXX Eek, it's totally changed with preempt_count() and such - // if (softirq_pending(cpu)) - // do_softirq(); -} - -int sparc_floppy_request_irq(int irq, unsigned long flags, - irq_handler_t irq_handler) -{ - floppy_irq_handler = irq_handler; - return request_fast_irq(irq, floppy_hardint, flags, "floppy"); } -EXPORT_SYMBOL(sparc_floppy_request_irq); - #endif -int request_irq(unsigned int irq, - irq_handler_t handler, - unsigned long irqflags, const char * devname, void *dev_id) -{ - struct irqaction * action, **actionp; - unsigned long flags; - unsigned int cpu_irq; - int ret; - - if (sparc_cpu_model == sun4d) { - extern int sun4d_request_irq(unsigned int, - irq_handler_t , - unsigned long, const char *, void *); - return sun4d_request_irq(irq, handler, irqflags, devname, dev_id); - } - cpu_irq = irq & (NR_IRQS - 1); - if(cpu_irq > 14) { - ret = -EINVAL; - goto out; - } - if (!handler) { - ret = -EINVAL; - goto out; - } - - spin_lock_irqsave(&irq_action_lock, flags); - - actionp = &sparc_irq[cpu_irq].action; - action = *actionp; - if (action) { - if (!(action->flags & IRQF_SHARED) || !(irqflags & IRQF_SHARED)) { - ret = -EBUSY; - goto out_unlock; - } - if ((action->flags & IRQF_DISABLED) != (irqflags & IRQF_DISABLED)) { - printk("Attempt to mix fast and slow interrupts on IRQ%d denied\n", irq); - ret = -EBUSY; - goto out_unlock; - } - for ( ; action; action = *actionp) - actionp = &action->next; - } - - /* If this is flagged as statically allocated then we use our - * private struct which is never freed. - */ - if (irqflags & SA_STATIC_ALLOC) { - if (static_irq_count < MAX_STATIC_ALLOC) - action = &static_irqaction[static_irq_count++]; - else - printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed using kmalloc\n", irq, devname); - } - - if (action == NULL) - action = kmalloc(sizeof(struct irqaction), - GFP_ATOMIC); - - if (!action) { - ret = -ENOMEM; - goto out_unlock; - } - - action->handler = handler; - action->flags = irqflags; - action->name = devname; - action->next = NULL; - action->dev_id = dev_id; - - *actionp = action; - - __enable_irq(irq); - - ret = 0; -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); -out: - return ret; -} - -EXPORT_SYMBOL(request_irq); - -void disable_irq_nosync(unsigned int irq) -{ - __disable_irq(irq); -} -EXPORT_SYMBOL(disable_irq_nosync); - -void disable_irq(unsigned int irq) -{ - __disable_irq(irq); -} -EXPORT_SYMBOL(disable_irq); - -void enable_irq(unsigned int irq) -{ - __enable_irq(irq); -} - -EXPORT_SYMBOL(enable_irq); - -/* We really don't need these at all on the Sparc. We only have - * stubs here because they are exported to modules. - */ -unsigned long probe_irq_on(void) -{ - return 0; -} - -EXPORT_SYMBOL(probe_irq_on); - -int probe_irq_off(unsigned long mask) -{ - return 0; -} - -EXPORT_SYMBOL(probe_irq_off); - /* djhr * This could probably be made indirect too and assigned in the CPU * bits of the code. That would be much nicer I think and would also @@ -636,27 +337,15 @@ EXPORT_SYMBOL(probe_irq_off); void __init init_IRQ(void) { - extern void sun4c_init_IRQ( void ); - extern void sun4m_init_IRQ( void ); - extern void sun4d_init_IRQ( void ); - - switch(sparc_cpu_model) { - case sun4c: - case sun4: - sun4c_init_IRQ(); - break; - + switch (sparc_cpu_model) { case sun4m: -#ifdef CONFIG_PCI pcic_probe(); - if (pcic_present()) { + if (pcic_present()) sun4m_pci_init_IRQ(); - break; - } -#endif - sun4m_init_IRQ(); + else + sun4m_init_IRQ(); break; - + case sun4d: sun4d_init_IRQ(); break; @@ -669,12 +358,5 @@ void __init init_IRQ(void) prom_printf("Cannot initialize IRQs on this Sun machine..."); break; } - btfixup(); } -#ifdef CONFIG_PROC_FS -void init_irq_proc(void) -{ - /* For now, nothing... */ -} -#endif /* CONFIG_PROC_FS */ diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 830d70a3e20..666193f4e8b 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -5,7 +5,6 @@ * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) */ -#include <linux/module.h> #include <linux/sched.h> #include <linux/linkage.h> #include <linux/ptrace.h> @@ -26,8 +25,7 @@ #include <asm/ptrace.h> #include <asm/processor.h> -#include <asm/atomic.h> -#include <asm/system.h> +#include <linux/atomic.h> #include <asm/irq.h> #include <asm/io.h> #include <asm/iommu.h> @@ -82,7 +80,7 @@ static void bucket_clear_chain_pa(unsigned long bucket_pa) "i" (ASI_PHYS_USE_EC)); } -static unsigned int bucket_get_virt_irq(unsigned long bucket_pa) +static unsigned int bucket_get_irq(unsigned long bucket_pa) { unsigned int ret; @@ -90,21 +88,20 @@ static unsigned int bucket_get_virt_irq(unsigned long bucket_pa) : "=&r" (ret) : "r" (bucket_pa + offsetof(struct ino_bucket, - __virt_irq)), + __irq)), "i" (ASI_PHYS_USE_EC)); return ret; } -static void bucket_set_virt_irq(unsigned long bucket_pa, - unsigned int virt_irq) +static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq) { __asm__ __volatile__("stwa %0, [%1] %2" : /* no outputs */ - : "r" (virt_irq), + : "r" (irq), "r" (bucket_pa + offsetof(struct ino_bucket, - __virt_irq)), + __irq)), "i" (ASI_PHYS_USE_EC)); } @@ -114,97 +111,63 @@ static struct { unsigned int dev_handle; unsigned int dev_ino; unsigned int in_use; -} virt_irq_table[NR_IRQS]; -static DEFINE_SPINLOCK(virt_irq_alloc_lock); +} irq_table[NR_IRQS]; +static DEFINE_SPINLOCK(irq_alloc_lock); -unsigned char virt_irq_alloc(unsigned int dev_handle, - unsigned int dev_ino) +unsigned char irq_alloc(unsigned int dev_handle, unsigned int dev_ino) { unsigned long flags; unsigned char ent; BUILD_BUG_ON(NR_IRQS >= 256); - spin_lock_irqsave(&virt_irq_alloc_lock, flags); + spin_lock_irqsave(&irq_alloc_lock, flags); for (ent = 1; ent < NR_IRQS; ent++) { - if (!virt_irq_table[ent].in_use) + if (!irq_table[ent].in_use) break; } if (ent >= NR_IRQS) { printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); ent = 0; } else { - virt_irq_table[ent].dev_handle = dev_handle; - virt_irq_table[ent].dev_ino = dev_ino; - virt_irq_table[ent].in_use = 1; + irq_table[ent].dev_handle = dev_handle; + irq_table[ent].dev_ino = dev_ino; + irq_table[ent].in_use = 1; } - spin_unlock_irqrestore(&virt_irq_alloc_lock, flags); + spin_unlock_irqrestore(&irq_alloc_lock, flags); return ent; } #ifdef CONFIG_PCI_MSI -void virt_irq_free(unsigned int virt_irq) +void irq_free(unsigned int irq) { unsigned long flags; - if (virt_irq >= NR_IRQS) + if (irq >= NR_IRQS) return; - spin_lock_irqsave(&virt_irq_alloc_lock, flags); + spin_lock_irqsave(&irq_alloc_lock, flags); - virt_irq_table[virt_irq].in_use = 0; + irq_table[irq].in_use = 0; - spin_unlock_irqrestore(&virt_irq_alloc_lock, flags); + spin_unlock_irqrestore(&irq_alloc_lock, flags); } #endif /* * /proc/interrupts printing: */ - -int show_interrupts(struct seq_file *p, void *v) +int arch_show_interrupts(struct seq_file *p, int prec) { - int i = *(loff_t *) v, j; - struct irqaction * action; - unsigned long flags; - - if (i == 0) { - seq_printf(p, " "); - for_each_online_cpu(j) - seq_printf(p, "CPU%d ",j); - seq_putc(p, '\n'); - } + int j; - if (i < NR_IRQS) { - raw_spin_lock_irqsave(&irq_desc[i].lock, flags); - action = irq_desc[i].action; - if (!action) - goto skip; - seq_printf(p, "%3d: ",i); -#ifndef CONFIG_SMP - seq_printf(p, "%10u ", kstat_irqs(i)); -#else - for_each_online_cpu(j) - seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); -#endif - seq_printf(p, " %9s", irq_desc[i].chip->name); - seq_printf(p, " %s", action->name); - - for (action=action->next; action; action = action->next) - seq_printf(p, ", %s", action->name); - - seq_putc(p, '\n'); -skip: - raw_spin_unlock_irqrestore(&irq_desc[i].lock, flags); - } else if (i == NR_IRQS) { - seq_printf(p, "NMI: "); - for_each_online_cpu(j) - seq_printf(p, "%10u ", cpu_data(j).__nmi_count); - seq_printf(p, " Non-maskable interrupts\n"); - } + seq_printf(p, "NMI: "); + for_each_online_cpu(j) + seq_printf(p, "%10u ", cpu_data(j).__nmi_count); + seq_printf(p, " Non-maskable interrupts\n"); return 0; } @@ -253,39 +216,38 @@ struct irq_handler_data { }; #ifdef CONFIG_SMP -static int irq_choose_cpu(unsigned int virt_irq, const struct cpumask *affinity) +static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) { cpumask_t mask; int cpuid; cpumask_copy(&mask, affinity); - if (cpus_equal(mask, cpu_online_map)) { - cpuid = map_to_cpu(virt_irq); + if (cpumask_equal(&mask, cpu_online_mask)) { + cpuid = map_to_cpu(irq); } else { cpumask_t tmp; - cpus_and(tmp, cpu_online_map, mask); - cpuid = cpus_empty(tmp) ? map_to_cpu(virt_irq) : first_cpu(tmp); + cpumask_and(&tmp, cpu_online_mask, &mask); + cpuid = cpumask_empty(&tmp) ? map_to_cpu(irq) : cpumask_first(&tmp); } return cpuid; } #else -#define irq_choose_cpu(virt_irq, affinity) \ +#define irq_choose_cpu(irq, affinity) \ real_hard_smp_processor_id() #endif -static void sun4u_irq_enable(unsigned int virt_irq) +static void sun4u_irq_enable(struct irq_data *data) { - struct irq_handler_data *data = get_irq_chip_data(virt_irq); + struct irq_handler_data *handler_data = data->handler_data; - if (likely(data)) { + if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; - cpuid = irq_choose_cpu(virt_irq, - irq_desc[virt_irq].affinity); - imap = data->imap; + cpuid = irq_choose_cpu(data->irq, data->affinity); + imap = handler_data->imap; tid = sun4u_compute_tid(imap, cpuid); @@ -294,21 +256,21 @@ static void sun4u_irq_enable(unsigned int virt_irq) IMAP_AID_SAFARI | IMAP_NID_SAFARI); val |= tid | IMAP_VALID; upa_writeq(val, imap); - upa_writeq(ICLR_IDLE, data->iclr); + upa_writeq(ICLR_IDLE, handler_data->iclr); } } -static int sun4u_set_affinity(unsigned int virt_irq, - const struct cpumask *mask) +static int sun4u_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) { - struct irq_handler_data *data = get_irq_chip_data(virt_irq); + struct irq_handler_data *handler_data = data->handler_data; - if (likely(data)) { + if (likely(handler_data)) { unsigned long cpuid, imap, val; unsigned int tid; - cpuid = irq_choose_cpu(virt_irq, mask); - imap = data->imap; + cpuid = irq_choose_cpu(data->irq, mask); + imap = handler_data->imap; tid = sun4u_compute_tid(imap, cpuid); @@ -317,7 +279,7 @@ static int sun4u_set_affinity(unsigned int virt_irq, IMAP_AID_SAFARI | IMAP_NID_SAFARI); val |= tid | IMAP_VALID; upa_writeq(val, imap); - upa_writeq(ICLR_IDLE, data->iclr); + upa_writeq(ICLR_IDLE, handler_data->iclr); } return 0; @@ -340,27 +302,22 @@ static int sun4u_set_affinity(unsigned int virt_irq, * sees that, it also hooks up a default ->shutdown method which * invokes ->mask() which we do not want. See irq_chip_set_defaults(). */ -static void sun4u_irq_disable(unsigned int virt_irq) +static void sun4u_irq_disable(struct irq_data *data) { } -static void sun4u_irq_eoi(unsigned int virt_irq) +static void sun4u_irq_eoi(struct irq_data *data) { - struct irq_handler_data *data = get_irq_chip_data(virt_irq); - struct irq_desc *desc = irq_desc + virt_irq; + struct irq_handler_data *handler_data = data->handler_data; - if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS))) - return; - - if (likely(data)) - upa_writeq(ICLR_IDLE, data->iclr); + if (likely(handler_data)) + upa_writeq(ICLR_IDLE, handler_data->iclr); } -static void sun4v_irq_enable(unsigned int virt_irq) +static void sun4v_irq_enable(struct irq_data *data) { - unsigned int ino = virt_irq_table[virt_irq].dev_ino; - unsigned long cpuid = irq_choose_cpu(virt_irq, - irq_desc[virt_irq].affinity); + unsigned int ino = irq_table[data->irq].dev_ino; + unsigned long cpuid = irq_choose_cpu(data->irq, data->affinity); int err; err = sun4v_intr_settarget(ino, cpuid); @@ -377,11 +334,11 @@ static void sun4v_irq_enable(unsigned int virt_irq) ino, err); } -static int sun4v_set_affinity(unsigned int virt_irq, - const struct cpumask *mask) +static int sun4v_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) { - unsigned int ino = virt_irq_table[virt_irq].dev_ino; - unsigned long cpuid = irq_choose_cpu(virt_irq, mask); + unsigned int ino = irq_table[data->irq].dev_ino; + unsigned long cpuid = irq_choose_cpu(data->irq, mask); int err; err = sun4v_intr_settarget(ino, cpuid); @@ -392,9 +349,9 @@ static int sun4v_set_affinity(unsigned int virt_irq, return 0; } -static void sun4v_irq_disable(unsigned int virt_irq) +static void sun4v_irq_disable(struct irq_data *data) { - unsigned int ino = virt_irq_table[virt_irq].dev_ino; + unsigned int ino = irq_table[data->irq].dev_ino; int err; err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); @@ -403,30 +360,26 @@ static void sun4v_irq_disable(unsigned int virt_irq) "err(%d)\n", ino, err); } -static void sun4v_irq_eoi(unsigned int virt_irq) +static void sun4v_irq_eoi(struct irq_data *data) { - unsigned int ino = virt_irq_table[virt_irq].dev_ino; - struct irq_desc *desc = irq_desc + virt_irq; + unsigned int ino = irq_table[data->irq].dev_ino; int err; - if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS))) - return; - err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); if (err != HV_EOK) printk(KERN_ERR "sun4v_intr_setstate(%x): " "err(%d)\n", ino, err); } -static void sun4v_virq_enable(unsigned int virt_irq) +static void sun4v_virq_enable(struct irq_data *data) { unsigned long cpuid, dev_handle, dev_ino; int err; - cpuid = irq_choose_cpu(virt_irq, irq_desc[virt_irq].affinity); + cpuid = irq_choose_cpu(data->irq, data->affinity); - dev_handle = virt_irq_table[virt_irq].dev_handle; - dev_ino = virt_irq_table[virt_irq].dev_ino; + dev_handle = irq_table[data->irq].dev_handle; + dev_ino = irq_table[data->irq].dev_ino; err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); if (err != HV_EOK) @@ -447,16 +400,16 @@ static void sun4v_virq_enable(unsigned int virt_irq) dev_handle, dev_ino, err); } -static int sun4v_virt_set_affinity(unsigned int virt_irq, - const struct cpumask *mask) +static int sun4v_virt_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) { unsigned long cpuid, dev_handle, dev_ino; int err; - cpuid = irq_choose_cpu(virt_irq, mask); + cpuid = irq_choose_cpu(data->irq, mask); - dev_handle = virt_irq_table[virt_irq].dev_handle; - dev_ino = virt_irq_table[virt_irq].dev_ino; + dev_handle = irq_table[data->irq].dev_handle; + dev_ino = irq_table[data->irq].dev_ino; err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); if (err != HV_EOK) @@ -467,13 +420,13 @@ static int sun4v_virt_set_affinity(unsigned int virt_irq, return 0; } -static void sun4v_virq_disable(unsigned int virt_irq) +static void sun4v_virq_disable(struct irq_data *data) { unsigned long dev_handle, dev_ino; int err; - dev_handle = virt_irq_table[virt_irq].dev_handle; - dev_ino = virt_irq_table[virt_irq].dev_ino; + dev_handle = irq_table[data->irq].dev_handle; + dev_ino = irq_table[data->irq].dev_ino; err = sun4v_vintr_set_valid(dev_handle, dev_ino, HV_INTR_DISABLED); @@ -483,17 +436,13 @@ static void sun4v_virq_disable(unsigned int virt_irq) dev_handle, dev_ino, err); } -static void sun4v_virq_eoi(unsigned int virt_irq) +static void sun4v_virq_eoi(struct irq_data *data) { - struct irq_desc *desc = irq_desc + virt_irq; unsigned long dev_handle, dev_ino; int err; - if (unlikely(desc->status & (IRQ_DISABLED|IRQ_INPROGRESS))) - return; - - dev_handle = virt_irq_table[virt_irq].dev_handle; - dev_ino = virt_irq_table[virt_irq].dev_ino; + dev_handle = irq_table[data->irq].dev_handle; + dev_ino = irq_table[data->irq].dev_ino; err = sun4v_vintr_set_state(dev_handle, dev_ino, HV_INTR_STATE_IDLE); @@ -504,132 +453,128 @@ static void sun4v_virq_eoi(unsigned int virt_irq) } static struct irq_chip sun4u_irq = { - .name = "sun4u", - .enable = sun4u_irq_enable, - .disable = sun4u_irq_disable, - .eoi = sun4u_irq_eoi, - .set_affinity = sun4u_set_affinity, + .name = "sun4u", + .irq_enable = sun4u_irq_enable, + .irq_disable = sun4u_irq_disable, + .irq_eoi = sun4u_irq_eoi, + .irq_set_affinity = sun4u_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, }; static struct irq_chip sun4v_irq = { - .name = "sun4v", - .enable = sun4v_irq_enable, - .disable = sun4v_irq_disable, - .eoi = sun4v_irq_eoi, - .set_affinity = sun4v_set_affinity, + .name = "sun4v", + .irq_enable = sun4v_irq_enable, + .irq_disable = sun4v_irq_disable, + .irq_eoi = sun4v_irq_eoi, + .irq_set_affinity = sun4v_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, }; static struct irq_chip sun4v_virq = { - .name = "vsun4v", - .enable = sun4v_virq_enable, - .disable = sun4v_virq_disable, - .eoi = sun4v_virq_eoi, - .set_affinity = sun4v_virt_set_affinity, + .name = "vsun4v", + .irq_enable = sun4v_virq_enable, + .irq_disable = sun4v_virq_disable, + .irq_eoi = sun4v_virq_eoi, + .irq_set_affinity = sun4v_virt_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, }; -static void pre_flow_handler(unsigned int virt_irq, - struct irq_desc *desc) +static void pre_flow_handler(struct irq_data *d) { - struct irq_handler_data *data = get_irq_chip_data(virt_irq); - unsigned int ino = virt_irq_table[virt_irq].dev_ino; - - data->pre_handler(ino, data->arg1, data->arg2); + struct irq_handler_data *handler_data = irq_data_get_irq_handler_data(d); + unsigned int ino = irq_table[d->irq].dev_ino; - handle_fasteoi_irq(virt_irq, desc); + handler_data->pre_handler(ino, handler_data->arg1, handler_data->arg2); } -void irq_install_pre_handler(int virt_irq, +void irq_install_pre_handler(int irq, void (*func)(unsigned int, void *, void *), void *arg1, void *arg2) { - struct irq_handler_data *data = get_irq_chip_data(virt_irq); - struct irq_desc *desc = irq_desc + virt_irq; + struct irq_handler_data *handler_data = irq_get_handler_data(irq); - data->pre_handler = func; - data->arg1 = arg1; - data->arg2 = arg2; + handler_data->pre_handler = func; + handler_data->arg1 = arg1; + handler_data->arg2 = arg2; - desc->handle_irq = pre_flow_handler; + __irq_set_preflow_handler(irq, pre_flow_handler); } unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) { struct ino_bucket *bucket; - struct irq_handler_data *data; - unsigned int virt_irq; + struct irq_handler_data *handler_data; + unsigned int irq; int ino; BUG_ON(tlb_type == hypervisor); ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup; bucket = &ivector_table[ino]; - virt_irq = bucket_get_virt_irq(__pa(bucket)); - if (!virt_irq) { - virt_irq = virt_irq_alloc(0, ino); - bucket_set_virt_irq(__pa(bucket), virt_irq); - set_irq_chip_and_handler_name(virt_irq, - &sun4u_irq, - handle_fasteoi_irq, - "IVEC"); + irq = bucket_get_irq(__pa(bucket)); + if (!irq) { + irq = irq_alloc(0, ino); + bucket_set_irq(__pa(bucket), irq); + irq_set_chip_and_handler_name(irq, &sun4u_irq, + handle_fasteoi_irq, "IVEC"); } - data = get_irq_chip_data(virt_irq); - if (unlikely(data)) + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) goto out; - data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); - if (unlikely(!data)) { + handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); + if (unlikely(!handler_data)) { prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); prom_halt(); } - set_irq_chip_data(virt_irq, data); + irq_set_handler_data(irq, handler_data); - data->imap = imap; - data->iclr = iclr; + handler_data->imap = imap; + handler_data->iclr = iclr; out: - return virt_irq; + return irq; } static unsigned int sun4v_build_common(unsigned long sysino, struct irq_chip *chip) { struct ino_bucket *bucket; - struct irq_handler_data *data; - unsigned int virt_irq; + struct irq_handler_data *handler_data; + unsigned int irq; BUG_ON(tlb_type != hypervisor); bucket = &ivector_table[sysino]; - virt_irq = bucket_get_virt_irq(__pa(bucket)); - if (!virt_irq) { - virt_irq = virt_irq_alloc(0, sysino); - bucket_set_virt_irq(__pa(bucket), virt_irq); - set_irq_chip_and_handler_name(virt_irq, chip, - handle_fasteoi_irq, + irq = bucket_get_irq(__pa(bucket)); + if (!irq) { + irq = irq_alloc(0, sysino); + bucket_set_irq(__pa(bucket), irq); + irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC"); } - data = get_irq_chip_data(virt_irq); - if (unlikely(data)) + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) goto out; - data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); - if (unlikely(!data)) { + handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); + if (unlikely(!handler_data)) { prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); prom_halt(); } - set_irq_chip_data(virt_irq, data); + irq_set_handler_data(irq, handler_data); /* Catch accidental accesses to these things. IMAP/ICLR handling * is done by hypervisor calls on sun4v platforms, not by direct * register accesses. */ - data->imap = ~0UL; - data->iclr = ~0UL; + handler_data->imap = ~0UL; + handler_data->iclr = ~0UL; out: - return virt_irq; + return irq; } unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) @@ -641,11 +586,10 @@ unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) { - struct irq_handler_data *data; + struct irq_handler_data *handler_data; unsigned long hv_err, cookie; struct ino_bucket *bucket; - struct irq_desc *desc; - unsigned int virt_irq; + unsigned int irq; bucket = kzalloc(sizeof(struct ino_bucket), GFP_ATOMIC); if (unlikely(!bucket)) @@ -662,32 +606,29 @@ unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) ((unsigned long) bucket + sizeof(struct ino_bucket))); - virt_irq = virt_irq_alloc(devhandle, devino); - bucket_set_virt_irq(__pa(bucket), virt_irq); + irq = irq_alloc(devhandle, devino); + bucket_set_irq(__pa(bucket), irq); - set_irq_chip_and_handler_name(virt_irq, &sun4v_virq, - handle_fasteoi_irq, + irq_set_chip_and_handler_name(irq, &sun4v_virq, handle_fasteoi_irq, "IVEC"); - data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); - if (unlikely(!data)) + handler_data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); + if (unlikely(!handler_data)) return 0; /* In order to make the LDC channel startup sequence easier, * especially wrt. locking, we do not let request_irq() enable * the interrupt. */ - desc = irq_desc + virt_irq; - desc->status |= IRQ_NOAUTOEN; - - set_irq_chip_data(virt_irq, data); + irq_set_status_flags(irq, IRQ_NOAUTOEN); + irq_set_handler_data(irq, handler_data); /* Catch accidental accesses to these things. IMAP/ICLR handling * is done by hypervisor calls on sun4v platforms, not by direct * register accesses. */ - data->imap = ~0UL; - data->iclr = ~0UL; + handler_data->imap = ~0UL; + handler_data->iclr = ~0UL; cookie = ~__pa(bucket); hv_err = sun4v_vintr_set_cookie(devhandle, devino, cookie); @@ -697,30 +638,30 @@ unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) prom_halt(); } - return virt_irq; + return irq; } -void ack_bad_irq(unsigned int virt_irq) +void ack_bad_irq(unsigned int irq) { - unsigned int ino = virt_irq_table[virt_irq].dev_ino; + unsigned int ino = irq_table[irq].dev_ino; if (!ino) ino = 0xdeadbeef; - printk(KERN_CRIT "Unexpected IRQ from ino[%x] virt_irq[%u]\n", - ino, virt_irq); + printk(KERN_CRIT "Unexpected IRQ from ino[%x] irq[%u]\n", + ino, irq); } void *hardirq_stack[NR_CPUS]; void *softirq_stack[NR_CPUS]; -void __irq_entry handler_irq(int irq, struct pt_regs *regs) +void __irq_entry handler_irq(int pil, struct pt_regs *regs) { unsigned long pstate, bucket_pa; struct pt_regs *old_regs; void *orig_sp; - clear_softint(1 << irq); + clear_softint(1 << pil); old_regs = set_irq_regs(regs); irq_enter(); @@ -739,18 +680,14 @@ void __irq_entry handler_irq(int irq, struct pt_regs *regs) orig_sp = set_hardirq_stack(); while (bucket_pa) { - struct irq_desc *desc; unsigned long next_pa; - unsigned int virt_irq; + unsigned int irq; next_pa = bucket_get_chain_pa(bucket_pa); - virt_irq = bucket_get_virt_irq(bucket_pa); + irq = bucket_get_irq(bucket_pa); bucket_clear_chain_pa(bucket_pa); - desc = irq_desc + virt_irq; - - if (!(desc->status & IRQ_DISABLED)) - desc->handle_irq(virt_irq, desc); + generic_handle_irq(irq); bucket_pa = next_pa; } @@ -761,30 +698,19 @@ void __irq_entry handler_irq(int irq, struct pt_regs *regs) set_irq_regs(old_regs); } -void do_softirq(void) +void do_softirq_own_stack(void) { - unsigned long flags; + void *orig_sp, *sp = softirq_stack[smp_processor_id()]; - if (in_interrupt()) - return; - - local_irq_save(flags); - - if (local_softirq_pending()) { - void *orig_sp, *sp = softirq_stack[smp_processor_id()]; - - sp += THREAD_SIZE - 192 - STACK_BIAS; - - __asm__ __volatile__("mov %%sp, %0\n\t" - "mov %1, %%sp" - : "=&r" (orig_sp) - : "r" (sp)); - __do_softirq(); - __asm__ __volatile__("mov %0, %%sp" - : : "r" (orig_sp)); - } + sp += THREAD_SIZE - 192 - STACK_BIAS; - local_irq_restore(flags); + __asm__ __volatile__("mov %%sp, %0\n\t" + "mov %1, %%sp" + : "=&r" (orig_sp) + : "r" (sp)); + __do_softirq(); + __asm__ __volatile__("mov %0, %%sp" + : : "r" (orig_sp)); } #ifdef CONFIG_HOTPLUG_CPU @@ -793,16 +719,18 @@ void fixup_irqs(void) unsigned int irq; for (irq = 0; irq < NR_IRQS; irq++) { + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *data = irq_desc_get_irq_data(desc); unsigned long flags; - raw_spin_lock_irqsave(&irq_desc[irq].lock, flags); - if (irq_desc[irq].action && - !(irq_desc[irq].status & IRQ_PER_CPU)) { - if (irq_desc[irq].chip->set_affinity) - irq_desc[irq].chip->set_affinity(irq, - irq_desc[irq].affinity); + raw_spin_lock_irqsave(&desc->lock, flags); + if (desc->action && !irqd_is_per_cpu(data)) { + if (data->chip->irq_set_affinity) + data->chip->irq_set_affinity(data, + data->affinity, + false); } - raw_spin_unlock_irqrestore(&irq_desc[irq].lock, flags); + raw_spin_unlock_irqrestore(&desc->lock, flags); } tick_ops->disable_irq(); @@ -860,7 +788,7 @@ static void kill_prom_timer(void) prom_limit0 = prom_timers->limit0; prom_limit1 = prom_timers->limit1; - /* Just as in sun4c/sun4m PROM uses timer which ticks at IRQ 14. + /* Just as in sun4c PROM uses timer which ticks at IRQ 14. * We turn both off here just to be paranoid. */ prom_timers->limit0 = 0; @@ -896,7 +824,8 @@ void notrace init_irqwork_curcpu(void) * Therefore you cannot make any OBP calls, not even prom_printf, * from these two routines. */ -static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) +static void notrace register_one_mondo(unsigned long paddr, unsigned long type, + unsigned long qmask) { unsigned long num_entries = (qmask + 1) / 64; unsigned long status; @@ -909,7 +838,7 @@ static void __cpuinit notrace register_one_mondo(unsigned long paddr, unsigned l } } -void __cpuinit notrace sun4v_register_mondo_queues(int this_cpu) +void notrace sun4v_register_mondo_queues(int this_cpu) { struct trap_per_cpu *tb = &trap_block[this_cpu]; @@ -1040,5 +969,5 @@ void __init init_IRQ(void) : "i" (PSTATE_IE) : "g1"); - irq_desc[0].action = &timer_irq_action; + irq_to_desc(0)->action = &timer_irq_action; } diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c index ea2dafc93d7..48565c11e82 100644 --- a/arch/sparc/kernel/jump_label.c +++ b/arch/sparc/kernel/jump_label.c @@ -6,6 +6,8 @@ #include <linux/jump_label.h> #include <linux/memory.h> +#include <asm/cacheflush.h> + #ifdef HAVE_JUMP_LABEL void arch_jump_label_transform(struct jump_entry *entry, @@ -36,12 +38,4 @@ void arch_jump_label_transform(struct jump_entry *entry, put_online_cpus(); } -void arch_jump_label_text_poke_early(jump_label_t addr) -{ - u32 *insn_p = (u32 *) (unsigned long) addr; - - *insn_p = 0x01000000; - flushi(insn_p); -} - #endif diff --git a/arch/sparc/kernel/kernel.h b/arch/sparc/kernel/kernel.h index 15d8a3f645c..e7f652be9e6 100644 --- a/arch/sparc/kernel/kernel.h +++ b/arch/sparc/kernel/kernel.h @@ -2,30 +2,184 @@ #define __SPARC_KERNEL_H #include <linux/interrupt.h> +#include <linux/ftrace.h> + +#include <asm/traps.h> +#include <asm/head.h> +#include <asm/io.h> /* cpu.c */ -extern const char *sparc_cpu_type; extern const char *sparc_pmu_type; -extern const char *sparc_fpu_type; - extern unsigned int fsr_storage; +extern int ncpus_probed; + +#ifdef CONFIG_SPARC64 +/* setup_64.c */ +struct seq_file; +void cpucap_info(struct seq_file *); + +static inline unsigned long kimage_addr_to_ra(const void *p) +{ + unsigned long val = (unsigned long) p; + + return kern_base + (val - KERNBASE); +} + +/* sys_sparc_64.c */ +asmlinkage long sys_kern_features(void); + +/* unaligned_64.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +int handle_popc(u32 insn, struct pt_regs *regs); +void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); +void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); + +/* smp_64.c */ +void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs); +void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs); +void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs); +void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs); +void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs); + +/* kgdb_64.c */ +void __irq_entry smp_kgdb_capture_client(int irq, struct pt_regs *regs); + +/* pci.c */ +int pci64_dma_supported(struct pci_dev *pdev, u64 device_mask); + +/* signal32.c */ +void do_sigreturn32(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn32(struct pt_regs *regs); +void do_signal32(struct pt_regs * regs); +asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp); + +/* compat_audit.c */ +extern unsigned sparc32_dir_class[]; +extern unsigned sparc32_chattr_class[]; +extern unsigned sparc32_write_class[]; +extern unsigned sparc32_read_class[]; +extern unsigned sparc32_signal_class[]; +int sparc32_classify_syscall(unsigned syscall); +#endif #ifdef CONFIG_SPARC32 +/* setup_32.c */ +struct linux_romvec; +void sparc32_start_kernel(struct linux_romvec *rp); + /* cpu.c */ -extern void cpu_probe(void); +void cpu_probe(void); /* traps_32.c */ -extern void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, - unsigned long npc, unsigned long psr); -/* muldiv.c */ -extern int do_user_muldiv (struct pt_regs *, unsigned long); - +void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); /* irq_32.c */ extern struct irqaction static_irqaction[]; extern int static_irq_count; extern spinlock_t irq_action_lock; -extern void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs); +void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs); +void init_IRQ(void); + +/* sun4m_irq.c */ +void sun4m_init_IRQ(void); +void sun4m_unmask_profile_irq(void); +void sun4m_clear_profile_irq(int cpu); + +/* sun4m_smp.c */ +void sun4m_cpu_pre_starting(void *arg); +void sun4m_cpu_pre_online(void *arg); +void __init smp4m_boot_cpus(void); +int smp4m_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4m_smp_done(void); +void smp4m_cross_call_irq(void); +void smp4m_percpu_timer_interrupt(struct pt_regs *regs); + +/* sun4d_irq.c */ +extern spinlock_t sun4d_imsk_lock; + +void sun4d_init_IRQ(void); +int sun4d_request_irq(unsigned int irq, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id); +int show_sun4d_interrupts(struct seq_file *, void *); +void sun4d_distribute_irqs(void); +void sun4d_free_irq(unsigned int irq, void *dev_id); + +/* sun4d_smp.c */ +void sun4d_cpu_pre_starting(void *arg); +void sun4d_cpu_pre_online(void *arg); +void __init smp4d_boot_cpus(void); +int smp4d_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4d_smp_done(void); +void smp4d_cross_call_irq(void); +void smp4d_percpu_timer_interrupt(struct pt_regs *regs); + +/* leon_smp.c */ +void leon_cpu_pre_starting(void *arg); +void leon_cpu_pre_online(void *arg); +void leonsmp_ipi_interrupt(void); +void leon_cross_call_irq(void); + +/* head_32.S */ +extern unsigned int t_nmi[]; +extern unsigned int linux_trap_ipi15_sun4d[]; +extern unsigned int linux_trap_ipi15_sun4m[]; + +extern struct tt_entry trapbase_cpu1; +extern struct tt_entry trapbase_cpu2; +extern struct tt_entry trapbase_cpu3; + +extern char cputypval[]; + +/* entry.S */ +extern unsigned long lvl14_save[4]; +extern unsigned int real_irq_entry[]; +extern unsigned int smp4d_ticker[]; +extern unsigned int patchme_maybe_smp_msg[]; + +void floppy_hardint(void); + +/* trampoline_32.S */ +extern unsigned long sun4m_cpu_startup; +extern unsigned long sun4d_cpu_startup; + +/* process_32.c */ +asmlinkage int sparc_do_fork(unsigned long clone_flags, + unsigned long stack_start, + struct pt_regs *regs, + unsigned long stack_size); + +/* signal_32.c */ +asmlinkage void do_sigreturn(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn(struct pt_regs *regs); +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, + unsigned long thread_info_flags); +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp); + +/* ptrace_32.c */ +asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p); + +/* unaligned_32.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn); + +/* windows.c */ +void try_to_clear_window_buffer(struct pt_regs *regs, int who); + +/* auxio_32.c */ +void __init auxio_probe(void); +void __init auxio_power_probe(void); + +/* pcic.c */ +extern void __iomem *pcic_regs; +void pcic_nmi(unsigned int pend, struct pt_regs *regs); + +/* time_32.c */ +void __init time_init(void); #else /* CONFIG_SPARC32 */ #endif /* CONFIG_SPARC32 */ diff --git a/arch/sparc/kernel/kgdb_32.c b/arch/sparc/kernel/kgdb_32.c index 539243b236f..dcf210811af 100644 --- a/arch/sparc/kernel/kgdb_32.c +++ b/arch/sparc/kernel/kgdb_32.c @@ -5,10 +5,12 @@ #include <linux/kgdb.h> #include <linux/kdebug.h> +#include <linux/sched.h> #include <asm/kdebug.h> #include <asm/ptrace.h> #include <asm/irq.h> +#include <asm/cacheflush.h> extern unsigned long trapbase; diff --git a/arch/sparc/kernel/kgdb_64.c b/arch/sparc/kernel/kgdb_64.c index 768290a6c02..cbf21d0870e 100644 --- a/arch/sparc/kernel/kgdb_64.c +++ b/arch/sparc/kernel/kgdb_64.c @@ -6,11 +6,15 @@ #include <linux/kgdb.h> #include <linux/kdebug.h> #include <linux/ftrace.h> +#include <linux/context_tracking.h> +#include <asm/cacheflush.h> #include <asm/kdebug.h> #include <asm/ptrace.h> #include <asm/irq.h> +#include "kernel.h" + void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) { struct reg_window *win; @@ -41,7 +45,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) { struct thread_info *t = task_thread_info(p); extern unsigned int switch_to_pc; - extern unsigned int ret_from_syscall; + extern unsigned int ret_from_fork; struct reg_window *win; unsigned long pc, cwp; int i; @@ -65,7 +69,7 @@ void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) gdb_regs[i] = 0; if (t->new_child) - pc = (unsigned long) &ret_from_syscall; + pc = (unsigned long) &ret_from_fork; else pc = (unsigned long) &switch_to_pc; @@ -158,11 +162,12 @@ int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); unsigned long flags; if (user_mode(regs)) { bad_trap(regs, trap_level); - return; + goto out; } flushw_all(); @@ -170,6 +175,8 @@ asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) local_irq_save(flags); kgdb_handle_exception(0x172, SIGTRAP, 0, regs); local_irq_restore(flags); +out: + exception_exit(prev_state); } int kgdb_arch_init(void) diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index a39d1ba5a11..98d71284341 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c @@ -8,6 +8,7 @@ #include <linux/module.h> #include <linux/kdebug.h> #include <linux/slab.h> +#include <linux/context_tracking.h> #include <asm/signal.h> #include <asm/cacheflush.h> #include <asm/uaccess.h> @@ -349,7 +350,7 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) case KPROBE_HIT_SSDONE: /* * We increment the nmissed count for accounting, - * we can also use npre/npostfault count for accouting + * we can also use npre/npostfault count for accounting * these specific fault cases. */ kprobes_inc_nmissed_count(cur); @@ -418,12 +419,14 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self, asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); + BUG_ON(trap_level != 0x170 && trap_level != 0x171); if (user_mode(regs)) { local_irq_enable(); bad_trap(regs, trap_level); - return; + goto out; } /* trap_level == 0x170 --> ta 0x70 @@ -433,6 +436,8 @@ asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, (trap_level == 0x170) ? "debug" : "debug_2", regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) bad_trap(regs, trap_level); +out: + exception_exit(prev_state); } /* Jprobes support. */ @@ -507,11 +512,12 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, /* * Called when the probe at kretprobe trampoline is hit */ -int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) +static int __kprobes trampoline_probe_handler(struct kprobe *p, + struct pt_regs *regs) { struct kretprobe_instance *ri = NULL; struct hlist_head *head, empty_rp; - struct hlist_node *node, *tmp; + struct hlist_node *tmp; unsigned long flags, orig_ret_address = 0; unsigned long trampoline_address =(unsigned long)&kretprobe_trampoline; @@ -531,7 +537,7 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) * real return address, and all the rest will point to * kretprobe_trampoline */ - hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { + hlist_for_each_entry_safe(ri, tmp, head, hlist) { if (ri->task != current) /* another task is sharing our hash bucket */ continue; @@ -559,7 +565,7 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) kretprobe_hash_unlock(current, &flags); preempt_enable_no_resched(); - hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { + hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) { hlist_del(&ri->hlist); kfree(ri); } @@ -571,7 +577,7 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs) return 1; } -void kretprobe_trampoline_holder(void) +static void __used kretprobe_trampoline_holder(void) { asm volatile(".global kretprobe_trampoline\n" "kretprobe_trampoline:\n" diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S index 1d361477d7d..605d4920458 100644 --- a/arch/sparc/kernel/ktlb.S +++ b/arch/sparc/kernel/ktlb.S @@ -25,11 +25,10 @@ kvmap_itlb: */ kvmap_itlb_4v: -kvmap_itlb_nonlinear: /* Catch kernel NULL pointer calls. */ sethi %hi(PAGE_SIZE), %g5 cmp %g4, %g5 - bleu,pn %xcc, kvmap_dtlb_longpath + blu,pn %xcc, kvmap_itlb_longpath nop KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) @@ -47,16 +46,16 @@ kvmap_itlb_tsb_miss: kvmap_itlb_vmalloc_addr: KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) - KTSB_LOCK_TAG(%g1, %g2, %g7) + TSB_LOCK_TAG(%g1, %g2, %g7) /* Load and check PTE. */ ldxa [%g5] ASI_PHYS_USE_EC, %g5 mov 1, %g7 sllx %g7, TSB_TAG_INVALID_BIT, %g7 brgez,a,pn %g5, kvmap_itlb_longpath - KTSB_STORE(%g1, %g7) + TSB_STORE(%g1, %g7) - KTSB_WRITE(%g1, %g5, %g6) + TSB_WRITE(%g1, %g5, %g6) /* fallthrough to TLB load */ @@ -102,9 +101,9 @@ kvmap_itlb_longpath: kvmap_itlb_obp: OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) - KTSB_LOCK_TAG(%g1, %g2, %g7) + TSB_LOCK_TAG(%g1, %g2, %g7) - KTSB_WRITE(%g1, %g5, %g6) + TSB_WRITE(%g1, %g5, %g6) ba,pt %xcc, kvmap_itlb_load nop @@ -112,17 +111,17 @@ kvmap_itlb_obp: kvmap_dtlb_obp: OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) - KTSB_LOCK_TAG(%g1, %g2, %g7) + TSB_LOCK_TAG(%g1, %g2, %g7) - KTSB_WRITE(%g1, %g5, %g6) + TSB_WRITE(%g1, %g5, %g6) ba,pt %xcc, kvmap_dtlb_load nop .align 32 kvmap_dtlb_tsb4m_load: - KTSB_LOCK_TAG(%g1, %g2, %g7) - KTSB_WRITE(%g1, %g5, %g6) + TSB_LOCK_TAG(%g1, %g2, %g7) + TSB_WRITE(%g1, %g5, %g6) ba,pt %xcc, kvmap_dtlb_load nop @@ -154,12 +153,19 @@ kvmap_dtlb_tsb4m_miss: /* Clear the PAGE_OFFSET top virtual bits, shift * down to get PFN, and make sure PFN is in range. */ - sllx %g4, 21, %g5 +661: sllx %g4, 0, %g5 + .section .page_offset_shift_patch, "ax" + .word 661b + .previous /* Check to see if we know about valid memory at the 4MB * chunk this physical address will reside within. */ - srlx %g5, 21 + 41, %g2 +661: srlx %g5, MAX_PHYS_ADDRESS_BITS, %g2 + .section .page_offset_shift_patch, "ax" + .word 661b + .previous + brnz,pn %g2, kvmap_dtlb_longpath nop @@ -177,7 +183,11 @@ valid_addr_bitmap_patch: or %g7, %lo(sparc64_valid_addr_bitmap), %g7 .previous - srlx %g5, 21 + 22, %g2 +661: srlx %g5, ILOG2_4MB, %g2 + .section .page_offset_shift_patch, "ax" + .word 661b + .previous + srlx %g2, 6, %g5 and %g2, 63, %g2 sllx %g5, 3, %g5 @@ -188,31 +198,35 @@ valid_addr_bitmap_patch: be,pn %xcc, kvmap_dtlb_longpath 2: sethi %hi(kpte_linear_bitmap), %g2 - or %g2, %lo(kpte_linear_bitmap), %g2 /* Get the 256MB physical address index. */ - sllx %g4, 21, %g5 - mov 1, %g7 - srlx %g5, 21 + 28, %g5 +661: sllx %g4, 0, %g5 + .section .page_offset_shift_patch, "ax" + .word 661b + .previous - /* Don't try this at home kids... this depends upon srlx - * only taking the low 6 bits of the shift count in %g5. - */ - sllx %g7, %g5, %g7 + or %g2, %lo(kpte_linear_bitmap), %g2 - /* Divide by 64 to get the offset into the bitmask. */ - srlx %g5, 6, %g5 +661: srlx %g5, ILOG2_256MB, %g5 + .section .page_offset_shift_patch, "ax" + .word 661b + .previous + + and %g5, (32 - 1), %g7 + + /* Divide by 32 to get the offset into the bitmask. */ + srlx %g5, 5, %g5 + add %g7, %g7, %g7 sllx %g5, 3, %g5 - /* kern_linear_pte_xor[((mask & bit) ? 1 : 0)] */ + /* kern_linear_pte_xor[(mask >> shift) & 3)] */ ldx [%g2 + %g5], %g2 - andcc %g2, %g7, %g0 + srlx %g2, %g7, %g7 sethi %hi(kern_linear_pte_xor), %g5 + and %g7, 3, %g7 or %g5, %lo(kern_linear_pte_xor), %g5 - bne,a,pt %xcc, 1f - add %g5, 8, %g5 - -1: ldx [%g5], %g2 + sllx %g7, 3, %g7 + ldx [%g5 + %g7], %g2 .globl kvmap_linear_patch kvmap_linear_patch: @@ -222,16 +236,16 @@ kvmap_linear_patch: kvmap_dtlb_vmalloc_addr: KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) - KTSB_LOCK_TAG(%g1, %g2, %g7) + TSB_LOCK_TAG(%g1, %g2, %g7) /* Load and check PTE. */ ldxa [%g5] ASI_PHYS_USE_EC, %g5 mov 1, %g7 sllx %g7, TSB_TAG_INVALID_BIT, %g7 brgez,a,pn %g5, kvmap_dtlb_longpath - KTSB_STORE(%g1, %g7) + TSB_STORE(%g1, %g7) - KTSB_WRITE(%g1, %g5, %g6) + TSB_WRITE(%g1, %g5, %g6) /* fallthrough to TLB load */ @@ -263,7 +277,7 @@ kvmap_dtlb_load: #ifdef CONFIG_SPARSEMEM_VMEMMAP kvmap_vmemmap: sub %g4, %g5, %g5 - srlx %g5, 22, %g5 + srlx %g5, ILOG2_4MB, %g5 sethi %hi(vmemmap_table), %g1 sllx %g5, 3, %g5 or %g1, %lo(vmemmap_table), %g1 diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c index df39a0f0d27..e01d75d4032 100644 --- a/arch/sparc/kernel/ldc.c +++ b/arch/sparc/kernel/ldc.c @@ -4,7 +4,7 @@ */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/delay.h> @@ -27,7 +27,7 @@ #define DRV_MODULE_VERSION "1.1" #define DRV_MODULE_RELDATE "July 22, 2008" -static char version[] __devinitdata = +static char version[] = DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; #define LDC_PACKET_SIZE 64 @@ -790,16 +790,20 @@ static void send_events(struct ldc_channel *lp, unsigned int event_mask) static irqreturn_t ldc_rx(int irq, void *dev_id) { struct ldc_channel *lp = dev_id; - unsigned long orig_state, hv_err, flags; + unsigned long orig_state, flags; unsigned int event_mask; spin_lock_irqsave(&lp->lock, flags); orig_state = lp->chan_state; - hv_err = sun4v_ldc_rx_get_state(lp->id, - &lp->rx_head, - &lp->rx_tail, - &lp->chan_state); + + /* We should probably check for hypervisor errors here and + * reset the LDC channel if we get one. + */ + sun4v_ldc_rx_get_state(lp->id, + &lp->rx_head, + &lp->rx_tail, + &lp->chan_state); ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); @@ -904,16 +908,20 @@ out: static irqreturn_t ldc_tx(int irq, void *dev_id) { struct ldc_channel *lp = dev_id; - unsigned long flags, hv_err, orig_state; + unsigned long flags, orig_state; unsigned int event_mask = 0; spin_lock_irqsave(&lp->lock, flags); orig_state = lp->chan_state; - hv_err = sun4v_ldc_tx_get_state(lp->id, - &lp->tx_head, - &lp->tx_tail, - &lp->chan_state); + + /* We should probably check for hypervisor errors here and + * reset the LDC channel if we get one. + */ + sun4v_ldc_tx_get_state(lp->id, + &lp->tx_head, + &lp->tx_tail, + &lp->chan_state); ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); @@ -945,9 +953,8 @@ static HLIST_HEAD(ldc_channel_list); static int __ldc_channel_exists(unsigned long id) { struct ldc_channel *lp; - struct hlist_node *n; - hlist_for_each_entry(lp, n, &ldc_channel_list, list) { + hlist_for_each_entry(lp, &ldc_channel_list, list) { if (lp->id == id) return 1; } @@ -1242,14 +1249,12 @@ int ldc_bind(struct ldc_channel *lp, const char *name) snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); - err = request_irq(lp->cfg.rx_irq, ldc_rx, - IRQF_SAMPLE_RANDOM | IRQF_DISABLED, + err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, lp->rx_irq_name, lp); if (err) return err; - err = request_irq(lp->cfg.tx_irq, ldc_tx, - IRQF_SAMPLE_RANDOM | IRQF_DISABLED, + err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, lp->tx_irq_name, lp); if (err) { free_irq(lp->cfg.rx_irq, lp); diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c index fdab7f854f8..683c4af999d 100644 --- a/arch/sparc/kernel/leon_kernel.c +++ b/arch/sparc/kernel/leon_kernel.c @@ -4,13 +4,14 @@ */ #include <linux/kernel.h> -#include <linux/module.h> #include <linux/errno.h> #include <linux/mutex.h> #include <linux/of.h> #include <linux/of_platform.h> #include <linux/interrupt.h> #include <linux/of_device.h> +#include <linux/clocksource.h> +#include <linux/clockchips.h> #include <asm/oplib.h> #include <asm/timer.h> @@ -19,55 +20,75 @@ #include <asm/leon_amba.h> #include <asm/traps.h> #include <asm/cacheflush.h> +#include <asm/smp.h> +#include <asm/setup.h> +#include "kernel.h" #include "prom.h" #include "irq.h" struct leon3_irqctrl_regs_map *leon3_irqctrl_regs; /* interrupt controller base address */ struct leon3_gptimer_regs_map *leon3_gptimer_regs; /* timer controller base address */ -struct amba_apb_device leon_percpu_timer_dev[16]; int leondebug_irq_disable; int leon_debug_irqout; -static int dummy_master_l10_counter; +static volatile u32 dummy_master_l10_counter; +unsigned long amba_system_id; +static DEFINE_SPINLOCK(leon_irq_lock); +static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ unsigned long leon3_gptimer_irq; /* interrupt controller irq number */ -unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ unsigned int sparc_leon_eirq; -#define LEON_IMASK ((&leon3_irqctrl_regs->mask[0])) +#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu]) +#define LEON_IACK (&leon3_irqctrl_regs->iclear) +#define LEON_DO_ACK_HW 1 -/* Return the IRQ of the pending IRQ on the extended IRQ controller */ -int sparc_leon_eirq_get(int eirq, int cpu) +/* Return the last ACKed IRQ by the Extended IRQ controller. It has already + * been (automatically) ACKed when the CPU takes the trap. + */ +static inline unsigned int leon_eirq_get(int cpu) { return LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->intid[cpu]) & 0x1f; } -irqreturn_t sparc_leon_eirq_isr(int dummy, void *dev_id) +/* Handle one or multiple IRQs from the extended interrupt controller */ +static void leon_handle_ext_irq(unsigned int irq, struct irq_desc *desc) { - printk(KERN_ERR "sparc_leon_eirq_isr: ERROR EXTENDED IRQ\n"); - return IRQ_HANDLED; + unsigned int eirq; + struct irq_bucket *p; + int cpu = sparc_leon3_cpuid(); + + eirq = leon_eirq_get(cpu); + p = irq_map[eirq]; + if ((eirq & 0x10) && p && p->irq) /* bit4 tells if IRQ happened */ + generic_handle_irq(p->irq); } /* The extended IRQ controller has been found, this function registers it */ -void sparc_leon_eirq_register(int eirq) +static void leon_eirq_setup(unsigned int eirq) { - int irq; + unsigned long mask, oldmask; + unsigned int veirq; - /* Register a "BAD" handler for this interrupt, it should never happen */ - irq = request_irq(eirq, sparc_leon_eirq_isr, - (IRQF_DISABLED | SA_STATIC_ALLOC), "extirq", NULL); - - if (irq) { - printk(KERN_ERR - "sparc_leon_eirq_register: unable to attach IRQ%d\n", - eirq); - } else { - sparc_leon_eirq = eirq; + if (eirq < 1 || eirq > 0xf) { + printk(KERN_ERR "LEON EXT IRQ NUMBER BAD: %d\n", eirq); + return; } + veirq = leon_build_device_irq(eirq, leon_handle_ext_irq, "extirq", 0); + + /* + * Unmask the Extended IRQ, the IRQs routed through the Ext-IRQ + * controller have a mask-bit of their own, so this is safe. + */ + irq_link(veirq); + mask = 1 << eirq; + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(boot_cpu_id)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(boot_cpu_id), (oldmask | mask)); + sparc_leon_eirq = eirq; } -static inline unsigned long get_irqmask(unsigned int irq) +unsigned long leon_get_irqmask(unsigned int irq) { unsigned long mask; @@ -82,45 +103,229 @@ static inline unsigned long get_irqmask(unsigned int irq) return mask; } -static void leon_enable_irq(unsigned int irq_nr) +#ifdef CONFIG_SMP +static int irq_choose_cpu(const struct cpumask *affinity) +{ + cpumask_t mask; + + cpumask_and(&mask, cpu_online_mask, affinity); + if (cpumask_equal(&mask, cpu_online_mask) || cpumask_empty(&mask)) + return boot_cpu_id; + else + return cpumask_first(&mask); +} +#else +#define irq_choose_cpu(affinity) boot_cpu_id +#endif + +static int leon_set_affinity(struct irq_data *data, const struct cpumask *dest, + bool force) +{ + unsigned long mask, oldmask, flags; + int oldcpu, newcpu; + + mask = (unsigned long)data->chip_data; + oldcpu = irq_choose_cpu(data->affinity); + newcpu = irq_choose_cpu(dest); + + if (oldcpu == newcpu) + goto out; + + /* unmask on old CPU first before enabling on the selected CPU */ + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(oldcpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(oldcpu), (oldmask & ~mask)); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(newcpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(newcpu), (oldmask | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +out: + return IRQ_SET_MASK_OK; +} + +static void leon_unmask_irq(struct irq_data *data) +{ + unsigned long mask, oldmask, flags; + int cpu; + + mask = (unsigned long)data->chip_data; + cpu = irq_choose_cpu(data->affinity); + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +} + +static void leon_mask_irq(struct irq_data *data) +{ + unsigned long mask, oldmask, flags; + int cpu; + + mask = (unsigned long)data->chip_data; + cpu = irq_choose_cpu(data->affinity); + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask & ~mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +} + +static unsigned int leon_startup_irq(struct irq_data *data) +{ + irq_link(data->irq); + leon_unmask_irq(data); + return 0; +} + +static void leon_shutdown_irq(struct irq_data *data) +{ + leon_mask_irq(data); + irq_unlink(data->irq); +} + +/* Used by external level sensitive IRQ handlers on the LEON: ACK IRQ ctrl */ +static void leon_eoi_irq(struct irq_data *data) +{ + unsigned long mask = (unsigned long)data->chip_data; + + if (mask & LEON_DO_ACK_HW) + LEON3_BYPASS_STORE_PA(LEON_IACK, mask & ~LEON_DO_ACK_HW); +} + +static struct irq_chip leon_irq = { + .name = "leon", + .irq_startup = leon_startup_irq, + .irq_shutdown = leon_shutdown_irq, + .irq_mask = leon_mask_irq, + .irq_unmask = leon_unmask_irq, + .irq_eoi = leon_eoi_irq, + .irq_set_affinity = leon_set_affinity, +}; + +/* + * Build a LEON IRQ for the edge triggered LEON IRQ controller: + * Edge (normal) IRQ - handle_simple_irq, ack=DONT-CARE, never ack + * Level IRQ (PCI|Level-GPIO) - handle_fasteoi_irq, ack=1, ack after ISR + * Per-CPU Edge - handle_percpu_irq, ack=0 + */ +unsigned int leon_build_device_irq(unsigned int real_irq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack) { - unsigned long mask, flags; - mask = get_irqmask(irq_nr); - local_irq_save(flags); - LEON3_BYPASS_STORE_PA(LEON_IMASK, - (LEON3_BYPASS_LOAD_PA(LEON_IMASK) | (mask))); - local_irq_restore(flags); + unsigned int irq; + unsigned long mask; + struct irq_desc *desc; + + irq = 0; + mask = leon_get_irqmask(real_irq); + if (mask == 0) + goto out; + + irq = irq_alloc(real_irq, real_irq); + if (irq == 0) + goto out; + + if (do_ack) + mask |= LEON_DO_ACK_HW; + + desc = irq_to_desc(irq); + if (!desc || !desc->handle_irq || desc->handle_irq == handle_bad_irq) { + irq_set_chip_and_handler_name(irq, &leon_irq, + flow_handler, name); + irq_set_chip_data(irq, (void *)mask); + } + +out: + return irq; } -static void leon_disable_irq(unsigned int irq_nr) +static unsigned int _leon_build_device_irq(struct platform_device *op, + unsigned int real_irq) { - unsigned long mask, flags; - mask = get_irqmask(irq_nr); - local_irq_save(flags); - LEON3_BYPASS_STORE_PA(LEON_IMASK, - (LEON3_BYPASS_LOAD_PA(LEON_IMASK) & ~(mask))); - local_irq_restore(flags); + return leon_build_device_irq(real_irq, handle_simple_irq, "edge", 0); +} + +void leon_update_virq_handling(unsigned int virq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack) +{ + unsigned long mask = (unsigned long)irq_get_chip_data(virq); + + mask &= ~LEON_DO_ACK_HW; + if (do_ack) + mask |= LEON_DO_ACK_HW; + irq_set_chip_and_handler_name(virq, &leon_irq, + flow_handler, name); + irq_set_chip_data(virq, (void *)mask); } -void __init leon_init_timers(irq_handler_t counter_fn) +static u32 leon_cycles_offset(void) { - int irq; + u32 rld, val, off; + rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld); + val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); + off = rld - val; + return rld - val; +} + +#ifdef CONFIG_SMP + +/* smp clockevent irq */ +static irqreturn_t leon_percpu_timer_ce_interrupt(int irq, void *unused) +{ + struct clock_event_device *ce; + int cpu = smp_processor_id(); + + leon_clear_profile_irq(cpu); + + if (cpu == boot_cpu_id) + timer_interrupt(irq, NULL); + + ce = &per_cpu(sparc32_clockevent, cpu); + + irq_enter(); + if (ce->event_handler) + ce->event_handler(ce); + irq_exit(); + + return IRQ_HANDLED; +} + +#endif /* CONFIG_SMP */ + +void __init leon_init_timers(void) +{ + int irq, eirq; struct device_node *rootnp, *np, *nnp; struct property *pp; int len; - int cpu, icsel; + int icsel; int ampopts; + int err; + u32 config; + + sparc_config.get_cycles_offset = leon_cycles_offset; + sparc_config.cs_period = 1000000 / HZ; + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + +#ifndef CONFIG_SMP + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif leondebug_irq_disable = 0; leon_debug_irqout = 0; - master_l10_counter = (unsigned int *)&dummy_master_l10_counter; + master_l10_counter = (u32 __iomem *)&dummy_master_l10_counter; dummy_master_l10_counter = 0; - /*Find IRQMP IRQ Controller Registers base address otherwise bail out.*/ rootnp = of_find_node_by_path("/ambapp0"); if (!rootnp) goto bad; + + /* Find System ID: GRLIB build ID and optional CHIP ID */ + pp = of_find_property(rootnp, "systemid", &len); + if (pp) + amba_system_id = *(unsigned long *)pp->value; + + /* Find IRQMP IRQ Controller Registers base adr otherwise bail out */ np = of_find_node_by_name(rootnp, "GAISLER_IRQMP"); if (!np) { np = of_find_node_by_name(rootnp, "01_00d"); @@ -166,98 +371,78 @@ void __init leon_init_timers(irq_handler_t counter_fn) leon3_gptimer_irq = *(unsigned int *)pp->value; } while (0); - if (leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq) { - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx].rld, - (((1000000 / HZ) - 1))); - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, 0); - -#ifdef CONFIG_SMP - leon_percpu_timer_dev[0].start = (int)leon3_gptimer_regs; - leon_percpu_timer_dev[0].irq = leon3_gptimer_irq + 1 + - leon3_gptimer_idx; - - if (!(LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->config) & - (1<<LEON3_GPTIMER_SEPIRQ))) { - prom_printf("irq timer not configured with separate irqs\n"); - BUG(); - } - - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx+1].val, 0); - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx+1].rld, - (((1000000/HZ) - 1))); - LEON3_BYPASS_STORE_PA( - &leon3_gptimer_regs->e[leon3_gptimer_idx+1].ctrl, 0); -# endif - - /* - * The IRQ controller may (if implemented) consist of multiple - * IRQ controllers, each mapped on a 4Kb boundary. - * Each CPU may be routed to different IRQCTRLs, however - * we assume that all CPUs (in SMP system) is routed to the - * same IRQ Controller, and for non-SMP only one IRQCTRL is - * accessed anyway. - * In AMP systems, Linux must run on CPU0 for the time being. - */ - cpu = sparc_leon3_cpuid(); - icsel = LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->icsel[cpu/8]); - icsel = (icsel >> ((7 - (cpu&0x7)) * 4)) & 0xf; - leon3_irqctrl_regs += icsel; - } else { + if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq)) goto bad; - } - irq = request_irq(leon3_gptimer_irq+leon3_gptimer_idx, - counter_fn, - (IRQF_DISABLED | SA_STATIC_ALLOC), "timer", NULL); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld, + (((1000000 / HZ) - 1))); + LEON3_BYPASS_STORE_PA( + &leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, 0); - if (irq) { - printk(KERN_ERR "leon_time_init: unable to attach IRQ%d\n", - LEON_INTERRUPT_TIMER1); - prom_halt(); - } + /* + * The IRQ controller may (if implemented) consist of multiple + * IRQ controllers, each mapped on a 4Kb boundary. + * Each CPU may be routed to different IRQCTRLs, however + * we assume that all CPUs (in SMP system) is routed to the + * same IRQ Controller, and for non-SMP only one IRQCTRL is + * accessed anyway. + * In AMP systems, Linux must run on CPU0 for the time being. + */ + icsel = LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->icsel[boot_cpu_id/8]); + icsel = (icsel >> ((7 - (boot_cpu_id&0x7)) * 4)) & 0xf; + leon3_irqctrl_regs += icsel; + + /* Mask all IRQs on boot-cpu IRQ controller */ + LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->mask[boot_cpu_id], 0); + + /* Probe extended IRQ controller */ + eirq = (LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->mpstatus) + >> 16) & 0xf; + if (eirq != 0) + leon_eirq_setup(eirq); -# ifdef CONFIG_SMP +#ifdef CONFIG_SMP { unsigned long flags; - struct tt_entry *trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (leon_percpu_timer_dev[0].irq - 1)]; - /* For SMP we use the level 14 ticker, however the bootup code - * has copied the firmwares level 14 vector into boot cpu's - * trap table, we must fix this now or we get squashed. + /* + * In SMP, sun4m adds a IPI handler to IRQ trap handler that + * LEON never must take, sun4d and LEON overwrites the branch + * with a NOP. */ local_irq_save(flags); - patchme_maybe_smp_msg[0] = 0x01000000; /* NOP out the branch */ - - /* Adjust so that we jump directly to smpleon_ticker */ - trap_table->inst_three += smpleon_ticker - real_irq_entry; - - local_flush_cache_all(); + local_ops->cache_all(); local_irq_restore(flags); } -# endif +#endif - if (leon3_gptimer_regs) { - LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, - LEON3_GPTIMER_EN | - LEON3_GPTIMER_RL | - LEON3_GPTIMER_LD | LEON3_GPTIMER_IRQEN); + config = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->config); + if (config & (1 << LEON3_GPTIMER_SEPIRQ)) + leon3_gptimer_irq += leon3_gptimer_idx; + else if ((config & LEON3_GPTIMER_TIMERS) > 1) + pr_warn("GPTIMER uses shared irqs, using other timers of the same core will fail.\n"); #ifdef CONFIG_SMP - LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx+1].ctrl, - LEON3_GPTIMER_EN | - LEON3_GPTIMER_RL | - LEON3_GPTIMER_LD | - LEON3_GPTIMER_IRQEN); + /* Install per-cpu IRQ handler for broadcasted ticker */ + irq = leon_build_device_irq(leon3_gptimer_irq, handle_percpu_irq, + "per-cpu", 0); + err = request_irq(irq, leon_percpu_timer_ce_interrupt, + IRQF_PERCPU | IRQF_TIMER, "timer", NULL); +#else + irq = _leon_build_device_irq(NULL, leon3_gptimer_irq); + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); #endif - + if (err) { + pr_err("Unable to attach timer IRQ%d\n", irq); + prom_halt(); } + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + LEON3_GPTIMER_EN | + LEON3_GPTIMER_RL | + LEON3_GPTIMER_LD | + LEON3_GPTIMER_IRQEN); return; bad: printk(KERN_ERR "No Timer/irqctrl found\n"); @@ -265,18 +450,14 @@ bad: return; } -void leon_clear_clock_irq(void) +static void leon_clear_clock_irq(void) { } -void leon_load_profile_irq(int cpu, unsigned int limit) +static void leon_load_profile_irq(int cpu, unsigned int limit) { - BUG(); } - - - void __init leon_trans_init(struct device_node *dp) { if (strcmp(dp->type, "cpu") == 0 && strcmp(dp->name, "<NULL>") == 0) { @@ -291,37 +472,7 @@ void __init leon_trans_init(struct device_node *dp) } } -void __initdata (*prom_amba_init)(struct device_node *dp, struct device_node ***nextp) = 0; - -void __init leon_node_init(struct device_node *dp, struct device_node ***nextp) -{ - if (prom_amba_init && - strcmp(dp->type, "ambapp") == 0 && - strcmp(dp->name, "ambapp0") == 0) { - prom_amba_init(dp, nextp); - } -} - #ifdef CONFIG_SMP - -void leon_set_cpu_int(int cpu, int level) -{ - unsigned long mask; - mask = get_irqmask(level); - LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->force[cpu], mask); -} - -static void leon_clear_ipi(int cpu, int level) -{ - unsigned long mask; - mask = get_irqmask(level); - LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->force[cpu], mask<<16); -} - -static void leon_set_udt(int cpu) -{ -} - void leon_clear_profile_irq(int cpu) { } @@ -329,38 +480,20 @@ void leon_clear_profile_irq(int cpu) void leon_enable_irq_cpu(unsigned int irq_nr, unsigned int cpu) { unsigned long mask, flags, *addr; - mask = get_irqmask(irq_nr); - local_irq_save(flags); - addr = (unsigned long *)&(leon3_irqctrl_regs->mask[cpu]); - LEON3_BYPASS_STORE_PA(addr, (LEON3_BYPASS_LOAD_PA(addr) | (mask))); - local_irq_restore(flags); + mask = leon_get_irqmask(irq_nr); + spin_lock_irqsave(&leon_irq_lock, flags); + addr = (unsigned long *)LEON_IMASK(cpu); + LEON3_BYPASS_STORE_PA(addr, (LEON3_BYPASS_LOAD_PA(addr) | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); } #endif void __init leon_init_IRQ(void) { - sparc_init_timers = leon_init_timers; - - BTFIXUPSET_CALL(enable_irq, leon_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_irq, leon_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(enable_pil_irq, leon_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_pil_irq, leon_disable_irq, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(clear_clock_irq, leon_clear_clock_irq, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(load_profile_irq, leon_load_profile_irq, - BTFIXUPCALL_NOP); - -#ifdef CONFIG_SMP - BTFIXUPSET_CALL(set_cpu_int, leon_set_cpu_int, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_cpu_int, leon_clear_ipi, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(set_irq_udt, leon_set_udt, BTFIXUPCALL_NORM); -#endif - -} - -void __init leon_init(void) -{ - of_pdt_build_more = &leon_node_init; + sparc_config.init_timers = leon_init_timers; + sparc_config.build_device_irq = _leon_build_device_irq; + sparc_config.clock_rate = 1000000; + sparc_config.clear_clock_irq = leon_clear_clock_irq; + sparc_config.load_profile_irq = leon_load_profile_irq; } diff --git a/arch/sparc/kernel/leon_pci.c b/arch/sparc/kernel/leon_pci.c new file mode 100644 index 00000000000..899b7203a4e --- /dev/null +++ b/arch/sparc/kernel/leon_pci.c @@ -0,0 +1,100 @@ +/* + * leon_pci.c: LEON Host PCI support + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + * + * Code is partially derived from pcic.c + */ + +#include <linux/of_device.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/export.h> +#include <asm/leon.h> +#include <asm/leon_pci.h> + +/* The LEON architecture does not rely on a BIOS or bootloader to setup + * PCI for us. The Linux generic routines are used to setup resources, + * reset values of configuration-space register settings are preserved. + * + * PCI Memory and Prefetchable Memory is direct-mapped. However I/O Space is + * accessed through a Window which is translated to low 64KB in PCI space, the + * first 4KB is not used so 60KB is available. + */ +void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info) +{ + LIST_HEAD(resources); + struct pci_bus *root_bus; + + pci_add_resource_offset(&resources, &info->io_space, + info->io_space.start - 0x1000); + pci_add_resource(&resources, &info->mem_space); + info->busn.flags = IORESOURCE_BUS; + pci_add_resource(&resources, &info->busn); + + root_bus = pci_scan_root_bus(&ofdev->dev, 0, info->ops, info, + &resources); + if (root_bus) { + /* Setup IRQs of all devices using custom routines */ + pci_fixup_irqs(pci_common_swizzle, info->map_irq); + + /* Assign devices with resources */ + pci_assign_unassigned_resources(); + } else { + pci_free_resource_list(&resources); + } +} + +void pcibios_fixup_bus(struct pci_bus *pbus) +{ + struct pci_dev *dev; + int i, has_io, has_mem; + u16 cmd; + + list_for_each_entry(dev, &pbus->devices, bus_list) { + /* + * We can not rely on that the bootloader has enabled I/O + * or memory access to PCI devices. Instead we enable it here + * if the device has BARs of respective type. + */ + has_io = has_mem = 0; + for (i = 0; i < PCI_ROM_RESOURCE; i++) { + unsigned long f = dev->resource[i].flags; + if (f & IORESOURCE_IO) + has_io = 1; + else if (f & IORESOURCE_MEM) + has_mem = 1; + } + /* ROM BARs are mapped into 32-bit memory space */ + if (dev->resource[PCI_ROM_RESOURCE].end != 0) { + dev->resource[PCI_ROM_RESOURCE].flags |= + IORESOURCE_ROM_ENABLE; + has_mem = 1; + } + pci_bus_read_config_word(pbus, dev->devfn, PCI_COMMAND, &cmd); + if (has_io && !(cmd & PCI_COMMAND_IO)) { +#ifdef CONFIG_PCI_DEBUG + printk(KERN_INFO "LEONPCI: Enabling I/O for dev %s\n", + pci_name(dev)); +#endif + cmd |= PCI_COMMAND_IO; + pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND, + cmd); + } + if (has_mem && !(cmd & PCI_COMMAND_MEMORY)) { +#ifdef CONFIG_PCI_DEBUG + printk(KERN_INFO "LEONPCI: Enabling MEMORY for dev" + "%s\n", pci_name(dev)); +#endif + cmd |= PCI_COMMAND_MEMORY; + pci_bus_write_config_word(pbus, dev->devfn, PCI_COMMAND, + cmd); + } + } +} + +resource_size_t pcibios_align_resource(void *data, const struct resource *res, + resource_size_t size, resource_size_t align) +{ + return res->start; +} diff --git a/arch/sparc/kernel/leon_pci_grpci1.c b/arch/sparc/kernel/leon_pci_grpci1.c new file mode 100644 index 00000000000..c8bf26edfa7 --- /dev/null +++ b/arch/sparc/kernel/leon_pci_grpci1.c @@ -0,0 +1,722 @@ +/* + * leon_pci_grpci1.c: GRPCI1 Host PCI driver + * + * Copyright (C) 2013 Aeroflex Gaisler AB + * + * This GRPCI1 driver does not support PCI interrupts taken from + * GPIO pins. Interrupt generation at PCI parity and system error + * detection is by default turned off since some GRPCI1 cores does + * not support detection. It can be turned on from the bootloader + * using the all_pci_errors property. + * + * Contributors: Daniel Hellstrom <daniel@gaisler.com> + */ + +#include <linux/of_device.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/of_irq.h> +#include <linux/delay.h> +#include <linux/pci.h> + +#include <asm/leon_pci.h> +#include <asm/sections.h> +#include <asm/vaddrs.h> +#include <asm/leon.h> +#include <asm/io.h> + +#include "irq.h" + +/* Enable/Disable Debugging Configuration Space Access */ +#undef GRPCI1_DEBUG_CFGACCESS + +/* + * GRPCI1 APB Register MAP + */ +struct grpci1_regs { + unsigned int cfg_stat; /* 0x00 Configuration / Status */ + unsigned int bar0; /* 0x04 BAR0 (RO) */ + unsigned int page0; /* 0x08 PAGE0 (RO) */ + unsigned int bar1; /* 0x0C BAR1 (RO) */ + unsigned int page1; /* 0x10 PAGE1 */ + unsigned int iomap; /* 0x14 IO Map */ + unsigned int stat_cmd; /* 0x18 PCI Status & Command (RO) */ + unsigned int irq; /* 0x1C Interrupt register */ +}; + +#define REGLOAD(a) (be32_to_cpu(__raw_readl(&(a)))) +#define REGSTORE(a, v) (__raw_writel(cpu_to_be32(v), &(a))) + +#define PAGE0_BTEN_BIT 0 +#define PAGE0_BTEN (1 << PAGE0_BTEN_BIT) + +#define CFGSTAT_HOST_BIT 13 +#define CFGSTAT_CTO_BIT 8 +#define CFGSTAT_HOST (1 << CFGSTAT_HOST_BIT) +#define CFGSTAT_CTO (1 << CFGSTAT_CTO_BIT) + +#define IRQ_DPE (1 << 9) +#define IRQ_SSE (1 << 8) +#define IRQ_RMA (1 << 7) +#define IRQ_RTA (1 << 6) +#define IRQ_STA (1 << 5) +#define IRQ_DPED (1 << 4) +#define IRQ_INTD (1 << 3) +#define IRQ_INTC (1 << 2) +#define IRQ_INTB (1 << 1) +#define IRQ_INTA (1 << 0) +#define IRQ_DEF_ERRORS (IRQ_RMA | IRQ_RTA | IRQ_STA) +#define IRQ_ALL_ERRORS (IRQ_DPED | IRQ_DEF_ERRORS | IRQ_SSE | IRQ_DPE) +#define IRQ_INTX (IRQ_INTA | IRQ_INTB | IRQ_INTC | IRQ_INTD) +#define IRQ_MASK_BIT 16 + +#define DEF_PCI_ERRORS (PCI_STATUS_SIG_TARGET_ABORT | \ + PCI_STATUS_REC_TARGET_ABORT | \ + PCI_STATUS_REC_MASTER_ABORT) +#define ALL_PCI_ERRORS (PCI_STATUS_PARITY | PCI_STATUS_DETECTED_PARITY | \ + PCI_STATUS_SIG_SYSTEM_ERROR | DEF_PCI_ERRORS) + +#define TGT 256 + +struct grpci1_priv { + struct leon_pci_info info; /* must be on top of this structure */ + struct grpci1_regs __iomem *regs; /* GRPCI register map */ + struct device *dev; + int pci_err_mask; /* STATUS register error mask */ + int irq; /* LEON irqctrl GRPCI IRQ */ + unsigned char irq_map[4]; /* GRPCI nexus PCI INTX# IRQs */ + unsigned int irq_err; /* GRPCI nexus Virt Error IRQ */ + + /* AHB PCI Windows */ + unsigned long pci_area; /* MEMORY */ + unsigned long pci_area_end; + unsigned long pci_io; /* I/O */ + unsigned long pci_conf; /* CONFIGURATION */ + unsigned long pci_conf_end; + unsigned long pci_io_va; +}; + +static struct grpci1_priv *grpci1priv; + +static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val); + +static int grpci1_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct grpci1_priv *priv = dev->bus->sysdata; + int irq_group; + + /* Use default IRQ decoding on PCI BUS0 according slot numbering */ + irq_group = slot & 0x3; + pin = ((pin - 1) + irq_group) & 0x3; + + return priv->irq_map[pin]; +} + +static int grpci1_cfg_r32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 *pci_conf, tmp, cfg; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + cfg = REGLOAD(priv->regs->cfg_stat); + REGSTORE(priv->regs->cfg_stat, (cfg & ~(0xf << 23)) | (bus << 23)); + + /* do read access */ + pci_conf = (u32 *) (priv->pci_conf | (devfn << 8) | (where & 0xfc)); + tmp = LEON3_BYPASS_LOAD_PA(pci_conf); + + /* check if master abort was received */ + if (REGLOAD(priv->regs->cfg_stat) & CFGSTAT_CTO) { + *val = 0xffffffff; + /* Clear Master abort bit in PCI cfg space (is set) */ + tmp = REGLOAD(priv->regs->stat_cmd); + grpci1_cfg_w32(priv, TGT, 0, PCI_COMMAND, tmp); + } else { + /* Bus always little endian (unaffected by byte-swapping) */ + *val = swab32(tmp); + } + + return 0; +} + +static int grpci1_cfg_r16(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + if (where & 0x1) + return -EINVAL; + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xffff & (v >> (8 * (where & 0x3))); + return ret; +} + +static int grpci1_cfg_r8(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xff & (v >> (8 * (where & 3))); + + return ret; +} + +static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + unsigned int *pci_conf; + u32 cfg; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + cfg = REGLOAD(priv->regs->cfg_stat); + REGSTORE(priv->regs->cfg_stat, (cfg & ~(0xf << 23)) | (bus << 23)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); + + return 0; +} + +static int grpci1_cfg_w16(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + if (where & 0x1) + return -EINVAL; + ret = grpci1_cfg_r32(priv, bus, devfn, where&~3, &v); + if (ret) + return ret; + v = (v & ~(0xffff << (8 * (where & 0x3)))) | + ((0xffff & val) << (8 * (where & 0x3))); + return grpci1_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +static int grpci1_cfg_w8(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + if (ret != 0) + return ret; + v = (v & ~(0xff << (8 * (where & 0x3)))) | + ((0xff & val) << (8 * (where & 0x3))); + return grpci1_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +/* Read from Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci1_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct grpci1_priv *priv = grpci1priv; + unsigned int busno = bus->number; + int ret; + + if (PCI_SLOT(devfn) > 15 || busno > 15) { + *val = ~0; + return 0; + } + + switch (size) { + case 1: + ret = grpci1_cfg_r8(priv, busno, devfn, where, val); + break; + case 2: + ret = grpci1_cfg_r16(priv, busno, devfn, where, val); + break; + case 4: + ret = grpci1_cfg_r32(priv, busno, devfn, where, val); + break; + default: + ret = -EINVAL; + break; + } + +#ifdef GRPCI1_DEBUG_CFGACCESS + printk(KERN_INFO + "grpci1_read_config: [%02x:%02x:%x] ofs=%d val=%x size=%d\n", + busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, *val, size); +#endif + + return ret; +} + +/* Write to Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci1_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct grpci1_priv *priv = grpci1priv; + unsigned int busno = bus->number; + + if (PCI_SLOT(devfn) > 15 || busno > 15) + return 0; + +#ifdef GRPCI1_DEBUG_CFGACCESS + printk(KERN_INFO + "grpci1_write_config: [%02x:%02x:%x] ofs=%d size=%d val=%x\n", + busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val); +#endif + + switch (size) { + default: + return -EINVAL; + case 1: + return grpci1_cfg_w8(priv, busno, devfn, where, val); + case 2: + return grpci1_cfg_w16(priv, busno, devfn, where, val); + case 4: + return grpci1_cfg_w32(priv, busno, devfn, where, val); + } +} + +static struct pci_ops grpci1_ops = { + .read = grpci1_read_config, + .write = grpci1_write_config, +}; + +/* GENIRQ IRQ chip implementation for grpci1 irqmode=0..2. In configuration + * 3 where all PCI Interrupts has a separate IRQ on the system IRQ controller + * this is not needed and the standard IRQ controller can be used. + */ + +static void grpci1_mask_irq(struct irq_data *data) +{ + u32 irqidx; + struct grpci1_priv *priv = grpci1priv; + + irqidx = (u32)data->chip_data - 1; + if (irqidx > 3) /* only mask PCI interrupts here */ + return; + irqidx += IRQ_MASK_BIT; + + REGSTORE(priv->regs->irq, REGLOAD(priv->regs->irq) & ~(1 << irqidx)); +} + +static void grpci1_unmask_irq(struct irq_data *data) +{ + u32 irqidx; + struct grpci1_priv *priv = grpci1priv; + + irqidx = (u32)data->chip_data - 1; + if (irqidx > 3) /* only unmask PCI interrupts here */ + return; + irqidx += IRQ_MASK_BIT; + + REGSTORE(priv->regs->irq, REGLOAD(priv->regs->irq) | (1 << irqidx)); +} + +static unsigned int grpci1_startup_irq(struct irq_data *data) +{ + grpci1_unmask_irq(data); + return 0; +} + +static void grpci1_shutdown_irq(struct irq_data *data) +{ + grpci1_mask_irq(data); +} + +static struct irq_chip grpci1_irq = { + .name = "grpci1", + .irq_startup = grpci1_startup_irq, + .irq_shutdown = grpci1_shutdown_irq, + .irq_mask = grpci1_mask_irq, + .irq_unmask = grpci1_unmask_irq, +}; + +/* Handle one or multiple IRQs from the PCI core */ +static void grpci1_pci_flow_irq(unsigned int irq, struct irq_desc *desc) +{ + struct grpci1_priv *priv = grpci1priv; + int i, ack = 0; + unsigned int irqreg; + + irqreg = REGLOAD(priv->regs->irq); + irqreg = (irqreg >> IRQ_MASK_BIT) & irqreg; + + /* Error Interrupt? */ + if (irqreg & IRQ_ALL_ERRORS) { + generic_handle_irq(priv->irq_err); + ack = 1; + } + + /* PCI Interrupt? */ + if (irqreg & IRQ_INTX) { + /* Call respective PCI Interrupt handler */ + for (i = 0; i < 4; i++) { + if (irqreg & (1 << i)) + generic_handle_irq(priv->irq_map[i]); + } + ack = 1; + } + + /* + * Call "first level" IRQ chip end-of-irq handler. It will ACK LEON IRQ + * Controller, this must be done after IRQ sources have been handled to + * avoid double IRQ generation + */ + if (ack) + desc->irq_data.chip->irq_eoi(&desc->irq_data); +} + +/* Create a virtual IRQ */ +static unsigned int grpci1_build_device_irq(unsigned int irq) +{ + unsigned int virq = 0, pil; + + pil = 1 << 8; + virq = irq_alloc(irq, pil); + if (virq == 0) + goto out; + + irq_set_chip_and_handler_name(virq, &grpci1_irq, handle_simple_irq, + "pcilvl"); + irq_set_chip_data(virq, (void *)irq); + +out: + return virq; +} + +/* + * Initialize mappings AMBA<->PCI, clear IRQ state, setup PCI interface + * + * Target BARs: + * BAR0: unused in this implementation + * BAR1: peripheral DMA to host's memory (size at least 256MByte) + * BAR2..BAR5: not implemented in hardware + */ +static void grpci1_hw_init(struct grpci1_priv *priv) +{ + u32 ahbadr, bar_sz, data, pciadr; + struct grpci1_regs __iomem *regs = priv->regs; + + /* set 1:1 mapping between AHB -> PCI memory space */ + REGSTORE(regs->cfg_stat, priv->pci_area & 0xf0000000); + + /* map PCI accesses to target BAR1 to Linux kernel memory 1:1 */ + ahbadr = 0xf0000000 & (u32)__pa(PAGE_ALIGN((unsigned long) &_end)); + REGSTORE(regs->page1, ahbadr); + + /* translate I/O accesses to 0, I/O Space always @ PCI low 64Kbytes */ + REGSTORE(regs->iomap, REGLOAD(regs->iomap) & 0x0000ffff); + + /* disable and clear pending interrupts */ + REGSTORE(regs->irq, 0); + + /* Setup BAR0 outside access range so that it does not conflict with + * peripheral DMA. There is no need to set up the PAGE0 register. + */ + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0, 0xffffffff); + grpci1_cfg_r32(priv, TGT, 0, PCI_BASE_ADDRESS_0, &bar_sz); + bar_sz = ~bar_sz + 1; + pciadr = priv->pci_area - bar_sz; + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0, pciadr); + + /* + * Setup the Host's PCI Target BAR1 for other peripherals to access, + * and do DMA to the host's memory. + */ + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_1, ahbadr); + + /* + * Setup Latency Timer and cache line size. Default cache line + * size will result in poor performance (256 word fetches), 0xff + * will set it according to the max size of the PCI FIFO. + */ + grpci1_cfg_w8(priv, TGT, 0, PCI_CACHE_LINE_SIZE, 0xff); + grpci1_cfg_w8(priv, TGT, 0, PCI_LATENCY_TIMER, 0x40); + + /* set as bus master, enable pci memory responses, clear status bits */ + grpci1_cfg_r32(priv, TGT, 0, PCI_COMMAND, &data); + data |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + grpci1_cfg_w32(priv, TGT, 0, PCI_COMMAND, data); +} + +static irqreturn_t grpci1_jump_interrupt(int irq, void *arg) +{ + struct grpci1_priv *priv = arg; + dev_err(priv->dev, "Jump IRQ happened\n"); + return IRQ_NONE; +} + +/* Handle GRPCI1 Error Interrupt */ +static irqreturn_t grpci1_err_interrupt(int irq, void *arg) +{ + struct grpci1_priv *priv = arg; + u32 status; + + grpci1_cfg_r16(priv, TGT, 0, PCI_STATUS, &status); + status &= priv->pci_err_mask; + + if (status == 0) + return IRQ_NONE; + + if (status & PCI_STATUS_PARITY) + dev_err(priv->dev, "Data Parity Error\n"); + + if (status & PCI_STATUS_SIG_TARGET_ABORT) + dev_err(priv->dev, "Signalled Target Abort\n"); + + if (status & PCI_STATUS_REC_TARGET_ABORT) + dev_err(priv->dev, "Received Target Abort\n"); + + if (status & PCI_STATUS_REC_MASTER_ABORT) + dev_err(priv->dev, "Received Master Abort\n"); + + if (status & PCI_STATUS_SIG_SYSTEM_ERROR) + dev_err(priv->dev, "Signalled System Error\n"); + + if (status & PCI_STATUS_DETECTED_PARITY) + dev_err(priv->dev, "Parity Error\n"); + + /* Clear handled INT TYPE IRQs */ + grpci1_cfg_w16(priv, TGT, 0, PCI_STATUS, status); + + return IRQ_HANDLED; +} + +static int grpci1_of_probe(struct platform_device *ofdev) +{ + struct grpci1_regs __iomem *regs; + struct grpci1_priv *priv; + int err, len; + const int *tmp; + u32 cfg, size, err_mask; + struct resource *res; + + if (grpci1priv) { + dev_err(&ofdev->dev, "only one GRPCI1 supported\n"); + return -ENODEV; + } + + if (ofdev->num_resources < 3) { + dev_err(&ofdev->dev, "not enough APB/AHB resources\n"); + return -EIO; + } + + priv = devm_kzalloc(&ofdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + dev_err(&ofdev->dev, "memory allocation failed\n"); + return -ENOMEM; + } + platform_set_drvdata(ofdev, priv); + priv->dev = &ofdev->dev; + + /* find device register base address */ + res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); + regs = devm_ioremap_resource(&ofdev->dev, res); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + /* + * check that we're in Host Slot and that we can act as a Host Bridge + * and not only as target/peripheral. + */ + cfg = REGLOAD(regs->cfg_stat); + if ((cfg & CFGSTAT_HOST) == 0) { + dev_err(&ofdev->dev, "not in host system slot\n"); + return -EIO; + } + + /* check that BAR1 support 256 MByte so that we can map kernel space */ + REGSTORE(regs->page1, 0xffffffff); + size = ~REGLOAD(regs->page1) + 1; + if (size < 0x10000000) { + dev_err(&ofdev->dev, "BAR1 must be at least 256MByte\n"); + return -EIO; + } + + /* hardware must support little-endian PCI (byte-twisting) */ + if ((REGLOAD(regs->page0) & PAGE0_BTEN) == 0) { + dev_err(&ofdev->dev, "byte-twisting is required\n"); + return -EIO; + } + + priv->regs = regs; + priv->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); + dev_info(&ofdev->dev, "host found at 0x%p, irq%d\n", regs, priv->irq); + + /* Find PCI Memory, I/O and Configuration Space Windows */ + priv->pci_area = ofdev->resource[1].start; + priv->pci_area_end = ofdev->resource[1].end+1; + priv->pci_io = ofdev->resource[2].start; + priv->pci_conf = ofdev->resource[2].start + 0x10000; + priv->pci_conf_end = priv->pci_conf + 0x10000; + priv->pci_io_va = (unsigned long)ioremap(priv->pci_io, 0x10000); + if (!priv->pci_io_va) { + dev_err(&ofdev->dev, "unable to map PCI I/O area\n"); + return -EIO; + } + + printk(KERN_INFO + "GRPCI1: MEMORY SPACE [0x%08lx - 0x%08lx]\n" + " I/O SPACE [0x%08lx - 0x%08lx]\n" + " CONFIG SPACE [0x%08lx - 0x%08lx]\n", + priv->pci_area, priv->pci_area_end-1, + priv->pci_io, priv->pci_conf-1, + priv->pci_conf, priv->pci_conf_end-1); + + /* + * I/O Space resources in I/O Window mapped into Virtual Adr Space + * We never use low 4KB because some devices seem have problems using + * address 0. + */ + priv->info.io_space.name = "GRPCI1 PCI I/O Space"; + priv->info.io_space.start = priv->pci_io_va + 0x1000; + priv->info.io_space.end = priv->pci_io_va + 0x10000 - 1; + priv->info.io_space.flags = IORESOURCE_IO; + + /* + * grpci1 has no prefetchable memory, map everything as + * non-prefetchable memory + */ + priv->info.mem_space.name = "GRPCI1 PCI MEM Space"; + priv->info.mem_space.start = priv->pci_area; + priv->info.mem_space.end = priv->pci_area_end - 1; + priv->info.mem_space.flags = IORESOURCE_MEM; + + if (request_resource(&iomem_resource, &priv->info.mem_space) < 0) { + dev_err(&ofdev->dev, "unable to request PCI memory area\n"); + err = -ENOMEM; + goto err1; + } + + if (request_resource(&ioport_resource, &priv->info.io_space) < 0) { + dev_err(&ofdev->dev, "unable to request PCI I/O area\n"); + err = -ENOMEM; + goto err2; + } + + /* setup maximum supported PCI buses */ + priv->info.busn.name = "GRPCI1 busn"; + priv->info.busn.start = 0; + priv->info.busn.end = 15; + + grpci1priv = priv; + + /* Initialize hardware */ + grpci1_hw_init(priv); + + /* + * Get PCI Interrupt to System IRQ mapping and setup IRQ handling + * Error IRQ. All PCI and PCI-Error interrupts are shared using the + * same system IRQ. + */ + leon_update_virq_handling(priv->irq, grpci1_pci_flow_irq, "pcilvl", 0); + + priv->irq_map[0] = grpci1_build_device_irq(1); + priv->irq_map[1] = grpci1_build_device_irq(2); + priv->irq_map[2] = grpci1_build_device_irq(3); + priv->irq_map[3] = grpci1_build_device_irq(4); + priv->irq_err = grpci1_build_device_irq(5); + + printk(KERN_INFO " PCI INTA..D#: IRQ%d, IRQ%d, IRQ%d, IRQ%d\n", + priv->irq_map[0], priv->irq_map[1], priv->irq_map[2], + priv->irq_map[3]); + + /* Enable IRQs on LEON IRQ controller */ + err = devm_request_irq(&ofdev->dev, priv->irq, grpci1_jump_interrupt, 0, + "GRPCI1_JUMP", priv); + if (err) { + dev_err(&ofdev->dev, "ERR IRQ request failed: %d\n", err); + goto err3; + } + + /* Setup IRQ handler for access errors */ + err = devm_request_irq(&ofdev->dev, priv->irq_err, + grpci1_err_interrupt, IRQF_SHARED, "GRPCI1_ERR", + priv); + if (err) { + dev_err(&ofdev->dev, "ERR VIRQ request failed: %d\n", err); + goto err3; + } + + tmp = of_get_property(ofdev->dev.of_node, "all_pci_errors", &len); + if (tmp && (len == 4)) { + priv->pci_err_mask = ALL_PCI_ERRORS; + err_mask = IRQ_ALL_ERRORS << IRQ_MASK_BIT; + } else { + priv->pci_err_mask = DEF_PCI_ERRORS; + err_mask = IRQ_DEF_ERRORS << IRQ_MASK_BIT; + } + + /* + * Enable Error Interrupts. PCI interrupts are unmasked once request_irq + * is called by the PCI Device drivers + */ + REGSTORE(regs->irq, err_mask); + + /* Init common layer and scan buses */ + priv->info.ops = &grpci1_ops; + priv->info.map_irq = grpci1_map_irq; + leon_pci_init(ofdev, &priv->info); + + return 0; + +err3: + release_resource(&priv->info.io_space); +err2: + release_resource(&priv->info.mem_space); +err1: + iounmap((void __iomem *)priv->pci_io_va); + grpci1priv = NULL; + return err; +} + +static struct of_device_id grpci1_of_match[] = { + { + .name = "GAISLER_PCIFBRG", + }, + { + .name = "01_014", + }, + {}, +}; + +static struct platform_driver grpci1_of_driver = { + .driver = { + .name = "grpci1", + .owner = THIS_MODULE, + .of_match_table = grpci1_of_match, + }, + .probe = grpci1_of_probe, +}; + +static int __init grpci1_init(void) +{ + return platform_driver_register(&grpci1_of_driver); +} + +subsys_initcall(grpci1_init); diff --git a/arch/sparc/kernel/leon_pci_grpci2.c b/arch/sparc/kernel/leon_pci_grpci2.c new file mode 100644 index 00000000000..e433a4d69fe --- /dev/null +++ b/arch/sparc/kernel/leon_pci_grpci2.c @@ -0,0 +1,914 @@ +/* + * leon_pci_grpci2.c: GRPCI2 Host PCI driver + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + * + */ + +#include <linux/of_device.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/export.h> +#include <asm/io.h> +#include <asm/leon.h> +#include <asm/vaddrs.h> +#include <asm/sections.h> +#include <asm/leon_pci.h> + +#include "irq.h" + +struct grpci2_barcfg { + unsigned long pciadr; /* PCI Space Address */ + unsigned long ahbadr; /* PCI Base address mapped to this AHB addr */ +}; + +/* Device Node Configuration options: + * - barcfgs : Custom Configuration of Host's 6 target BARs + * - irq_mask : Limit which PCI interrupts are enabled + * - do_reset : Force PCI Reset on startup + * + * barcfgs + * ======= + * + * Optional custom Target BAR configuration (see struct grpci2_barcfg). All + * addresses are physical. Array always contains 6 elements (len=2*4*6 bytes) + * + * -1 means not configured (let host driver do default setup). + * + * [i*2+0] = PCI Address of BAR[i] on target interface + * [i*2+1] = Accessing PCI address of BAR[i] result in this AMBA address + * + * + * irq_mask + * ======== + * + * Limit which PCI interrupts are enabled. 0=Disable, 1=Enable. By default + * all are enabled. Use this when PCI interrupt pins are floating on PCB. + * int, len=4. + * bit0 = PCI INTA# + * bit1 = PCI INTB# + * bit2 = PCI INTC# + * bit3 = PCI INTD# + * + * + * reset + * ===== + * + * Force PCI reset on startup. int, len=4 + */ + +/* Enable Debugging Configuration Space Access */ +#undef GRPCI2_DEBUG_CFGACCESS + +/* + * GRPCI2 APB Register MAP + */ +struct grpci2_regs { + unsigned int ctrl; /* 0x00 Control */ + unsigned int sts_cap; /* 0x04 Status / Capabilities */ + int res1; /* 0x08 */ + unsigned int io_map; /* 0x0C I/O Map address */ + unsigned int dma_ctrl; /* 0x10 DMA */ + unsigned int dma_bdbase; /* 0x14 DMA */ + int res2[2]; /* 0x18 */ + unsigned int bars[6]; /* 0x20 read-only PCI BARs */ + int res3[2]; /* 0x38 */ + unsigned int ahbmst_map[16]; /* 0x40 AHB->PCI Map per AHB Master */ + + /* PCI Trace Buffer Registers (OPTIONAL) */ + unsigned int t_ctrl; /* 0x80 */ + unsigned int t_cnt; /* 0x84 */ + unsigned int t_adpat; /* 0x88 */ + unsigned int t_admask; /* 0x8C */ + unsigned int t_sigpat; /* 0x90 */ + unsigned int t_sigmask; /* 0x94 */ + unsigned int t_adstate; /* 0x98 */ + unsigned int t_sigstate; /* 0x9C */ +}; + +#define REGLOAD(a) (be32_to_cpu(__raw_readl(&(a)))) +#define REGSTORE(a, v) (__raw_writel(cpu_to_be32(v), &(a))) + +#define CTRL_BUS_BIT 16 + +#define CTRL_RESET (1<<31) +#define CTRL_SI (1<<27) +#define CTRL_PE (1<<26) +#define CTRL_EI (1<<25) +#define CTRL_ER (1<<24) +#define CTRL_BUS (0xff<<CTRL_BUS_BIT) +#define CTRL_HOSTINT 0xf + +#define STS_HOST_BIT 31 +#define STS_MST_BIT 30 +#define STS_TAR_BIT 29 +#define STS_DMA_BIT 28 +#define STS_DI_BIT 27 +#define STS_HI_BIT 26 +#define STS_IRQMODE_BIT 24 +#define STS_TRACE_BIT 23 +#define STS_CFGERRVALID_BIT 20 +#define STS_CFGERR_BIT 19 +#define STS_INTTYPE_BIT 12 +#define STS_INTSTS_BIT 8 +#define STS_FDEPTH_BIT 2 +#define STS_FNUM_BIT 0 + +#define STS_HOST (1<<STS_HOST_BIT) +#define STS_MST (1<<STS_MST_BIT) +#define STS_TAR (1<<STS_TAR_BIT) +#define STS_DMA (1<<STS_DMA_BIT) +#define STS_DI (1<<STS_DI_BIT) +#define STS_HI (1<<STS_HI_BIT) +#define STS_IRQMODE (0x3<<STS_IRQMODE_BIT) +#define STS_TRACE (1<<STS_TRACE_BIT) +#define STS_CFGERRVALID (1<<STS_CFGERRVALID_BIT) +#define STS_CFGERR (1<<STS_CFGERR_BIT) +#define STS_INTTYPE (0x3f<<STS_INTTYPE_BIT) +#define STS_INTSTS (0xf<<STS_INTSTS_BIT) +#define STS_FDEPTH (0x7<<STS_FDEPTH_BIT) +#define STS_FNUM (0x3<<STS_FNUM_BIT) + +#define STS_ISYSERR (1<<17) +#define STS_IDMA (1<<16) +#define STS_IDMAERR (1<<15) +#define STS_IMSTABRT (1<<14) +#define STS_ITGTABRT (1<<13) +#define STS_IPARERR (1<<12) + +#define STS_ERR_IRQ (STS_ISYSERR | STS_IMSTABRT | STS_ITGTABRT | STS_IPARERR) + +struct grpci2_bd_chan { + unsigned int ctrl; /* 0x00 DMA Control */ + unsigned int nchan; /* 0x04 Next DMA Channel Address */ + unsigned int nbd; /* 0x08 Next Data Descriptor in chan */ + unsigned int res; /* 0x0C Reserved */ +}; + +#define BD_CHAN_EN 0x80000000 +#define BD_CHAN_TYPE 0x00300000 +#define BD_CHAN_BDCNT 0x0000ffff +#define BD_CHAN_EN_BIT 31 +#define BD_CHAN_TYPE_BIT 20 +#define BD_CHAN_BDCNT_BIT 0 + +struct grpci2_bd_data { + unsigned int ctrl; /* 0x00 DMA Data Control */ + unsigned int pci_adr; /* 0x04 PCI Start Address */ + unsigned int ahb_adr; /* 0x08 AHB Start address */ + unsigned int next; /* 0x0C Next Data Descriptor in chan */ +}; + +#define BD_DATA_EN 0x80000000 +#define BD_DATA_IE 0x40000000 +#define BD_DATA_DR 0x20000000 +#define BD_DATA_TYPE 0x00300000 +#define BD_DATA_ER 0x00080000 +#define BD_DATA_LEN 0x0000ffff +#define BD_DATA_EN_BIT 31 +#define BD_DATA_IE_BIT 30 +#define BD_DATA_DR_BIT 29 +#define BD_DATA_TYPE_BIT 20 +#define BD_DATA_ER_BIT 19 +#define BD_DATA_LEN_BIT 0 + +/* GRPCI2 Capability */ +struct grpci2_cap_first { + unsigned int ctrl; + unsigned int pci2ahb_map[6]; + unsigned int ext2ahb_map; + unsigned int io_map; + unsigned int pcibar_size[6]; +}; +#define CAP9_CTRL_OFS 0 +#define CAP9_BAR_OFS 0x4 +#define CAP9_IOMAP_OFS 0x20 +#define CAP9_BARSIZE_OFS 0x24 + +#define TGT 256 + +struct grpci2_priv { + struct leon_pci_info info; /* must be on top of this structure */ + struct grpci2_regs __iomem *regs; + char irq; + char irq_mode; /* IRQ Mode from CAPSTS REG */ + char bt_enabled; + char do_reset; + char irq_mask; + u32 pciid; /* PCI ID of Host */ + unsigned char irq_map[4]; + + /* Virtual IRQ numbers */ + unsigned int virq_err; + unsigned int virq_dma; + + /* AHB PCI Windows */ + unsigned long pci_area; /* MEMORY */ + unsigned long pci_area_end; + unsigned long pci_io; /* I/O */ + unsigned long pci_conf; /* CONFIGURATION */ + unsigned long pci_conf_end; + unsigned long pci_io_va; + + struct grpci2_barcfg tgtbars[6]; +}; + +static DEFINE_SPINLOCK(grpci2_dev_lock); +static struct grpci2_priv *grpci2priv; + +static int grpci2_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct grpci2_priv *priv = dev->bus->sysdata; + int irq_group; + + /* Use default IRQ decoding on PCI BUS0 according slot numbering */ + irq_group = slot & 0x3; + pin = ((pin - 1) + irq_group) & 0x3; + + return priv->irq_map[pin]; +} + +static int grpci2_cfg_r32(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + unsigned int *pci_conf; + unsigned long flags; + u32 tmp; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, (REGLOAD(priv->regs->ctrl) & ~(0xff << 16)) | + (bus << 16)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); + + /* clear old status */ + REGSTORE(priv->regs->sts_cap, (STS_CFGERR | STS_CFGERRVALID)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + tmp = LEON3_BYPASS_LOAD_PA(pci_conf); + + /* Wait until GRPCI2 signals that CFG access is done, it should be + * done instantaneously unless a DMA operation is ongoing... + */ + while ((REGLOAD(priv->regs->sts_cap) & STS_CFGERRVALID) == 0) + ; + + if (REGLOAD(priv->regs->sts_cap) & STS_CFGERR) { + *val = 0xffffffff; + } else { + /* Bus always little endian (unaffected by byte-swapping) */ + *val = swab32(tmp); + } + + return 0; +} + +static int grpci2_cfg_r16(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + if (where & 0x1) + return -EINVAL; + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xffff & (v >> (8 * (where & 0x3))); + return ret; +} + +static int grpci2_cfg_r8(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xff & (v >> (8 * (where & 3))); + + return ret; +} + +static int grpci2_cfg_w32(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + unsigned int *pci_conf; + unsigned long flags; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, (REGLOAD(priv->regs->ctrl) & ~(0xff << 16)) | + (bus << 16)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); + + /* clear old status */ + REGSTORE(priv->regs->sts_cap, (STS_CFGERR | STS_CFGERRVALID)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); + + /* Wait until GRPCI2 signals that CFG access is done, it should be + * done instantaneously unless a DMA operation is ongoing... + */ + while ((REGLOAD(priv->regs->sts_cap) & STS_CFGERRVALID) == 0) + ; + + return 0; +} + +static int grpci2_cfg_w16(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + if (where & 0x1) + return -EINVAL; + ret = grpci2_cfg_r32(priv, bus, devfn, where&~3, &v); + if (ret) + return ret; + v = (v & ~(0xffff << (8 * (where & 0x3)))) | + ((0xffff & val) << (8 * (where & 0x3))); + return grpci2_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +static int grpci2_cfg_w8(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + if (ret != 0) + return ret; + v = (v & ~(0xff << (8 * (where & 0x3)))) | + ((0xff & val) << (8 * (where & 0x3))); + return grpci2_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +/* Read from Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci2_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct grpci2_priv *priv = grpci2priv; + unsigned int busno = bus->number; + int ret; + + if (PCI_SLOT(devfn) > 15 || busno > 255) { + *val = ~0; + return 0; + } + + switch (size) { + case 1: + ret = grpci2_cfg_r8(priv, busno, devfn, where, val); + break; + case 2: + ret = grpci2_cfg_r16(priv, busno, devfn, where, val); + break; + case 4: + ret = grpci2_cfg_r32(priv, busno, devfn, where, val); + break; + default: + ret = -EINVAL; + break; + } + +#ifdef GRPCI2_DEBUG_CFGACCESS + printk(KERN_INFO "grpci2_read_config: [%02x:%02x:%x] ofs=%d val=%x " + "size=%d\n", busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, + *val, size); +#endif + + return ret; +} + +/* Write to Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci2_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct grpci2_priv *priv = grpci2priv; + unsigned int busno = bus->number; + + if (PCI_SLOT(devfn) > 15 || busno > 255) + return 0; + +#ifdef GRPCI2_DEBUG_CFGACCESS + printk(KERN_INFO "grpci2_write_config: [%02x:%02x:%x] ofs=%d size=%d " + "val=%x\n", busno, PCI_SLOT(devfn), PCI_FUNC(devfn), + where, size, val); +#endif + + switch (size) { + default: + return -EINVAL; + case 1: + return grpci2_cfg_w8(priv, busno, devfn, where, val); + case 2: + return grpci2_cfg_w16(priv, busno, devfn, where, val); + case 4: + return grpci2_cfg_w32(priv, busno, devfn, where, val); + } +} + +static struct pci_ops grpci2_ops = { + .read = grpci2_read_config, + .write = grpci2_write_config, +}; + +/* GENIRQ IRQ chip implementation for GRPCI2 irqmode=0..2. In configuration + * 3 where all PCI Interrupts has a separate IRQ on the system IRQ controller + * this is not needed and the standard IRQ controller can be used. + */ + +static void grpci2_mask_irq(struct irq_data *data) +{ + unsigned long flags; + unsigned int irqidx; + struct grpci2_priv *priv = grpci2priv; + + irqidx = (unsigned int)data->chip_data - 1; + if (irqidx > 3) /* only mask PCI interrupts here */ + return; + + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, REGLOAD(priv->regs->ctrl) & ~(1 << irqidx)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); +} + +static void grpci2_unmask_irq(struct irq_data *data) +{ + unsigned long flags; + unsigned int irqidx; + struct grpci2_priv *priv = grpci2priv; + + irqidx = (unsigned int)data->chip_data - 1; + if (irqidx > 3) /* only unmask PCI interrupts here */ + return; + + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, REGLOAD(priv->regs->ctrl) | (1 << irqidx)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); +} + +static unsigned int grpci2_startup_irq(struct irq_data *data) +{ + grpci2_unmask_irq(data); + return 0; +} + +static void grpci2_shutdown_irq(struct irq_data *data) +{ + grpci2_mask_irq(data); +} + +static struct irq_chip grpci2_irq = { + .name = "grpci2", + .irq_startup = grpci2_startup_irq, + .irq_shutdown = grpci2_shutdown_irq, + .irq_mask = grpci2_mask_irq, + .irq_unmask = grpci2_unmask_irq, +}; + +/* Handle one or multiple IRQs from the PCI core */ +static void grpci2_pci_flow_irq(unsigned int irq, struct irq_desc *desc) +{ + struct grpci2_priv *priv = grpci2priv; + int i, ack = 0; + unsigned int ctrl, sts_cap, pci_ints; + + ctrl = REGLOAD(priv->regs->ctrl); + sts_cap = REGLOAD(priv->regs->sts_cap); + + /* Error Interrupt? */ + if (sts_cap & STS_ERR_IRQ) { + generic_handle_irq(priv->virq_err); + ack = 1; + } + + /* PCI Interrupt? */ + pci_ints = ((~sts_cap) >> STS_INTSTS_BIT) & ctrl & CTRL_HOSTINT; + if (pci_ints) { + /* Call respective PCI Interrupt handler */ + for (i = 0; i < 4; i++) { + if (pci_ints & (1 << i)) + generic_handle_irq(priv->irq_map[i]); + } + ack = 1; + } + + /* + * Decode DMA Interrupt only when shared with Err and PCI INTX#, when + * the DMA is a unique IRQ the DMA interrupts doesn't end up here, they + * goes directly to DMA ISR. + */ + if ((priv->irq_mode == 0) && (sts_cap & (STS_IDMA | STS_IDMAERR))) { + generic_handle_irq(priv->virq_dma); + ack = 1; + } + + /* + * Call "first level" IRQ chip end-of-irq handler. It will ACK LEON IRQ + * Controller, this must be done after IRQ sources have been handled to + * avoid double IRQ generation + */ + if (ack) + desc->irq_data.chip->irq_eoi(&desc->irq_data); +} + +/* Create a virtual IRQ */ +static unsigned int grpci2_build_device_irq(unsigned int irq) +{ + unsigned int virq = 0, pil; + + pil = 1 << 8; + virq = irq_alloc(irq, pil); + if (virq == 0) + goto out; + + irq_set_chip_and_handler_name(virq, &grpci2_irq, handle_simple_irq, + "pcilvl"); + irq_set_chip_data(virq, (void *)irq); + +out: + return virq; +} + +static void grpci2_hw_init(struct grpci2_priv *priv) +{ + u32 ahbadr, pciadr, bar_sz, capptr, io_map, data; + struct grpci2_regs __iomem *regs = priv->regs; + int i; + struct grpci2_barcfg *barcfg = priv->tgtbars; + + /* Reset any earlier setup */ + if (priv->do_reset) { + printk(KERN_INFO "GRPCI2: Resetting PCI bus\n"); + REGSTORE(regs->ctrl, CTRL_RESET); + ssleep(1); /* Wait for boards to settle */ + } + REGSTORE(regs->ctrl, 0); + REGSTORE(regs->sts_cap, ~0); /* Clear Status */ + REGSTORE(regs->dma_ctrl, 0); + REGSTORE(regs->dma_bdbase, 0); + + /* Translate I/O accesses to 0, I/O Space always @ PCI low 64Kbytes */ + REGSTORE(regs->io_map, REGLOAD(regs->io_map) & 0x0000ffff); + + /* set 1:1 mapping between AHB -> PCI memory space, for all Masters + * Each AHB master has it's own mapping registers. Max 16 AHB masters. + */ + for (i = 0; i < 16; i++) + REGSTORE(regs->ahbmst_map[i], priv->pci_area); + + /* Get the GRPCI2 Host PCI ID */ + grpci2_cfg_r32(priv, TGT, 0, PCI_VENDOR_ID, &priv->pciid); + + /* Get address to first (always defined) capability structure */ + grpci2_cfg_r8(priv, TGT, 0, PCI_CAPABILITY_LIST, &capptr); + + /* Enable/Disable Byte twisting */ + grpci2_cfg_r32(priv, TGT, 0, capptr+CAP9_IOMAP_OFS, &io_map); + io_map = (io_map & ~0x1) | (priv->bt_enabled ? 1 : 0); + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_IOMAP_OFS, io_map); + + /* Setup the Host's PCI Target BARs for other peripherals to access, + * and do DMA to the host's memory. The target BARs can be sized and + * enabled individually. + * + * User may set custom target BARs, but default is: + * The first BARs is used to map kernel low (DMA is part of normal + * region on sparc which is SRMMU_MAXMEM big) main memory 1:1 to the + * PCI bus, the other BARs are disabled. We assume that the first BAR + * is always available. + */ + for (i = 0; i < 6; i++) { + if (barcfg[i].pciadr != ~0 && barcfg[i].ahbadr != ~0) { + /* Target BARs must have the proper alignment */ + ahbadr = barcfg[i].ahbadr; + pciadr = barcfg[i].pciadr; + bar_sz = ((pciadr - 1) & ~pciadr) + 1; + } else { + if (i == 0) { + /* Map main memory */ + bar_sz = 0xf0000008; /* 256MB prefetchable */ + ahbadr = 0xf0000000 & (u32)__pa(PAGE_ALIGN( + (unsigned long) &_end)); + pciadr = ahbadr; + } else { + bar_sz = 0; + ahbadr = 0; + pciadr = 0; + } + } + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_BARSIZE_OFS+i*4, + bar_sz); + grpci2_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0+i*4, pciadr); + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_BAR_OFS+i*4, ahbadr); + printk(KERN_INFO " TGT BAR[%d]: 0x%08x (PCI)-> 0x%08x\n", + i, pciadr, ahbadr); + } + + /* set as bus master and enable pci memory responses */ + grpci2_cfg_r32(priv, TGT, 0, PCI_COMMAND, &data); + data |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + grpci2_cfg_w32(priv, TGT, 0, PCI_COMMAND, data); + + /* Enable Error respone (CPU-TRAP) on illegal memory access. */ + REGSTORE(regs->ctrl, CTRL_ER | CTRL_PE); +} + +static irqreturn_t grpci2_jump_interrupt(int irq, void *arg) +{ + printk(KERN_ERR "GRPCI2: Jump IRQ happened\n"); + return IRQ_NONE; +} + +/* Handle GRPCI2 Error Interrupt */ +static irqreturn_t grpci2_err_interrupt(int irq, void *arg) +{ + struct grpci2_priv *priv = arg; + struct grpci2_regs __iomem *regs = priv->regs; + unsigned int status; + + status = REGLOAD(regs->sts_cap); + if ((status & STS_ERR_IRQ) == 0) + return IRQ_NONE; + + if (status & STS_IPARERR) + printk(KERN_ERR "GRPCI2: Parity Error\n"); + + if (status & STS_ITGTABRT) + printk(KERN_ERR "GRPCI2: Target Abort\n"); + + if (status & STS_IMSTABRT) + printk(KERN_ERR "GRPCI2: Master Abort\n"); + + if (status & STS_ISYSERR) + printk(KERN_ERR "GRPCI2: System Error\n"); + + /* Clear handled INT TYPE IRQs */ + REGSTORE(regs->sts_cap, status & STS_ERR_IRQ); + + return IRQ_HANDLED; +} + +static int grpci2_of_probe(struct platform_device *ofdev) +{ + struct grpci2_regs __iomem *regs; + struct grpci2_priv *priv; + int err, i, len; + const int *tmp; + unsigned int capability; + + if (grpci2priv) { + printk(KERN_ERR "GRPCI2: only one GRPCI2 core supported\n"); + return -ENODEV; + } + + if (ofdev->num_resources < 3) { + printk(KERN_ERR "GRPCI2: not enough APB/AHB resources\n"); + return -EIO; + } + + /* Find Device Address */ + regs = of_ioremap(&ofdev->resource[0], 0, + resource_size(&ofdev->resource[0]), + "grlib-grpci2 regs"); + if (regs == NULL) { + printk(KERN_ERR "GRPCI2: ioremap failed\n"); + return -EIO; + } + + /* + * Check that we're in Host Slot and that we can act as a Host Bridge + * and not only as target. + */ + capability = REGLOAD(regs->sts_cap); + if ((capability & STS_HOST) || !(capability & STS_MST)) { + printk(KERN_INFO "GRPCI2: not in host system slot\n"); + err = -EIO; + goto err1; + } + + priv = grpci2priv = kzalloc(sizeof(struct grpci2_priv), GFP_KERNEL); + if (grpci2priv == NULL) { + err = -ENOMEM; + goto err1; + } + memset(grpci2priv, 0, sizeof(*grpci2priv)); + priv->regs = regs; + priv->irq = ofdev->archdata.irqs[0]; /* BASE IRQ */ + priv->irq_mode = (capability & STS_IRQMODE) >> STS_IRQMODE_BIT; + + printk(KERN_INFO "GRPCI2: host found at %p, irq%d\n", regs, priv->irq); + + /* Byte twisting should be made configurable from kernel command line */ + priv->bt_enabled = 1; + + /* Let user do custom Target BAR assignment */ + tmp = of_get_property(ofdev->dev.of_node, "barcfg", &len); + if (tmp && (len == 2*4*6)) + memcpy(priv->tgtbars, tmp, 2*4*6); + else + memset(priv->tgtbars, -1, 2*4*6); + + /* Limit IRQ unmasking in irq_mode 2 and 3 */ + tmp = of_get_property(ofdev->dev.of_node, "irq_mask", &len); + if (tmp && (len == 4)) + priv->do_reset = *tmp; + else + priv->irq_mask = 0xf; + + /* Optional PCI reset. Force PCI reset on startup */ + tmp = of_get_property(ofdev->dev.of_node, "reset", &len); + if (tmp && (len == 4)) + priv->do_reset = *tmp; + else + priv->do_reset = 0; + + /* Find PCI Memory, I/O and Configuration Space Windows */ + priv->pci_area = ofdev->resource[1].start; + priv->pci_area_end = ofdev->resource[1].end+1; + priv->pci_io = ofdev->resource[2].start; + priv->pci_conf = ofdev->resource[2].start + 0x10000; + priv->pci_conf_end = priv->pci_conf + 0x10000; + priv->pci_io_va = (unsigned long)ioremap(priv->pci_io, 0x10000); + if (!priv->pci_io_va) { + err = -EIO; + goto err2; + } + + printk(KERN_INFO + "GRPCI2: MEMORY SPACE [0x%08lx - 0x%08lx]\n" + " I/O SPACE [0x%08lx - 0x%08lx]\n" + " CONFIG SPACE [0x%08lx - 0x%08lx]\n", + priv->pci_area, priv->pci_area_end-1, + priv->pci_io, priv->pci_conf-1, + priv->pci_conf, priv->pci_conf_end-1); + + /* + * I/O Space resources in I/O Window mapped into Virtual Adr Space + * We never use low 4KB because some devices seem have problems using + * address 0. + */ + memset(&priv->info.io_space, 0, sizeof(struct resource)); + priv->info.io_space.name = "GRPCI2 PCI I/O Space"; + priv->info.io_space.start = priv->pci_io_va + 0x1000; + priv->info.io_space.end = priv->pci_io_va + 0x10000 - 1; + priv->info.io_space.flags = IORESOURCE_IO; + + /* + * GRPCI2 has no prefetchable memory, map everything as + * non-prefetchable memory + */ + memset(&priv->info.mem_space, 0, sizeof(struct resource)); + priv->info.mem_space.name = "GRPCI2 PCI MEM Space"; + priv->info.mem_space.start = priv->pci_area; + priv->info.mem_space.end = priv->pci_area_end - 1; + priv->info.mem_space.flags = IORESOURCE_MEM; + + if (request_resource(&iomem_resource, &priv->info.mem_space) < 0) + goto err3; + if (request_resource(&ioport_resource, &priv->info.io_space) < 0) + goto err4; + + /* setup maximum supported PCI buses */ + priv->info.busn.name = "GRPCI2 busn"; + priv->info.busn.start = 0; + priv->info.busn.end = 255; + + grpci2_hw_init(priv); + + /* + * Get PCI Interrupt to System IRQ mapping and setup IRQ handling + * Error IRQ always on PCI INTA. + */ + if (priv->irq_mode < 2) { + /* All PCI interrupts are shared using the same system IRQ */ + leon_update_virq_handling(priv->irq, grpci2_pci_flow_irq, + "pcilvl", 0); + + priv->irq_map[0] = grpci2_build_device_irq(1); + priv->irq_map[1] = grpci2_build_device_irq(2); + priv->irq_map[2] = grpci2_build_device_irq(3); + priv->irq_map[3] = grpci2_build_device_irq(4); + + priv->virq_err = grpci2_build_device_irq(5); + if (priv->irq_mode & 1) + priv->virq_dma = ofdev->archdata.irqs[1]; + else + priv->virq_dma = grpci2_build_device_irq(6); + + /* Enable IRQs on LEON IRQ controller */ + err = request_irq(priv->irq, grpci2_jump_interrupt, 0, + "GRPCI2_JUMP", priv); + if (err) + printk(KERN_ERR "GRPCI2: ERR IRQ request failed\n"); + } else { + /* All PCI interrupts have an unique IRQ interrupt */ + for (i = 0; i < 4; i++) { + /* Make LEON IRQ layer handle level IRQ by acking */ + leon_update_virq_handling(ofdev->archdata.irqs[i], + handle_fasteoi_irq, "pcilvl", + 1); + priv->irq_map[i] = ofdev->archdata.irqs[i]; + } + priv->virq_err = priv->irq_map[0]; + if (priv->irq_mode & 1) + priv->virq_dma = ofdev->archdata.irqs[4]; + else + priv->virq_dma = priv->irq_map[0]; + + /* Unmask all PCI interrupts, request_irq will not do that */ + REGSTORE(regs->ctrl, REGLOAD(regs->ctrl)|(priv->irq_mask&0xf)); + } + + /* Setup IRQ handler for non-configuration space access errors */ + err = request_irq(priv->virq_err, grpci2_err_interrupt, IRQF_SHARED, + "GRPCI2_ERR", priv); + if (err) { + printk(KERN_DEBUG "GRPCI2: ERR VIRQ request failed: %d\n", err); + goto err5; + } + + /* + * Enable Error Interrupts. PCI interrupts are unmasked once request_irq + * is called by the PCI Device drivers + */ + REGSTORE(regs->ctrl, REGLOAD(regs->ctrl) | CTRL_EI | CTRL_SI); + + /* Init common layer and scan buses */ + priv->info.ops = &grpci2_ops; + priv->info.map_irq = grpci2_map_irq; + leon_pci_init(ofdev, &priv->info); + + return 0; + +err5: + release_resource(&priv->info.io_space); +err4: + release_resource(&priv->info.mem_space); +err3: + err = -ENOMEM; + iounmap((void __iomem *)priv->pci_io_va); +err2: + kfree(priv); +err1: + of_iounmap(&ofdev->resource[0], regs, + resource_size(&ofdev->resource[0])); + return err; +} + +static struct of_device_id grpci2_of_match[] = { + { + .name = "GAISLER_GRPCI2", + }, + { + .name = "01_07c", + }, + {}, +}; + +static struct platform_driver grpci2_of_driver = { + .driver = { + .name = "grpci2", + .owner = THIS_MODULE, + .of_match_table = grpci2_of_match, + }, + .probe = grpci2_of_probe, +}; + +static int __init grpci2_init(void) +{ + return platform_driver_register(&grpci2_of_driver); +} + +subsys_initcall(grpci2_init); diff --git a/arch/sparc/kernel/leon_pmc.c b/arch/sparc/kernel/leon_pmc.c new file mode 100644 index 00000000000..ddcf950282e --- /dev/null +++ b/arch/sparc/kernel/leon_pmc.c @@ -0,0 +1,93 @@ +/* leon_pmc.c: LEON Power-down cpu_idle() handler + * + * Copyright (C) 2011 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + */ + +#include <linux/init.h> +#include <linux/pm.h> + +#include <asm/leon_amba.h> +#include <asm/cpu_type.h> +#include <asm/leon.h> +#include <asm/processor.h> + +/* List of Systems that need fixup instructions around power-down instruction */ +static unsigned int pmc_leon_fixup_ids[] = { + AEROFLEX_UT699, + GAISLER_GR712RC, + LEON4_NEXTREME1, + 0 +}; + +static int pmc_leon_need_fixup(void) +{ + unsigned int systemid = amba_system_id >> 16; + unsigned int *id; + + id = &pmc_leon_fixup_ids[0]; + while (*id != 0) { + if (*id == systemid) + return 1; + id++; + } + + return 0; +} + +/* + * CPU idle callback function for systems that need some extra handling + * See .../arch/sparc/kernel/process.c + */ +static void pmc_leon_idle_fixup(void) +{ + /* Prepare an address to a non-cachable region. APB is always + * none-cachable. One instruction is executed after the Sleep + * instruction, we make sure to read the bus and throw away the + * value by accessing a non-cachable area, also we make sure the + * MMU does not get a TLB miss here by using the MMU BYPASS ASI. + */ + register unsigned int address = (unsigned int)leon3_irqctrl_regs; + + /* Interrupts need to be enabled to not hang the CPU */ + local_irq_enable(); + + __asm__ __volatile__ ( + "wr %%g0, %%asr19\n" + "lda [%0] %1, %%g0\n" + : + : "r"(address), "i"(ASI_LEON_BYPASS)); +} + +/* + * CPU idle callback function + * See .../arch/sparc/kernel/process.c + */ +static void pmc_leon_idle(void) +{ + /* Interrupts need to be enabled to not hang the CPU */ + local_irq_enable(); + + /* For systems without power-down, this will be no-op */ + __asm__ __volatile__ ("wr %g0, %asr19\n\t"); +} + +/* Install LEON Power Down function */ +static int __init leon_pmc_install(void) +{ + if (sparc_cpu_model == sparc_leon) { + /* Assign power management IDLE handler */ + if (pmc_leon_need_fixup()) + sparc_idle = pmc_leon_idle_fixup; + else + sparc_idle = pmc_leon_idle; + + printk(KERN_INFO "leon: power management initialized\n"); + } + + return 0; +} + +/* This driver is not critical to the boot process, don't care + * if initialized late. + */ +late_initcall(leon_pmc_install); diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c index 16582d85368..018ef11f57d 100644 --- a/arch/sparc/kernel/leon_smp.c +++ b/arch/sparc/kernel/leon_smp.c @@ -14,6 +14,7 @@ #include <linux/smp.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> +#include <linux/of.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/mm.h> @@ -22,13 +23,16 @@ #include <linux/pm.h> #include <linux/delay.h> #include <linux/gfp.h> +#include <linux/cpu.h> +#include <linux/clockchips.h> #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <asm/ptrace.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/irq_regs.h> +#include <asm/traps.h> #include <asm/delay.h> #include <asm/irq.h> @@ -40,17 +44,21 @@ #include <asm/asi.h> #include <asm/leon.h> #include <asm/leon_amba.h> +#include <asm/timer.h> -#ifdef CONFIG_SPARC_LEON +#include "kernel.h" #include "irq.h" extern ctxd_t *srmmu_ctx_table_phys; static int smp_processors_ready; extern volatile unsigned long cpu_callin_map[NR_CPUS]; -extern unsigned char boot_cpu_id; extern cpumask_t smp_commenced_mask; -void __init leon_configure_cache_smp(void); +void leon_configure_cache_smp(void); +static void leon_ipi_init(void); + +/* IRQ number of LEON IPIs */ +int leon_ipi_irq = LEON3_IRQ_IPI_DEFAULT; static inline unsigned long do_swap(volatile unsigned long *ptr, unsigned long val) @@ -61,38 +69,24 @@ static inline unsigned long do_swap(volatile unsigned long *ptr, return val; } -static void smp_setup_percpu_timer(void); - -void __cpuinit leon_callin(void) +void leon_cpu_pre_starting(void *arg) { - int cpuid = hard_smpleon_processor_id(); - - local_flush_cache_all(); - local_flush_tlb_all(); leon_configure_cache_smp(); +} - /* Get our local ticker going. */ - smp_setup_percpu_timer(); - - calibrate_delay(); - smp_store_cpu_info(cpuid); - - local_flush_cache_all(); - local_flush_tlb_all(); +void leon_cpu_pre_online(void *arg) +{ + int cpuid = hard_smp_processor_id(); - /* - * Unblock the master CPU _only_ when the scheduler state - * of all secondary CPUs will be up-to-date, so after - * the SMP initialization the master will be just allowed - * to call the scheduler code. - * Allow master to continue. + /* Allow master to continue. The master will then give us the + * go-ahead by setting the smp_commenced_mask and will wait without + * timeouts until our setup is completed fully (signified by + * our bit being set in the cpu_online_mask). */ do_swap(&cpu_callin_map[cpuid], 1); - local_flush_cache_all(); - local_flush_tlb_all(); - - cpu_probe(); + local_ops->cache_all(); + local_ops->tlb_all(); /* Fix idle thread fields. */ __asm__ __volatile__("ld [%0], %%g6\n\t" : : "r"(¤t_set[cpuid]) @@ -102,11 +96,8 @@ void __cpuinit leon_callin(void) atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; - while (!cpu_isset(cpuid, smp_commenced_mask)) + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) mb(); - - local_irq_enable(); - cpu_set(cpuid, cpu_online_map); } /* @@ -115,7 +106,7 @@ void __cpuinit leon_callin(void) extern struct linux_prom_registers smp_penguin_ctable; -void __init leon_configure_cache_smp(void) +void leon_configure_cache_smp(void) { unsigned long cfg = sparc_leon3_get_dcachecfg(); int me = smp_processor_id(); @@ -135,11 +126,11 @@ void __init leon_configure_cache_smp(void) } } - local_flush_cache_all(); - local_flush_tlb_all(); + local_ops->cache_all(); + local_ops->tlb_all(); } -void leon_smp_setbroadcast(unsigned int mask) +static void leon_smp_setbroadcast(unsigned int mask) { int broadcast = ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> @@ -157,13 +148,6 @@ void leon_smp_setbroadcast(unsigned int mask) LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpbroadcast), mask); } -unsigned int leon_smp_getbroadcast(void) -{ - unsigned int mask; - mask = LEON_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpbroadcast)); - return mask; -} - int leon_smp_nrcpus(void) { int nrcpu = @@ -177,32 +161,29 @@ void __init leon_boot_cpus(void) int nrcpu = leon_smp_nrcpus(); int me = smp_processor_id(); + /* Setup IPI */ + leon_ipi_init(); + printk(KERN_INFO "%d:(%d:%d) cpus mpirq at 0x%x\n", (unsigned int)me, (unsigned int)nrcpu, (unsigned int)NR_CPUS, (unsigned int)&(leon3_irqctrl_regs->mpstatus)); leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, me); leon_enable_irq_cpu(LEON3_IRQ_TICKER, me); - leon_enable_irq_cpu(LEON3_IRQ_RESCHEDULE, me); + leon_enable_irq_cpu(leon_ipi_irq, me); leon_smp_setbroadcast(1 << LEON3_IRQ_TICKER); leon_configure_cache_smp(); - smp_setup_percpu_timer(); - local_flush_cache_all(); + local_ops->cache_all(); } -int __cpuinit leon_boot_one_cpu(int i) +int leon_boot_one_cpu(int i, struct task_struct *idle) { - - struct task_struct *p; int timeout; - /* Cook up an idler for this guy. */ - p = fork_idle(i); - - current_set[i] = task_thread_info(p); + current_set[i] = task_thread_info(idle); /* See trampoline.S:leon_smp_cpu_startup for details... * Initialize the contexts table @@ -216,8 +197,12 @@ int __cpuinit leon_boot_one_cpu(int i) /* whirrr, whirrr, whirrrrrrrrr... */ printk(KERN_INFO "Starting CPU %d : (irqmp: 0x%x)\n", (unsigned int)i, (unsigned int)&leon3_irqctrl_regs->mpstatus); - local_flush_cache_all(); + local_ops->cache_all(); + /* Make sure all IRQs are of from the start for this new CPU */ + LEON_BYPASS_STORE_PA(&leon3_irqctrl_regs->mask[i], 0); + + /* Wake one CPU */ LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpstatus), 1 << i); /* wheee... it's going... */ @@ -234,10 +219,10 @@ int __cpuinit leon_boot_one_cpu(int i) } else { leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, i); leon_enable_irq_cpu(LEON3_IRQ_TICKER, i); - leon_enable_irq_cpu(LEON3_IRQ_RESCHEDULE, i); + leon_enable_irq_cpu(leon_ipi_irq, i); } - local_flush_cache_all(); + local_ops->cache_all(); return 0; } @@ -257,37 +242,121 @@ void __init leon_smp_done(void) } } *prev = first; - local_flush_cache_all(); + local_ops->cache_all(); /* Free unneeded trap tables */ - if (!cpu_isset(1, cpu_present_map)) { - ClearPageReserved(virt_to_page(trapbase_cpu1)); - init_page_count(virt_to_page(trapbase_cpu1)); - free_page((unsigned long)trapbase_cpu1); - totalram_pages++; - num_physpages++; + if (!cpu_present(1)) { + free_reserved_page(virt_to_page(&trapbase_cpu1)); } - if (!cpu_isset(2, cpu_present_map)) { - ClearPageReserved(virt_to_page(trapbase_cpu2)); - init_page_count(virt_to_page(trapbase_cpu2)); - free_page((unsigned long)trapbase_cpu2); - totalram_pages++; - num_physpages++; + if (!cpu_present(2)) { + free_reserved_page(virt_to_page(&trapbase_cpu2)); } - if (!cpu_isset(3, cpu_present_map)) { - ClearPageReserved(virt_to_page(trapbase_cpu3)); - init_page_count(virt_to_page(trapbase_cpu3)); - free_page((unsigned long)trapbase_cpu3); - totalram_pages++; - num_physpages++; + if (!cpu_present(3)) { + free_reserved_page(virt_to_page(&trapbase_cpu3)); } /* Ok, they are spinning and ready to go. */ smp_processors_ready = 1; } -void leon_irq_rotate(int cpu) +struct leon_ipi_work { + int single; + int msk; + int resched; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct leon_ipi_work, leon_ipi_work); + +/* Initialize IPIs on the LEON, in order to save IRQ resources only one IRQ + * is used for all three types of IPIs. + */ +static void __init leon_ipi_init(void) +{ + int cpu, len; + struct leon_ipi_work *work; + struct property *pp; + struct device_node *rootnp; + struct tt_entry *trap_table; + unsigned long flags; + + /* Find IPI IRQ or stick with default value */ + rootnp = of_find_node_by_path("/ambapp0"); + if (rootnp) { + pp = of_find_property(rootnp, "ipi_num", &len); + if (pp && (*(int *)pp->value)) + leon_ipi_irq = *(int *)pp->value; + } + printk(KERN_INFO "leon: SMP IPIs at IRQ %d\n", leon_ipi_irq); + + /* Adjust so that we jump directly to smpleon_ipi */ + local_irq_save(flags); + trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (leon_ipi_irq - 1)]; + trap_table->inst_three += smpleon_ipi - real_irq_entry; + local_ops->cache_all(); + local_irq_restore(flags); + + for_each_possible_cpu(cpu) { + work = &per_cpu(leon_ipi_work, cpu); + work->single = work->msk = work->resched = 0; + } +} + +static void leon_send_ipi(int cpu, int level) +{ + unsigned long mask; + mask = leon_get_irqmask(level); + LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->force[cpu], mask); +} + +static void leon_ipi_single(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->single = 1; + + /* Generate IRQ on the CPU */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +static void leon_ipi_mask_one(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->msk = 1; + + /* Generate IRQ on the CPU */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +static void leon_ipi_resched(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->resched = 1; + + /* Generate IRQ on the CPU (any IRQ will cause resched) */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +void leonsmp_ipi_interrupt(void) { + struct leon_ipi_work *work = &__get_cpu_var(leon_ipi_work); + + if (work->single) { + work->single = 0; + smp_call_function_single_interrupt(); + } + if (work->msk) { + work->msk = 0; + smp_call_function_interrupt(); + } + if (work->resched) { + work->resched = 0; + smp_resched_interrupt(); + } } static struct smp_funcall { @@ -335,13 +404,13 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, { register int i; - cpu_clear(smp_processor_id(), mask); - cpus_and(mask, cpu_online_map, mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); for (i = 0; i <= high; i++) { - if (cpu_isset(i, mask)) { + if (cpumask_test_cpu(i, &mask)) { ccall_info.processors_in[i] = 0; ccall_info.processors_out[i] = 0; - set_cpu_int(i, LEON3_IRQ_CROSS_CALL); + leon_send_ipi(i, LEON3_IRQ_CROSS_CALL); } } @@ -352,7 +421,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; while (!ccall_info.processors_in[i]) @@ -361,7 +430,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; while (!ccall_info.processors_out[i]) @@ -384,85 +453,17 @@ void leon_cross_call_irq(void) ccall_info.processors_out[i] = 1; } -void leon_percpu_timer_interrupt(struct pt_regs *regs) -{ - struct pt_regs *old_regs; - int cpu = smp_processor_id(); - - old_regs = set_irq_regs(regs); - - leon_clear_profile_irq(cpu); - - profile_tick(CPU_PROFILING); - - if (!--prof_counter(cpu)) { - int user = user_mode(regs); - - irq_enter(); - update_process_times(user); - irq_exit(); - - prof_counter(cpu) = prof_multiplier(cpu); - } - set_irq_regs(old_regs); -} - -static void __init smp_setup_percpu_timer(void) -{ - int cpu = smp_processor_id(); - - prof_counter(cpu) = prof_multiplier(cpu) = 1; -} - -void __init leon_blackbox_id(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - int rs1 = rd >> 11; - - /* patch places where ___b_hard_smp_processor_id appears */ - addr[0] = 0x81444000 | rd; /* rd %asr17, reg */ - addr[1] = 0x8130201c | rd | rs1; /* srl reg, 0x1c, reg */ - addr[2] = 0x01000000; /* nop */ -} - -void __init leon_blackbox_current(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - int rs1 = rd >> 11; - - /* patch LOAD_CURRENT macro where ___b_load_current appears */ - addr[0] = 0x81444000 | rd; /* rd %asr17, reg */ - addr[2] = 0x8130201c | rd | rs1; /* srl reg, 0x1c, reg */ - addr[4] = 0x81282002 | rd | rs1; /* sll reg, 0x2, reg */ - -} - -/* - * CPU idle callback function - * See .../arch/sparc/kernel/process.c - */ -void pmc_leon_idle(void) -{ - __asm__ volatile ("mov %g0, %asr19"); -} +static const struct sparc32_ipi_ops leon_ipi_ops = { + .cross_call = leon_cross_call, + .resched = leon_ipi_resched, + .single = leon_ipi_single, + .mask_one = leon_ipi_mask_one, +}; void __init leon_init_smp(void) { /* Patch ipi15 trap table */ t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_leon - linux_trap_ipi15_sun4m); - BTFIXUPSET_BLACKBOX(hard_smp_processor_id, leon_blackbox_id); - BTFIXUPSET_BLACKBOX(load_current, leon_blackbox_current); - BTFIXUPSET_CALL(smp_cross_call, leon_cross_call, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__hard_smp_processor_id, __leon_processor_id, - BTFIXUPCALL_NORM); - -#ifndef PMC_NO_IDLE - /* Assign power management IDLE handler */ - pm_idle = pmc_leon_idle; - printk(KERN_INFO "leon: power management initialized\n"); -#endif - + sparc32_ipi_ops = &leon_ipi_ops; } - -#endif /* CONFIG_SPARC_LEON */ diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c index 6addb914fcc..a1a4400d402 100644 --- a/arch/sparc/kernel/mdesc.c +++ b/arch/sparc/kernel/mdesc.c @@ -11,11 +11,13 @@ #include <linux/mm.h> #include <linux/miscdevice.h> #include <linux/bootmem.h> +#include <linux/export.h> #include <asm/cpudata.h> #include <asm/hypervisor.h> #include <asm/mdesc.h> #include <asm/prom.h> +#include <asm/uaccess.h> #include <asm/oplib.h> #include <asm/smp.h> @@ -107,7 +109,7 @@ static struct mdesc_handle * __init mdesc_memblock_alloc(unsigned int mdesc_size return hp; } -static void mdesc_memblock_free(struct mdesc_handle *hp) +static void __init mdesc_memblock_free(struct mdesc_handle *hp) { unsigned int alloc_size; unsigned long start; @@ -508,6 +510,8 @@ const char *mdesc_node_name(struct mdesc_handle *hp, u64 node) } EXPORT_SYMBOL(mdesc_node_name); +static u64 max_cpus = 64; + static void __init report_platform_properties(void) { struct mdesc_handle *hp = mdesc_grab(); @@ -543,8 +547,10 @@ static void __init report_platform_properties(void) if (v) printk("PLATFORM: watchdog-max-timeout [%llu ms]\n", *v); v = mdesc_get_property(hp, pn, "max-cpus", NULL); - if (v) - printk("PLATFORM: max-cpus [%llu]\n", *v); + if (v) { + max_cpus = *v; + printk("PLATFORM: max-cpus [%llu]\n", max_cpus); + } #ifdef CONFIG_SMP { @@ -565,9 +571,7 @@ static void __init report_platform_properties(void) mdesc_release(hp); } -static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, - struct mdesc_handle *hp, - u64 mp) +static void fill_in_one_cache(cpuinfo_sparc *c, struct mdesc_handle *hp, u64 mp) { const u64 *level = mdesc_get_property(hp, mp, "level", NULL); const u64 *size = mdesc_get_property(hp, mp, "size", NULL); @@ -610,7 +614,7 @@ static void __cpuinit fill_in_one_cache(cpuinfo_sparc *c, } } -static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) +static void mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id) { u64 a; @@ -643,7 +647,7 @@ static void __cpuinit mark_core_ids(struct mdesc_handle *hp, u64 mp, int core_id } } -static void __cpuinit set_core_ids(struct mdesc_handle *hp) +static void set_core_ids(struct mdesc_handle *hp) { int idx; u64 mp; @@ -668,7 +672,7 @@ static void __cpuinit set_core_ids(struct mdesc_handle *hp) } } -static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) +static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) { u64 a; @@ -687,7 +691,7 @@ static void __cpuinit mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id } } -static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) +static void __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) { int idx; u64 mp; @@ -708,14 +712,14 @@ static void __cpuinit __set_proc_ids(struct mdesc_handle *hp, const char *exec_u } } -static void __cpuinit set_proc_ids(struct mdesc_handle *hp) +static void set_proc_ids(struct mdesc_handle *hp) { __set_proc_ids(hp, "exec_unit"); __set_proc_ids(hp, "exec-unit"); } -static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, - unsigned char def) +static void get_one_mondo_bits(const u64 *p, unsigned int *mask, + unsigned long def, unsigned long max) { u64 val; @@ -726,6 +730,9 @@ static void __cpuinit get_one_mondo_bits(const u64 *p, unsigned int *mask, if (!val || val >= 64) goto use_default; + if (val > max) + val = max; + *mask = ((1U << val) * 64U) - 1U; return; @@ -733,25 +740,34 @@ use_default: *mask = ((1U << def) * 64U) - 1U; } -static void __cpuinit get_mondo_data(struct mdesc_handle *hp, u64 mp, - struct trap_per_cpu *tb) +static void get_mondo_data(struct mdesc_handle *hp, u64 mp, + struct trap_per_cpu *tb) { + static int printed; const u64 *val; val = mdesc_get_property(hp, mp, "q-cpu-mondo-#bits", NULL); - get_one_mondo_bits(val, &tb->cpu_mondo_qmask, 7); + get_one_mondo_bits(val, &tb->cpu_mondo_qmask, 7, ilog2(max_cpus * 2)); val = mdesc_get_property(hp, mp, "q-dev-mondo-#bits", NULL); - get_one_mondo_bits(val, &tb->dev_mondo_qmask, 7); + get_one_mondo_bits(val, &tb->dev_mondo_qmask, 7, 8); val = mdesc_get_property(hp, mp, "q-resumable-#bits", NULL); - get_one_mondo_bits(val, &tb->resum_qmask, 6); + get_one_mondo_bits(val, &tb->resum_qmask, 6, 7); val = mdesc_get_property(hp, mp, "q-nonresumable-#bits", NULL); - get_one_mondo_bits(val, &tb->nonresum_qmask, 2); + get_one_mondo_bits(val, &tb->nonresum_qmask, 2, 2); + if (!printed++) { + pr_info("SUN4V: Mondo queue sizes " + "[cpu(%u) dev(%u) r(%u) nr(%u)]\n", + tb->cpu_mondo_qmask + 1, + tb->dev_mondo_qmask + 1, + tb->resum_qmask + 1, + tb->nonresum_qmask + 1); + } } -static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) +static void *mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) { struct mdesc_handle *hp = mdesc_grab(); void *ret = NULL; @@ -768,7 +784,7 @@ static void * __cpuinit mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handl cpuid, NR_CPUS); continue; } - if (!cpu_isset(cpuid, *mask)) + if (!cpumask_test_cpu(cpuid, mask)) continue; #endif @@ -781,7 +797,8 @@ out: return ret; } -static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void *record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) { ncpus_probed++; #ifdef CONFIG_SMP @@ -790,7 +807,7 @@ static void * __cpuinit record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpui return NULL; } -void __cpuinit mdesc_populate_present_mask(cpumask_t *mask) +void mdesc_populate_present_mask(cpumask_t *mask) { if (tlb_type != hypervisor) return; @@ -799,7 +816,32 @@ void __cpuinit mdesc_populate_present_mask(cpumask_t *mask) mdesc_iterate_over_cpus(record_one_cpu, NULL, mask); } -static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +static void * __init check_one_pgsz(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +{ + const u64 *pgsz_prop = mdesc_get_property(hp, mp, "mmu-page-size-list", NULL); + unsigned long *pgsz_mask = arg; + u64 val; + + val = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K | + HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB); + if (pgsz_prop) + val = *pgsz_prop; + + if (!*pgsz_mask) + *pgsz_mask = val; + else + *pgsz_mask &= val; + return NULL; +} + +void __init mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask) +{ + *pgsz_mask = 0; + mdesc_iterate_over_cpus(check_one_pgsz, pgsz_mask, mask); +} + +static void *fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) { const u64 *cfreq = mdesc_get_property(hp, mp, "clock-frequency", NULL); struct trap_per_cpu *tb; @@ -848,16 +890,12 @@ static void * __cpuinit fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpu return NULL; } -void __cpuinit mdesc_fill_in_cpu_data(cpumask_t *mask) +void mdesc_fill_in_cpu_data(cpumask_t *mask) { struct mdesc_handle *hp; mdesc_iterate_over_cpus(fill_in_one_cpu, NULL, mask); -#ifdef CONFIG_SMP - sparc64_multi_core = 1; -#endif - hp = mdesc_grab(); set_core_ids(hp); diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c index 8d348c474a2..97655e0fd24 100644 --- a/arch/sparc/kernel/module.c +++ b/arch/sparc/kernel/module.c @@ -16,6 +16,9 @@ #include <asm/processor.h> #include <asm/spitfire.h> +#include <asm/cacheflush.h> + +#include "entry.h" #ifdef CONFIG_SPARC64 @@ -26,54 +29,27 @@ static void *module_map(unsigned long size) if (PAGE_ALIGN(size) > MODULES_LEN) return NULL; return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, - GFP_KERNEL, PAGE_KERNEL, -1, + GFP_KERNEL, PAGE_KERNEL, NUMA_NO_NODE, __builtin_return_address(0)); } - -static char *dot2underscore(char *name) -{ - return name; -} #else static void *module_map(unsigned long size) { return vmalloc(size); } - -/* Replace references to .func with _Func */ -static char *dot2underscore(char *name) -{ - if (name[0] == '.') { - name[0] = '_'; - name[1] = toupper(name[1]); - } - return name; -} #endif /* CONFIG_SPARC64 */ void *module_alloc(unsigned long size) { void *ret; - /* We handle the zero case fine, unlike vmalloc */ - if (size == 0) - return NULL; - ret = module_map(size); - if (!ret) - ret = ERR_PTR(-ENOMEM); - else + if (ret) memset(ret, 0, size); return ret; } -/* Free memory returned from module_core_alloc/module_init_alloc */ -void module_free(struct module *mod, void *module_region) -{ - vfree(module_region); -} - /* Make generic code ignore STT_REGISTER dummy undefined symbols. */ int module_frob_arch_sections(Elf_Ehdr *hdr, Elf_Shdr *sechdrs, @@ -96,28 +72,13 @@ int module_frob_arch_sections(Elf_Ehdr *hdr, for (i = 1; i < sechdrs[symidx].sh_size / sizeof(Elf_Sym); i++) { if (sym[i].st_shndx == SHN_UNDEF) { - if (ELF_ST_TYPE(sym[i].st_info) == STT_REGISTER) { + if (ELF_ST_TYPE(sym[i].st_info) == STT_REGISTER) sym[i].st_shndx = SHN_ABS; - } else { - char *name = strtab + sym[i].st_name; - dot2underscore(name); - } } } return 0; } -int apply_relocate(Elf_Shdr *sechdrs, - const char *strtab, - unsigned int symindex, - unsigned int relsec, - struct module *me) -{ - printk(KERN_ERR "module %s: non-ADD RELOCATION unsupported\n", - me->name); - return -ENOEXEC; -} - int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, unsigned int symindex, @@ -149,6 +110,10 @@ int apply_relocate_add(Elf_Shdr *sechdrs, v = sym->st_value + rel[i].r_addend; switch (ELF_R_TYPE(rel[i].r_info) & 0xff) { + case R_SPARC_DISP32: + v -= (Elf_Addr) location; + *loc32 = v; + break; #ifdef CONFIG_SPARC64 case R_SPARC_64: location[0] = v >> 56; @@ -161,11 +126,6 @@ int apply_relocate_add(Elf_Shdr *sechdrs, location[7] = v >> 0; break; - case R_SPARC_DISP32: - v -= (Elf_Addr) location; - *loc32 = v; - break; - case R_SPARC_WDISP19: v -= (Elf_Addr) location; *loc32 = (*loc32 & ~0x7ffff) | @@ -214,12 +174,35 @@ int apply_relocate_add(Elf_Shdr *sechdrs, me->name, (int) (ELF_R_TYPE(rel[i].r_info) & 0xff)); return -ENOEXEC; - }; + } } return 0; } #ifdef CONFIG_SPARC64 +static void do_patch_sections(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs) +{ + const Elf_Shdr *s, *sun4v_1insn = NULL, *sun4v_2insn = NULL; + char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { + if (!strcmp(".sun4v_1insn_patch", secstrings + s->sh_name)) + sun4v_1insn = s; + if (!strcmp(".sun4v_2insn_patch", secstrings + s->sh_name)) + sun4v_2insn = s; + } + + if (sun4v_1insn && tlb_type == hypervisor) { + void *p = (void *) sun4v_1insn->sh_addr; + sun4v_patch_1insn_range(p, p + sun4v_1insn->sh_size); + } + if (sun4v_2insn && tlb_type == hypervisor) { + void *p = (void *) sun4v_2insn->sh_addr; + sun4v_patch_2insn_range(p, p + sun4v_2insn->sh_size); + } +} + int module_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, struct module *me) @@ -227,6 +210,8 @@ int module_finalize(const Elf_Ehdr *hdr, /* make jump label nops */ jump_label_apply_nops(me); + do_patch_sections(hdr, sechdrs); + /* Cheetah's I-cache is fully coherent. */ if (tlb_type == spitfire) { unsigned long va; @@ -239,15 +224,4 @@ int module_finalize(const Elf_Ehdr *hdr, return 0; } -#else -int module_finalize(const Elf_Ehdr *hdr, - const Elf_Shdr *sechdrs, - struct module *me) -{ - return 0; -} #endif /* CONFIG_SPARC64 */ - -void module_arch_cleanup(struct module *mod) -{ -} diff --git a/arch/sparc/kernel/muldiv.c b/arch/sparc/kernel/muldiv.c deleted file mode 100644 index 6ce1021d487..00000000000 --- a/arch/sparc/kernel/muldiv.c +++ /dev/null @@ -1,239 +0,0 @@ -/* - * muldiv.c: Hardware multiply/division illegal instruction trap - * for sun4c/sun4 (which do not have those instructions) - * - * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * - * 2004-12-25 Krzysztof Helt (krzysztof.h1@wp.pl) - * - fixed registers constrains in inline assembly declarations - */ - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/mm.h> -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/system.h> -#include <asm/uaccess.h> - -#include "kernel.h" - -/* #define DEBUG_MULDIV */ - -static inline int has_imm13(int insn) -{ - return (insn & 0x2000); -} - -static inline int is_foocc(int insn) -{ - return (insn & 0x800000); -} - -static inline int sign_extend_imm13(int imm) -{ - return imm << 19 >> 19; -} - -static inline void advance(struct pt_regs *regs) -{ - regs->pc = regs->npc; - regs->npc += 4; -} - -static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, - unsigned int rd) -{ - if(rs2 >= 16 || rs1 >= 16 || rd >= 16) { - /* Wheee... */ - __asm__ __volatile__("save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "save %sp, -0x40, %sp\n\t" - "restore; restore; restore; restore;\n\t" - "restore; restore; restore;\n\t"); - } -} - -#define fetch_reg(reg, regs) ({ \ - struct reg_window32 __user *win; \ - register unsigned long ret; \ - \ - if (!(reg)) ret = 0; \ - else if ((reg) < 16) { \ - ret = regs->u_regs[(reg)]; \ - } else { \ - /* Ho hum, the slightly complicated case. */ \ - win = (struct reg_window32 __user *)regs->u_regs[UREG_FP];\ - if (get_user (ret, &win->locals[(reg) - 16])) return -1;\ - } \ - ret; \ -}) - -static inline int -store_reg(unsigned int result, unsigned int reg, struct pt_regs *regs) -{ - struct reg_window32 __user *win; - - if (!reg) - return 0; - if (reg < 16) { - regs->u_regs[reg] = result; - return 0; - } else { - /* need to use put_user() in this case: */ - win = (struct reg_window32 __user *) regs->u_regs[UREG_FP]; - return (put_user(result, &win->locals[reg - 16])); - } -} - -/* Should return 0 if mul/div emulation succeeded and SIGILL should - * not be issued. - */ -int do_user_muldiv(struct pt_regs *regs, unsigned long pc) -{ - unsigned int insn; - int inst; - unsigned int rs1, rs2, rdv; - - if (!pc) - return -1; /* This happens to often, I think */ - if (get_user (insn, (unsigned int __user *)pc)) - return -1; - if ((insn & 0xc1400000) != 0x80400000) - return -1; - inst = ((insn >> 19) & 0xf); - if ((inst & 0xe) != 10 && (inst & 0xe) != 14) - return -1; - - /* Now we know we have to do something with umul, smul, udiv or sdiv */ - rs1 = (insn >> 14) & 0x1f; - rs2 = insn & 0x1f; - rdv = (insn >> 25) & 0x1f; - if (has_imm13(insn)) { - maybe_flush_windows(rs1, 0, rdv); - rs2 = sign_extend_imm13(insn); - } else { - maybe_flush_windows(rs1, rs2, rdv); - rs2 = fetch_reg(rs2, regs); - } - rs1 = fetch_reg(rs1, regs); - switch (inst) { - case 10: /* umul */ -#ifdef DEBUG_MULDIV - printk ("unsigned muldiv: 0x%x * 0x%x = ", rs1, rs2); -#endif - __asm__ __volatile__ ("\n\t" - "mov %0, %%o0\n\t" - "call .umul\n\t" - " mov %1, %%o1\n\t" - "mov %%o0, %0\n\t" - "mov %%o1, %1\n\t" - : "=r" (rs1), "=r" (rs2) - : "0" (rs1), "1" (rs2) - : "o0", "o1", "o2", "o3", "o4", "o5", "o7", "cc"); -#ifdef DEBUG_MULDIV - printk ("0x%x%08x\n", rs2, rs1); -#endif - if (store_reg(rs1, rdv, regs)) - return -1; - regs->y = rs2; - break; - case 11: /* smul */ -#ifdef DEBUG_MULDIV - printk ("signed muldiv: 0x%x * 0x%x = ", rs1, rs2); -#endif - __asm__ __volatile__ ("\n\t" - "mov %0, %%o0\n\t" - "call .mul\n\t" - " mov %1, %%o1\n\t" - "mov %%o0, %0\n\t" - "mov %%o1, %1\n\t" - : "=r" (rs1), "=r" (rs2) - : "0" (rs1), "1" (rs2) - : "o0", "o1", "o2", "o3", "o4", "o5", "o7", "cc"); -#ifdef DEBUG_MULDIV - printk ("0x%x%08x\n", rs2, rs1); -#endif - if (store_reg(rs1, rdv, regs)) - return -1; - regs->y = rs2; - break; - case 14: /* udiv */ -#ifdef DEBUG_MULDIV - printk ("unsigned muldiv: 0x%x%08x / 0x%x = ", regs->y, rs1, rs2); -#endif - if (!rs2) { -#ifdef DEBUG_MULDIV - printk ("DIVISION BY ZERO\n"); -#endif - handle_hw_divzero (regs, pc, regs->npc, regs->psr); - return 0; - } - __asm__ __volatile__ ("\n\t" - "mov %2, %%o0\n\t" - "mov %0, %%o1\n\t" - "mov %%g0, %%o2\n\t" - "call __udivdi3\n\t" - " mov %1, %%o3\n\t" - "mov %%o1, %0\n\t" - "mov %%o0, %1\n\t" - : "=r" (rs1), "=r" (rs2) - : "r" (regs->y), "0" (rs1), "1" (rs2) - : "o0", "o1", "o2", "o3", "o4", "o5", "o7", - "g1", "g2", "g3", "cc"); -#ifdef DEBUG_MULDIV - printk ("0x%x\n", rs1); -#endif - if (store_reg(rs1, rdv, regs)) - return -1; - break; - case 15: /* sdiv */ -#ifdef DEBUG_MULDIV - printk ("signed muldiv: 0x%x%08x / 0x%x = ", regs->y, rs1, rs2); -#endif - if (!rs2) { -#ifdef DEBUG_MULDIV - printk ("DIVISION BY ZERO\n"); -#endif - handle_hw_divzero (regs, pc, regs->npc, regs->psr); - return 0; - } - __asm__ __volatile__ ("\n\t" - "mov %2, %%o0\n\t" - "mov %0, %%o1\n\t" - "mov %%g0, %%o2\n\t" - "call __divdi3\n\t" - " mov %1, %%o3\n\t" - "mov %%o1, %0\n\t" - "mov %%o0, %1\n\t" - : "=r" (rs1), "=r" (rs2) - : "r" (regs->y), "0" (rs1), "1" (rs2) - : "o0", "o1", "o2", "o3", "o4", "o5", "o7", - "g1", "g2", "g3", "cc"); -#ifdef DEBUG_MULDIV - printk ("0x%x\n", rs1); -#endif - if (store_reg(rs1, rdv, regs)) - return -1; - break; - } - if (is_foocc (insn)) { - regs->psr &= ~PSR_ICC; - if ((inst & 0xe) == 14) { - /* ?div */ - if (rs2) regs->psr |= PSR_V; - } - if (!rs1) regs->psr |= PSR_Z; - if (((int)rs1) < 0) regs->psr |= PSR_N; -#ifdef DEBUG_MULDIV - printk ("psr muldiv: %08x\n", regs->psr); -#endif - } - advance(regs); - return 0; -} diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c index 300f810142f..33709455691 100644 --- a/arch/sparc/kernel/nmi.c +++ b/arch/sparc/kernel/nmi.c @@ -10,7 +10,7 @@ #include <linux/init.h> #include <linux/percpu.h> #include <linux/nmi.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/kprobes.h> #include <linux/kernel_stat.h> #include <linux/reboot.h> @@ -68,27 +68,16 @@ EXPORT_SYMBOL(touch_nmi_watchdog); static void die_nmi(const char *str, struct pt_regs *regs, int do_panic) { + int this_cpu = smp_processor_id(); + if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, pt_regs_trap_type(regs), SIGINT) == NOTIFY_STOP) return; - console_verbose(); - bust_spinlocks(1); - - printk(KERN_EMERG "%s", str); - printk(" on CPU%d, ip %08lx, registers:\n", - smp_processor_id(), regs->tpc); - show_regs(regs); - dump_stack(); - - bust_spinlocks(0); - if (do_panic || panic_on_oops) - panic("Non maskable interrupt"); - - nmi_exit(); - local_irq_enable(); - do_exit(SIGBUS); + panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu); + else + WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); } notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) @@ -108,7 +97,7 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) pt_regs_trap_type(regs), SIGINT) == NOTIFY_STOP) touched = 1; else - pcr_ops->write(PCR_PIC_PRIV); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); sum = local_cpu_data().irq0_irqs; if (__get_cpu_var(nmi_touch)) { @@ -125,8 +114,8 @@ notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) __this_cpu_write(alert_counter, 0); } if (__get_cpu_var(wd_enabled)) { - write_pic(picl_value(nmi_hz)); - pcr_ops->write(pcr_enable); + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); } restore_hardirq_stack(orig_sp); @@ -165,7 +154,7 @@ static void report_broken_nmi(int cpu, int *prev_nmi_count) void stop_nmi_watchdog(void *unused) { - pcr_ops->write(PCR_PIC_PRIV); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); __get_cpu_var(wd_enabled) = 0; atomic_dec(&nmi_active); } @@ -222,10 +211,10 @@ void start_nmi_watchdog(void *unused) __get_cpu_var(wd_enabled) = 1; atomic_inc(&nmi_active); - pcr_ops->write(PCR_PIC_PRIV); - write_pic(picl_value(nmi_hz)); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); - pcr_ops->write(pcr_enable); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); } static void nmi_adjust_hz_one(void *unused) @@ -233,10 +222,10 @@ static void nmi_adjust_hz_one(void *unused) if (!__get_cpu_var(wd_enabled)) return; - pcr_ops->write(PCR_PIC_PRIV); - write_pic(picl_value(nmi_hz)); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); - pcr_ops->write(pcr_enable); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); } void nmi_adjust_hz(unsigned int new_hz) diff --git a/arch/sparc/kernel/of_device_32.c b/arch/sparc/kernel/of_device_32.c index 2d055a1e9cc..185aa96fa5b 100644 --- a/arch/sparc/kernel/of_device_32.c +++ b/arch/sparc/kernel/of_device_32.c @@ -2,7 +2,6 @@ #include <linux/kernel.h> #include <linux/of.h> #include <linux/init.h> -#include <linux/module.h> #include <linux/mod_devicetable.h> #include <linux/slab.h> #include <linux/errno.h> @@ -13,6 +12,7 @@ #include <asm/leon_amba.h> #include "of_device_common.h" +#include "irq.h" /* * PCI bus specific translator @@ -355,7 +355,8 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, if (intr) { op->archdata.num_irqs = len / sizeof(struct linux_prom_irqs); for (i = 0; i < op->archdata.num_irqs; i++) - op->archdata.irqs[i] = intr[i].pri; + op->archdata.irqs[i] = + sparc_config.build_device_irq(op, intr[i].pri); } else { const unsigned int *irq = of_get_property(dp, "interrupts", &len); @@ -363,64 +364,13 @@ static struct platform_device * __init scan_one_device(struct device_node *dp, if (irq) { op->archdata.num_irqs = len / sizeof(unsigned int); for (i = 0; i < op->archdata.num_irqs; i++) - op->archdata.irqs[i] = irq[i]; + op->archdata.irqs[i] = + sparc_config.build_device_irq(op, irq[i]); } else { op->archdata.num_irqs = 0; } } - if (sparc_cpu_model == sun4d) { - static int pil_to_sbus[] = { - 0, 0, 1, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 0, - }; - struct device_node *io_unit, *sbi = dp->parent; - const struct linux_prom_registers *regs; - int board, slot; - - while (sbi) { - if (!strcmp(sbi->name, "sbi")) - break; - - sbi = sbi->parent; - } - if (!sbi) - goto build_resources; - - regs = of_get_property(dp, "reg", NULL); - if (!regs) - goto build_resources; - - slot = regs->which_io; - - /* If SBI's parent is not io-unit or the io-unit lacks - * a "board#" property, something is very wrong. - */ - if (!sbi->parent || strcmp(sbi->parent->name, "io-unit")) { - printk("%s: Error, parent is not io-unit.\n", - sbi->full_name); - goto build_resources; - } - io_unit = sbi->parent; - board = of_getintprop_default(io_unit, "board#", -1); - if (board == -1) { - printk("%s: Error, lacks board# property.\n", - io_unit->full_name); - goto build_resources; - } - - for (i = 0; i < op->archdata.num_irqs; i++) { - int this_irq = op->archdata.irqs[i]; - int sbusl = pil_to_sbus[this_irq]; - - if (sbusl) - this_irq = (((board + 1) << 5) + - (sbusl << 2) + - slot); - - op->archdata.irqs[i] = this_irq; - } - } -build_resources: build_device_resources(op, parent); op->dev.parent = parent; diff --git a/arch/sparc/kernel/of_device_64.c b/arch/sparc/kernel/of_device_64.c index 63cd4e5d47c..7bbdc26d951 100644 --- a/arch/sparc/kernel/of_device_64.c +++ b/arch/sparc/kernel/of_device_64.c @@ -2,13 +2,14 @@ #include <linux/kernel.h> #include <linux/of.h> #include <linux/init.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/mod_devicetable.h> #include <linux/slab.h> #include <linux/errno.h> #include <linux/irq.h> #include <linux/of_device.h> #include <linux/of_platform.h> +#include <asm/spitfire.h> #include "of_device_common.h" @@ -459,7 +460,7 @@ apply_interrupt_map(struct device_node *dp, struct device_node *pp, * * Handle this by deciding that, if we didn't get a * match in the parent's 'interrupt-map', and the - * parent is an IRQ translater, then use the parent as + * parent is an IRQ translator, then use the parent as * our IRQ controller. */ if (pp->irq_trans) @@ -579,7 +580,7 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, printk("%s: Apply [%s:%x] imap --> [%s:%x]\n", op->dev.of_node->full_name, pp->full_name, this_orig_irq, - (iret ? iret->full_name : "NULL"), irq); + of_node_full_name(iret), irq); if (!iret) break; @@ -622,8 +623,9 @@ static unsigned int __init build_one_device_irq(struct platform_device *op, out: nid = of_node_to_nid(dp); if (nid != -1) { - cpumask_t numa_mask = *cpumask_of_node(nid); + cpumask_t numa_mask; + cpumask_copy(&numa_mask, cpumask_of_node(nid)); irq_set_affinity(irq, &numa_mask); } diff --git a/arch/sparc/kernel/of_device_common.c b/arch/sparc/kernel/of_device_common.c index 49ddff56cb0..de0ee3971f0 100644 --- a/arch/sparc/kernel/of_device_common.c +++ b/arch/sparc/kernel/of_device_common.c @@ -1,13 +1,14 @@ #include <linux/string.h> #include <linux/kernel.h> #include <linux/of.h> -#include <linux/init.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/mod_devicetable.h> #include <linux/errno.h> #include <linux/irq.h> -#include <linux/of_device.h> #include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_irq.h> #include "of_device_common.h" @@ -22,6 +23,33 @@ unsigned int irq_of_parse_and_map(struct device_node *node, int index) } EXPORT_SYMBOL(irq_of_parse_and_map); +int of_address_to_resource(struct device_node *node, int index, + struct resource *r) +{ + struct platform_device *op = of_find_device_by_node(node); + + if (!op || index >= op->num_resources) + return -EINVAL; + + memcpy(r, &op->archdata.resource[index], sizeof(*r)); + return 0; +} +EXPORT_SYMBOL_GPL(of_address_to_resource); + +void __iomem *of_iomap(struct device_node *node, int index) +{ + struct platform_device *op = of_find_device_by_node(node); + struct resource *r; + + if (!op || index >= op->num_resources) + return NULL; + + r = &op->archdata.resource[index]; + + return of_ioremap(r, 0, resource_size(r), (char *) r->name); +} +EXPORT_SYMBOL(of_iomap); + /* Take the archdata values for IOMMU, STC, and HOSTDATA found in * BUS and propagate to all child platform_device objects. */ diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index 4137579d9ad..539babf00bb 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -8,7 +8,7 @@ * with minor modifications, see there for credits. */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/sched.h> @@ -28,6 +28,7 @@ #include <asm/apb.h> #include "pci_impl.h" +#include "kernel.h" /* List of all PCI controllers found in the system. */ struct pci_pbm_info *pci_pbm_root = NULL; @@ -230,7 +231,8 @@ static void pci_parse_of_addrs(struct platform_device *op, res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2]; } else if (i == dev->rom_base_reg) { res = &dev->resource[PCI_ROM_RESOURCE]; - flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE; + flags |= IORESOURCE_READONLY | IORESOURCE_CACHEABLE + | IORESOURCE_SIZEALIGN; } else { printk(KERN_ERR "PCI: bad cfg reg num 0x%x\n", i); continue; @@ -253,7 +255,7 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, const char *type; u32 class; - dev = alloc_pci_dev(); + dev = pci_alloc_dev(bus); if (!dev) return NULL; @@ -280,11 +282,10 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, printk(" create device, devfn: %x, type: %s\n", devfn, type); - dev->bus = bus; dev->sysdata = node; dev->dev.parent = bus->bridge; dev->dev.bus = &pci_bus_type; - dev->dev.of_node = node; + dev->dev.of_node = of_node_get(node); dev->devfn = devfn; dev->multifunction = 0; /* maybe a lie? */ set_pcie_port_type(dev); @@ -326,7 +327,7 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, if ((dev->class >> 8) == PCI_CLASS_STORAGE_IDE) pci_set_master(dev); - dev->current_state = 4; /* unknown power state */ + dev->current_state = PCI_UNKNOWN; /* unknown power state */ dev->error_state = pci_channel_io_normal; dev->dma_mask = 0xffffffff; @@ -355,7 +356,7 @@ static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, return dev; } -static void __devinit apb_calc_first_last(u8 map, u32 *first_p, u32 *last_p) +static void apb_calc_first_last(u8 map, u32 *first_p, u32 *last_p) { u32 idx, first, last; @@ -374,103 +375,14 @@ static void __devinit apb_calc_first_last(u8 map, u32 *first_p, u32 *last_p) *last_p = last; } -static void pci_resource_adjust(struct resource *res, - struct resource *root) -{ - res->start += root->start; - res->end += root->start; -} - -/* For PCI bus devices which lack a 'ranges' property we interrogate - * the config space values to set the resources, just like the generic - * Linux PCI probing code does. - */ -static void __devinit pci_cfg_fake_ranges(struct pci_dev *dev, - struct pci_bus *bus, - struct pci_pbm_info *pbm) -{ - struct resource *res; - u8 io_base_lo, io_limit_lo; - u16 mem_base_lo, mem_limit_lo; - unsigned long base, limit; - - pci_read_config_byte(dev, PCI_IO_BASE, &io_base_lo); - pci_read_config_byte(dev, PCI_IO_LIMIT, &io_limit_lo); - base = (io_base_lo & PCI_IO_RANGE_MASK) << 8; - limit = (io_limit_lo & PCI_IO_RANGE_MASK) << 8; - - if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) { - u16 io_base_hi, io_limit_hi; - - pci_read_config_word(dev, PCI_IO_BASE_UPPER16, &io_base_hi); - pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, &io_limit_hi); - base |= (io_base_hi << 16); - limit |= (io_limit_hi << 16); - } - - res = bus->resource[0]; - if (base <= limit) { - res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO; - if (!res->start) - res->start = base; - if (!res->end) - res->end = limit + 0xfff; - pci_resource_adjust(res, &pbm->io_space); - } - - pci_read_config_word(dev, PCI_MEMORY_BASE, &mem_base_lo); - pci_read_config_word(dev, PCI_MEMORY_LIMIT, &mem_limit_lo); - base = (mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16; - limit = (mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16; - - res = bus->resource[1]; - if (base <= limit) { - res->flags = ((mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | - IORESOURCE_MEM); - res->start = base; - res->end = limit + 0xfffff; - pci_resource_adjust(res, &pbm->mem_space); - } - - pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, &mem_base_lo); - pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, &mem_limit_lo); - base = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16; - limit = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16; - - if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) { - u32 mem_base_hi, mem_limit_hi; - - pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, &mem_base_hi); - pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, &mem_limit_hi); - - /* - * Some bridges set the base > limit by default, and some - * (broken) BIOSes do not initialize them. If we find - * this, just assume they are not being used. - */ - if (mem_base_hi <= mem_limit_hi) { - base |= ((long) mem_base_hi) << 32; - limit |= ((long) mem_limit_hi) << 32; - } - } - - res = bus->resource[2]; - if (base <= limit) { - res->flags = ((mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | - IORESOURCE_MEM | IORESOURCE_PREFETCH); - res->start = base; - res->end = limit + 0xfffff; - pci_resource_adjust(res, &pbm->mem_space); - } -} - /* Cook up fake bus resources for SUNW,simba PCI bridges which lack * a proper 'ranges' property. */ -static void __devinit apb_fake_ranges(struct pci_dev *dev, - struct pci_bus *bus, - struct pci_pbm_info *pbm) +static void apb_fake_ranges(struct pci_dev *dev, + struct pci_bus *bus, + struct pci_pbm_info *pbm) { + struct pci_bus_region region; struct resource *res; u32 first, last; u8 map; @@ -478,33 +390,34 @@ static void __devinit apb_fake_ranges(struct pci_dev *dev, pci_read_config_byte(dev, APB_IO_ADDRESS_MAP, &map); apb_calc_first_last(map, &first, &last); res = bus->resource[0]; - res->start = (first << 21); - res->end = (last << 21) + ((1 << 21) - 1); res->flags = IORESOURCE_IO; - pci_resource_adjust(res, &pbm->io_space); + region.start = (first << 21); + region.end = (last << 21) + ((1 << 21) - 1); + pcibios_bus_to_resource(dev->bus, res, ®ion); pci_read_config_byte(dev, APB_MEM_ADDRESS_MAP, &map); apb_calc_first_last(map, &first, &last); res = bus->resource[1]; - res->start = (first << 21); - res->end = (last << 21) + ((1 << 21) - 1); res->flags = IORESOURCE_MEM; - pci_resource_adjust(res, &pbm->mem_space); + region.start = (first << 29); + region.end = (last << 29) + ((1 << 29) - 1); + pcibios_bus_to_resource(dev->bus, res, ®ion); } -static void __devinit pci_of_scan_bus(struct pci_pbm_info *pbm, - struct device_node *node, - struct pci_bus *bus); +static void pci_of_scan_bus(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_bus *bus); #define GET_64BIT(prop, i) ((((u64) (prop)[(i)]) << 32) | (prop)[(i)+1]) -static void __devinit of_scan_pci_bridge(struct pci_pbm_info *pbm, - struct device_node *node, - struct pci_dev *dev) +static void of_scan_pci_bridge(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_dev *dev) { struct pci_bus *bus; const u32 *busrange, *ranges; int len, i, simba; + struct pci_bus_region region; struct resource *res; unsigned int flags; u64 size; @@ -535,7 +448,7 @@ static void __devinit of_scan_pci_bridge(struct pci_pbm_info *pbm, } bus->primary = dev->bus->number; - bus->subordinate = busrange[1]; + pci_bus_insert_busn_res(bus, busrange[0], busrange[1]); bus->bridge_ctl = 0; /* parse ranges property, or cook one up by hand for Simba */ @@ -550,13 +463,11 @@ static void __devinit of_scan_pci_bridge(struct pci_pbm_info *pbm, apb_fake_ranges(dev, bus, pbm); goto after_ranges; } else if (ranges == NULL) { - pci_cfg_fake_ranges(dev, bus, pbm); + pci_read_bridge_bases(bus); goto after_ranges; } i = 1; for (; len >= 32; len -= 32, ranges += 8) { - struct resource *root; - flags = pci_parse_of_flags(ranges[0]); size = GET_64BIT(ranges, 6); if (flags == 0 || size == 0) @@ -568,7 +479,6 @@ static void __devinit of_scan_pci_bridge(struct pci_pbm_info *pbm, " for bridge %s\n", node->full_name); continue; } - root = &pbm->io_space; } else { if (i >= PCI_NUM_RESOURCES - PCI_BRIDGE_RESOURCES) { printk(KERN_ERR "PCI: too many memory ranges" @@ -577,18 +487,12 @@ static void __devinit of_scan_pci_bridge(struct pci_pbm_info *pbm, } res = bus->resource[i]; ++i; - root = &pbm->mem_space; } - res->start = GET_64BIT(ranges, 1); - res->end = res->start + size - 1; res->flags = flags; - - /* Another way to implement this would be to add an of_device - * layer routine that can calculate a resource for a given - * range property value in a PCI device. - */ - pci_resource_adjust(res, root); + region.start = GET_64BIT(ranges, 1); + region.end = region.start + size - 1; + pcibios_bus_to_resource(dev->bus, res, ®ion); } after_ranges: sprintf(bus->name, "PCI Bus %04x:%02x", pci_domain_nr(bus), @@ -599,9 +503,9 @@ after_ranges: pci_of_scan_bus(pbm, node, bus); } -static void __devinit pci_of_scan_bus(struct pci_pbm_info *pbm, - struct device_node *node, - struct pci_bus *bus) +static void pci_of_scan_bus(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_bus *bus) { struct device_node *child; const u32 *reg; @@ -640,8 +544,7 @@ static void __devinit pci_of_scan_bus(struct pci_pbm_info *pbm, printk("PCI: dev header type: %x\n", dev->hdr_type); - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE || - dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) + if (pci_is_bridge(dev)) of_scan_pci_bridge(pbm, child, dev); } } @@ -660,7 +563,7 @@ show_pciobppath_attr(struct device * dev, struct device_attribute * attr, char * static DEVICE_ATTR(obppath, S_IRUSR | S_IRGRP | S_IROTH, show_pciobppath_attr, NULL); -static void __devinit pci_bus_register_of_sysfs(struct pci_bus *bus) +static void pci_bus_register_of_sysfs(struct pci_bus *bus) { struct pci_dev *dev; struct pci_bus *child_bus; @@ -675,30 +578,37 @@ static void __devinit pci_bus_register_of_sysfs(struct pci_bus *bus) * humanoid. */ err = sysfs_create_file(&dev->dev.kobj, &dev_attr_obppath.attr); + (void) err; } list_for_each_entry(child_bus, &bus->children, node) pci_bus_register_of_sysfs(child_bus); } -struct pci_bus * __devinit pci_scan_one_pbm(struct pci_pbm_info *pbm, - struct device *parent) +struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, + struct device *parent) { + LIST_HEAD(resources); struct device_node *node = pbm->op->dev.of_node; struct pci_bus *bus; printk("PCI: Scanning PBM %s\n", node->full_name); - bus = pci_create_bus(parent, pbm->pci_first_busno, pbm->pci_ops, pbm); + pci_add_resource_offset(&resources, &pbm->io_space, + pbm->io_space.start); + pci_add_resource_offset(&resources, &pbm->mem_space, + pbm->mem_space.start); + pbm->busn.start = pbm->pci_first_busno; + pbm->busn.end = pbm->pci_last_busno; + pbm->busn.flags = IORESOURCE_BUS; + pci_add_resource(&resources, &pbm->busn); + bus = pci_create_root_bus(parent, pbm->pci_first_busno, pbm->pci_ops, + pbm, &resources); if (!bus) { printk(KERN_ERR "Failed to create bus for %s\n", node->full_name); + pci_free_resource_list(&resources); return NULL; } - bus->secondary = pbm->pci_first_busno; - bus->subordinate = pbm->pci_last_busno; - - bus->resource[0] = &pbm->io_space; - bus->resource[1] = &pbm->mem_space; pci_of_scan_bus(pbm, node, bus); pci_bus_add_devices(bus); @@ -707,18 +617,7 @@ struct pci_bus * __devinit pci_scan_one_pbm(struct pci_pbm_info *pbm, return bus; } -void __devinit pcibios_fixup_bus(struct pci_bus *pbus) -{ - struct pci_pbm_info *pbm = pbus->sysdata; - - /* Generic PCI bus probing sets these to point at - * &io{port,mem}_resouce which is wrong for us. - */ - pbus->resource[0] = &pbm->io_space; - pbus->resource[1] = &pbm->mem_space; -} - -void pcibios_update_irq(struct pci_dev *pdev, int irq) +void pcibios_fixup_bus(struct pci_bus *pbus) { } @@ -758,51 +657,6 @@ int pcibios_enable_device(struct pci_dev *dev, int mask) return 0; } -void pcibios_resource_to_bus(struct pci_dev *pdev, struct pci_bus_region *region, - struct resource *res) -{ - struct pci_pbm_info *pbm = pdev->bus->sysdata; - struct resource zero_res, *root; - - zero_res.start = 0; - zero_res.end = 0; - zero_res.flags = res->flags; - - if (res->flags & IORESOURCE_IO) - root = &pbm->io_space; - else - root = &pbm->mem_space; - - pci_resource_adjust(&zero_res, root); - - region->start = res->start - zero_res.start; - region->end = res->end - zero_res.start; -} -EXPORT_SYMBOL(pcibios_resource_to_bus); - -void pcibios_bus_to_resource(struct pci_dev *pdev, struct resource *res, - struct pci_bus_region *region) -{ - struct pci_pbm_info *pbm = pdev->bus->sysdata; - struct resource *root; - - res->start = region->start; - res->end = region->end; - - if (res->flags & IORESOURCE_IO) - root = &pbm->io_space; - else - root = &pbm->mem_space; - - pci_resource_adjust(res, root); -} -EXPORT_SYMBOL(pcibios_bus_to_resource); - -char * __devinit pcibios_setup(char *str) -{ - return str; -} - /* Platform support for /proc/bus/pci/X/Y mmap()s. */ /* If the user uses a host-bridge as the PCI device, he may use @@ -819,11 +673,9 @@ static int __pci_mmap_make_offset_bus(struct pci_dev *pdev, struct vm_area_struc unsigned long space_size, user_offset, user_size; if (mmap_state == pci_mmap_io) { - space_size = (pbm->io_space.end - - pbm->io_space.start) + 1; + space_size = resource_size(&pbm->io_space); } else { - space_size = (pbm->mem_space.end - - pbm->mem_space.start) + 1; + space_size = resource_size(&pbm->mem_space); } /* Make sure the request is in range. */ @@ -920,15 +772,6 @@ static int __pci_mmap_make_offset(struct pci_dev *pdev, return 0; } -/* Set vm_flags of VMA, as appropriate for this architecture, for a pci device - * mapping. - */ -static void __pci_mmap_set_flags(struct pci_dev *dev, struct vm_area_struct *vma, - enum pci_mmap_state mmap_state) -{ - vma->vm_flags |= (VM_IO | VM_RESERVED); -} - /* Set vm_page_prot of VMA, as appropriate for this architecture, for a pci * device mapping. */ @@ -956,7 +799,6 @@ int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, if (ret < 0) return ret; - __pci_mmap_set_flags(dev, vma, mmap_state); __pci_mmap_set_pgprot(dev, vma, mmap_state); vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -1001,31 +843,25 @@ EXPORT_SYMBOL(pci_domain_nr); int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) { struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; - unsigned int virt_irq; + unsigned int irq; if (!pbm->setup_msi_irq) return -EINVAL; - return pbm->setup_msi_irq(&virt_irq, pdev, desc); + return pbm->setup_msi_irq(&irq, pdev, desc); } -void arch_teardown_msi_irq(unsigned int virt_irq) +void arch_teardown_msi_irq(unsigned int irq) { - struct msi_desc *entry = get_irq_msi(virt_irq); + struct msi_desc *entry = irq_get_msi_desc(irq); struct pci_dev *pdev = entry->dev; struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; if (pbm->teardown_msi_irq) - pbm->teardown_msi_irq(virt_irq, pdev); + pbm->teardown_msi_irq(irq, pdev); } #endif /* !(CONFIG_PCI_MSI) */ -struct device_node *pci_device_to_OF_node(struct pci_dev *pdev) -{ - return pdev->dev.of_node; -} -EXPORT_SYMBOL(pci_device_to_OF_node); - static void ali_sound_dma_hack(struct pci_dev *pdev, int set_bit) { struct pci_dev *ali_isa_bridge; @@ -1089,6 +925,11 @@ void pci_resource_to_user(const struct pci_dev *pdev, int bar, *end = rp->end - offset; } +void pcibios_set_master(struct pci_dev *dev) +{ + /* No special bus mastering setup handling */ +} + static int __init pcibios_init(void) { pci_dfl_cache_line_size = 64 >> 2; @@ -1097,8 +938,7 @@ static int __init pcibios_init(void) subsys_initcall(pcibios_init); #ifdef CONFIG_SYSFS -static void __devinit pci_bus_slot_names(struct device_node *node, - struct pci_bus *bus) +static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus) { const struct pci_slot_names { u32 slot_mask; @@ -1165,6 +1005,5 @@ static int __init of_pci_slot_init(void) return 0; } - -module_init(of_pci_slot_init); +device_initcall(of_pci_slot_init); #endif diff --git a/arch/sparc/kernel/pci_common.c b/arch/sparc/kernel/pci_common.c index 6c7a33af3ba..944a06536ec 100644 --- a/arch/sparc/kernel/pci_common.c +++ b/arch/sparc/kernel/pci_common.c @@ -5,7 +5,6 @@ #include <linux/string.h> #include <linux/slab.h> -#include <linux/init.h> #include <linux/pci.h> #include <linux/device.h> #include <linux/of_device.h> @@ -281,7 +280,7 @@ static int sun4v_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, case 4: *value = ret & 0xffffffff; break; - }; + } return PCIBIOS_SUCCESSFUL; @@ -295,14 +294,17 @@ static int sun4v_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, unsigned int bus = bus_dev->number; unsigned int device = PCI_SLOT(devfn); unsigned int func = PCI_FUNC(devfn); - unsigned long ret; if (config_out_of_range(pbm, bus, devfn, where)) { /* Do nothing. */ } else { - ret = pci_sun4v_config_put(devhandle, - HV_PCI_DEVICE_BUILD(bus, device, func), - where, size, value); + /* We don't check for hypervisor errors here, but perhaps + * we should and influence our return value depending upon + * what kind of error is thrown. + */ + pci_sun4v_config_put(devhandle, + HV_PCI_DEVICE_BUILD(bus, device, func), + where, size, value); } return PCIBIOS_SUCCESSFUL; } @@ -453,7 +455,7 @@ void pci_determine_mem_io_space(struct pci_pbm_info *pbm) default: break; - }; + } } if (!saw_io || !saw_mem) { diff --git a/arch/sparc/kernel/pci_fire.c b/arch/sparc/kernel/pci_fire.c index efb896d6875..e60fc6a67e9 100644 --- a/arch/sparc/kernel/pci_fire.c +++ b/arch/sparc/kernel/pci_fire.c @@ -7,6 +7,7 @@ #include <linux/slab.h> #include <linux/init.h> #include <linux/msi.h> +#include <linux/export.h> #include <linux/irq.h> #include <linux/of_device.h> @@ -214,11 +215,9 @@ static int pci_fire_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, static int pci_fire_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) { - unsigned long msiqid; u64 val; val = upa_readq(pbm->pbm_regs + MSI_MAP(msi)); - msiqid = (val & MSI_MAP_EQNUM); val &= ~MSI_MAP_VALID; @@ -277,7 +276,7 @@ static int pci_fire_msiq_build_irq(struct pci_pbm_info *pbm, { unsigned long cregs = (unsigned long) pbm->pbm_regs; unsigned long imap_reg, iclr_reg, int_ctrlr; - unsigned int virt_irq; + unsigned int irq; int fixup; u64 val; @@ -293,14 +292,14 @@ static int pci_fire_msiq_build_irq(struct pci_pbm_info *pbm, fixup = ((pbm->portid << 6) | devino) - int_ctrlr; - virt_irq = build_irq(fixup, iclr_reg, imap_reg); - if (!virt_irq) + irq = build_irq(fixup, iclr_reg, imap_reg); + if (!irq) return -ENOMEM; upa_writeq(EVENT_QUEUE_CONTROL_SET_EN, pbm->pbm_regs + EVENT_QUEUE_CONTROL_SET(msiqid)); - return virt_irq; + return irq; } static const struct sparc64_msiq_ops pci_fire_msiq_ops = { @@ -409,8 +408,8 @@ static void pci_fire_hw_init(struct pci_pbm_info *pbm) upa_writeq(~(u64)0, pbm->pbm_regs + FIRE_PEC_IENAB); } -static int __devinit pci_fire_pbm_init(struct pci_pbm_info *pbm, - struct platform_device *op, u32 portid) +static int pci_fire_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 portid) { const struct linux_prom64_registers *regs; struct device_node *dp = op->dev.of_node; @@ -455,8 +454,7 @@ static int __devinit pci_fire_pbm_init(struct pci_pbm_info *pbm, return 0; } -static int __devinit fire_probe(struct platform_device *op, - const struct of_device_id *match) +static int fire_probe(struct platform_device *op) { struct device_node *dp = op->dev.of_node; struct pci_pbm_info *pbm; @@ -499,7 +497,7 @@ out_err: return err; } -static struct of_device_id __initdata fire_match[] = { +static const struct of_device_id fire_match[] = { { .name = "pci", .compatible = "pciex108e,80f0", @@ -507,7 +505,7 @@ static struct of_device_id __initdata fire_match[] = { {}, }; -static struct of_platform_driver fire_driver = { +static struct platform_driver fire_driver = { .driver = { .name = DRIVER_NAME, .owner = THIS_MODULE, @@ -518,7 +516,7 @@ static struct of_platform_driver fire_driver = { static int __init fire_init(void) { - return of_register_platform_driver(&fire_driver); + return platform_driver_register(&fire_driver); } subsys_initcall(fire_init); diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h index e20ed5f06e9..75803c780af 100644 --- a/arch/sparc/kernel/pci_impl.h +++ b/arch/sparc/kernel/pci_impl.h @@ -48,8 +48,8 @@ struct sparc64_msiq_ops { unsigned long devino); }; -extern void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, - const struct sparc64_msiq_ops *ops); +void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops); struct sparc64_msiq_cookie { struct pci_pbm_info *pbm; @@ -88,7 +88,7 @@ struct pci_pbm_info { int chip_revision; /* Name used for top-level resources. */ - char *name; + const char *name; /* OBP specific information. */ struct platform_device *op; @@ -97,6 +97,7 @@ struct pci_pbm_info { /* PBM I/O and Memory space resources. */ struct resource io_space; struct resource mem_space; + struct resource busn; /* Base of PCI Config space, can be per-PBM or shared. */ unsigned long config_space; @@ -131,9 +132,9 @@ struct pci_pbm_info { void *msi_queues; unsigned long *msi_bitmap; unsigned int *msi_irq_table; - int (*setup_msi_irq)(unsigned int *virt_irq_p, struct pci_dev *pdev, + int (*setup_msi_irq)(unsigned int *irq_p, struct pci_dev *pdev, struct msi_desc *entry); - void (*teardown_msi_irq)(unsigned int virt_irq, struct pci_dev *pdev); + void (*teardown_msi_irq)(unsigned int irq, struct pci_dev *pdev); const struct sparc64_msiq_ops *msi_ops; #endif /* !(CONFIG_PCI_MSI) */ @@ -157,23 +158,23 @@ extern struct pci_pbm_info *pci_pbm_root; extern int pci_num_pbms; /* PCI bus scanning and fixup support. */ -extern void pci_get_pbm_props(struct pci_pbm_info *pbm); -extern struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, - struct device *parent); -extern void pci_determine_mem_io_space(struct pci_pbm_info *pbm); +void pci_get_pbm_props(struct pci_pbm_info *pbm); +struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, + struct device *parent); +void pci_determine_mem_io_space(struct pci_pbm_info *pbm); /* Error reporting support. */ -extern void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); -extern void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); -extern void pci_scan_for_parity_error(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_parity_error(struct pci_pbm_info *, struct pci_bus *); /* Configuration space access. */ -extern void pci_config_read8(u8 *addr, u8 *ret); -extern void pci_config_read16(u16 *addr, u16 *ret); -extern void pci_config_read32(u32 *addr, u32 *ret); -extern void pci_config_write8(u8 *addr, u8 val); -extern void pci_config_write16(u16 *addr, u16 val); -extern void pci_config_write32(u32 *addr, u32 val); +void pci_config_read8(u8 *addr, u8 *ret); +void pci_config_read16(u16 *addr, u16 *ret); +void pci_config_read32(u32 *addr, u32 *ret); +void pci_config_write8(u8 *addr, u8 val); +void pci_config_write16(u16 *addr, u16 val); +void pci_config_write32(u32 *addr, u32 val); extern struct pci_ops sun4u_pci_ops; extern struct pci_ops sun4v_pci_ops; diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index b210416ace7..580651af73f 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -30,13 +30,10 @@ static irqreturn_t sparc64_msiq_interrupt(int irq, void *cookie) err = ops->dequeue_msi(pbm, msiqid, &head, &msi); if (likely(err > 0)) { - struct irq_desc *desc; - unsigned int virt_irq; + unsigned int irq; - virt_irq = pbm->msi_irq_table[msi - pbm->msi_first]; - desc = irq_desc + virt_irq; - - desc->handle_irq(virt_irq, desc); + irq = pbm->msi_irq_table[msi - pbm->msi_first]; + generic_handle_irq(irq); } if (unlikely(err < 0)) @@ -121,7 +118,7 @@ static struct irq_chip msi_irq = { /* XXX affinity XXX */ }; -static int sparc64_setup_msi_irq(unsigned int *virt_irq_p, +static int sparc64_setup_msi_irq(unsigned int *irq_p, struct pci_dev *pdev, struct msi_desc *entry) { @@ -131,17 +128,17 @@ static int sparc64_setup_msi_irq(unsigned int *virt_irq_p, int msi, err; u32 msiqid; - *virt_irq_p = virt_irq_alloc(0, 0); + *irq_p = irq_alloc(0, 0); err = -ENOMEM; - if (!*virt_irq_p) + if (!*irq_p) goto out_err; - set_irq_chip_and_handler_name(*virt_irq_p, &msi_irq, - handle_simple_irq, "MSI"); + irq_set_chip_and_handler_name(*irq_p, &msi_irq, handle_simple_irq, + "MSI"); err = alloc_msi(pbm); if (unlikely(err < 0)) - goto out_virt_irq_free; + goto out_irq_free; msi = err; @@ -152,7 +149,7 @@ static int sparc64_setup_msi_irq(unsigned int *virt_irq_p, if (err) goto out_msi_free; - pbm->msi_irq_table[msi - pbm->msi_first] = *virt_irq_p; + pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; if (entry->msi_attrib.is_64) { msg.address_hi = pbm->msi64_start >> 32; @@ -163,24 +160,24 @@ static int sparc64_setup_msi_irq(unsigned int *virt_irq_p, } msg.data = msi; - set_irq_msi(*virt_irq_p, entry); - write_msi_msg(*virt_irq_p, &msg); + irq_set_msi_desc(*irq_p, entry); + write_msi_msg(*irq_p, &msg); return 0; out_msi_free: free_msi(pbm, msi); -out_virt_irq_free: - set_irq_chip(*virt_irq_p, NULL); - virt_irq_free(*virt_irq_p); - *virt_irq_p = 0; +out_irq_free: + irq_set_chip(*irq_p, NULL); + irq_free(*irq_p); + *irq_p = 0; out_err: return err; } -static void sparc64_teardown_msi_irq(unsigned int virt_irq, +static void sparc64_teardown_msi_irq(unsigned int irq, struct pci_dev *pdev) { struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; @@ -189,12 +186,12 @@ static void sparc64_teardown_msi_irq(unsigned int virt_irq, int i, err; for (i = 0; i < pbm->msi_num; i++) { - if (pbm->msi_irq_table[i] == virt_irq) + if (pbm->msi_irq_table[i] == irq) break; } if (i >= pbm->msi_num) { printk(KERN_ERR "%s: teardown: No MSI for irq %u\n", - pbm->name, virt_irq); + pbm->name, irq); return; } @@ -205,14 +202,14 @@ static void sparc64_teardown_msi_irq(unsigned int virt_irq, if (err) { printk(KERN_ERR "%s: teardown: ops->teardown() on MSI %u, " "irq %u, gives error %d\n", - pbm->name, msi_num, virt_irq, err); + pbm->name, msi_num, irq, err); return; } free_msi(pbm, msi_num); - set_irq_chip(virt_irq, NULL); - virt_irq_free(virt_irq); + irq_set_chip(irq, NULL); + irq_free(irq); } static int msi_bitmap_alloc(struct pci_pbm_info *pbm) @@ -287,8 +284,9 @@ static int bringup_one_msi_queue(struct pci_pbm_info *pbm, nid = pbm->numa_node; if (nid != -1) { - cpumask_t numa_mask = *cpumask_of_node(nid); + cpumask_t numa_mask; + cpumask_copy(&numa_mask, cpumask_of_node(nid)); irq_set_affinity(irq, &numa_mask); } err = request_irq(irq, sparc64_msiq_interrupt, 0, diff --git a/arch/sparc/kernel/pci_psycho.c b/arch/sparc/kernel/pci_psycho.c index 22eab7cf3b1..c647634ead2 100644 --- a/arch/sparc/kernel/pci_psycho.c +++ b/arch/sparc/kernel/pci_psycho.c @@ -9,6 +9,7 @@ #include <linux/types.h> #include <linux/pci.h> #include <linux/init.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/interrupt.h> #include <linux/of_device.h> @@ -365,8 +366,8 @@ static void pbm_config_busmastering(struct pci_pbm_info *pbm) pci_config_write8(addr, 64); } -static void __devinit psycho_scan_bus(struct pci_pbm_info *pbm, - struct device *parent) +static void psycho_scan_bus(struct pci_pbm_info *pbm, + struct device *parent) { pbm_config_busmastering(pbm); pbm->is_66mhz_capable = 0; @@ -482,15 +483,15 @@ static void psycho_pbm_strbuf_init(struct pci_pbm_info *pbm, #define PSYCHO_MEMSPACE_B 0x180000000UL #define PSYCHO_MEMSPACE_SIZE 0x07fffffffUL -static void __devinit psycho_pbm_init(struct pci_pbm_info *pbm, - struct platform_device *op, int is_pbm_a) +static void psycho_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, int is_pbm_a) { psycho_pbm_init_common(pbm, op, "PSYCHO", PBM_CHIP_TYPE_PSYCHO); psycho_pbm_strbuf_init(pbm, is_pbm_a); psycho_scan_bus(pbm, &op->dev); } -static struct pci_pbm_info * __devinit psycho_find_sibling(u32 upa_portid) +static struct pci_pbm_info *psycho_find_sibling(u32 upa_portid) { struct pci_pbm_info *pbm; @@ -503,8 +504,7 @@ static struct pci_pbm_info * __devinit psycho_find_sibling(u32 upa_portid) #define PSYCHO_CONFIGSPACE 0x001000000UL -static int __devinit psycho_probe(struct platform_device *op, - const struct of_device_id *match) +static int psycho_probe(struct platform_device *op) { const struct linux_prom64_registers *pr_regs; struct device_node *dp = op->dev.of_node; @@ -593,7 +593,7 @@ out_err: return err; } -static struct of_device_id __initdata psycho_match[] = { +static const struct of_device_id psycho_match[] = { { .name = "pci", .compatible = "pci108e,8000", @@ -601,7 +601,7 @@ static struct of_device_id __initdata psycho_match[] = { {}, }; -static struct of_platform_driver psycho_driver = { +static struct platform_driver psycho_driver = { .driver = { .name = DRIVER_NAME, .owner = THIS_MODULE, @@ -612,7 +612,7 @@ static struct of_platform_driver psycho_driver = { static int __init psycho_init(void) { - return of_register_platform_driver(&psycho_driver); + return platform_driver_register(&psycho_driver); } subsys_initcall(psycho_init); diff --git a/arch/sparc/kernel/pci_sabre.c b/arch/sparc/kernel/pci_sabre.c index 5c3f5ec4cab..6f00d27e8da 100644 --- a/arch/sparc/kernel/pci_sabre.c +++ b/arch/sparc/kernel/pci_sabre.c @@ -9,6 +9,7 @@ #include <linux/types.h> #include <linux/pci.h> #include <linux/init.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/interrupt.h> #include <linux/of_device.h> @@ -402,8 +403,7 @@ static void apb_init(struct pci_bus *sabre_bus) } } -static void __devinit sabre_scan_bus(struct pci_pbm_info *pbm, - struct device *parent) +static void sabre_scan_bus(struct pci_pbm_info *pbm, struct device *parent) { static int once; @@ -442,8 +442,8 @@ static void __devinit sabre_scan_bus(struct pci_pbm_info *pbm, sabre_register_error_handlers(pbm); } -static void __devinit sabre_pbm_init(struct pci_pbm_info *pbm, - struct platform_device *op) +static void sabre_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op) { psycho_pbm_init_common(pbm, op, "SABRE", PBM_CHIP_TYPE_SABRE); pbm->pci_afsr = pbm->controller_regs + SABRE_PIOAFSR; @@ -452,9 +452,10 @@ static void __devinit sabre_pbm_init(struct pci_pbm_info *pbm, sabre_scan_bus(pbm, &op->dev); } -static int __devinit sabre_probe(struct platform_device *op, - const struct of_device_id *match) +static const struct of_device_id sabre_match[]; +static int sabre_probe(struct platform_device *op) { + const struct of_device_id *match; const struct linux_prom64_registers *pr_regs; struct device_node *dp = op->dev.of_node; struct pci_pbm_info *pbm; @@ -464,7 +465,8 @@ static int __devinit sabre_probe(struct platform_device *op, const u32 *vdma; u64 clear_irq; - hummingbird_p = (match->data != NULL); + match = of_match_device(sabre_match, &op->dev); + hummingbird_p = match && (match->data != NULL); if (!hummingbird_p) { struct device_node *cpu_dp; @@ -582,7 +584,7 @@ out_err: return err; } -static struct of_device_id __initdata sabre_match[] = { +static const struct of_device_id sabre_match[] = { { .name = "pci", .compatible = "pci108e,a001", @@ -595,7 +597,7 @@ static struct of_device_id __initdata sabre_match[] = { {}, }; -static struct of_platform_driver sabre_driver = { +static struct platform_driver sabre_driver = { .driver = { .name = DRIVER_NAME, .owner = THIS_MODULE, @@ -606,7 +608,7 @@ static struct of_platform_driver sabre_driver = { static int __init sabre_init(void) { - return of_register_platform_driver(&sabre_driver); + return platform_driver_register(&sabre_driver); } subsys_initcall(sabre_init); diff --git a/arch/sparc/kernel/pci_schizo.c b/arch/sparc/kernel/pci_schizo.c index 445a47a2fb3..8f76f23dac3 100644 --- a/arch/sparc/kernel/pci_schizo.c +++ b/arch/sparc/kernel/pci_schizo.c @@ -8,6 +8,7 @@ #include <linux/pci.h> #include <linux/init.h> #include <linux/slab.h> +#include <linux/export.h> #include <linux/interrupt.h> #include <linux/of_device.h> @@ -264,7 +265,7 @@ static void schizo_check_iommu_error_pbm(struct pci_pbm_info *pbm, default: type_string = "ECC Error"; break; - }; + } printk("%s: IOMMU Error, type[%s]\n", pbm->name, type_string); @@ -319,7 +320,7 @@ static void schizo_check_iommu_error_pbm(struct pci_pbm_info *pbm, default: type_string = "ECC Error"; break; - }; + } printk("%s: IOMMU TAG(%d)[error(%s) ctx(%x) wr(%d) str(%d) " "sz(%dK) vpg(%08lx)]\n", pbm->name, i, type_string, @@ -1063,8 +1064,7 @@ static void pbm_config_busmastering(struct pci_pbm_info *pbm) pci_config_write8(addr, 64); } -static void __devinit schizo_scan_bus(struct pci_pbm_info *pbm, - struct device *parent) +static void schizo_scan_bus(struct pci_pbm_info *pbm, struct device *parent) { pbm_config_busmastering(pbm); pbm->is_66mhz_capable = @@ -1306,14 +1306,14 @@ static void schizo_pbm_hw_init(struct pci_pbm_info *pbm) } } -static int __devinit schizo_pbm_init(struct pci_pbm_info *pbm, - struct platform_device *op, u32 portid, - int chip_type) +static int schizo_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 portid, + int chip_type) { const struct linux_prom64_registers *regs; struct device_node *dp = op->dev.of_node; const char *chipset_name; - int is_pbm_a, err; + int err; switch (chip_type) { case PBM_CHIP_TYPE_TOMATILLO: @@ -1328,7 +1328,7 @@ static int __devinit schizo_pbm_init(struct pci_pbm_info *pbm, default: chipset_name = "SCHIZO"; break; - }; + } /* For SCHIZO, three OBP regs: * 1) PBM controller regs @@ -1343,8 +1343,6 @@ static int __devinit schizo_pbm_init(struct pci_pbm_info *pbm, */ regs = of_get_property(dp, "reg", NULL); - is_pbm_a = ((regs[0].phys_addr & 0x00700000) == 0x00600000); - pbm->next = pci_pbm_root; pci_pbm_root = pbm; @@ -1401,8 +1399,7 @@ static inline int portid_compare(u32 x, u32 y, int chip_type) return (x == y); } -static struct pci_pbm_info * __devinit schizo_find_sibling(u32 portid, - int chip_type) +static struct pci_pbm_info *schizo_find_sibling(u32 portid, int chip_type) { struct pci_pbm_info *pbm; @@ -1413,7 +1410,7 @@ static struct pci_pbm_info * __devinit schizo_find_sibling(u32 portid, return NULL; } -static int __devinit __schizo_init(struct platform_device *op, unsigned long chip_type) +static int __schizo_init(struct platform_device *op, unsigned long chip_type) { struct device_node *dp = op->dev.of_node; struct pci_pbm_info *pbm; @@ -1460,10 +1457,15 @@ out_err: return err; } -static int __devinit schizo_probe(struct platform_device *op, - const struct of_device_id *match) +static const struct of_device_id schizo_match[]; +static int schizo_probe(struct platform_device *op) { - return __schizo_init(op, (unsigned long) match->data); + const struct of_device_id *match; + + match = of_match_device(schizo_match, &op->dev); + if (!match) + return -EINVAL; + return __schizo_init(op, (unsigned long)match->data); } /* The ordering of this table is very important. Some Tomatillo @@ -1471,7 +1473,7 @@ static int __devinit schizo_probe(struct platform_device *op, * and pci108e,8001. So list the chips in reverse chronological * order. */ -static struct of_device_id __initdata schizo_match[] = { +static const struct of_device_id schizo_match[] = { { .name = "pci", .compatible = "pci108e,a801", @@ -1490,7 +1492,7 @@ static struct of_device_id __initdata schizo_match[] = { {}, }; -static struct of_platform_driver schizo_driver = { +static struct platform_driver schizo_driver = { .driver = { .name = DRIVER_NAME, .owner = THIS_MODULE, @@ -1501,7 +1503,7 @@ static struct of_platform_driver schizo_driver = { static int __init schizo_init(void) { - return of_register_platform_driver(&schizo_driver); + return platform_driver_register(&schizo_driver); } subsys_initcall(schizo_init); diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 743344aa6d8..d07f6b29aed 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -12,6 +12,7 @@ #include <linux/percpu.h> #include <linux/irq.h> #include <linux/msi.h> +#include <linux/export.h> #include <linux/log2.h> #include <linux/of_device.h> @@ -127,7 +128,8 @@ static inline long iommu_batch_end(void) } static void *dma_4v_alloc_coherent(struct device *dev, size_t size, - dma_addr_t *dma_addrp, gfp_t gfp) + dma_addr_t *dma_addrp, gfp_t gfp, + struct dma_attrs *attrs) { unsigned long flags, order, first_page, npages, n; struct iommu *iommu; @@ -197,7 +199,7 @@ range_alloc_fail: } static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, - dma_addr_t dvma) + dma_addr_t dvma, struct dma_attrs *attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -526,16 +528,15 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, } static struct dma_map_ops sun4v_dma_ops = { - .alloc_coherent = dma_4v_alloc_coherent, - .free_coherent = dma_4v_free_coherent, + .alloc = dma_4v_alloc_coherent, + .free = dma_4v_free_coherent, .map_page = dma_4v_map_page, .unmap_page = dma_4v_unmap_page, .map_sg = dma_4v_map_sg, .unmap_sg = dma_4v_unmap_sg, }; -static void __devinit pci_sun4v_scan_bus(struct pci_pbm_info *pbm, - struct device *parent) +static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) { struct property *prop; struct device_node *dp; @@ -548,8 +549,8 @@ static void __devinit pci_sun4v_scan_bus(struct pci_pbm_info *pbm, /* XXX register error interrupt handlers XXX */ } -static unsigned long __devinit probe_existing_entries(struct pci_pbm_info *pbm, - struct iommu *iommu) +static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, + struct iommu *iommu) { struct iommu_arena *arena = &iommu->arena; unsigned long i, cnt = 0; @@ -576,11 +577,11 @@ static unsigned long __devinit probe_existing_entries(struct pci_pbm_info *pbm, return cnt; } -static int __devinit pci_sun4v_iommu_init(struct pci_pbm_info *pbm) +static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) { static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; struct iommu *iommu = pbm->iommu; - unsigned long num_tsb_entries, sz, tsbsize; + unsigned long num_tsb_entries, sz; u32 dma_mask, dma_offset; const u32 *vdma; @@ -592,11 +593,10 @@ static int __devinit pci_sun4v_iommu_init(struct pci_pbm_info *pbm) printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", vdma[0], vdma[1]); return -EINVAL; - }; + } dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); num_tsb_entries = vdma[1] / IO_PAGE_SIZE; - tsbsize = num_tsb_entries * sizeof(iopte_t); dma_offset = vdma[0]; @@ -844,17 +844,17 @@ static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, unsigned long msiqid, unsigned long devino) { - unsigned int virt_irq = sun4v_build_irq(pbm->devhandle, devino); + unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); - if (!virt_irq) + if (!irq) return -ENOMEM; - if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) - return -EINVAL; if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) return -EINVAL; + if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) + return -EINVAL; - return virt_irq; + return irq; } static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { @@ -878,8 +878,8 @@ static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) } #endif /* !(CONFIG_PCI_MSI) */ -static int __devinit pci_sun4v_pbm_init(struct pci_pbm_info *pbm, - struct platform_device *op, u32 devhandle) +static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 devhandle) { struct device_node *dp = op->dev.of_node; int err; @@ -918,8 +918,7 @@ static int __devinit pci_sun4v_pbm_init(struct pci_pbm_info *pbm, return 0; } -static int __devinit pci_sun4v_probe(struct platform_device *op, - const struct of_device_id *match) +static int pci_sun4v_probe(struct platform_device *op) { const struct linux_prom64_registers *regs; static int hvapi_negotiated = 0; @@ -1000,7 +999,7 @@ out_err: return err; } -static struct of_device_id __initdata pci_sun4v_match[] = { +static const struct of_device_id pci_sun4v_match[] = { { .name = "pci", .compatible = "SUNW,sun4v-pci", @@ -1008,7 +1007,7 @@ static struct of_device_id __initdata pci_sun4v_match[] = { {}, }; -static struct of_platform_driver pci_sun4v_driver = { +static struct platform_driver pci_sun4v_driver = { .driver = { .name = DRIVER_NAME, .owner = THIS_MODULE, @@ -1019,7 +1018,7 @@ static struct of_platform_driver pci_sun4v_driver = { static int __init pci_sun4v_init(void) { - return of_register_platform_driver(&pci_sun4v_driver); + return platform_driver_register(&pci_sun4v_driver); } subsys_initcall(pci_sun4v_init); diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h index 8e9fc3a5b4f..5642212390b 100644 --- a/arch/sparc/kernel/pci_sun4v.h +++ b/arch/sparc/kernel/pci_sun4v.h @@ -6,87 +6,87 @@ #ifndef _PCI_SUN4V_H #define _PCI_SUN4V_H -extern long pci_sun4v_iommu_map(unsigned long devhandle, - unsigned long tsbid, - unsigned long num_ttes, - unsigned long io_attributes, - unsigned long io_page_list_pa); -extern unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, - unsigned long tsbid, - unsigned long num_ttes); -extern unsigned long pci_sun4v_iommu_getmap(unsigned long devhandle, - unsigned long tsbid, - unsigned long *io_attributes, - unsigned long *real_address); -extern unsigned long pci_sun4v_config_get(unsigned long devhandle, - unsigned long pci_device, - unsigned long config_offset, - unsigned long size); -extern int pci_sun4v_config_put(unsigned long devhandle, - unsigned long pci_device, - unsigned long config_offset, - unsigned long size, - unsigned long data); +long pci_sun4v_iommu_map(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes, + unsigned long io_attributes, + unsigned long io_page_list_pa); +unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes); +unsigned long pci_sun4v_iommu_getmap(unsigned long devhandle, + unsigned long tsbid, + unsigned long *io_attributes, + unsigned long *real_address); +unsigned long pci_sun4v_config_get(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size); +int pci_sun4v_config_put(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size, + unsigned long data); -extern unsigned long pci_sun4v_msiq_conf(unsigned long devhandle, +unsigned long pci_sun4v_msiq_conf(unsigned long devhandle, unsigned long msiqid, unsigned long msiq_paddr, unsigned long num_entries); -extern unsigned long pci_sun4v_msiq_info(unsigned long devhandle, - unsigned long msiqid, - unsigned long *msiq_paddr, - unsigned long *num_entries); -extern unsigned long pci_sun4v_msiq_getvalid(unsigned long devhandle, - unsigned long msiqid, - unsigned long *valid); -extern unsigned long pci_sun4v_msiq_setvalid(unsigned long devhandle, - unsigned long msiqid, - unsigned long valid); -extern unsigned long pci_sun4v_msiq_getstate(unsigned long devhandle, - unsigned long msiqid, - unsigned long *state); -extern unsigned long pci_sun4v_msiq_setstate(unsigned long devhandle, - unsigned long msiqid, - unsigned long state); -extern unsigned long pci_sun4v_msiq_gethead(unsigned long devhandle, - unsigned long msiqid, - unsigned long *head); -extern unsigned long pci_sun4v_msiq_sethead(unsigned long devhandle, - unsigned long msiqid, - unsigned long head); -extern unsigned long pci_sun4v_msiq_gettail(unsigned long devhandle, - unsigned long msiqid, - unsigned long *head); -extern unsigned long pci_sun4v_msi_getvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long *valid); -extern unsigned long pci_sun4v_msi_setvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long valid); -extern unsigned long pci_sun4v_msi_getmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long *msiq); -extern unsigned long pci_sun4v_msi_setmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long msiq, - unsigned long msitype); -extern unsigned long pci_sun4v_msi_getstate(unsigned long devhandle, - unsigned long msinum, - unsigned long *state); -extern unsigned long pci_sun4v_msi_setstate(unsigned long devhandle, - unsigned long msinum, - unsigned long state); -extern unsigned long pci_sun4v_msg_getmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long *msiq); -extern unsigned long pci_sun4v_msg_setmsiq(unsigned long devhandle, - unsigned long msinum, - unsigned long msiq); -extern unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long *valid); -extern unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, - unsigned long msinum, - unsigned long valid); +unsigned long pci_sun4v_msiq_info(unsigned long devhandle, + unsigned long msiqid, + unsigned long *msiq_paddr, + unsigned long *num_entries); +unsigned long pci_sun4v_msiq_getvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long *valid); +unsigned long pci_sun4v_msiq_setvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long valid); +unsigned long pci_sun4v_msiq_getstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long *state); +unsigned long pci_sun4v_msiq_setstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long state); +unsigned long pci_sun4v_msiq_gethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msiq_sethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long head); +unsigned long pci_sun4v_msiq_gettail(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msi_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msi_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); +unsigned long pci_sun4v_msi_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msi_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq, + unsigned long msitype); +unsigned long pci_sun4v_msi_getstate(unsigned long devhandle, + unsigned long msinum, + unsigned long *state); +unsigned long pci_sun4v_msi_setstate(unsigned long devhandle, + unsigned long msinum, + unsigned long state); +unsigned long pci_sun4v_msg_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msg_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq); +unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); #endif /* !(_PCI_SUN4V_H) */ diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c index aeaa09a3c65..6cc78c213c0 100644 --- a/arch/sparc/kernel/pcic.c +++ b/arch/sparc/kernel/pcic.c @@ -25,6 +25,7 @@ #include <linux/time.h> #include <linux/timex.h> #include <linux/interrupt.h> +#include <linux/export.h> #include <asm/irq.h> #include <asm/oplib.h> @@ -35,6 +36,7 @@ #include <asm/uaccess.h> #include <asm/irq_regs.h> +#include "kernel.h" #include "irq.h" /* @@ -161,9 +163,12 @@ static int pcic0_up; static struct linux_pcic pcic0; void __iomem *pcic_regs; -volatile int pcic_speculative; -volatile int pcic_trapped; +static volatile int pcic_speculative; +static volatile int pcic_trapped; +/* forward */ +unsigned int pcic_build_device_irq(struct platform_device *op, + unsigned int real_irq); #define CONFIG_CMD(bus, device_fn, where) (0x80000000 | (((unsigned int)bus) << 16) | (((unsigned int)device_fn) << 8) | (where & ~3)) @@ -325,7 +330,7 @@ int __init pcic_probe(void) pcic->pcic_res_cfg_addr.name = "pcic_cfg_addr"; if ((pcic->pcic_config_space_addr = - ioremap(regs[2].phys_addr, regs[2].reg_size * 2)) == 0) { + ioremap(regs[2].phys_addr, regs[2].reg_size * 2)) == NULL) { prom_printf("PCIC: Error, cannot map " "PCI Configuration Space Address.\n"); prom_halt(); @@ -337,7 +342,7 @@ int __init pcic_probe(void) */ pcic->pcic_res_cfg_data.name = "pcic_cfg_data"; if ((pcic->pcic_config_space_data = - ioremap(regs[3].phys_addr, regs[3].reg_size * 2)) == 0) { + ioremap(regs[3].phys_addr, regs[3].reg_size * 2)) == NULL) { prom_printf("PCIC: Error, cannot map " "PCI Configuration Space Data.\n"); prom_halt(); @@ -349,8 +354,7 @@ int __init pcic_probe(void) strcpy(pbm->prom_name, namebuf); { - extern volatile int t_nmi[1]; - extern int pcic_nmi_trap_patch[1]; + extern int pcic_nmi_trap_patch[4]; t_nmi[0] = pcic_nmi_trap_patch[0]; t_nmi[1] = pcic_nmi_trap_patch[1]; @@ -435,8 +439,7 @@ int pcic_present(void) return pcic0_up; } -static int __devinit pdev_to_pnode(struct linux_pbm_info *pbm, - struct pci_dev *pdev) +static int pdev_to_pnode(struct linux_pbm_info *pbm, struct pci_dev *pdev) { struct linux_prom_pci_registers regs[PROMREG_MAX]; int err; @@ -523,6 +526,7 @@ static void pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) { struct pcic_ca2irq *p; + unsigned int real_irq; int i, ivec; char namebuf[64]; @@ -532,7 +536,7 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) prom_getstring(node, "name", namebuf, sizeof(namebuf)); } - if ((p = pcic->pcic_imap) == 0) { + if ((p = pcic->pcic_imap) == NULL) { dev->irq = 0; return; } @@ -551,26 +555,25 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) i = p->pin; if (i >= 0 && i < 4) { ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_LO); - dev->irq = ivec >> (i << 2) & 0xF; + real_irq = ivec >> (i << 2) & 0xF; } else if (i >= 4 && i < 8) { ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); - dev->irq = ivec >> ((i-4) << 2) & 0xF; + real_irq = ivec >> ((i-4) << 2) & 0xF; } else { /* Corrupted map */ printk("PCIC: BAD PIN %d\n", i); for (;;) {} } /* P3 */ /* printk("PCIC: device %s pin %d ivec 0x%x irq %x\n", namebuf, i, ivec, dev->irq); */ - /* - * dev->irq=0 means PROM did not bother to program the upper + /* real_irq means PROM did not bother to program the upper * half of PCIC. This happens on JS-E with PROM 3.11, for instance. */ - if (dev->irq == 0 || p->force) { + if (real_irq == 0 || p->force) { if (p->irq == 0 || p->irq >= 15) { /* Corrupted map */ printk("PCIC: BAD IRQ %d\n", p->irq); for (;;) {} } printk("PCIC: setting irq %d at pin %d for device %02x:%02x\n", p->irq, p->pin, dev->bus->number, dev->devfn); - dev->irq = p->irq; + real_irq = p->irq; i = p->pin; if (i >= 4) { @@ -584,13 +587,14 @@ pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) ivec |= p->irq << (i << 2); writew(ivec, pcic->pcic_regs+PCI_INT_SELECT_LO); } - } + } + dev->irq = pcic_build_device_irq(NULL, real_irq); } /* * Normally called from {do_}pci_scan_bus... */ -void __devinit pcibios_fixup_bus(struct pci_bus *bus) +void pcibios_fixup_bus(struct pci_bus *bus) { struct pci_dev *dev; int i, has_io, has_mem; @@ -666,30 +670,6 @@ void __devinit pcibios_fixup_bus(struct pci_bus *bus) } } -/* - * pcic_pin_to_irq() is exported to bus probing code - */ -unsigned int -pcic_pin_to_irq(unsigned int pin, const char *name) -{ - struct linux_pcic *pcic = &pcic0; - unsigned int irq; - unsigned int ivec; - - if (pin < 4) { - ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_LO); - irq = ivec >> (pin << 2) & 0xF; - } else if (pin < 8) { - ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); - irq = ivec >> ((pin-4) << 2) & 0xF; - } else { /* Corrupted map */ - printk("PCIC: BAD PIN %d FOR %s\n", pin, name); - for (;;) {} /* XXX Cannot panic properly in case of PROLL */ - } -/* P3 */ /* printk("PCIC: dev %s pin %d ivec 0x%x irq %x\n", name, pin, ivec, irq); */ - return irq; -} - /* Makes compiler happy */ static volatile int pcic_timer_dummy; @@ -698,43 +678,46 @@ static void pcic_clear_clock_irq(void) pcic_timer_dummy = readl(pcic0.pcic_regs+PCI_SYS_LIMIT); } -static irqreturn_t pcic_timer_handler (int irq, void *h) +/* CPU frequency is 100 MHz, timer increments every 4 CPU clocks */ +#define USECS_PER_JIFFY (1000000 / HZ) +#define TICK_TIMER_LIMIT ((100 * 1000000 / 4) / HZ) + +static unsigned int pcic_cycles_offset(void) { - write_seqlock(&xtime_lock); /* Dummy, to show that we remember */ - pcic_clear_clock_irq(); - do_timer(1); - write_sequnlock(&xtime_lock); -#ifndef CONFIG_SMP - update_process_times(user_mode(get_irq_regs())); -#endif - return IRQ_HANDLED; -} + u32 value, count; -#define USECS_PER_JIFFY 10000 /* We have 100HZ "standard" timer for sparc */ -#define TICK_TIMER_LIMIT ((100*1000000/4)/100) + value = readl(pcic0.pcic_regs + PCI_SYS_COUNTER); + count = value & ~PCI_SYS_COUNTER_OVERFLOW; -u32 pci_gettimeoffset(void) -{ + if (value & PCI_SYS_COUNTER_OVERFLOW) + count += TICK_TIMER_LIMIT; /* - * We divide all by 100 + * We divide all by HZ * to have microsecond resolution and to avoid overflow */ - unsigned long count = - readl(pcic0.pcic_regs+PCI_SYS_COUNTER) & ~PCI_SYS_COUNTER_OVERFLOW; - count = ((count/100)*USECS_PER_JIFFY) / (TICK_TIMER_LIMIT/100); - return count * 1000; -} + count = ((count / HZ) * USECS_PER_JIFFY) / (TICK_TIMER_LIMIT / HZ); + /* Coordinate with the sparc_config.clock_rate setting */ + return count * 2; +} void __init pci_time_init(void) { struct linux_pcic *pcic = &pcic0; unsigned long v; int timer_irq, irq; + int err; - do_arch_gettimeoffset = pci_gettimeoffset; - - btfixup(); +#ifndef CONFIG_SMP + /* + * The clock_rate is in SBUS dimension. + * We take into account this in pcic_cycles_offset() + */ + sparc_config.clock_rate = SBUS_CLOCK_RATE / HZ; + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sparc_config.get_cycles_offset = pcic_cycles_offset; writel (TICK_TIMER_LIMIT, pcic->pcic_regs+PCI_SYS_LIMIT); /* PROM should set appropriate irq */ @@ -742,9 +725,10 @@ void __init pci_time_init(void) timer_irq = PCI_COUNTER_IRQ_SYS(v); writel (PCI_COUNTER_IRQ_SET(timer_irq, 0), pcic->pcic_regs+PCI_COUNTER_IRQ); - irq = request_irq(timer_irq, pcic_timer_handler, - (IRQF_DISABLED | SA_STATIC_ALLOC), "timer", NULL); - if (irq) { + irq = pcic_build_device_irq(NULL, timer_irq); + err = request_irq(irq, timer_interrupt, + IRQF_TIMER, "timer", NULL); + if (err) { prom_printf("time_init: unable to attach IRQ%d\n", timer_irq); prom_halt(); } @@ -758,14 +742,6 @@ static void watchdog_reset() { } #endif -/* - * Other archs parse arguments here. - */ -char * __devinit pcibios_setup(char *str) -{ - return str; -} - resource_size_t pcibios_align_resource(void *data, const struct resource *res, resource_size_t size, resource_size_t align) { @@ -783,7 +759,7 @@ int pcibios_enable_device(struct pci_dev *pdev, int mask) void pcic_nmi(unsigned int pend, struct pt_regs *regs) { - pend = flip_dword(pend); + pend = swab32(pend); if (!pcic_speculative || (pend & PCI_SYS_INT_PENDING_PIO) == 0) { /* @@ -805,143 +781,74 @@ static inline unsigned long get_irqmask(int irq_nr) return 1 << irq_nr; } -static void pcic_disable_irq(unsigned int irq_nr) +static void pcic_mask_irq(struct irq_data *data) { unsigned long mask, flags; - mask = get_irqmask(irq_nr); + mask = (unsigned long)data->chip_data; local_irq_save(flags); writel(mask, pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_SET); local_irq_restore(flags); } -static void pcic_enable_irq(unsigned int irq_nr) +static void pcic_unmask_irq(struct irq_data *data) { unsigned long mask, flags; - mask = get_irqmask(irq_nr); + mask = (unsigned long)data->chip_data; local_irq_save(flags); writel(mask, pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_CLEAR); local_irq_restore(flags); } -static void pcic_load_profile_irq(int cpu, unsigned int limit) -{ - printk("PCIC: unimplemented code: FILE=%s LINE=%d", __FILE__, __LINE__); -} - -/* We assume the caller has disabled local interrupts when these are called, - * or else very bizarre behavior will result. - */ -static void pcic_disable_pil_irq(unsigned int pil) +static unsigned int pcic_startup_irq(struct irq_data *data) { - writel(get_irqmask(pil), pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_SET); + irq_link(data->irq); + pcic_unmask_irq(data); + return 0; } -static void pcic_enable_pil_irq(unsigned int pil) -{ - writel(get_irqmask(pil), pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_CLEAR); -} +static struct irq_chip pcic_irq = { + .name = "pcic", + .irq_startup = pcic_startup_irq, + .irq_mask = pcic_mask_irq, + .irq_unmask = pcic_unmask_irq, +}; -void __init sun4m_pci_init_IRQ(void) +unsigned int pcic_build_device_irq(struct platform_device *op, + unsigned int real_irq) { - BTFIXUPSET_CALL(enable_irq, pcic_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_irq, pcic_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(enable_pil_irq, pcic_enable_pil_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_pil_irq, pcic_disable_pil_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_clock_irq, pcic_clear_clock_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(load_profile_irq, pcic_load_profile_irq, BTFIXUPCALL_NORM); -} + unsigned int irq; + unsigned long mask; -int pcibios_assign_resource(struct pci_dev *pdev, int resource) -{ - return -ENXIO; -} + irq = 0; + mask = get_irqmask(real_irq); + if (mask == 0) + goto out; -struct device_node *pci_device_to_OF_node(struct pci_dev *pdev) -{ - struct pcidev_cookie *pc = pdev->sysdata; + irq = irq_alloc(real_irq, real_irq); + if (irq == 0) + goto out; - return pc->prom_node; -} -EXPORT_SYMBOL(pci_device_to_OF_node); + irq_set_chip_and_handler_name(irq, &pcic_irq, + handle_level_irq, "PCIC"); + irq_set_chip_data(irq, (void *)mask); -/* - * This probably belongs here rather than ioport.c because - * we do not want this crud linked into SBus kernels. - * Also, think for a moment about likes of floppy.c that - * include architecture specific parts. They may want to redefine ins/outs. - * - * We do not use horrible macros here because we want to - * advance pointer by sizeof(size). - */ -void outsb(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 1; - outb(*(const char *)src, addr); - src += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(outsb); - -void outsw(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 2; - outw(*(const short *)src, addr); - src += 2; - /* addr += 2; */ - } -} -EXPORT_SYMBOL(outsw); - -void outsl(unsigned long addr, const void *src, unsigned long count) -{ - while (count) { - count -= 4; - outl(*(const long *)src, addr); - src += 4; - /* addr += 4; */ - } +out: + return irq; } -EXPORT_SYMBOL(outsl); -void insb(unsigned long addr, void *dst, unsigned long count) -{ - while (count) { - count -= 1; - *(unsigned char *)dst = inb(addr); - dst += 1; - /* addr += 1; */ - } -} -EXPORT_SYMBOL(insb); -void insw(unsigned long addr, void *dst, unsigned long count) +static void pcic_load_profile_irq(int cpu, unsigned int limit) { - while (count) { - count -= 2; - *(unsigned short *)dst = inw(addr); - dst += 2; - /* addr += 2; */ - } + printk("PCIC: unimplemented code: FILE=%s LINE=%d", __FILE__, __LINE__); } -EXPORT_SYMBOL(insw); -void insl(unsigned long addr, void *dst, unsigned long count) +void __init sun4m_pci_init_IRQ(void) { - while (count) { - count -= 4; - /* - * XXX I am sure we are in for an unaligned trap here. - */ - *(unsigned long *)dst = inl(addr); - dst += 4; - /* addr += 4; */ - } + sparc_config.build_device_irq = pcic_build_device_irq; + sparc_config.clear_clock_irq = pcic_clear_clock_irq; + sparc_config.load_profile_irq = pcic_load_profile_irq; } -EXPORT_SYMBOL(insl); subsys_initcall(pcic_init); diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c index 7c2ced612b8..269af58497a 100644 --- a/arch/sparc/kernel/pcr.c +++ b/arch/sparc/kernel/pcr.c @@ -3,7 +3,7 @@ * Copyright (C) 2009 David S. Miller (davem@davemloft.net) */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/irq.h> @@ -13,21 +13,14 @@ #include <asm/pil.h> #include <asm/pcr.h> #include <asm/nmi.h> +#include <asm/asi.h> +#include <asm/spitfire.h> /* This code is shared between various users of the performance * counters. Users will be oprofile, pseudo-NMI watchdog, and the * perf_event support layer. */ -#define PCR_SUN4U_ENABLE (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE) -#define PCR_N2_ENABLE (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE | \ - PCR_N2_TOE_OV1 | \ - (2 << PCR_N2_SL1_SHIFT) | \ - (0xff << PCR_N2_MASK1_SHIFT)) - -u64 pcr_enable; -unsigned int picl_shift; - /* Performance counter interrupts run unmasked at PIL level 15. * Therefore we can't do things like wakeups and other work * that expects IRQ disabling to be adhered to in locking etc. @@ -58,36 +51,144 @@ void arch_irq_work_raise(void) const struct pcr_ops *pcr_ops; EXPORT_SYMBOL_GPL(pcr_ops); -static u64 direct_pcr_read(void) +static u64 direct_pcr_read(unsigned long reg_num) +{ + u64 val; + + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("rd %%pcr, %0" : "=r" (val)); + return val; +} + +static void direct_pcr_write(unsigned long reg_num, u64 val) +{ + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("wr %0, 0x0, %%pcr" : : "r" (val)); +} + +static u64 direct_pic_read(unsigned long reg_num) { u64 val; - read_pcr(val); + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("rd %%pic, %0" : "=r" (val)); return val; } -static void direct_pcr_write(u64 val) +static void direct_pic_write(unsigned long reg_num, u64 val) +{ + WARN_ON_ONCE(reg_num != 0); + + /* Blackbird errata workaround. See commentary in + * arch/sparc64/kernel/smp.c:smp_percpu_timer_interrupt() + * for more information. + */ + __asm__ __volatile__("ba,pt %%xcc, 99f\n\t" + " nop\n\t" + ".align 64\n" + "99:wr %0, 0x0, %%pic\n\t" + "rd %%pic, %%g0" : : "r" (val)); +} + +static u64 direct_picl_value(unsigned int nmi_hz) { - write_pcr(val); + u32 delta = local_cpu_data().clock_tick / nmi_hz; + + return ((u64)((0 - delta) & 0xffffffff)) << 32; } static const struct pcr_ops direct_pcr_ops = { - .read = direct_pcr_read, - .write = direct_pcr_write, + .read_pcr = direct_pcr_read, + .write_pcr = direct_pcr_write, + .read_pic = direct_pic_read, + .write_pic = direct_pic_write, + .nmi_picl_value = direct_picl_value, + .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE), + .pcr_nmi_disable = PCR_PIC_PRIV, }; -static void n2_pcr_write(u64 val) +static void n2_pcr_write(unsigned long reg_num, u64 val) { unsigned long ret; - ret = sun4v_niagara2_setperf(HV_N2_PERF_SPARC_CTL, val); - if (val != HV_EOK) - write_pcr(val); + WARN_ON_ONCE(reg_num != 0); + if (val & PCR_N2_HTRACE) { + ret = sun4v_niagara2_setperf(HV_N2_PERF_SPARC_CTL, val); + if (ret != HV_EOK) + direct_pcr_write(reg_num, val); + } else + direct_pcr_write(reg_num, val); +} + +static u64 n2_picl_value(unsigned int nmi_hz) +{ + u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); + + return ((u64)((0 - delta) & 0xffffffff)) << 32; } static const struct pcr_ops n2_pcr_ops = { - .read = direct_pcr_read, - .write = n2_pcr_write, + .read_pcr = direct_pcr_read, + .write_pcr = n2_pcr_write, + .read_pic = direct_pic_read, + .write_pic = direct_pic_write, + .nmi_picl_value = n2_picl_value, + .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE | + PCR_N2_TOE_OV1 | + (2 << PCR_N2_SL1_SHIFT) | + (0xff << PCR_N2_MASK1_SHIFT)), + .pcr_nmi_disable = PCR_PIC_PRIV, +}; + +static u64 n4_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_vt_get_perfreg(reg_num, &val); + + return val; +} + +static void n4_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_vt_set_perfreg(reg_num, val); +} + +static u64 n4_pic_read(unsigned long reg_num) +{ + unsigned long val; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (val) + : "r" (reg_num * 0x8UL), "i" (ASI_PIC)); + + return val; +} + +static void n4_pic_write(unsigned long reg_num, u64 val) +{ + __asm__ __volatile__("stxa %0, [%1] %2" + : /* no outputs */ + : "r" (val), "r" (reg_num * 0x8UL), "i" (ASI_PIC)); +} + +static u64 n4_picl_value(unsigned int nmi_hz) +{ + u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); + + return ((u64)((0 - delta) & 0xffffffff)); +} + +static const struct pcr_ops n4_pcr_ops = { + .read_pcr = n4_pcr_read, + .write_pcr = n4_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, }; static unsigned long perf_hsvc_group; @@ -106,6 +207,14 @@ static int __init register_perf_hsvc(void) perf_hsvc_group = HV_GRP_N2_CPU; break; + case SUN4V_CHIP_NIAGARA3: + perf_hsvc_group = HV_GRP_KT_CPU; + break; + + case SUN4V_CHIP_NIAGARA4: + perf_hsvc_group = HV_GRP_VT_CPU; + break; + default: return -ENODEV; } @@ -130,6 +239,29 @@ static void __init unregister_perf_hsvc(void) sun4v_hvapi_unregister(perf_hsvc_group); } +static int __init setup_sun4v_pcr_ops(void) +{ + int ret = 0; + + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + case SUN4V_CHIP_NIAGARA3: + pcr_ops = &n2_pcr_ops; + break; + + case SUN4V_CHIP_NIAGARA4: + pcr_ops = &n4_pcr_ops; + break; + + default: + ret = -ENODEV; + break; + } + + return ret; +} + int __init pcr_arch_init(void) { int err = register_perf_hsvc(); @@ -139,15 +271,14 @@ int __init pcr_arch_init(void) switch (tlb_type) { case hypervisor: - pcr_ops = &n2_pcr_ops; - pcr_enable = PCR_N2_ENABLE; - picl_shift = 2; + err = setup_sun4v_pcr_ops(); + if (err) + goto out_unregister; break; case cheetah: case cheetah_plus: pcr_ops = &direct_pcr_ops; - pcr_enable = PCR_SUN4U_ENABLE; break; case spitfire: diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c index 760578687e7..8efd33753ad 100644 --- a/arch/sparc/kernel/perf_event.c +++ b/arch/sparc/kernel/perf_event.c @@ -22,36 +22,51 @@ #include <asm/stacktrace.h> #include <asm/cpudata.h> #include <asm/uaccess.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/nmi.h> #include <asm/pcr.h> +#include <asm/cacheflush.h> +#include "kernel.h" #include "kstack.h" -/* Sparc64 chips have two performance counters, 32-bits each, with - * overflow interrupts generated on transition from 0xffffffff to 0. - * The counters are accessed in one go using a 64-bit register. +/* Two classes of sparc64 chips currently exist. All of which have + * 32-bit counters which can generate overflow interrupts on the + * transition from 0xffffffff to 0. * - * Both counters are controlled using a single control register. The - * only way to stop all sampling is to clear all of the context (user, - * supervisor, hypervisor) sampling enable bits. But these bits apply - * to both counters, thus the two counters can't be enabled/disabled - * individually. + * All chips upto and including SPARC-T3 have two performance + * counters. The two 32-bit counters are accessed in one go using a + * single 64-bit register. * - * The control register has two event fields, one for each of the two - * counters. It's thus nearly impossible to have one counter going - * while keeping the other one stopped. Therefore it is possible to - * get overflow interrupts for counters not currently "in use" and - * that condition must be checked in the overflow interrupt handler. + * On these older chips both counters are controlled using a single + * control register. The only way to stop all sampling is to clear + * all of the context (user, supervisor, hypervisor) sampling enable + * bits. But these bits apply to both counters, thus the two counters + * can't be enabled/disabled individually. + * + * Furthermore, the control register on these older chips have two + * event fields, one for each of the two counters. It's thus nearly + * impossible to have one counter going while keeping the other one + * stopped. Therefore it is possible to get overflow interrupts for + * counters not currently "in use" and that condition must be checked + * in the overflow interrupt handler. * * So we use a hack, in that we program inactive counters with the * "sw_count0" and "sw_count1" events. These count how many times * the instruction "sethi %hi(0xfc000), %g0" is executed. It's an * unusual way to encode a NOP and therefore will not trigger in * normal code. + * + * Starting with SPARC-T4 we have one control register per counter. + * And the counters are stored in individual registers. The registers + * for the counters are 64-bit but only a 32-bit counter is + * implemented. The event selections on SPARC-T4 lack any + * restrictions, therefore we can elide all of the complicated + * conflict resolution code we have for SPARC-T3 and earlier chips. */ -#define MAX_HWEVENTS 2 +#define MAX_HWEVENTS 4 +#define MAX_PCRS 4 #define MAX_PERIOD ((1UL << 32) - 1) #define PIC_UPPER_INDEX 0 @@ -87,19 +102,21 @@ struct cpu_hw_events { */ int current_idx[MAX_HWEVENTS]; - /* Software copy of %pcr register on this cpu. */ - u64 pcr; + /* Software copy of %pcr register(s) on this cpu. */ + u64 pcr[MAX_HWEVENTS]; /* Enabled/disable state. */ int enabled; unsigned int group_flag; }; -DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; /* An event map describes the characteristics of a performance * counter event. In particular it gives the encoding as well as * a mask telling which counters the event can be measured on. + * + * The mask is unused on SPARC-T4 and later. */ struct perf_event_map { u16 encoding; @@ -139,15 +156,53 @@ struct sparc_pmu { const struct perf_event_map *(*event_map)(int); const cache_map_t *cache_map; int max_events; + u32 (*read_pmc)(int); + void (*write_pmc)(int, u64); int upper_shift; int lower_shift; int event_mask; + int user_bit; + int priv_bit; int hv_bit; int irq_bit; int upper_nop; int lower_nop; + unsigned int flags; +#define SPARC_PMU_ALL_EXCLUDES_SAME 0x00000001 +#define SPARC_PMU_HAS_CONFLICTS 0x00000002 + int max_hw_events; + int num_pcrs; + int num_pic_regs; }; +static u32 sparc_default_read_pmc(int idx) +{ + u64 val; + + val = pcr_ops->read_pic(0); + if (idx == PIC_UPPER_INDEX) + val >>= 32; + + return val & 0xffffffff; +} + +static void sparc_default_write_pmc(int idx, u64 val) +{ + u64 shift, mask, pic; + + shift = 0; + if (idx == PIC_UPPER_INDEX) + shift = 32; + + mask = ((u64) 0xffffffff) << shift; + val <<= shift; + + pic = pcr_ops->read_pic(0); + pic &= ~mask; + pic |= val; + pcr_ops->write_pic(0, pic); +} + static const struct perf_event_map ultra3_perfmon_event_map[] = { [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER }, [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER }, @@ -245,17 +300,40 @@ static const cache_map_t ultra3_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu ultra3_pmu = { .event_map = ultra3_event_map, .cache_map = &ultra3_cache_map, .max_events = ARRAY_SIZE(ultra3_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, .upper_shift = 11, .lower_shift = 4, .event_mask = 0x3f, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, .upper_nop = 0x1c, .lower_nop = 0x14, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, }; /* Niagara1 is very limited. The upper PIC is hard-locked to count @@ -360,17 +438,40 @@ static const cache_map_t niagara1_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu niagara1_pmu = { .event_map = niagara1_event_map, .cache_map = &niagara1_cache_map, .max_events = ARRAY_SIZE(niagara1_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, .upper_shift = 0, .lower_shift = 4, .event_mask = 0x7, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, .upper_nop = 0x0, .lower_nop = 0x0, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, }; static const struct perf_event_map niagara2_perfmon_event_map[] = { @@ -472,19 +573,223 @@ static const cache_map_t niagara2_cache_map = { [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, }, }, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, }; static const struct sparc_pmu niagara2_pmu = { .event_map = niagara2_event_map, .cache_map = &niagara2_cache_map, .max_events = ARRAY_SIZE(niagara2_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, .upper_shift = 19, .lower_shift = 6, .event_mask = 0xfff, - .hv_bit = 0x8, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, + .hv_bit = PCR_N2_HTRACE, .irq_bit = 0x30, .upper_nop = 0x220, .lower_nop = 0x220, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, +}; + +static const struct perf_event_map niagara4_perfmon_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = { (26 << 6) }, + [PERF_COUNT_HW_INSTRUCTIONS] = { (3 << 6) | 0x3f }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { (3 << 6) | 0x04 }, + [PERF_COUNT_HW_CACHE_MISSES] = { (16 << 6) | 0x07 }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { (4 << 6) | 0x01 }, + [PERF_COUNT_HW_BRANCH_MISSES] = { (25 << 6) | 0x0f }, +}; + +static const struct perf_event_map *niagara4_event_map(int event_id) +{ + return &niagara4_perfmon_event_map[event_id]; +} + +static const cache_map_t niagara4_cache_map = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 }, + [C(RESULT_MISS)] = { (16 << 6) | 0x07 }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 }, + [C(RESULT_MISS)] = { (16 << 6) | 0x07 }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x3f }, + [C(RESULT_MISS)] = { (11 << 6) | 0x03 }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE }, + [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { (17 << 6) | 0x3f }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { (6 << 6) | 0x3f }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +}; + +static u32 sparc_vt_read_pmc(int idx) +{ + u64 val = pcr_ops->read_pic(idx); + + return val & 0xffffffff; +} + +static void sparc_vt_write_pmc(int idx, u64 val) +{ + u64 pcr; + + /* There seems to be an internal latch on the overflow event + * on SPARC-T4 that prevents it from triggering unless you + * update the PIC exactly as we do here. The requirement + * seems to be that you have to turn off event counting in the + * PCR around the PIC update. + * + * For example, after the following sequence: + * + * 1) set PIC to -1 + * 2) enable event counting and overflow reporting in PCR + * 3) overflow triggers, softint 15 handler invoked + * 4) clear OV bit in PCR + * 5) write PIC to -1 + * + * a subsequent overflow event will not trigger. This + * sequence works on SPARC-T3 and previous chips. + */ + pcr = pcr_ops->read_pcr(idx); + pcr_ops->write_pcr(idx, PCR_N4_PICNPT); + + pcr_ops->write_pic(idx, val & 0xffffffff); + + pcr_ops->write_pcr(idx, pcr); +} + +static const struct sparc_pmu niagara4_pmu = { + .event_map = niagara4_event_map, + .cache_map = &niagara4_cache_map, + .max_events = ARRAY_SIZE(niagara4_perfmon_event_map), + .read_pmc = sparc_vt_read_pmc, + .write_pmc = sparc_vt_write_pmc, + .upper_shift = 5, + .lower_shift = 5, + .event_mask = 0x7ff, + .user_bit = PCR_N4_UTRACE, + .priv_bit = PCR_N4_STRACE, + + /* We explicitly don't support hypervisor tracing. The T4 + * generates the overflow event for precise events via a trap + * which will not be generated (ie. it's completely lost) if + * we happen to be in the hypervisor when the event triggers. + * Essentially, the overflow event reporting is completely + * unusable when you have hypervisor mode tracing enabled. + */ + .hv_bit = 0, + + .irq_bit = PCR_N4_TOE, + .upper_nop = 0, + .lower_nop = 0, + .flags = 0, + .max_hw_events = 4, + .num_pcrs = 4, + .num_pic_regs = 4, }; static const struct sparc_pmu *sparc_pmu __read_mostly; @@ -512,56 +817,38 @@ static u64 nop_for_index(int idx) static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx) { - u64 val, mask = mask_for_index(idx); + u64 enc, val, mask = mask_for_index(idx); + int pcr_index = 0; + + if (sparc_pmu->num_pcrs > 1) + pcr_index = idx; - val = cpuc->pcr; + enc = perf_event_get_enc(cpuc->events[idx]); + + val = cpuc->pcr[pcr_index]; val &= ~mask; - val |= hwc->config; - cpuc->pcr = val; + val |= event_encoding(enc, idx); + cpuc->pcr[pcr_index] = val; - pcr_ops->write(cpuc->pcr); + pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]); } static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx) { u64 mask = mask_for_index(idx); u64 nop = nop_for_index(idx); + int pcr_index = 0; u64 val; - val = cpuc->pcr; + if (sparc_pmu->num_pcrs > 1) + pcr_index = idx; + + val = cpuc->pcr[pcr_index]; val &= ~mask; val |= nop; - cpuc->pcr = val; - - pcr_ops->write(cpuc->pcr); -} + cpuc->pcr[pcr_index] = val; -static u32 read_pmc(int idx) -{ - u64 val; - - read_pic(val); - if (idx == PIC_UPPER_INDEX) - val >>= 32; - - return val & 0xffffffff; -} - -static void write_pmc(int idx, u64 val) -{ - u64 shift, mask, pic; - - shift = 0; - if (idx == PIC_UPPER_INDEX) - shift = 32; - - mask = ((u64) 0xffffffff) << shift; - val <<= shift; - - read_pic(pic); - pic &= ~mask; - pic |= val; - write_pic(pic); + pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]); } static u64 sparc_perf_event_update(struct perf_event *event, @@ -573,7 +860,7 @@ static u64 sparc_perf_event_update(struct perf_event *event, again: prev_raw_count = local64_read(&hwc->prev_count); - new_raw_count = read_pmc(idx); + new_raw_count = sparc_pmu->read_pmc(idx); if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, new_raw_count) != prev_raw_count) @@ -613,25 +900,17 @@ static int sparc_perf_event_set_period(struct perf_event *event, local64_set(&hwc->prev_count, (u64)-left); - write_pmc(idx, (u64)(-left) & 0xffffffff); + sparc_pmu->write_pmc(idx, (u64)(-left) & 0xffffffff); perf_event_update_userpage(event); return ret; } -/* If performance event entries have been added, move existing - * events around (if necessary) and then assign new entries to - * counters. - */ -static u64 maybe_change_configuration(struct cpu_hw_events *cpuc, u64 pcr) +static void read_in_all_counters(struct cpu_hw_events *cpuc) { int i; - if (!cpuc->n_added) - goto out; - - /* Read in the counters which are moving. */ for (i = 0; i < cpuc->n_events; i++) { struct perf_event *cp = cpuc->event[i]; @@ -642,6 +921,20 @@ static u64 maybe_change_configuration(struct cpu_hw_events *cpuc, u64 pcr) cpuc->current_idx[i] = PIC_NO_INDEX; } } +} + +/* On this PMU all PICs are programmed using a single PCR. Calculate + * the combined control register value. + * + * For such chips we require that all of the events have the same + * configuration, so just fetch the settings from the first entry. + */ +static void calculate_single_pcr(struct cpu_hw_events *cpuc) +{ + int i; + + if (!cpuc->n_added) + goto out; /* Assign to counters all unassigned events. */ for (i = 0; i < cpuc->n_events; i++) { @@ -657,20 +950,71 @@ static u64 maybe_change_configuration(struct cpu_hw_events *cpuc, u64 pcr) cpuc->current_idx[i] = idx; enc = perf_event_get_enc(cpuc->events[i]); - pcr &= ~mask_for_index(idx); + cpuc->pcr[0] &= ~mask_for_index(idx); if (hwc->state & PERF_HES_STOPPED) - pcr |= nop_for_index(idx); + cpuc->pcr[0] |= nop_for_index(idx); else - pcr |= event_encoding(enc, idx); + cpuc->pcr[0] |= event_encoding(enc, idx); } out: - return pcr; + cpuc->pcr[0] |= cpuc->event[0]->hw.config_base; +} + +/* On this PMU each PIC has it's own PCR control register. */ +static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc) +{ + int i; + + if (!cpuc->n_added) + goto out; + + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + struct hw_perf_event *hwc = &cp->hw; + int idx = hwc->idx; + u64 enc; + + if (cpuc->current_idx[i] != PIC_NO_INDEX) + continue; + + sparc_perf_event_set_period(cp, hwc, idx); + cpuc->current_idx[i] = idx; + + enc = perf_event_get_enc(cpuc->events[i]); + cpuc->pcr[idx] &= ~mask_for_index(idx); + if (hwc->state & PERF_HES_STOPPED) + cpuc->pcr[idx] |= nop_for_index(idx); + else + cpuc->pcr[idx] |= event_encoding(enc, idx); + } +out: + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + int idx = cp->hw.idx; + + cpuc->pcr[idx] |= cp->hw.config_base; + } +} + +/* If performance event entries have been added, move existing events + * around (if necessary) and then assign new entries to counters. + */ +static void update_pcrs_for_enable(struct cpu_hw_events *cpuc) +{ + if (cpuc->n_added) + read_in_all_counters(cpuc); + + if (sparc_pmu->num_pcrs == 1) { + calculate_single_pcr(cpuc); + } else { + calculate_multiple_pcrs(cpuc); + } } static void sparc_pmu_enable(struct pmu *pmu) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - u64 pcr; + int i; if (cpuc->enabled) return; @@ -678,26 +1022,17 @@ static void sparc_pmu_enable(struct pmu *pmu) cpuc->enabled = 1; barrier(); - pcr = cpuc->pcr; - if (!cpuc->n_events) { - pcr = 0; - } else { - pcr = maybe_change_configuration(cpuc, pcr); + if (cpuc->n_events) + update_pcrs_for_enable(cpuc); - /* We require that all of the events have the same - * configuration, so just fetch the settings from the - * first entry. - */ - cpuc->pcr = pcr | cpuc->event[0]->hw.config_base; - } - - pcr_ops->write(cpuc->pcr); + for (i = 0; i < sparc_pmu->num_pcrs; i++) + pcr_ops->write_pcr(i, cpuc->pcr[i]); } static void sparc_pmu_disable(struct pmu *pmu) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); - u64 val; + int i; if (!cpuc->enabled) return; @@ -705,12 +1040,14 @@ static void sparc_pmu_disable(struct pmu *pmu) cpuc->enabled = 0; cpuc->n_added = 0; - val = cpuc->pcr; - val &= ~(PCR_UTRACE | PCR_STRACE | - sparc_pmu->hv_bit | sparc_pmu->irq_bit); - cpuc->pcr = val; + for (i = 0; i < sparc_pmu->num_pcrs; i++) { + u64 val = cpuc->pcr[i]; - pcr_ops->write(cpuc->pcr); + val &= ~(sparc_pmu->user_bit | sparc_pmu->priv_bit | + sparc_pmu->hv_bit | sparc_pmu->irq_bit); + cpuc->pcr[i] = val; + pcr_ops->write_pcr(i, cpuc->pcr[i]); + } } static int active_event_index(struct cpu_hw_events *cpuc, @@ -809,12 +1146,14 @@ static DEFINE_MUTEX(pmc_grab_mutex); static void perf_stop_nmi_watchdog(void *unused) { struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + int i; stop_nmi_watchdog(NULL); - cpuc->pcr = pcr_ops->read(); + for (i = 0; i < sparc_pmu->num_pcrs; i++) + cpuc->pcr[i] = pcr_ops->read_pcr(i); } -void perf_event_grab_pmc(void) +static void perf_event_grab_pmc(void) { if (atomic_inc_not_zero(&active_events)) return; @@ -830,7 +1169,7 @@ void perf_event_grab_pmc(void) mutex_unlock(&pmc_grab_mutex); } -void perf_event_release_pmc(void) +static void perf_event_release_pmc(void) { if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) { if (atomic_read(&nmi_active) == 0) @@ -897,9 +1236,17 @@ static int sparc_check_constraints(struct perf_event **evts, if (!n_ev) return 0; - if (n_ev > MAX_HWEVENTS) + if (n_ev > sparc_pmu->max_hw_events) return -1; + if (!(sparc_pmu->flags & SPARC_PMU_HAS_CONFLICTS)) { + int i; + + for (i = 0; i < n_ev; i++) + evts[i]->hw.idx = i; + return 0; + } + msk0 = perf_event_get_msk(events[0]); if (n_ev == 1) { if (msk0 & PIC_LOWER) @@ -955,6 +1302,9 @@ static int check_excludes(struct perf_event **evts, int n_prev, int n_new) struct perf_event *event; int i, n, first; + if (!(sparc_pmu->flags & SPARC_PMU_ALL_EXCLUDES_SAME)) + return 0; + n = n_prev + n_new; if (n <= 1) return 0; @@ -1014,7 +1364,7 @@ static int sparc_pmu_add(struct perf_event *event, int ef_flags) perf_pmu_disable(event->pmu); n0 = cpuc->n_events; - if (n0 >= MAX_HWEVENTS) + if (n0 >= sparc_pmu->max_hw_events) goto out; cpuc->event[n0] = event; @@ -1027,7 +1377,7 @@ static int sparc_pmu_add(struct perf_event *event, int ef_flags) /* * If group events scheduling transaction was started, - * skip the schedulability test here, it will be peformed + * skip the schedulability test here, it will be performed * at commit time(->commit_txn) as a whole */ if (cpuc->group_flag & PERF_EVENT_TXN) @@ -1062,6 +1412,10 @@ static int sparc_pmu_event_init(struct perf_event *event) if (atomic_read(&nmi_active) < 0) return -ENODEV; + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + switch (attr->type) { case PERF_TYPE_HARDWARE: if (attr->config >= sparc_pmu->max_events) @@ -1097,16 +1451,16 @@ static int sparc_pmu_event_init(struct perf_event *event) /* We save the enable bits in the config_base. */ hwc->config_base = sparc_pmu->irq_bit; if (!attr->exclude_user) - hwc->config_base |= PCR_UTRACE; + hwc->config_base |= sparc_pmu->user_bit; if (!attr->exclude_kernel) - hwc->config_base |= PCR_STRACE; + hwc->config_base |= sparc_pmu->priv_bit; if (!attr->exclude_hv) hwc->config_base |= sparc_pmu->hv_bit; n = 0; if (event->group_leader != event) { n = collect_events(event->group_leader, - MAX_HWEVENTS - 1, + sparc_pmu->max_hw_events - 1, evts, events, current_idx_dmy); if (n < 0) return -EINVAL; @@ -1205,8 +1559,7 @@ static struct pmu pmu = { void perf_event_print_debug(void) { unsigned long flags; - u64 pcr, pic; - int cpu; + int cpu, i; if (!sparc_pmu) return; @@ -1215,12 +1568,13 @@ void perf_event_print_debug(void) cpu = smp_processor_id(); - pcr = pcr_ops->read(); - read_pic(pic); - pr_info("\n"); - pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n", - cpu, pcr, pic); + for (i = 0; i < sparc_pmu->num_pcrs; i++) + pr_info("CPU#%d: PCR%d[%016llx]\n", + cpu, i, pcr_ops->read_pcr(i)); + for (i = 0; i < sparc_pmu->num_pic_regs; i++) + pr_info("CPU#%d: PIC%d[%016llx]\n", + cpu, i, pcr_ops->read_pic(i)); local_irq_restore(flags); } @@ -1247,8 +1601,6 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, regs = args->regs; - perf_sample_data_init(&data, 0); - cpuc = &__get_cpu_var(cpu_hw_events); /* If the PMU has the TOE IRQ enable bits, we need to do a @@ -1258,8 +1610,9 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, * Do this before we peek at the counters to determine * overflow so we don't lose any events. */ - if (sparc_pmu->irq_bit) - pcr_ops->write(cpuc->pcr); + if (sparc_pmu->irq_bit && + sparc_pmu->num_pcrs == 1) + pcr_ops->write_pcr(0, cpuc->pcr[0]); for (i = 0; i < cpuc->n_events; i++) { struct perf_event *event = cpuc->event[i]; @@ -1267,16 +1620,20 @@ static int __kprobes perf_event_nmi_handler(struct notifier_block *self, struct hw_perf_event *hwc; u64 val; + if (sparc_pmu->irq_bit && + sparc_pmu->num_pcrs > 1) + pcr_ops->write_pcr(idx, cpuc->pcr[idx]); + hwc = &event->hw; val = sparc_perf_event_update(event, hwc, idx); if (val & (1ULL << 31)) continue; - data.period = event->hw.last_period; + perf_sample_data_init(&data, 0, hwc->last_period); if (!sparc_perf_event_set_period(event, hwc, idx)) continue; - if (perf_event_overflow(event, 1, &data, regs)) + if (perf_event_overflow(event, &data, regs)) sparc_pmu_stop(event, 0); } @@ -1300,14 +1657,19 @@ static bool __init supported_pmu(void) sparc_pmu = &niagara1_pmu; return true; } - if (!strcmp(sparc_pmu_type, "niagara2")) { + if (!strcmp(sparc_pmu_type, "niagara2") || + !strcmp(sparc_pmu_type, "niagara3")) { sparc_pmu = &niagara2_pmu; return true; } + if (!strcmp(sparc_pmu_type, "niagara4")) { + sparc_pmu = &niagara4_pmu; + return true; + } return false; } -int __init init_hw_perf_events(void) +static int __init init_hw_perf_events(void) { pr_info("Performance events: "); @@ -1378,14 +1740,13 @@ static void perf_callchain_user_64(struct perf_callchain_entry *entry, { unsigned long ufp; - perf_callchain_store(entry, regs->tpc); - ufp = regs->u_regs[UREG_I6] + STACK_BIAS; do { - struct sparc_stackf *usf, sf; + struct sparc_stackf __user *usf; + struct sparc_stackf sf; unsigned long pc; - usf = (struct sparc_stackf *) ufp; + usf = (struct sparc_stackf __user *)ufp; if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) break; @@ -1400,19 +1761,29 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, { unsigned long ufp; - perf_callchain_store(entry, regs->tpc); - ufp = regs->u_regs[UREG_I6] & 0xffffffffUL; do { - struct sparc_stackf32 *usf, sf; unsigned long pc; - usf = (struct sparc_stackf32 *) ufp; - if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) - break; + if (thread32_stack_is_64bit(ufp)) { + struct sparc_stackf __user *usf; + struct sparc_stackf sf; - pc = sf.callers_pc; - ufp = (unsigned long)sf.fp; + ufp += STACK_BIAS; + usf = (struct sparc_stackf __user *)ufp; + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) + break; + pc = sf.callers_pc & 0xffffffff; + ufp = ((unsigned long) sf.fp) & 0xffffffff; + } else { + struct sparc_stackf32 __user *usf; + struct sparc_stackf32 sf; + usf = (struct sparc_stackf32 __user *)ufp; + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) + break; + pc = sf.callers_pc; + ufp = (unsigned long)sf.fp; + } perf_callchain_store(entry, pc); } while (entry->nr < PERF_MAX_STACK_DEPTH); } @@ -1420,6 +1791,11 @@ static void perf_callchain_user_32(struct perf_callchain_entry *entry, void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) { + perf_callchain_store(entry, regs->tpc); + + if (!current->mm) + return; + flushw_user(); if (test_thread_flag(TIF_32BIT)) perf_callchain_user_32(entry, regs); diff --git a/arch/sparc/kernel/pmc.c b/arch/sparc/kernel/pmc.c index 94536a85f16..8b7297faca7 100644 --- a/arch/sparc/kernel/pmc.c +++ b/arch/sparc/kernel/pmc.c @@ -11,11 +11,13 @@ #include <linux/pm.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/module.h> #include <asm/io.h> #include <asm/oplib.h> #include <asm/uaccess.h> #include <asm/auxio.h> +#include <asm/processor.h> /* Debug * @@ -51,8 +53,7 @@ static void pmc_swift_idle(void) #endif } -static int __devinit pmc_probe(struct platform_device *op, - const struct of_device_id *match) +static int pmc_probe(struct platform_device *op) { regs = of_ioremap(&op->resource[0], 0, resource_size(&op->resource[0]), PMC_OBPNAME); @@ -63,14 +64,14 @@ static int __devinit pmc_probe(struct platform_device *op, #ifndef PMC_NO_IDLE /* Assign power management IDLE handler */ - pm_idle = pmc_swift_idle; + sparc_idle = pmc_swift_idle; #endif printk(KERN_INFO "%s: power management initialized\n", PMC_DEVNAME); return 0; } -static struct of_device_id __initdata pmc_match[] = { +static struct of_device_id pmc_match[] = { { .name = PMC_OBPNAME, }, @@ -78,7 +79,7 @@ static struct of_device_id __initdata pmc_match[] = { }; MODULE_DEVICE_TABLE(of, pmc_match); -static struct of_platform_driver pmc_driver = { +static struct platform_driver pmc_driver = { .driver = { .name = "pmc", .owner = THIS_MODULE, @@ -89,7 +90,7 @@ static struct of_platform_driver pmc_driver = { static int __init pmc_init(void) { - return of_register_platform_driver(&pmc_driver); + return platform_driver_register(&pmc_driver); } /* This driver is not critical to the boot process diff --git a/arch/sparc/kernel/power.c b/arch/sparc/kernel/power.c index 2c59f4d387d..4cb23c41553 100644 --- a/arch/sparc/kernel/power.c +++ b/arch/sparc/kernel/power.c @@ -4,7 +4,7 @@ */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/reboot.h> @@ -23,7 +23,7 @@ static irqreturn_t power_handler(int irq, void *dev_id) return IRQ_HANDLED; } -static int __devinit has_button_interrupt(unsigned int irq, struct device_node *dp) +static int has_button_interrupt(unsigned int irq, struct device_node *dp) { if (irq == 0xffffffff) return 0; @@ -33,7 +33,7 @@ static int __devinit has_button_interrupt(unsigned int irq, struct device_node * return 1; } -static int __devinit power_probe(struct platform_device *op, const struct of_device_id *match) +static int power_probe(struct platform_device *op) { struct resource *res = &op->resource[0]; unsigned int irq = op->archdata.irqs[0]; @@ -52,14 +52,14 @@ static int __devinit power_probe(struct platform_device *op, const struct of_dev return 0; } -static struct of_device_id __initdata power_match[] = { +static const struct of_device_id power_match[] = { { .name = "power", }, {}, }; -static struct of_platform_driver power_driver = { +static struct platform_driver power_driver = { .probe = power_probe, .driver = { .name = "power", @@ -70,7 +70,7 @@ static struct of_platform_driver power_driver = { static int __init power_init(void) { - return of_register_platform_driver(&power_driver); + return platform_driver_register(&power_driver); } device_initcall(power_init); diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index 17529298c50..50e7b626afe 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -10,6 +10,7 @@ #include <stdarg.h> +#include <linux/elfcore.h> #include <linux/errno.h> #include <linux/module.h> #include <linux/sched.h> @@ -22,13 +23,12 @@ #include <linux/reboot.h> #include <linux/delay.h> #include <linux/pm.h> -#include <linux/init.h> #include <linux/slab.h> +#include <linux/cpu.h> #include <asm/auxio.h> #include <asm/oplib.h> #include <asm/uaccess.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> @@ -38,13 +38,15 @@ #include <asm/elf.h> #include <asm/prom.h> #include <asm/unistd.h> +#include <asm/setup.h> + +#include "kernel.h" /* * Power management idle function * Set in pm platform drivers (apc.c and pmc.c) */ -void (*pm_idle)(void); -EXPORT_SYMBOL(pm_idle); +void (*sparc_idle)(void); /* * Power-off handler instantiation for pm.h compliance @@ -65,80 +67,14 @@ extern void fpsave(unsigned long *, unsigned long *, void *, unsigned long *); struct task_struct *last_task_used_math = NULL; struct thread_info *current_set[NR_CPUS]; -#ifndef CONFIG_SMP - -#define SUN4C_FAULT_HIGH 100 - -/* - * the idle loop on a Sparc... ;) - */ -void cpu_idle(void) -{ - /* endless idle loop with no priority at all */ - for (;;) { - if (ARCH_SUN4C) { - static int count = HZ; - static unsigned long last_jiffies; - static unsigned long last_faults; - static unsigned long fps; - unsigned long now; - unsigned long faults; - - extern unsigned long sun4c_kernel_faults; - extern void sun4c_grow_kernel_ring(void); - - local_irq_disable(); - now = jiffies; - count -= (now - last_jiffies); - last_jiffies = now; - if (count < 0) { - count += HZ; - faults = sun4c_kernel_faults; - fps = (fps + (faults - last_faults)) >> 1; - last_faults = faults; -#if 0 - printk("kernel faults / second = %ld\n", fps); -#endif - if (fps >= SUN4C_FAULT_HIGH) { - sun4c_grow_kernel_ring(); - } - } - local_irq_enable(); - } - - if (pm_idle) { - while (!need_resched()) - (*pm_idle)(); - } else { - while (!need_resched()) - cpu_relax(); - } - preempt_enable_no_resched(); - schedule(); - preempt_disable(); - check_pgt_cache(); - } -} - -#else - -/* This is being executed in task 0 'user space'. */ -void cpu_idle(void) +/* Idle loop support. */ +void arch_cpu_idle(void) { - set_thread_flag(TIF_POLLING_NRFLAG); - /* endless idle loop with no priority at all */ - while(1) { - while (!need_resched()) - cpu_relax(); - preempt_enable_no_resched(); - schedule(); - preempt_disable(); - check_pgt_cache(); - } + if (sparc_idle) + (*sparc_idle)(); + local_irq_enable(); } -#endif - /* XXX cli/sti -> local_irq_xxx here, check this works once SMP is fixed. */ void machine_halt(void) { @@ -170,97 +106,21 @@ void machine_restart(char * cmd) void machine_power_off(void) { if (auxio_power_register && - (strcmp(of_console_device->type, "serial") || scons_pwroff)) - *auxio_power_register |= AUXIO_POWER_OFF; - machine_halt(); -} - -#if 0 - -static DEFINE_SPINLOCK(sparc_backtrace_lock); - -void __show_backtrace(unsigned long fp) -{ - struct reg_window32 *rw; - unsigned long flags; - int cpu = smp_processor_id(); - - spin_lock_irqsave(&sparc_backtrace_lock, flags); - - rw = (struct reg_window32 *)fp; - while(rw && (((unsigned long) rw) >= PAGE_OFFSET) && - !(((unsigned long) rw) & 0x7)) { - printk("CPU[%d]: ARGS[%08lx,%08lx,%08lx,%08lx,%08lx,%08lx] " - "FP[%08lx] CALLER[%08lx]: ", cpu, - rw->ins[0], rw->ins[1], rw->ins[2], rw->ins[3], - rw->ins[4], rw->ins[5], - rw->ins[6], - rw->ins[7]); - printk("%pS\n", (void *) rw->ins[7]); - rw = (struct reg_window32 *) rw->ins[6]; + (strcmp(of_console_device->type, "serial") || scons_pwroff)) { + u8 power_register = sbus_readb(auxio_power_register); + power_register |= AUXIO_POWER_OFF; + sbus_writeb(power_register, auxio_power_register); } - spin_unlock_irqrestore(&sparc_backtrace_lock, flags); -} - -#define __SAVE __asm__ __volatile__("save %sp, -0x40, %sp\n\t") -#define __RESTORE __asm__ __volatile__("restore %g0, %g0, %g0\n\t") -#define __GET_FP(fp) __asm__ __volatile__("mov %%i6, %0" : "=r" (fp)) - -void show_backtrace(void) -{ - unsigned long fp; - - __SAVE; __SAVE; __SAVE; __SAVE; - __SAVE; __SAVE; __SAVE; __SAVE; - __RESTORE; __RESTORE; __RESTORE; __RESTORE; - __RESTORE; __RESTORE; __RESTORE; __RESTORE; - - __GET_FP(fp); - - __show_backtrace(fp); -} - -#ifdef CONFIG_SMP -void smp_show_backtrace_all_cpus(void) -{ - xc0((smpfunc_t) show_backtrace); - show_backtrace(); -} -#endif -void show_stackframe(struct sparc_stackf *sf) -{ - unsigned long size; - unsigned long *stk; - int i; - - printk("l0: %08lx l1: %08lx l2: %08lx l3: %08lx " - "l4: %08lx l5: %08lx l6: %08lx l7: %08lx\n", - sf->locals[0], sf->locals[1], sf->locals[2], sf->locals[3], - sf->locals[4], sf->locals[5], sf->locals[6], sf->locals[7]); - printk("i0: %08lx i1: %08lx i2: %08lx i3: %08lx " - "i4: %08lx i5: %08lx fp: %08lx i7: %08lx\n", - sf->ins[0], sf->ins[1], sf->ins[2], sf->ins[3], - sf->ins[4], sf->ins[5], (unsigned long)sf->fp, sf->callers_pc); - printk("sp: %08lx x0: %08lx x1: %08lx x2: %08lx " - "x3: %08lx x4: %08lx x5: %08lx xx: %08lx\n", - (unsigned long)sf->structptr, sf->xargs[0], sf->xargs[1], - sf->xargs[2], sf->xargs[3], sf->xargs[4], sf->xargs[5], - sf->xxargs[0]); - size = ((unsigned long)sf->fp) - ((unsigned long)sf); - size -= STACKFRAME_SZ; - stk = (unsigned long *)((unsigned long)sf + STACKFRAME_SZ); - i = 0; - do { - printk("s%d: %08lx\n", i++, *stk++); - } while ((size -= sizeof(unsigned long))); + machine_halt(); } -#endif void show_regs(struct pt_regs *r) { struct reg_window32 *rw = (struct reg_window32 *) r->u_regs[14]; + show_regs_print_info(KERN_DEFAULT); + printk("PSR: %08lx PC: %08lx NPC: %08lx Y: %08lx %s\n", r->psr, r->pc, r->npc, r->y, print_tainted()); printk("PC: <%pS>\n", (void *) r->pc); @@ -291,11 +151,13 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) struct reg_window32 *rw; int count = 0; - if (tsk != NULL) - task_base = (unsigned long) task_stack_page(tsk); - else - task_base = (unsigned long) current_thread_info(); + if (!tsk) + tsk = current; + + if (tsk == current && !_ksp) + __asm__ __volatile__("mov %%fp, %0" : "=r" (_ksp)); + task_base = (unsigned long) task_stack_page(tsk); fp = (unsigned long) _ksp; do { /* Bogus frame pointer? */ @@ -311,17 +173,6 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) printk("\n"); } -void dump_stack(void) -{ - unsigned long *ksp; - - __asm__ __volatile__("mov %%fp, %0" - : "=r" (ksp)); - show_stack(current, ksp); -} - -EXPORT_SYMBOL(dump_stack); - /* * Note: sparc64 has a pretty intricated thread_saved_pc, check it out. */ @@ -372,8 +223,7 @@ void flush_thread(void) #endif } - /* Now, this task is no longer a kernel thread. */ - current->thread.current_ds = USER_DS; + /* This task is no longer a kernel thread. */ if (current->thread.flags & SPARC_FLAG_KTHREAD) { current->thread.flags &= ~SPARC_FLAG_KTHREAD; @@ -424,8 +274,7 @@ asmlinkage int sparc_do_fork(unsigned long clone_flags, parent_tid_ptr = regs->u_regs[UREG_I2]; child_tid_ptr = regs->u_regs[UREG_I4]; - ret = do_fork(clone_flags, stack_start, - regs, stack_size, + ret = do_fork(clone_flags, stack_start, stack_size, (int __user *) parent_tid_ptr, (int __user *) child_tid_ptr); @@ -454,13 +303,13 @@ asmlinkage int sparc_do_fork(unsigned long clone_flags, * XXX See comment above sys_vfork in sparc64. todo. */ extern void ret_from_fork(void); +extern void ret_from_kernel_thread(void); int copy_thread(unsigned long clone_flags, unsigned long sp, - unsigned long unused, - struct task_struct *p, struct pt_regs *regs) + unsigned long arg, struct task_struct *p) { struct thread_info *ti = task_thread_info(p); - struct pt_regs *childregs; + struct pt_regs *childregs, *regs = current_pt_regs(); char *new_stack; #ifndef CONFIG_SMP @@ -471,22 +320,16 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, put_psr(get_psr() | PSR_EF); fpsave(&p->thread.float_regs[0], &p->thread.fsr, &p->thread.fpqueue[0], &p->thread.fpqdepth); -#ifdef CONFIG_SMP - clear_thread_flag(TIF_USEDFPU); -#endif } /* - * p->thread_info new_stack childregs - * ! ! ! {if(PSR_PS) } - * V V (stk.fr.) V (pt_regs) { (stk.fr.) } - * +----- - - - - - ------+===========+============={+==========}+ + * p->thread_info new_stack childregs stack bottom + * ! ! ! ! + * V V (stk.fr.) V (pt_regs) V + * +----- - - - - - ------+===========+=============+ */ new_stack = task_stack_page(p) + THREAD_SIZE; - if (regs->psr & PSR_PS) - new_stack -= STACKFRAME_SZ; new_stack -= STACKFRAME_SZ + TRACEREG_SZ; - memcpy(new_stack, (char *)regs - STACKFRAME_SZ, STACKFRAME_SZ + TRACEREG_SZ); childregs = (struct pt_regs *) (new_stack + STACKFRAME_SZ); /* @@ -497,60 +340,64 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, * Thus, kpsr|=PSR_PIL. */ ti->ksp = (unsigned long) new_stack; + p->thread.kregs = childregs; + + if (unlikely(p->flags & PF_KTHREAD)) { + extern int nwindows; + unsigned long psr; + memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); + p->thread.flags |= SPARC_FLAG_KTHREAD; + p->thread.current_ds = KERNEL_DS; + ti->kpc = (((unsigned long) ret_from_kernel_thread) - 0x8); + childregs->u_regs[UREG_G1] = sp; /* function */ + childregs->u_regs[UREG_G2] = arg; + psr = childregs->psr = get_psr(); + ti->kpsr = psr | PSR_PIL; + ti->kwim = 1 << (((psr & PSR_CWP) + 1) % nwindows); + return 0; + } + memcpy(new_stack, (char *)regs - STACKFRAME_SZ, STACKFRAME_SZ + TRACEREG_SZ); + childregs->u_regs[UREG_FP] = sp; + p->thread.flags &= ~SPARC_FLAG_KTHREAD; + p->thread.current_ds = USER_DS; ti->kpc = (((unsigned long) ret_from_fork) - 0x8); ti->kpsr = current->thread.fork_kpsr | PSR_PIL; ti->kwim = current->thread.fork_kwim; - if(regs->psr & PSR_PS) { - extern struct pt_regs fake_swapper_regs; + if (sp != regs->u_regs[UREG_FP]) { + struct sparc_stackf __user *childstack; + struct sparc_stackf __user *parentstack; - p->thread.kregs = &fake_swapper_regs; - new_stack += STACKFRAME_SZ + TRACEREG_SZ; - childregs->u_regs[UREG_FP] = (unsigned long) new_stack; - p->thread.flags |= SPARC_FLAG_KTHREAD; - p->thread.current_ds = KERNEL_DS; - memcpy(new_stack, (void *)regs->u_regs[UREG_FP], STACKFRAME_SZ); - childregs->u_regs[UREG_G6] = (unsigned long) ti; - } else { - p->thread.kregs = childregs; - childregs->u_regs[UREG_FP] = sp; - p->thread.flags &= ~SPARC_FLAG_KTHREAD; - p->thread.current_ds = USER_DS; - - if (sp != regs->u_regs[UREG_FP]) { - struct sparc_stackf __user *childstack; - struct sparc_stackf __user *parentstack; - - /* - * This is a clone() call with supplied user stack. - * Set some valid stack frames to give to the child. - */ - childstack = (struct sparc_stackf __user *) - (sp & ~0xfUL); - parentstack = (struct sparc_stackf __user *) - regs->u_regs[UREG_FP]; + /* + * This is a clone() call with supplied user stack. + * Set some valid stack frames to give to the child. + */ + childstack = (struct sparc_stackf __user *) + (sp & ~0xfUL); + parentstack = (struct sparc_stackf __user *) + regs->u_regs[UREG_FP]; #if 0 - printk("clone: parent stack:\n"); - show_stackframe(parentstack); + printk("clone: parent stack:\n"); + show_stackframe(parentstack); #endif - childstack = clone_stackframe(childstack, parentstack); - if (!childstack) - return -EFAULT; + childstack = clone_stackframe(childstack, parentstack); + if (!childstack) + return -EFAULT; #if 0 - printk("clone: child stack:\n"); - show_stackframe(childstack); + printk("clone: child stack:\n"); + show_stackframe(childstack); #endif - childregs->u_regs[UREG_FP] = (unsigned long)childstack; - } + childregs->u_regs[UREG_FP] = (unsigned long)childstack; } #ifdef CONFIG_SMP /* FPU must be disabled on SMP. */ childregs->psr &= ~PSR_EF; + clear_tsk_thread_flag(p, TIF_USEDFPU); #endif /* Set the return value for the child. */ @@ -615,69 +462,6 @@ int dump_fpu (struct pt_regs * regs, elf_fpregset_t * fpregs) return 1; } -/* - * sparc_execve() executes a new program after the asm stub has set - * things up for us. This should basically do what I want it to. - */ -asmlinkage int sparc_execve(struct pt_regs *regs) -{ - int error, base = 0; - char *filename; - - /* Check for indirect call. */ - if(regs->u_regs[UREG_G1] == 0) - base = 1; - - filename = getname((char __user *)regs->u_regs[base + UREG_I0]); - error = PTR_ERR(filename); - if(IS_ERR(filename)) - goto out; - error = do_execve(filename, - (const char __user *const __user *) - regs->u_regs[base + UREG_I1], - (const char __user *const __user *) - regs->u_regs[base + UREG_I2], - regs); - putname(filename); -out: - return error; -} - -/* - * This is the mechanism for creating a new kernel thread. - * - * NOTE! Only a kernel-only process(ie the swapper or direct descendants - * who haven't done an "execve()") should use this: it will work within - * a system call from a "real" process, but the process memory space will - * not be freed until both the parent and the child have exited. - */ -pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags) -{ - long retval; - - __asm__ __volatile__("mov %4, %%g2\n\t" /* Set aside fn ptr... */ - "mov %5, %%g3\n\t" /* and arg. */ - "mov %1, %%g1\n\t" - "mov %2, %%o0\n\t" /* Clone flags. */ - "mov 0, %%o1\n\t" /* usp arg == 0 */ - "t 0x10\n\t" /* Linux/Sparc clone(). */ - "cmp %%o1, 0\n\t" - "be 1f\n\t" /* The parent, just return. */ - " nop\n\t" /* Delay slot. */ - "jmpl %%g2, %%o7\n\t" /* Call the function. */ - " mov %%g3, %%o0\n\t" /* Get back the arg in delay. */ - "mov %3, %%g1\n\t" - "t 0x10\n\t" /* Linux/Sparc exit(). */ - /* Notreached by child. */ - "1: mov %%o0, %0\n\t" : - "=r" (retval) : - "i" (__NR_clone), "r" (flags | CLONE_VM | CLONE_UNTRACED), - "i" (__NR_exit), "r" (fn), "r" (arg) : - "g1", "g2", "g3", "o0", "o1", "memory", "cc"); - return retval; -} -EXPORT_SYMBOL(kernel_thread); - unsigned long get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index c158a95ec66..027e0998619 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -12,7 +12,7 @@ #include <stdarg.h> #include <linux/errno.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/mm.h> @@ -27,12 +27,13 @@ #include <linux/tick.h> #include <linux/init.h> #include <linux/cpu.h> +#include <linux/perf_event.h> #include <linux/elfcore.h> #include <linux/sysrq.h> #include <linux/nmi.h> +#include <linux/context_tracking.h> #include <asm/uaccess.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> @@ -48,23 +49,24 @@ #include <asm/syscalls.h> #include <asm/irq_regs.h> #include <asm/smp.h> +#include <asm/pcr.h> #include "kstack.h" -static void sparc64_yield(int cpu) +/* Idle loop support on sparc64. */ +void arch_cpu_idle(void) { if (tlb_type != hypervisor) { touch_nmi_watchdog(); - return; - } - - clear_thread_flag(TIF_POLLING_NRFLAG); - smp_mb__after_clear_bit(); - - while (!need_resched() && !cpu_is_offline(cpu)) { + local_irq_enable(); + } else { unsigned long pstate; - /* Disable interrupts. */ + local_irq_enable(); + + /* The sun4v sleeping code requires that we have PSTATE.IE cleared over + * the cpu sleep hypervisor call. + */ __asm__ __volatile__( "rdpr %%pstate, %0\n\t" "andn %0, %1, %0\n\t" @@ -72,7 +74,7 @@ static void sparc64_yield(int cpu) : "=&r" (pstate) : "i" (PSTATE_IE)); - if (!need_resched() && !cpu_is_offline(cpu)) + if (!need_resched() && !cpu_is_offline(smp_processor_id())) sun4v_cpu_yield(); /* Re-enable interrupts. */ @@ -83,36 +85,15 @@ static void sparc64_yield(int cpu) : "=&r" (pstate) : "i" (PSTATE_IE)); } - - set_thread_flag(TIF_POLLING_NRFLAG); } -/* The idle loop on sparc64. */ -void cpu_idle(void) -{ - int cpu = smp_processor_id(); - - set_thread_flag(TIF_POLLING_NRFLAG); - - while(1) { - tick_nohz_stop_sched_tick(1); - - while (!need_resched() && !cpu_is_offline(cpu)) - sparc64_yield(cpu); - - tick_nohz_restart_sched_tick(); - - preempt_enable_no_resched(); - #ifdef CONFIG_HOTPLUG_CPU - if (cpu_is_offline(cpu)) - cpu_play_dead(); -#endif - - schedule(); - preempt_disable(); - } +void arch_cpu_idle_dead(void) +{ + sched_preempt_enable_no_resched(); + cpu_play_dead(); } +#endif #ifdef CONFIG_COMPAT static void show_regwindow32(struct pt_regs *regs) @@ -185,6 +166,8 @@ static void show_regwindow(struct pt_regs *regs) void show_regs(struct pt_regs *regs) { + show_regs_print_info(KERN_DEFAULT); + printk("TSTATE: %016lx TPC: %016lx TNPC: %016lx Y: %08x %s\n", regs->tstate, regs->tpc, regs->tnpc, regs->y, print_tainted()); printk("TPC: <%pS>\n", (void *) regs->tpc); @@ -205,18 +188,22 @@ void show_regs(struct pt_regs *regs) show_stack(current, (unsigned long *) regs->u_regs[UREG_FP]); } -struct global_reg_snapshot global_reg_snapshot[NR_CPUS]; -static DEFINE_SPINLOCK(global_reg_snapshot_lock); +union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; +static DEFINE_SPINLOCK(global_cpu_snapshot_lock); static void __global_reg_self(struct thread_info *tp, struct pt_regs *regs, int this_cpu) { + struct global_reg_snapshot *rp; + flushw_all(); - global_reg_snapshot[this_cpu].tstate = regs->tstate; - global_reg_snapshot[this_cpu].tpc = regs->tpc; - global_reg_snapshot[this_cpu].tnpc = regs->tnpc; - global_reg_snapshot[this_cpu].o7 = regs->u_regs[UREG_I7]; + rp = &global_cpu_snapshot[this_cpu].reg; + + rp->tstate = regs->tstate; + rp->tpc = regs->tpc; + rp->tnpc = regs->tnpc; + rp->o7 = regs->u_regs[UREG_I7]; if (regs->tstate & TSTATE_PRIV) { struct reg_window *rw; @@ -224,17 +211,17 @@ static void __global_reg_self(struct thread_info *tp, struct pt_regs *regs, rw = (struct reg_window *) (regs->u_regs[UREG_FP] + STACK_BIAS); if (kstack_valid(tp, (unsigned long) rw)) { - global_reg_snapshot[this_cpu].i7 = rw->ins[7]; + rp->i7 = rw->ins[7]; rw = (struct reg_window *) (rw->ins[6] + STACK_BIAS); if (kstack_valid(tp, (unsigned long) rw)) - global_reg_snapshot[this_cpu].rpc = rw->ins[7]; + rp->rpc = rw->ins[7]; } } else { - global_reg_snapshot[this_cpu].i7 = 0; - global_reg_snapshot[this_cpu].rpc = 0; + rp->i7 = 0; + rp->rpc = 0; } - global_reg_snapshot[this_cpu].thread = tp; + rp->thread = tp; } /* In order to avoid hangs we do not try to synchronize with the @@ -252,7 +239,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void arch_trigger_all_cpu_backtrace(void) +void arch_trigger_all_cpu_backtrace(bool include_self) { struct thread_info *tp = current_thread_info(); struct pt_regs *regs = get_irq_regs(); @@ -262,18 +249,24 @@ void arch_trigger_all_cpu_backtrace(void) if (!regs) regs = tp->kregs; - spin_lock_irqsave(&global_reg_snapshot_lock, flags); - - memset(global_reg_snapshot, 0, sizeof(global_reg_snapshot)); + spin_lock_irqsave(&global_cpu_snapshot_lock, flags); this_cpu = raw_smp_processor_id(); - __global_reg_self(tp, regs, this_cpu); + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + if (include_self) + __global_reg_self(tp, regs, this_cpu); smp_fetch_global_regs(); for_each_online_cpu(cpu) { - struct global_reg_snapshot *gp = &global_reg_snapshot[cpu]; + struct global_reg_snapshot *gp; + + if (!include_self && cpu == this_cpu) + continue; + + gp = &global_cpu_snapshot[cpu].reg; __global_reg_poll(gp); @@ -296,30 +289,104 @@ void arch_trigger_all_cpu_backtrace(void) } } - memset(global_reg_snapshot, 0, sizeof(global_reg_snapshot)); + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); - spin_unlock_irqrestore(&global_reg_snapshot_lock, flags); + spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); } #ifdef CONFIG_MAGIC_SYSRQ static void sysrq_handle_globreg(int key) { - arch_trigger_all_cpu_backtrace(); + arch_trigger_all_cpu_backtrace(true); } static struct sysrq_key_op sparc_globalreg_op = { .handler = sysrq_handle_globreg, - .help_msg = "Globalregs", + .help_msg = "global-regs(y)", .action_msg = "Show Global CPU Regs", }; -static int __init sparc_globreg_init(void) +static void __global_pmu_self(int this_cpu) +{ + struct global_pmu_snapshot *pp; + int i, num; + + pp = &global_cpu_snapshot[this_cpu].pmu; + + num = 1; + if (tlb_type == hypervisor && + sun4v_chip_type >= SUN4V_CHIP_NIAGARA4) + num = 4; + + for (i = 0; i < num; i++) { + pp->pcr[i] = pcr_ops->read_pcr(i); + pp->pic[i] = pcr_ops->read_pic(i); + } +} + +static void __global_pmu_poll(struct global_pmu_snapshot *pp) +{ + int limit = 0; + + while (!pp->pcr[0] && ++limit < 100) { + barrier(); + udelay(1); + } +} + +static void pmu_snapshot_all_cpus(void) +{ + unsigned long flags; + int this_cpu, cpu; + + spin_lock_irqsave(&global_cpu_snapshot_lock, flags); + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + this_cpu = raw_smp_processor_id(); + + __global_pmu_self(this_cpu); + + smp_fetch_global_pmu(); + + for_each_online_cpu(cpu) { + struct global_pmu_snapshot *pp = &global_cpu_snapshot[cpu].pmu; + + __global_pmu_poll(pp); + + printk("%c CPU[%3d]: PCR[%08lx:%08lx:%08lx:%08lx] PIC[%08lx:%08lx:%08lx:%08lx]\n", + (cpu == this_cpu ? '*' : ' '), cpu, + pp->pcr[0], pp->pcr[1], pp->pcr[2], pp->pcr[3], + pp->pic[0], pp->pic[1], pp->pic[2], pp->pic[3]); + } + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); +} + +static void sysrq_handle_globpmu(int key) +{ + pmu_snapshot_all_cpus(); +} + +static struct sysrq_key_op sparc_globalpmu_op = { + .handler = sysrq_handle_globpmu, + .help_msg = "global-pmu(x)", + .action_msg = "Show Global PMU Regs", +}; + +static int __init sparc_sysrq_init(void) { - return register_sysrq_key('y', &sparc_globalreg_op); + int ret = register_sysrq_key('y', &sparc_globalreg_op); + + if (!ret) + ret = register_sysrq_key('x', &sparc_globalpmu_op); + return ret; } -core_initcall(sparc_globreg_init); +core_initcall(sparc_sysrq_init); #endif @@ -368,21 +435,21 @@ void flush_thread(void) /* Clear FPU register state. */ t->fpsaved[0] = 0; - - if (get_thread_current_ds() != ASI_AIUS) - set_fs(USER_DS); } /* It's a bit more tricky when 64-bit tasks are involved... */ static unsigned long clone_stackframe(unsigned long csp, unsigned long psp) { + bool stack_64bit = test_thread_64bit_stack(psp); unsigned long fp, distance, rval; - if (!(test_thread_flag(TIF_32BIT))) { + if (stack_64bit) { csp += STACK_BIAS; psp += STACK_BIAS; __get_user(fp, &(((struct reg_window __user *)psp)->ins[6])); fp += STACK_BIAS; + if (test_thread_flag(TIF_32BIT)) + fp &= 0xffffffff; } else __get_user(fp, &(((struct reg_window32 __user *)psp)->ins[6])); @@ -396,7 +463,7 @@ static unsigned long clone_stackframe(unsigned long csp, unsigned long psp) rval = (csp - distance); if (copy_in_user((void __user *) rval, (void __user *) psp, distance)) rval = 0; - else if (test_thread_flag(TIF_32BIT)) { + else if (!stack_64bit) { if (put_user(((u32)csp), &(((struct reg_window32 __user *)rval)->ins[6]))) rval = 0; @@ -431,18 +498,18 @@ void synchronize_user_stack(void) flush_user_windows(); if ((window = get_thread_wsaved()) != 0) { - int winsize = sizeof(struct reg_window); - int bias = 0; - - if (test_thread_flag(TIF_32BIT)) - winsize = sizeof(struct reg_window32); - else - bias = STACK_BIAS; - window -= 1; do { - unsigned long sp = (t->rwbuf_stkptrs[window] + bias); struct reg_window *rwin = &t->reg_window[window]; + int winsize = sizeof(struct reg_window); + unsigned long sp; + + sp = t->rwbuf_stkptrs[window]; + + if (test_thread_64bit_stack(sp)) + sp += STACK_BIAS; + else + winsize = sizeof(struct reg_window32); if (!copy_to_user((char __user *)sp, rwin, winsize)) { shift_window_buffer(window, get_thread_wsaved() - 1, t); @@ -468,13 +535,6 @@ void fault_in_user_windows(void) { struct thread_info *t = current_thread_info(); unsigned long window; - int winsize = sizeof(struct reg_window); - int bias = 0; - - if (test_thread_flag(TIF_32BIT)) - winsize = sizeof(struct reg_window32); - else - bias = STACK_BIAS; flush_user_windows(); window = get_thread_wsaved(); @@ -482,8 +542,16 @@ void fault_in_user_windows(void) if (likely(window != 0)) { window -= 1; do { - unsigned long sp = (t->rwbuf_stkptrs[window] + bias); struct reg_window *rwin = &t->reg_window[window]; + int winsize = sizeof(struct reg_window); + unsigned long sp; + + sp = t->rwbuf_stkptrs[window]; + + if (test_thread_64bit_stack(sp)) + sp += STACK_BIAS; + else + winsize = sizeof(struct reg_window32); if (unlikely(sp & 0x7UL)) stack_unaligned(sp); @@ -498,6 +566,7 @@ void fault_in_user_windows(void) barf: set_thread_wsaved(window + 1); + user_exit(); do_exit(SIGILL); } @@ -521,8 +590,7 @@ asmlinkage long sparc_do_fork(unsigned long clone_flags, child_tid_ptr = (int __user *) regs->u_regs[UREG_I4]; } - ret = do_fork(clone_flags, stack_start, - regs, stack_size, + ret = do_fork(clone_flags, stack_start, stack_size, parent_tid_ptr, child_tid_ptr); /* If we get an error and potentially restart the system @@ -542,64 +610,55 @@ asmlinkage long sparc_do_fork(unsigned long clone_flags, * Child --> %o0 == parents pid, %o1 == 1 */ int copy_thread(unsigned long clone_flags, unsigned long sp, - unsigned long unused, - struct task_struct *p, struct pt_regs *regs) + unsigned long arg, struct task_struct *p) { struct thread_info *t = task_thread_info(p); + struct pt_regs *regs = current_pt_regs(); struct sparc_stackf *parent_sf; unsigned long child_stack_sz; char *child_trap_frame; - int kernel_thread; - - kernel_thread = (regs->tstate & TSTATE_PRIV) ? 1 : 0; - parent_sf = ((struct sparc_stackf *) regs) - 1; /* Calculate offset to stack_frame & pt_regs */ - child_stack_sz = ((STACKFRAME_SZ + TRACEREG_SZ) + - (kernel_thread ? STACKFRAME_SZ : 0)); + child_stack_sz = (STACKFRAME_SZ + TRACEREG_SZ); child_trap_frame = (task_stack_page(p) + (THREAD_SIZE - child_stack_sz)); - memcpy(child_trap_frame, parent_sf, child_stack_sz); - t->flags = (t->flags & ~((0xffUL << TI_FLAG_CWP_SHIFT) | - (0xffUL << TI_FLAG_CURRENT_DS_SHIFT))) | - (((regs->tstate + 1) & TSTATE_CWP) << TI_FLAG_CWP_SHIFT); t->new_child = 1; t->ksp = ((unsigned long) child_trap_frame) - STACK_BIAS; t->kregs = (struct pt_regs *) (child_trap_frame + sizeof(struct sparc_stackf)); t->fpsaved[0] = 0; - if (kernel_thread) { - struct sparc_stackf *child_sf = (struct sparc_stackf *) - (child_trap_frame + (STACKFRAME_SZ + TRACEREG_SZ)); - - /* Zero terminate the stack backtrace. */ - child_sf->fp = NULL; - t->kregs->u_regs[UREG_FP] = - ((unsigned long) child_sf) - STACK_BIAS; + if (unlikely(p->flags & PF_KTHREAD)) { + memset(child_trap_frame, 0, child_stack_sz); + __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = + (current_pt_regs()->tstate + 1) & TSTATE_CWP; + t->current_ds = ASI_P; + t->kregs->u_regs[UREG_G1] = sp; /* function */ + t->kregs->u_regs[UREG_G2] = arg; + return 0; + } - t->flags |= ((long)ASI_P << TI_FLAG_CURRENT_DS_SHIFT); - t->kregs->u_regs[UREG_G6] = (unsigned long) t; - t->kregs->u_regs[UREG_G4] = (unsigned long) t->task; - } else { - if (t->flags & _TIF_32BIT) { - sp &= 0x00000000ffffffffUL; - regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; - } - t->kregs->u_regs[UREG_FP] = sp; - t->flags |= ((long)ASI_AIUS << TI_FLAG_CURRENT_DS_SHIFT); - if (sp != regs->u_regs[UREG_FP]) { - unsigned long csp; - - csp = clone_stackframe(sp, regs->u_regs[UREG_FP]); - if (!csp) - return -EFAULT; - t->kregs->u_regs[UREG_FP] = csp; - } - if (t->utraps) - t->utraps[0]++; + parent_sf = ((struct sparc_stackf *) regs) - 1; + memcpy(child_trap_frame, parent_sf, child_stack_sz); + if (t->flags & _TIF_32BIT) { + sp &= 0x00000000ffffffffUL; + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; } + t->kregs->u_regs[UREG_FP] = sp; + __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = + (regs->tstate + 1) & TSTATE_CWP; + t->current_ds = ASI_AIUS; + if (sp != regs->u_regs[UREG_FP]) { + unsigned long csp; + + csp = clone_stackframe(sp, regs->u_regs[UREG_FP]); + if (!csp) + return -EFAULT; + t->kregs->u_regs[UREG_FP] = csp; + } + if (t->utraps) + t->utraps[0]++; /* Set the return value for the child. */ t->kregs->u_regs[UREG_I0] = current->pid; @@ -614,45 +673,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, return 0; } -/* - * This is the mechanism for creating a new kernel thread. - * - * NOTE! Only a kernel-only process(ie the swapper or direct descendants - * who haven't done an "execve()") should use this: it will work within - * a system call from a "real" process, but the process memory space will - * not be freed until both the parent and the child have exited. - */ -pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags) -{ - long retval; - - /* If the parent runs before fn(arg) is called by the child, - * the input registers of this function can be clobbered. - * So we stash 'fn' and 'arg' into global registers which - * will not be modified by the parent. - */ - __asm__ __volatile__("mov %4, %%g2\n\t" /* Save FN into global */ - "mov %5, %%g3\n\t" /* Save ARG into global */ - "mov %1, %%g1\n\t" /* Clone syscall nr. */ - "mov %2, %%o0\n\t" /* Clone flags. */ - "mov 0, %%o1\n\t" /* usp arg == 0 */ - "t 0x6d\n\t" /* Linux/Sparc clone(). */ - "brz,a,pn %%o1, 1f\n\t" /* Parent, just return. */ - " mov %%o0, %0\n\t" - "jmpl %%g2, %%o7\n\t" /* Call the function. */ - " mov %%g3, %%o0\n\t" /* Set arg in delay. */ - "mov %3, %%g1\n\t" - "t 0x6d\n\t" /* Linux/Sparc exit(). */ - /* Notreached by child. */ - "1:" : - "=r" (retval) : - "i" (__NR_clone), "r" (flags | CLONE_VM | CLONE_UNTRACED), - "i" (__NR_exit), "r" (fn), "r" (arg) : - "g1", "g2", "g3", "o0", "o1", "memory", "cc"); - return retval; -} -EXPORT_SYMBOL(kernel_thread); - typedef struct { union { unsigned int pr_regs[32]; @@ -719,41 +739,6 @@ int dump_fpu (struct pt_regs * regs, elf_fpregset_t * fpregs) } EXPORT_SYMBOL(dump_fpu); -/* - * sparc_execve() executes a new program after the asm stub has set - * things up for us. This should basically do what I want it to. - */ -asmlinkage int sparc_execve(struct pt_regs *regs) -{ - int error, base = 0; - char *filename; - - /* User register window flush is done by entry.S */ - - /* Check for indirect call. */ - if (regs->u_regs[UREG_G1] == 0) - base = 1; - - filename = getname((char __user *)regs->u_regs[base + UREG_I0]); - error = PTR_ERR(filename); - if (IS_ERR(filename)) - goto out; - error = do_execve(filename, - (const char __user *const __user *) - regs->u_regs[base + UREG_I1], - (const char __user *const __user *) - regs->u_regs[base + UREG_I2], regs); - putname(filename); - if (!error) { - fprs_write(0); - current_thread_info()->xfsr[0] = 0; - current_thread_info()->fpsaved[0] = 0; - regs->tstate &= ~TSTATE_PEF; - } -out: - return error; -} - unsigned long get_wchan(struct task_struct *task) { unsigned long pc, fp, bias = 0; diff --git a/arch/sparc/kernel/prom.h b/arch/sparc/kernel/prom.h index cf5fe1c0b02..890281b12b2 100644 --- a/arch/sparc/kernel/prom.h +++ b/arch/sparc/kernel/prom.h @@ -4,7 +4,7 @@ #include <linux/spinlock.h> #include <asm/prom.h> -extern void of_console_init(void); +void of_console_init(void); extern unsigned int prom_early_allocated; diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c index 05fb2533058..b51cbb9e87d 100644 --- a/arch/sparc/kernel/prom_32.c +++ b/arch/sparc/kernel/prom_32.c @@ -20,7 +20,6 @@ #include <linux/string.h> #include <linux/mm.h> #include <linux/bootmem.h> -#include <linux/module.h> #include <asm/prom.h> #include <asm/oplib.h> @@ -326,7 +325,6 @@ void __init of_console_init(void) of_console_options = NULL; } - prom_printf(msg, of_console_path); printk(msg, of_console_path); } diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c index 86597d9867f..20cc5d80a47 100644 --- a/arch/sparc/kernel/prom_64.c +++ b/arch/sparc/kernel/prom_64.c @@ -15,12 +15,12 @@ * 2 of the License, or (at your option) any later version. */ +#include <linux/memblock.h> #include <linux/kernel.h> -#include <linux/types.h> #include <linux/string.h> +#include <linux/types.h> +#include <linux/cpu.h> #include <linux/mm.h> -#include <linux/module.h> -#include <linux/memblock.h> #include <linux/of.h> #include <asm/prom.h> @@ -38,7 +38,7 @@ void * __init prom_early_alloc(unsigned long size) void *ret; if (!paddr) { - prom_printf("prom_early_alloc(%lu) failed\n"); + prom_printf("prom_early_alloc(%lu) failed\n", size); prom_halt(); } @@ -374,6 +374,59 @@ static const char *get_mid_prop(void) return (tlb_type == spitfire ? "upa-portid" : "portid"); } +bool arch_find_n_match_cpu_physical_id(struct device_node *cpun, + int cpu, unsigned int *thread) +{ + const char *mid_prop = get_mid_prop(); + int this_cpu_id; + + /* On hypervisor based platforms we interrogate the 'reg' + * property. On everything else we look for a 'upa-portis', + * 'portid', or 'cpuid' property. + */ + + if (tlb_type == hypervisor) { + struct property *prop = of_find_property(cpun, "reg", NULL); + u32 *regs; + + if (!prop) { + pr_warn("CPU node missing reg property\n"); + return false; + } + regs = prop->value; + this_cpu_id = regs[0] & 0x0fffffff; + } else { + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + + if (this_cpu_id < 0) { + mid_prop = "cpuid"; + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + } + if (this_cpu_id < 0) { + pr_warn("CPU node missing cpu ID property\n"); + return false; + } + } + if (this_cpu_id == cpu) { + if (thread) { + int proc_id = cpu_data(cpu).proc_id; + + /* On sparc64, the cpu thread information is obtained + * either from OBP or the machine description. We've + * actually probed this information already long before + * this interface gets called so instead of interrogating + * both the OF node and the MDESC again, just use what + * we discovered already. + */ + if (proc_id < 0) + proc_id = 0; + *thread = proc_id; + } + return true; + } + return false; +} + static void *of_iterate_over_cpus(void *(*func)(struct device_node *, int, int), int arg) { struct device_node *dp; @@ -503,9 +556,6 @@ static void *fill_in_one_cpu(struct device_node *dp, int cpuid, int arg) cpu_data(cpuid).core_id = portid + 1; cpu_data(cpuid).proc_id = portid; -#ifdef CONFIG_SMP - sparc64_multi_core = 1; -#endif } else { cpu_data(cpuid).dcache_size = of_getintprop_default(dp, "dcache-size", 16 * 1024); diff --git a/arch/sparc/kernel/prom_common.c b/arch/sparc/kernel/prom_common.c index ed25834328f..79cc0d1a477 100644 --- a/arch/sparc/kernel/prom_common.c +++ b/arch/sparc/kernel/prom_common.c @@ -15,7 +15,7 @@ */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/errno.h> #include <linux/mutex.h> #include <linux/slab.h> @@ -23,7 +23,6 @@ #include <linux/of_pdt.h> #include <asm/prom.h> #include <asm/oplib.h> -#include <asm/leon.h> #include "prom.h" @@ -55,19 +54,18 @@ EXPORT_SYMBOL(of_set_property_mutex); int of_set_property(struct device_node *dp, const char *name, void *val, int len) { struct property **prevp; + unsigned long flags; void *new_val; int err; - new_val = kmalloc(len, GFP_KERNEL); + new_val = kmemdup(val, len, GFP_KERNEL); if (!new_val) return -ENOMEM; - memcpy(new_val, val, len); - err = -ENODEV; mutex_lock(&of_set_property_mutex); - write_lock(&devtree_lock); + raw_spin_lock_irqsave(&devtree_lock, flags); prevp = &dp->properties; while (*prevp) { struct property *prop = *prevp; @@ -94,7 +92,7 @@ int of_set_property(struct device_node *dp, const char *name, void *val, int len } prevp = &(*prevp)->next; } - write_unlock(&devtree_lock); + raw_spin_unlock_irqrestore(&devtree_lock, flags); mutex_unlock(&of_set_property_mutex); /* XXX Upate procfs if necessary... */ diff --git a/arch/sparc/kernel/prom_irqtrans.c b/arch/sparc/kernel/prom_irqtrans.c index ce651147fab..40e4936bd47 100644 --- a/arch/sparc/kernel/prom_irqtrans.c +++ b/arch/sparc/kernel/prom_irqtrans.c @@ -227,7 +227,7 @@ static unsigned int sabre_irq_build(struct device_node *dp, unsigned long imap, iclr; unsigned long imap_off, iclr_off; int inofixup = 0; - int virt_irq; + int irq; ino &= 0x3f; if (ino < SABRE_ONBOARD_IRQ_BASE) { @@ -247,7 +247,7 @@ static unsigned int sabre_irq_build(struct device_node *dp, if ((ino & 0x20) == 0) inofixup = ino & 0x03; - virt_irq = build_irq(inofixup, iclr, imap); + irq = build_irq(inofixup, iclr, imap); /* If the parent device is a PCI<->PCI bridge other than * APB, we have to install a pre-handler to ensure that @@ -256,13 +256,13 @@ static unsigned int sabre_irq_build(struct device_node *dp, */ regs = of_get_property(dp, "reg", NULL); if (regs && sabre_device_needs_wsync(dp)) { - irq_install_pre_handler(virt_irq, + irq_install_pre_handler(irq, sabre_wsync_handler, (void *) (long) regs->phys_hi, (void *) irq_data); } - return virt_irq; + return irq; } static void __init sabre_irq_trans_init(struct device_node *dp) @@ -382,7 +382,7 @@ static unsigned int schizo_irq_build(struct device_node *dp, unsigned long pbm_regs = irq_data->pbm_regs; unsigned long imap, iclr; int ign_fixup; - int virt_irq; + int irq; int is_tomatillo; ino &= 0x3f; @@ -409,17 +409,17 @@ static unsigned int schizo_irq_build(struct device_node *dp, ign_fixup = (1 << 6); } - virt_irq = build_irq(ign_fixup, iclr, imap); + irq = build_irq(ign_fixup, iclr, imap); if (is_tomatillo) { - irq_install_pre_handler(virt_irq, + irq_install_pre_handler(irq, tomatillo_wsync_handler, ((irq_data->chip_version <= 4) ? (void *) 1 : (void *) 0), (void *) irq_data->sync_reg); } - return virt_irq; + return irq; } static void __init __schizo_irq_trans_init(struct device_node *dp, @@ -694,7 +694,7 @@ static unsigned int sbus_of_build_irq(struct device_node *dp, case 3: iclr = reg_base + SYSIO_ICLR_SLOT3; break; - }; + } iclr += ((unsigned long)sbus_level - 1UL) * 8UL; } diff --git a/arch/sparc/kernel/psycho_common.c b/arch/sparc/kernel/psycho_common.c index fe2af66bb19..8db48e808ed 100644 --- a/arch/sparc/kernel/psycho_common.c +++ b/arch/sparc/kernel/psycho_common.c @@ -228,7 +228,7 @@ void psycho_check_iommu_error(struct pci_pbm_info *pbm, default: type_str = "ECC Error"; break; - }; + } printk(KERN_ERR "%s: IOMMU Error, type[%s]\n", pbm->name, type_str); diff --git a/arch/sparc/kernel/psycho_common.h b/arch/sparc/kernel/psycho_common.h index 590b4ed8ab5..05a6e30a928 100644 --- a/arch/sparc/kernel/psycho_common.h +++ b/arch/sparc/kernel/psycho_common.h @@ -30,19 +30,19 @@ enum psycho_error_type { UE_ERR, CE_ERR, PCI_ERR }; -extern void psycho_check_iommu_error(struct pci_pbm_info *pbm, - unsigned long afsr, - unsigned long afar, - enum psycho_error_type type); +void psycho_check_iommu_error(struct pci_pbm_info *pbm, + unsigned long afsr, + unsigned long afar, + enum psycho_error_type type); -extern irqreturn_t psycho_pcierr_intr(int irq, void *dev_id); +irqreturn_t psycho_pcierr_intr(int irq, void *dev_id); -extern int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, - u32 dvma_offset, u32 dma_mask, - unsigned long write_complete_offset); +int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, + u32 dvma_offset, u32 dma_mask, + unsigned long write_complete_offset); -extern void psycho_pbm_init_common(struct pci_pbm_info *pbm, - struct platform_device *op, - const char *chip_name, int chip_type); +void psycho_pbm_init_common(struct pci_pbm_info *pbm, + struct platform_device *op, + const char *chip_name, int chip_type); #endif /* _PSYCHO_COMMON_H */ diff --git a/arch/sparc/kernel/ptrace_32.c b/arch/sparc/kernel/ptrace_32.c index 27b9e93d012..a331fdc11a2 100644 --- a/arch/sparc/kernel/ptrace_32.c +++ b/arch/sparc/kernel/ptrace_32.c @@ -23,8 +23,10 @@ #include <linux/tracehook.h> #include <asm/pgtable.h> -#include <asm/system.h> #include <asm/uaccess.h> +#include <asm/cacheflush.h> + +#include "kernel.h" /* #define ALLOW_INIT_TRACING */ diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c index 9ccc812bc09..c13c9f25d83 100644 --- a/arch/sparc/kernel/ptrace_64.c +++ b/arch/sparc/kernel/ptrace_64.c @@ -14,6 +14,7 @@ #include <linux/sched.h> #include <linux/mm.h> #include <linux/errno.h> +#include <linux/export.h> #include <linux/ptrace.h> #include <linux/user.h> #include <linux/smp.h> @@ -26,10 +27,10 @@ #include <trace/syscall.h> #include <linux/compat.h> #include <linux/elf.h> +#include <linux/context_tracking.h> #include <asm/asi.h> #include <asm/pgtable.h> -#include <asm/system.h> #include <asm/uaccess.h> #include <asm/psrcompat.h> #include <asm/visasm.h> @@ -117,6 +118,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, preempt_enable(); } +EXPORT_SYMBOL_GPL(flush_ptrace_access); static int get_from_target(struct task_struct *target, unsigned long uaddr, void *kbuf, int len) @@ -152,7 +154,7 @@ static int regwindow64_get(struct task_struct *target, { unsigned long rw_addr = regs->u_regs[UREG_I6]; - if (test_tsk_thread_flag(current, TIF_32BIT)) { + if (!test_thread_64bit_stack(rw_addr)) { struct reg_window32 win32; int i; @@ -177,7 +179,7 @@ static int regwindow64_set(struct task_struct *target, { unsigned long rw_addr = regs->u_regs[UREG_I6]; - if (test_tsk_thread_flag(current, TIF_32BIT)) { + if (!test_thread_64bit_stack(rw_addr)) { struct reg_window32 win32; int i; @@ -1063,7 +1065,10 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) int ret = 0; /* do the secure computing check first */ - secure_computing(regs->u_regs[UREG_G1]); + secure_computing_strict(regs->u_regs[UREG_G1]); + + if (test_thread_flag(TIF_NOHZ)) + user_exit(); if (test_thread_flag(TIF_SYSCALL_TRACE)) ret = tracehook_report_syscall_entry(regs); @@ -1071,34 +1076,31 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) trace_sys_enter(regs, regs->u_regs[UREG_G1]); - if (unlikely(current->audit_context) && !ret) - audit_syscall_entry((test_thread_flag(TIF_32BIT) ? - AUDIT_ARCH_SPARC : - AUDIT_ARCH_SPARC64), - regs->u_regs[UREG_G1], - regs->u_regs[UREG_I0], - regs->u_regs[UREG_I1], - regs->u_regs[UREG_I2], - regs->u_regs[UREG_I3]); + audit_syscall_entry((test_thread_flag(TIF_32BIT) ? + AUDIT_ARCH_SPARC : + AUDIT_ARCH_SPARC64), + regs->u_regs[UREG_G1], + regs->u_regs[UREG_I0], + regs->u_regs[UREG_I1], + regs->u_regs[UREG_I2], + regs->u_regs[UREG_I3]); return ret; } asmlinkage void syscall_trace_leave(struct pt_regs *regs) { - if (unlikely(current->audit_context)) { - unsigned long tstate = regs->tstate; - int result = AUDITSC_SUCCESS; - - if (unlikely(tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) - result = AUDITSC_FAILURE; + if (test_thread_flag(TIF_NOHZ)) + user_exit(); - audit_syscall_exit(result, regs->u_regs[UREG_I0]); - } + audit_syscall_exit(regs); if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) - trace_sys_exit(regs, regs->u_regs[UREG_G1]); + trace_sys_exit(regs, regs->u_regs[UREG_I0]); if (test_thread_flag(TIF_SYSCALL_TRACE)) tracehook_report_syscall_exit(regs, 0); + + if (test_thread_flag(TIF_NOHZ)) + user_enter(); } diff --git a/arch/sparc/kernel/reboot.c b/arch/sparc/kernel/reboot.c index ef89d3d6974..eba7d918162 100644 --- a/arch/sparc/kernel/reboot.c +++ b/arch/sparc/kernel/reboot.c @@ -4,12 +4,12 @@ */ #include <linux/kernel.h> #include <linux/reboot.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/pm.h> -#include <asm/system.h> #include <asm/oplib.h> #include <asm/prom.h> +#include <asm/setup.h> /* sysctl - toggle power-off restriction for serial console * systems in machine_power_off() diff --git a/arch/sparc/kernel/rtrap_32.S b/arch/sparc/kernel/rtrap_32.S index 5f5f74c2c2c..6c34de0c2ab 100644 --- a/arch/sparc/kernel/rtrap_32.S +++ b/arch/sparc/kernel/rtrap_32.S @@ -128,13 +128,12 @@ rtrap_patch2: and %glob_tmp, 0xff, %glob_tmp wr %glob_tmp, 0x0, %wim - /* Here comes the architecture specific - * branch to the user stack checking routine - * for return from traps. - */ - .globl rtrap_mmu_patchme -rtrap_mmu_patchme: b sun4c_rett_stackchk - andcc %fp, 0x7, %g0 + /* Here comes the architecture specific + * branch to the user stack checking routine + * for return from traps. + */ + b srmmu_rett_stackchk + andcc %fp, 0x7, %g0 ret_trap_userwins_ok: LOAD_PT_PRIV(sp, t_psr, t_pc, t_npc) @@ -225,69 +224,6 @@ ret_trap_user_stack_is_bolixed: b signal_p ld [%curptr + TI_FLAGS], %g2 -sun4c_rett_stackchk: - be 1f - and %fp, 0xfff, %g1 ! delay slot - - b ret_trap_user_stack_is_bolixed + 0x4 - wr %t_wim, 0x0, %wim - - /* See if we have to check the sanity of one page or two */ -1: - add %g1, 0x38, %g1 - sra %fp, 29, %g2 - add %g2, 0x1, %g2 - andncc %g2, 0x1, %g0 - be 1f - andncc %g1, 0xff8, %g0 - - /* %sp is in vma hole, yuck */ - b ret_trap_user_stack_is_bolixed + 0x4 - wr %t_wim, 0x0, %wim - -1: - be sun4c_rett_onepage /* Only one page to check */ - lda [%fp] ASI_PTE, %g2 - -sun4c_rett_twopages: - add %fp, 0x38, %g1 - sra %g1, 29, %g2 - add %g2, 0x1, %g2 - andncc %g2, 0x1, %g0 - be 1f - lda [%g1] ASI_PTE, %g2 - - /* Second page is in vma hole */ - b ret_trap_user_stack_is_bolixed + 0x4 - wr %t_wim, 0x0, %wim - -1: - srl %g2, 29, %g2 - andcc %g2, 0x4, %g0 - bne sun4c_rett_onepage - lda [%fp] ASI_PTE, %g2 - - /* Second page has bad perms */ - b ret_trap_user_stack_is_bolixed + 0x4 - wr %t_wim, 0x0, %wim - -sun4c_rett_onepage: - srl %g2, 29, %g2 - andcc %g2, 0x4, %g0 - bne,a 1f - restore %g0, %g0, %g0 - - /* A page had bad page permissions, losing... */ - b ret_trap_user_stack_is_bolixed + 0x4 - wr %t_wim, 0x0, %wim - - /* Whee, things are ok, load the window and continue. */ -1: - LOAD_WINDOW(sp) - - b ret_trap_userwins_ok - save %g0, %g0, %g0 - .globl srmmu_rett_stackchk srmmu_rett_stackchk: bne ret_trap_user_stack_is_bolixed @@ -295,11 +231,14 @@ srmmu_rett_stackchk: cmp %g1, %fp bleu ret_trap_user_stack_is_bolixed mov AC_M_SFSR, %g1 - lda [%g1] ASI_M_MMUREGS, %g0 +LEON_PI(lda [%g1] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%g1] ASI_M_MMUREGS, %g0) - lda [%g0] ASI_M_MMUREGS, %g1 +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %g1) +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %g1) or %g1, 0x2, %g1 - sta %g1, [%g0] ASI_M_MMUREGS +LEON_PI(sta %g1, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g0] ASI_M_MMUREGS) restore %g0, %g0, %g0 @@ -308,13 +247,16 @@ srmmu_rett_stackchk: save %g0, %g0, %g0 andn %g1, 0x2, %g1 - sta %g1, [%g0] ASI_M_MMUREGS +LEON_PI(sta %g1, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g0] ASI_M_MMUREGS) mov AC_M_SFAR, %g2 - lda [%g2] ASI_M_MMUREGS, %g2 +LEON_PI(lda [%g2] ASI_LEON_MMUREGS, %g2) +SUN_PI_(lda [%g2] ASI_M_MMUREGS, %g2) mov AC_M_SFSR, %g1 - lda [%g1] ASI_M_MMUREGS, %g1 +LEON_PI(lda [%g1] ASI_LEON_MMUREGS, %g1) +SUN_PI_(lda [%g1] ASI_M_MMUREGS, %g1) andcc %g1, 0x2, %g0 be ret_trap_userwins_ok nop diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index 77f1b95e080..39f0c662f4c 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -18,15 +18,16 @@ #define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) #define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) +#ifdef CONFIG_CONTEXT_TRACKING +# define SCHEDULE_USER schedule_user +#else +# define SCHEDULE_USER schedule +#endif + .text .align 32 -__handle_softirq: - call do_softirq - nop - ba,a,pt %xcc, __handle_softirq_continue - nop __handle_preemption: - call schedule + call SCHEDULE_USER wrpr %g0, RTRAP_PSTATE, %pstate ba,pt %xcc, __handle_preemption_continue wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate @@ -78,20 +79,8 @@ rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 .globl rtrap_irq, rtrap, irqsz_patchme, rtrap_xcall rtrap_irq: rtrap: -#ifndef CONFIG_SMP - sethi %hi(__cpu_data), %l0 - lduw [%l0 + %lo(__cpu_data)], %l1 -#else - sethi %hi(__cpu_data), %l0 - or %l0, %lo(__cpu_data), %l0 - lduw [%l0 + %g5], %l1 -#endif - cmp %l1, 0 - /* mm/ultra.S:xcall_report_regs KNOWS about this load. */ - bne,pn %icc, __handle_softirq - ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 -__handle_softirq_continue: + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 rtrap_xcall: sethi %hi(0xf << 20), %l4 and %l1, %l4, %l4 @@ -323,12 +312,10 @@ to_kernel: nop cmp %l4, 0 bne,pn %xcc, kern_fpucheck - sethi %hi(PREEMPT_ACTIVE), %l6 - stw %l6, [%g6 + TI_PRE_COUNT] - call schedule + nop + call preempt_schedule_irq nop ba,pt %xcc, rtrap - stw %g0, [%g6 + TI_PRE_COUNT] #endif kern_fpucheck: ldub [%g6 + TI_FPDEPTH], %l5 brz,pt %l5, rt_continue diff --git a/arch/sparc/kernel/sbus.c b/arch/sparc/kernel/sbus.c index 2ca32d13abc..be5bdf93c76 100644 --- a/arch/sparc/kernel/sbus.c +++ b/arch/sparc/kernel/sbus.c @@ -9,6 +9,7 @@ #include <linux/mm.h> #include <linux/spinlock.h> #include <linux/slab.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/of.h> @@ -97,7 +98,7 @@ void sbus_set_sbus64(struct device *dev, int bursts) default: return; - }; + } val = upa_readq(cfg_reg); if (val & (1UL << 14UL)) { @@ -244,7 +245,7 @@ static unsigned int sbus_build_irq(struct platform_device *op, unsigned int ino) case 3: iclr = reg_base + SYSIO_ICLR_SLOT3; break; - }; + } iclr += ((unsigned long)sbus_level - 1UL) * 8UL; } @@ -553,10 +554,8 @@ static void __init sbus_iommu_init(struct platform_device *op) regs = pr->phys_addr; iommu = kzalloc(sizeof(*iommu), GFP_ATOMIC); - if (!iommu) - goto fatal_memory_error; strbuf = kzalloc(sizeof(*strbuf), GFP_ATOMIC); - if (!strbuf) + if (!iommu || !strbuf) goto fatal_memory_error; op->dev.archdata.iommu = iommu; @@ -655,6 +654,8 @@ static void __init sbus_iommu_init(struct platform_device *op) return; fatal_memory_error: + kfree(iommu); + kfree(strbuf); prom_printf("sbus_iommu_init: Fatal memory allocation error.\n"); } diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c index 648f2161b85..baef495c06b 100644 --- a/arch/sparc/kernel/setup_32.c +++ b/arch/sparc/kernel/setup_32.c @@ -31,8 +31,9 @@ #include <linux/root_dev.h> #include <linux/cpu.h> #include <linux/kdebug.h> +#include <linux/export.h> +#include <linux/start_kernel.h> -#include <asm/system.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/oplib.h> @@ -42,9 +43,10 @@ #include <asm/vaddrs.h> #include <asm/mbus.h> #include <asm/idprom.h> -#include <asm/machines.h> #include <asm/cpudata.h> #include <asm/setup.h> +#include <asm/cacheflush.h> +#include <asm/sections.h> #include "kernel.h" @@ -82,8 +84,8 @@ static void prom_sync_me(void) "nop\n\t" : : "r" (&trapbase)); prom_printf("PROM SYNC COMMAND...\n"); - show_free_areas(); - if(current->pid != 0) { + show_free_areas(0); + if (!is_idle_task(current)) { local_irq_enable(); sys_sync(); local_irq_disable(); @@ -103,16 +105,19 @@ static unsigned int boot_flags __initdata = 0; /* Exported for mm/init.c:paging_init. */ unsigned long cmdline_memory_size __initdata = 0; +/* which CPU booted us (0xff = not set) */ +unsigned char boot_cpu_id = 0xff; /* 0xff will make it into DATA section... */ + static void prom_console_write(struct console *con, const char *s, unsigned n) { prom_write(s, n); } -static struct console prom_debug_console = { - .name = "debug", +static struct console prom_early_console = { + .name = "earlyprom", .write = prom_console_write, - .flags = CON_PRINTBUFFER, + .flags = CON_PRINTBUFFER | CON_BOOT, .index = -1, }; @@ -133,8 +138,7 @@ static void __init process_switch(char c) prom_halt(); break; case 'p': - /* Use PROM debug console. */ - register_console(&prom_debug_console); + prom_early_console.flags &= ~CON_BOOT; break; default: printk("Unknown boot switch (-%c)\n", c); @@ -178,14 +182,6 @@ static void __init boot_flags_init(char *commands) } } -/* This routine will in the future do all the nasty prom stuff - * to probe for the mmu type and its parameters, etc. This will - * also be where SMP things happen. - */ - -extern void sun4c_probe_vac(void); -extern char cputypval; - extern unsigned short root_flags; extern unsigned short root_dev; extern unsigned short ram_flags; @@ -197,12 +193,107 @@ extern int root_mountflags; char reboot_command[COMMAND_LINE_SIZE]; +struct cpuid_patch_entry { + unsigned int addr; + unsigned int sun4d[3]; + unsigned int leon[3]; +}; +extern struct cpuid_patch_entry __cpuid_patch, __cpuid_patch_end; + +static void __init per_cpu_patch(void) +{ + struct cpuid_patch_entry *p; + + if (sparc_cpu_model == sun4m) { + /* Nothing to do, this is what the unpatched code + * targets. + */ + return; + } + + p = &__cpuid_patch; + while (p < &__cpuid_patch_end) { + unsigned long addr = p->addr; + unsigned int *insns; + + switch (sparc_cpu_model) { + case sun4d: + insns = &p->sun4d[0]; + break; + + case sparc_leon: + insns = &p->leon[0]; + break; + default: + prom_printf("Unknown cpu type, halting.\n"); + prom_halt(); + } + *(unsigned int *) (addr + 0) = insns[0]; + flushi(addr + 0); + *(unsigned int *) (addr + 4) = insns[1]; + flushi(addr + 4); + *(unsigned int *) (addr + 8) = insns[2]; + flushi(addr + 8); + + p++; + } +} + +struct leon_1insn_patch_entry { + unsigned int addr; + unsigned int insn; +}; + enum sparc_cpu sparc_cpu_model; EXPORT_SYMBOL(sparc_cpu_model); +static __init void leon_patch(void) +{ + struct leon_1insn_patch_entry *start = (void *)__leon_1insn_patch; + struct leon_1insn_patch_entry *end = (void *)__leon_1insn_patch_end; + + /* Default instruction is leon - no patching */ + if (sparc_cpu_model == sparc_leon) + return; + + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *)(addr) = start->insn; + flushi(addr); + + start++; + } +} + struct tt_entry *sparc_ttable; +static struct pt_regs fake_swapper_regs; -struct pt_regs fake_swapper_regs; +/* Called from head_32.S - before we have setup anything + * in the kernel. Be very careful with what you do here. + */ +void __init sparc32_start_kernel(struct linux_romvec *rp) +{ + prom_init(rp); + + /* Set sparc_cpu_model */ + sparc_cpu_model = sun_unknown; + if (!strcmp(&cputypval[0], "sun4m")) + sparc_cpu_model = sun4m; + if (!strcmp(&cputypval[0], "sun4s")) + sparc_cpu_model = sun4m; /* CP-1200 with PROM 2.30 -E */ + if (!strcmp(&cputypval[0], "sun4d")) + sparc_cpu_model = sun4d; + if (!strcmp(&cputypval[0], "sun4e")) + sparc_cpu_model = sun4e; + if (!strcmp(&cputypval[0], "sun4u")) + sparc_cpu_model = sun4u; + if (!strncmp(&cputypval[0], "leon" , 4)) + sparc_cpu_model = sparc_leon; + + leon_patch(); + start_kernel(); +} void __init setup_arch(char **cmdline_p) { @@ -213,36 +304,15 @@ void __init setup_arch(char **cmdline_p) /* Initialize PROM console and command line. */ *cmdline_p = prom_getbootargs(); - strcpy(boot_command_line, *cmdline_p); + strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); parse_early_param(); - /* Set sparc_cpu_model */ - sparc_cpu_model = sun_unknown; - if (!strcmp(&cputypval,"sun4 ")) - sparc_cpu_model = sun4; - if (!strcmp(&cputypval,"sun4c")) - sparc_cpu_model = sun4c; - if (!strcmp(&cputypval,"sun4m")) - sparc_cpu_model = sun4m; - if (!strcmp(&cputypval,"sun4s")) - sparc_cpu_model = sun4m; /* CP-1200 with PROM 2.30 -E */ - if (!strcmp(&cputypval,"sun4d")) - sparc_cpu_model = sun4d; - if (!strcmp(&cputypval,"sun4e")) - sparc_cpu_model = sun4e; - if (!strcmp(&cputypval,"sun4u")) - sparc_cpu_model = sun4u; - if (!strncmp(&cputypval, "leon" , 4)) - sparc_cpu_model = sparc_leon; + boot_flags_init(*cmdline_p); + + register_console(&prom_early_console); printk("ARCH: "); switch(sparc_cpu_model) { - case sun4: - printk("SUN4\n"); - break; - case sun4c: - printk("SUN4C\n"); - break; case sun4m: printk("SUN4M\n"); break; @@ -261,16 +331,13 @@ void __init setup_arch(char **cmdline_p) default: printk("UNKNOWN!\n"); break; - }; + } #ifdef CONFIG_DUMMY_CONSOLE conswitchp = &dummy_con; #endif - boot_flags_init(*cmdline_p); idprom_init(); - if (ARCH_SUN4C) - sun4c_probe_vac(); load_mmu(); phys_base = 0xffffffffUL; @@ -298,89 +365,22 @@ void __init setup_arch(char **cmdline_p) prom_setsync(prom_sync_me); - if((boot_flags&BOOTME_DEBUG) && (linux_dbvec!=0) && + if((boot_flags & BOOTME_DEBUG) && (linux_dbvec != NULL) && ((*(short *)linux_dbvec) != -1)) { printk("Booted under KADB. Syncing trap table.\n"); (*(linux_dbvec->teach_debugger))(); } - init_mm.context = (unsigned long) NO_CONTEXT; init_task.thread.kregs = &fake_swapper_regs; + /* Run-time patch instructions to match the cpu model */ + per_cpu_patch(); + paging_init(); smp_setup_cpu_possible_map(); } -static int ncpus_probed; - -static int show_cpuinfo(struct seq_file *m, void *__unused) -{ - seq_printf(m, - "cpu\t\t: %s\n" - "fpu\t\t: %s\n" - "promlib\t\t: Version %d Revision %d\n" - "prom\t\t: %d.%d\n" - "type\t\t: %s\n" - "ncpus probed\t: %d\n" - "ncpus active\t: %d\n" -#ifndef CONFIG_SMP - "CPU0Bogo\t: %lu.%02lu\n" - "CPU0ClkTck\t: %ld\n" -#endif - , - sparc_cpu_type, - sparc_fpu_type , - romvec->pv_romvers, - prom_rev, - romvec->pv_printrev >> 16, - romvec->pv_printrev & 0xffff, - &cputypval, - ncpus_probed, - num_online_cpus() -#ifndef CONFIG_SMP - , cpu_data(0).udelay_val/(500000/HZ), - (cpu_data(0).udelay_val/(5000/HZ)) % 100, - cpu_data(0).clock_tick -#endif - ); - -#ifdef CONFIG_SMP - smp_bogo(m); -#endif - mmu_info(m); -#ifdef CONFIG_SMP - smp_info(m); -#endif - return 0; -} - -static void *c_start(struct seq_file *m, loff_t *pos) -{ - /* The pointer we are returning is arbitrary, - * it just has to be non-NULL and not IS_ERR - * in the success case. - */ - return *pos == 0 ? &c_start : NULL; -} - -static void *c_next(struct seq_file *m, void *v, loff_t *pos) -{ - ++*pos; - return c_start(m, pos); -} - -static void c_stop(struct seq_file *m, void *v) -{ -} - -const struct seq_operations cpuinfo_op = { - .start =c_start, - .next = c_next, - .stop = c_stop, - .show = show_cpuinfo, -}; - extern int stop_a_enabled; void sun_do_break(void) diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 29bafe051bb..3fdb455e331 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -29,8 +29,8 @@ #include <linux/interrupt.h> #include <linux/cpu.h> #include <linux/initrd.h> +#include <linux/module.h> -#include <asm/system.h> #include <asm/io.h> #include <asm/processor.h> #include <asm/oplib.h> @@ -46,6 +46,9 @@ #include <asm/mmu.h> #include <asm/ns87303.h> #include <asm/btext.h> +#include <asm/elf.h> +#include <asm/mdesc.h> +#include <asm/cacheflush.h> #ifdef CONFIG_IP_PNP #include <net/ipconfig.h> @@ -103,7 +106,7 @@ static void __init process_switch(char c) prom_halt(); break; case 'p': - /* Just ignore, this behavior is now the default. */ + prom_early_console.flags &= ~CON_BOOT; break; case 'P': /* Force UltraSPARC-III P-Cache on. */ @@ -112,7 +115,7 @@ static void __init process_switch(char c) break; } cheetah_pcache_forced_on = 1; - add_taint(TAINT_MACHINE_CHECK); + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); cheetah_enable_pcache(); break; @@ -209,7 +212,7 @@ void __init per_cpu_patch(void) default: prom_printf("Unknown cpu type, halting.\n"); prom_halt(); - }; + } *(unsigned int *) (addr + 0) = insns[0]; wmb(); @@ -231,44 +234,107 @@ void __init per_cpu_patch(void) } } -void __init sun4v_patch(void) +void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *start, + struct sun4v_1insn_patch_entry *end) { - extern void sun4v_hvapi_init(void); - struct sun4v_1insn_patch_entry *p1; - struct sun4v_2insn_patch_entry *p2; - - if (tlb_type != hypervisor) - return; + while (start < end) { + unsigned long addr = start->addr; - p1 = &__sun4v_1insn_patch; - while (p1 < &__sun4v_1insn_patch_end) { - unsigned long addr = p1->addr; - - *(unsigned int *) (addr + 0) = p1->insn; + *(unsigned int *) (addr + 0) = start->insn; wmb(); __asm__ __volatile__("flush %0" : : "r" (addr + 0)); - p1++; + start++; } +} - p2 = &__sun4v_2insn_patch; - while (p2 < &__sun4v_2insn_patch_end) { - unsigned long addr = p2->addr; +void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *start, + struct sun4v_2insn_patch_entry *end) +{ + while (start < end) { + unsigned long addr = start->addr; - *(unsigned int *) (addr + 0) = p2->insns[0]; + *(unsigned int *) (addr + 0) = start->insns[0]; wmb(); __asm__ __volatile__("flush %0" : : "r" (addr + 0)); - *(unsigned int *) (addr + 4) = p2->insns[1]; + *(unsigned int *) (addr + 4) = start->insns[1]; wmb(); __asm__ __volatile__("flush %0" : : "r" (addr + 4)); - p2++; + start++; } +} + +void __init sun4v_patch(void) +{ + extern void sun4v_hvapi_init(void); + + if (tlb_type != hypervisor) + return; + + sun4v_patch_1insn_range(&__sun4v_1insn_patch, + &__sun4v_1insn_patch_end); + + sun4v_patch_2insn_range(&__sun4v_2insn_patch, + &__sun4v_2insn_patch_end); sun4v_hvapi_init(); } +static void __init popc_patch(void) +{ + struct popc_3insn_patch_entry *p3; + struct popc_6insn_patch_entry *p6; + + p3 = &__popc_3insn_patch; + while (p3 < &__popc_3insn_patch_end) { + unsigned long i, addr = p3->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p3->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p3++; + } + + p6 = &__popc_6insn_patch; + while (p6 < &__popc_6insn_patch_end) { + unsigned long i, addr = p6->addr; + + for (i = 0; i < 6; i++) { + *(unsigned int *) (addr + (i * 4)) = p6->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p6++; + } +} + +static void __init pause_patch(void) +{ + struct pause_patch_entry *p; + + p = &__pause_3insn_patch; + while (p < &__pause_3insn_patch_end) { + unsigned long i, addr = p->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p++; + } +} + #ifdef CONFIG_SMP void __init boot_cpu_id_too_large(int cpu) { @@ -278,11 +344,222 @@ void __init boot_cpu_id_too_large(int cpu) } #endif +/* On Ultra, we support all of the v8 capabilities. */ +unsigned long sparc64_elf_hwcap = (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | + HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV | + HWCAP_SPARC_V9); +EXPORT_SYMBOL(sparc64_elf_hwcap); + +static const char *hwcaps[] = { + "flush", "stbar", "swap", "muldiv", "v9", + "ultra3", "blkinit", "n2", + + /* These strings are as they appear in the machine description + * 'hwcap-list' property for cpu nodes. + */ + "mul32", "div32", "fsmuld", "v8plus", "popc", "vis", "vis2", + "ASIBlkInit", "fmaf", "vis3", "hpc", "random", "trans", "fjfmau", + "ima", "cspare", "pause", "cbcond", +}; + +static const char *crypto_hwcaps[] = { + "aes", "des", "kasumi", "camellia", "md5", "sha1", "sha256", + "sha512", "mpmul", "montmul", "montsqr", "crc32c", +}; + +void cpucap_info(struct seq_file *m) +{ + unsigned long caps = sparc64_elf_hwcap; + int i, printed = 0; + + seq_puts(m, "cpucaps\t\t: "); + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + if (caps & bit) { + seq_printf(m, "%s%s", + printed ? "," : "", hwcaps[i]); + printed++; + } + } + if (caps & HWCAP_SPARC_CRYPTO) { + unsigned long cfr; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + unsigned long bit = 1UL << i; + if (cfr & bit) { + seq_printf(m, "%s%s", + printed ? "," : "", crypto_hwcaps[i]); + printed++; + } + } + } + seq_putc(m, '\n'); +} + +static void __init report_one_hwcap(int *printed, const char *name) +{ + if ((*printed) == 0) + printk(KERN_INFO "CPU CAPS: ["); + printk(KERN_CONT "%s%s", + (*printed) ? "," : "", name); + if (++(*printed) == 8) { + printk(KERN_CONT "]\n"); + *printed = 0; + } +} + +static void __init report_crypto_hwcaps(int *printed) +{ + unsigned long cfr; + int i; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + unsigned long bit = 1UL << i; + if (cfr & bit) + report_one_hwcap(printed, crypto_hwcaps[i]); + } +} + +static void __init report_hwcaps(unsigned long caps) +{ + int i, printed = 0; + + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + if (caps & bit) + report_one_hwcap(&printed, hwcaps[i]); + } + if (caps & HWCAP_SPARC_CRYPTO) + report_crypto_hwcaps(&printed); + if (printed != 0) + printk(KERN_CONT "]\n"); +} + +static unsigned long __init mdesc_cpu_hwcap_list(void) +{ + struct mdesc_handle *hp; + unsigned long caps = 0; + const char *prop; + int len; + u64 pn; + + hp = mdesc_grab(); + if (!hp) + return 0; + + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu"); + if (pn == MDESC_NODE_NULL) + goto out; + + prop = mdesc_get_property(hp, pn, "hwcap-list", &len); + if (!prop) + goto out; + + while (len) { + int i, plen; + + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + + if (!strcmp(prop, hwcaps[i])) { + caps |= bit; + break; + } + } + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + if (!strcmp(prop, crypto_hwcaps[i])) + caps |= HWCAP_SPARC_CRYPTO; + } + + plen = strlen(prop) + 1; + prop += plen; + len -= plen; + } + +out: + mdesc_release(hp); + return caps; +} + +/* This yields a mask that user programs can use to figure out what + * instruction set this cpu supports. + */ +static void __init init_sparc64_elf_hwcap(void) +{ + unsigned long cap = sparc64_elf_hwcap; + unsigned long mdesc_caps; + + if (tlb_type == cheetah || tlb_type == cheetah_plus) + cap |= HWCAP_SPARC_ULTRA3; + else if (tlb_type == hypervisor) { + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= HWCAP_SPARC_BLKINIT; + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= HWCAP_SPARC_N2; + } + + cap |= (AV_SPARC_MUL32 | AV_SPARC_DIV32 | AV_SPARC_V8PLUS); + + mdesc_caps = mdesc_cpu_hwcap_list(); + if (!mdesc_caps) { + if (tlb_type == spitfire) + cap |= AV_SPARC_VIS; + if (tlb_type == cheetah || tlb_type == cheetah_plus) + cap |= AV_SPARC_VIS | AV_SPARC_VIS2; + if (tlb_type == cheetah_plus) { + unsigned long impl, ver; + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + impl = ((ver >> 32) & 0xffff); + if (impl == PANTHER_IMPL) + cap |= AV_SPARC_POPC; + } + if (tlb_type == hypervisor) { + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1) + cap |= AV_SPARC_ASI_BLK_INIT; + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 | + AV_SPARC_ASI_BLK_INIT | + AV_SPARC_POPC); + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC | + AV_SPARC_FMAF); + } + } + sparc64_elf_hwcap = cap | mdesc_caps; + + report_hwcaps(sparc64_elf_hwcap); + + if (sparc64_elf_hwcap & AV_SPARC_POPC) + popc_patch(); + if (sparc64_elf_hwcap & AV_SPARC_PAUSE) + pause_patch(); +} + void __init setup_arch(char **cmdline_p) { /* Initialize PROM console and command line. */ *cmdline_p = prom_getbootargs(); - strcpy(boot_command_line, *cmdline_p); + strlcpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); parse_early_param(); boot_flags_init(*cmdline_p); @@ -337,86 +614,9 @@ void __init setup_arch(char **cmdline_p) init_cur_cpu_trap(current_thread_info()); paging_init(); + init_sparc64_elf_hwcap(); } -/* BUFFER is PAGE_SIZE bytes long. */ - -extern void smp_info(struct seq_file *); -extern void smp_bogo(struct seq_file *); -extern void mmu_info(struct seq_file *); - -unsigned int dcache_parity_tl1_occurred; -unsigned int icache_parity_tl1_occurred; - -int ncpus_probed; - -static int show_cpuinfo(struct seq_file *m, void *__unused) -{ - seq_printf(m, - "cpu\t\t: %s\n" - "fpu\t\t: %s\n" - "pmu\t\t: %s\n" - "prom\t\t: %s\n" - "type\t\t: %s\n" - "ncpus probed\t: %d\n" - "ncpus active\t: %d\n" - "D$ parity tl1\t: %u\n" - "I$ parity tl1\t: %u\n" -#ifndef CONFIG_SMP - "Cpu0ClkTck\t: %016lx\n" -#endif - , - sparc_cpu_type, - sparc_fpu_type, - sparc_pmu_type, - prom_version, - ((tlb_type == hypervisor) ? - "sun4v" : - "sun4u"), - ncpus_probed, - num_online_cpus(), - dcache_parity_tl1_occurred, - icache_parity_tl1_occurred -#ifndef CONFIG_SMP - , cpu_data(0).clock_tick -#endif - ); -#ifdef CONFIG_SMP - smp_bogo(m); -#endif - mmu_info(m); -#ifdef CONFIG_SMP - smp_info(m); -#endif - return 0; -} - -static void *c_start(struct seq_file *m, loff_t *pos) -{ - /* The pointer we are returning is arbitrary, - * it just has to be non-NULL and not IS_ERR - * in the success case. - */ - return *pos == 0 ? &c_start : NULL; -} - -static void *c_next(struct seq_file *m, void *v, loff_t *pos) -{ - ++*pos; - return c_start(m, pos); -} - -static void c_stop(struct seq_file *m, void *v) -{ -} - -const struct seq_operations cpuinfo_op = { - .start =c_start, - .next = c_next, - .stop = c_stop, - .show = show_cpuinfo, -}; - extern int stop_a_enabled; void sun_do_break(void) diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c index 75fad425e24..62deba7be1a 100644 --- a/arch/sparc/kernel/signal32.c +++ b/arch/sparc/kernel/signal32.c @@ -28,8 +28,10 @@ #include <asm/fpumacro.h> #include <asm/visasm.h> #include <asm/compat_signal.h> +#include <asm/switch_to.h> -#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) +#include "sigutil.h" +#include "kernel.h" /* This magic should be in g_upper[0] for all upper parts * to be valid. @@ -44,86 +46,30 @@ typedef struct { struct signal_frame32 { struct sparc_stackf32 ss; __siginfo32_t info; - /* __siginfo_fpu32_t * */ u32 fpu_save; + /* __siginfo_fpu_t * */ u32 fpu_save; unsigned int insns[2]; unsigned int extramask[_COMPAT_NSIG_WORDS - 1]; unsigned int extra_size; /* Should be sizeof(siginfo_extra_v8plus_t) */ /* Only valid if (info.si_regs.psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS */ siginfo_extra_v8plus_t v8plus; - __siginfo_fpu_t fpu_state; -}; - -typedef struct compat_siginfo{ - int si_signo; - int si_errno; - int si_code; - - union { - int _pad[SI_PAD_SIZE32]; - - /* kill() */ - struct { - compat_pid_t _pid; /* sender's pid */ - unsigned int _uid; /* sender's uid */ - } _kill; - - /* POSIX.1b timers */ - struct { - compat_timer_t _tid; /* timer id */ - int _overrun; /* overrun count */ - compat_sigval_t _sigval; /* same as below */ - int _sys_private; /* not to be passed to user */ - } _timer; - - /* POSIX.1b signals */ - struct { - compat_pid_t _pid; /* sender's pid */ - unsigned int _uid; /* sender's uid */ - compat_sigval_t _sigval; - } _rt; - - /* SIGCHLD */ - struct { - compat_pid_t _pid; /* which child */ - unsigned int _uid; /* sender's uid */ - int _status; /* exit code */ - compat_clock_t _utime; - compat_clock_t _stime; - } _sigchld; - - /* SIGILL, SIGFPE, SIGSEGV, SIGBUS, SIGEMT */ - struct { - u32 _addr; /* faulting insn/memory ref. */ - int _trapno; - } _sigfault; - - /* SIGPOLL */ - struct { - int _band; /* POLL_IN, POLL_OUT, POLL_MSG */ - int _fd; - } _sigpoll; - } _sifields; -}compat_siginfo_t; + /* __siginfo_rwin_t * */u32 rwin_save; +} __attribute__((aligned(8))); struct rt_signal_frame32 { struct sparc_stackf32 ss; compat_siginfo_t info; struct pt_regs32 regs; compat_sigset_t mask; - /* __siginfo_fpu32_t * */ u32 fpu_save; + /* __siginfo_fpu_t * */ u32 fpu_save; unsigned int insns[2]; - stack_t32 stack; + compat_stack_t stack; unsigned int extra_size; /* Should be sizeof(siginfo_extra_v8plus_t) */ /* Only valid if (regs.psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS */ siginfo_extra_v8plus_t v8plus; - __siginfo_fpu_t fpu_state; -}; + /* __siginfo_rwin_t * */u32 rwin_save; +} __attribute__((aligned(8))); -/* Align macros */ -#define SF_ALIGNEDSZ (((sizeof(struct signal_frame32) + 15) & (~15))) -#define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame32) + 15) & (~15))) - -int copy_siginfo_to_user32(compat_siginfo_t __user *to, siginfo_t *from) +int copy_siginfo_to_user32(compat_siginfo_t __user *to, const siginfo_t *from) { int err; @@ -192,32 +138,15 @@ int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from) return 0; } -static int restore_fpu_state32(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - unsigned long *fpregs = current_thread_info()->fpregs; - unsigned long fprs; - int err; - - err = __get_user(fprs, &fpu->si_fprs); - fprs_write(0); - regs->tstate &= ~TSTATE_PEF; - if (fprs & FPRS_DL) - err |= copy_from_user(fpregs, &fpu->si_float_regs[0], (sizeof(unsigned int) * 32)); - if (fprs & FPRS_DU) - err |= copy_from_user(fpregs+16, &fpu->si_float_regs[32], (sizeof(unsigned int) * 32)); - err |= __get_user(current_thread_info()->xfsr[0], &fpu->si_fsr); - err |= __get_user(current_thread_info()->gsr[0], &fpu->si_gsr); - current_thread_info()->fpsaved[0] |= fprs; - return err; -} - void do_sigreturn32(struct pt_regs *regs) { struct signal_frame32 __user *sf; + compat_uptr_t fpu_save; + compat_uptr_t rwin_save; unsigned int psr; - unsigned pc, npc, fpu_save; + unsigned pc, npc; sigset_t set; - unsigned seta[_COMPAT_NSIG_WORDS]; + compat_sigset_t seta; int err, i; /* Always make any pending restarted system calls return -EINTR */ @@ -233,8 +162,9 @@ void do_sigreturn32(struct pt_regs *regs) (((unsigned long) sf) & 3)) goto segv; - get_user(pc, &sf->info.si_regs.pc); - __get_user(npc, &sf->info.si_regs.npc); + if (get_user(pc, &sf->info.si_regs.pc) || + __get_user(npc, &sf->info.si_regs.npc)) + goto segv; if ((pc | npc) & 3) goto segv; @@ -273,24 +203,21 @@ void do_sigreturn32(struct pt_regs *regs) pt_regs_clear_syscall(regs); err |= __get_user(fpu_save, &sf->fpu_save); - if (fpu_save) - err |= restore_fpu_state32(regs, &sf->fpu_state); - err |= __get_user(seta[0], &sf->info.si_mask); - err |= copy_from_user(seta+1, &sf->extramask, + if (!err && fpu_save) + err |= restore_fpu_state(regs, compat_ptr(fpu_save)); + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(compat_ptr(rwin_save))) + goto segv; + } + err |= __get_user(seta.sig[0], &sf->info.si_mask); + err |= copy_from_user(&seta.sig[1], &sf->extramask, (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); if (err) goto segv; - switch (_NSIG_WORDS) { - case 4: set.sig[3] = seta[6] + (((long)seta[7]) << 32); - case 3: set.sig[2] = seta[4] + (((long)seta[5]) << 32); - case 2: set.sig[1] = seta[2] + (((long)seta[3]) << 32); - case 1: set.sig[0] = seta[0] + (((long)seta[1]) << 32); - } - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + + set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); + set_current_blocked(&set); return; segv: @@ -300,11 +227,11 @@ segv: asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) { struct rt_signal_frame32 __user *sf; - unsigned int psr, pc, npc, fpu_save, u_ss_sp; - mm_segment_t old_fs; + unsigned int psr, pc, npc; + compat_uptr_t fpu_save; + compat_uptr_t rwin_save; sigset_t set; compat_sigset_t seta; - stack_t st; int err, i; /* Always make any pending restarted system calls return -EINTR */ @@ -319,8 +246,9 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) (((unsigned long) sf) & 3)) goto segv; - get_user(pc, &sf->regs.pc); - __get_user(npc, &sf->regs.npc); + if (get_user(pc, &sf->regs.pc) || + __get_user(npc, &sf->regs.npc)) + goto segv; if ((pc | npc) & 3) goto segv; @@ -359,34 +287,21 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) pt_regs_clear_syscall(regs); err |= __get_user(fpu_save, &sf->fpu_save); - if (fpu_save) - err |= restore_fpu_state32(regs, &sf->fpu_state); + if (!err && fpu_save) + err |= restore_fpu_state(regs, compat_ptr(fpu_save)); err |= copy_from_user(&seta, &sf->mask, sizeof(compat_sigset_t)); - err |= __get_user(u_ss_sp, &sf->stack.ss_sp); - st.ss_sp = compat_ptr(u_ss_sp); - err |= __get_user(st.ss_flags, &sf->stack.ss_flags); - err |= __get_user(st.ss_size, &sf->stack.ss_size); + err |= compat_restore_altstack(&sf->stack); if (err) goto segv; - /* It is more difficult to avoid calling this function than to - call it and ignore errors. */ - old_fs = get_fs(); - set_fs(KERNEL_DS); - do_sigaltstack((stack_t __user *) &st, NULL, (unsigned long)sf); - set_fs(old_fs); - - switch (_NSIG_WORDS) { - case 4: set.sig[3] = seta.sig[6] + (((long)seta.sig[7]) << 32); - case 3: set.sig[2] = seta.sig[4] + (((long)seta.sig[5]) << 32); - case 2: set.sig[1] = seta.sig[2] + (((long)seta.sig[3]) << 32); - case 1: set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(compat_ptr(rwin_save))) + goto segv; } - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + + set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); + set_current_blocked(&set); return; segv: force_sig(SIGSEGV, current); @@ -400,7 +315,7 @@ static int invalid_frame_pointer(void __user *fp, int fplen) return 0; } -static void __user *get_sigframe(struct sigaction *sa, struct pt_regs *regs, unsigned long framesize) +static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp; @@ -415,12 +330,7 @@ static void __user *get_sigframe(struct sigaction *sa, struct pt_regs *regs, uns return (void __user *) -1L; /* This is the X/Open sanctioned signal stack switching. */ - if (sa->sa_flags & SA_ONSTACK) { - if (sas_ss_flags(sp) == 0) - sp = current->sas_ss_sp + current->sas_ss_size; - } - - sp -= framesize; + sp = sigsp(sp, ksig) - framesize; /* Always align the stack frame. This handles two cases. First, * sigaltstack need not be mindful of platform specific stack @@ -433,26 +343,6 @@ static void __user *get_sigframe(struct sigaction *sa, struct pt_regs *regs, uns return (void __user *) sp; } -static int save_fpu_state32(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - unsigned long *fpregs = current_thread_info()->fpregs; - unsigned long fprs; - int err = 0; - - fprs = current_thread_info()->fpsaved[0]; - if (fprs & FPRS_DL) - err |= copy_to_user(&fpu->si_float_regs[0], fpregs, - (sizeof(unsigned int) * 32)); - if (fprs & FPRS_DU) - err |= copy_to_user(&fpu->si_float_regs[32], fpregs+16, - (sizeof(unsigned int) * 32)); - err |= __put_user(current_thread_info()->xfsr[0], &fpu->si_fsr); - err |= __put_user(current_thread_info()->gsr[0], &fpu->si_gsr); - err |= __put_user(fprs, &fpu->si_fprs); - - return err; -} - /* The I-cache flush instruction only works in the primary ASI, which * right now is the nucleus, aka. kernel space. * @@ -511,31 +401,37 @@ out_irqs_on: } -static int setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, - int signo, sigset_t *oldset) +static int setup_frame32(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) { struct signal_frame32 __user *sf; + int i, err, wsaved; + void __user *tail; int sigframe_size; u32 psr; - int i, err; - unsigned int seta[_COMPAT_NSIG_WORDS]; + compat_sigset_t seta; /* 1. Make sure everything is clean */ synchronize_user_stack(); save_and_clear_fpu(); - sigframe_size = SF_ALIGNEDSZ; - if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) - sigframe_size -= sizeof(__siginfo_fpu_t); + wsaved = get_thread_wsaved(); + + sigframe_size = sizeof(*sf); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); sf = (struct signal_frame32 __user *) - get_sigframe(&ka->sa, regs, sigframe_size); + get_sigframe(ksig, regs, sigframe_size); - if (invalid_frame_pointer(sf, sigframe_size)) - goto sigill; + if (invalid_frame_pointer(sf, sigframe_size)) { + do_exit(SIGILL); + return -EINVAL; + } - if (get_thread_wsaved() != 0) - goto sigill; + tail = (sf + 1); /* 2. Save the current process state */ if (test_thread_flag(TIF_32BIT)) { @@ -560,41 +456,59 @@ static int setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, &sf->v8plus.asi); if (psr & PSR_EF) { - err |= save_fpu_state32(regs, &sf->fpu_state); - err |= __put_user((u64)&sf->fpu_state, &sf->fpu_save); + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user((u64)fp, &sf->fpu_save); } else { err |= __put_user(0, &sf->fpu_save); } - - switch (_NSIG_WORDS) { - case 4: seta[7] = (oldset->sig[3] >> 32); - seta[6] = oldset->sig[3]; - case 3: seta[5] = (oldset->sig[2] >> 32); - seta[4] = oldset->sig[2]; - case 2: seta[3] = (oldset->sig[1] >> 32); - seta[2] = oldset->sig[1]; - case 1: seta[1] = (oldset->sig[0] >> 32); - seta[0] = oldset->sig[0]; + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user((u64)rwp, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); } - err |= __put_user(seta[0], &sf->info.si_mask); - err |= __copy_to_user(sf->extramask, seta + 1, + + /* If these change we need to know - assignments to seta relies on these sizes */ + BUILD_BUG_ON(_NSIG_WORDS != 1); + BUILD_BUG_ON(_COMPAT_NSIG_WORDS != 2); + seta.sig[1] = (oldset->sig[0] >> 32); + seta.sig[0] = oldset->sig[0]; + + err |= __put_user(seta.sig[0], &sf->info.si_mask); + err |= __copy_to_user(sf->extramask, &seta.sig[1], (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); - err |= copy_in_user((u32 __user *)sf, - (u32 __user *)(regs->u_regs[UREG_FP]), - sizeof(struct reg_window32)); - + if (!wsaved) { + err |= copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); + } else { + struct reg_window *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + for (i = 0; i < 8; i++) + err |= __put_user(rp->locals[i], &sf->ss.locals[i]); + for (i = 0; i < 6; i++) + err |= __put_user(rp->ins[i], &sf->ss.ins[i]); + err |= __put_user(rp->ins[6], &sf->ss.fp); + err |= __put_user(rp->ins[7], &sf->ss.callers_pc); + } if (err) - goto sigsegv; + return err; /* 3. signal handler back-trampoline and parameters */ regs->u_regs[UREG_FP] = (unsigned long) sf; - regs->u_regs[UREG_I0] = signo; + regs->u_regs[UREG_I0] = ksig->sig; regs->u_regs[UREG_I1] = (unsigned long) &sf->info; regs->u_regs[UREG_I2] = (unsigned long) &sf->info; /* 4. signal handler */ - regs->tpc = (unsigned long) ka->sa.sa_handler; + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; regs->tnpc = (regs->tpc + 4); if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; @@ -602,8 +516,8 @@ static int setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, } /* 5. return to kernel instructions */ - if (ka->ka_restorer) { - regs->u_regs[UREG_I7] = (unsigned long)ka->ka_restorer; + if (ksig->ka.ka_restorer) { + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; } else { unsigned long address = ((unsigned long)&(sf->insns[0])); @@ -612,47 +526,43 @@ static int setup_frame32(struct k_sigaction *ka, struct pt_regs *regs, err = __put_user(0x821020d8, &sf->insns[0]); /*mov __NR_sigreturn, %g1*/ err |= __put_user(0x91d02010, &sf->insns[1]); /*t 0x10*/ if (err) - goto sigsegv; - + return err; flush_signal_insns(address); } return 0; - -sigill: - do_exit(SIGILL); - return -EINVAL; - -sigsegv: - force_sigsegv(signo, current); - return -EFAULT; } -static int setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs, - unsigned long signr, sigset_t *oldset, - siginfo_t *info) +static int setup_rt_frame32(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) { struct rt_signal_frame32 __user *sf; + int i, err, wsaved; + void __user *tail; int sigframe_size; u32 psr; - int i, err; compat_sigset_t seta; /* 1. Make sure everything is clean */ synchronize_user_stack(); save_and_clear_fpu(); - sigframe_size = RT_ALIGNEDSZ; - if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) - sigframe_size -= sizeof(__siginfo_fpu_t); + wsaved = get_thread_wsaved(); + + sigframe_size = sizeof(*sf); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); sf = (struct rt_signal_frame32 __user *) - get_sigframe(&ka->sa, regs, sigframe_size); + get_sigframe(ksig, regs, sigframe_size); - if (invalid_frame_pointer(sf, sigframe_size)) - goto sigill; + if (invalid_frame_pointer(sf, sigframe_size)) { + do_exit(SIGILL); + return -EINVAL; + } - if (get_thread_wsaved() != 0) - goto sigill; + tail = (sf + 1); /* 2. Save the current process state */ if (test_thread_flag(TIF_32BIT)) { @@ -677,46 +587,59 @@ static int setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs, &sf->v8plus.asi); if (psr & PSR_EF) { - err |= save_fpu_state32(regs, &sf->fpu_state); - err |= __put_user((u64)&sf->fpu_state, &sf->fpu_save); + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user((u64)fp, &sf->fpu_save); } else { err |= __put_user(0, &sf->fpu_save); } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user((u64)rwp, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); + } /* Update the siginfo structure. */ - err |= copy_siginfo_to_user32(&sf->info, info); + err |= copy_siginfo_to_user32(&sf->info, &ksig->info); /* Setup sigaltstack */ - err |= __put_user(current->sas_ss_sp, &sf->stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->u_regs[UREG_FP]), &sf->stack.ss_flags); - err |= __put_user(current->sas_ss_size, &sf->stack.ss_size); - - switch (_NSIG_WORDS) { - case 4: seta.sig[7] = (oldset->sig[3] >> 32); - seta.sig[6] = oldset->sig[3]; - case 3: seta.sig[5] = (oldset->sig[2] >> 32); - seta.sig[4] = oldset->sig[2]; - case 2: seta.sig[3] = (oldset->sig[1] >> 32); - seta.sig[2] = oldset->sig[1]; - case 1: seta.sig[1] = (oldset->sig[0] >> 32); - seta.sig[0] = oldset->sig[0]; - } + err |= __compat_save_altstack(&sf->stack, regs->u_regs[UREG_FP]); + + seta.sig[1] = (oldset->sig[0] >> 32); + seta.sig[0] = oldset->sig[0]; err |= __copy_to_user(&sf->mask, &seta, sizeof(compat_sigset_t)); - err |= copy_in_user((u32 __user *)sf, - (u32 __user *)(regs->u_regs[UREG_FP]), - sizeof(struct reg_window32)); + if (!wsaved) { + err |= copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); + } else { + struct reg_window *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + for (i = 0; i < 8; i++) + err |= __put_user(rp->locals[i], &sf->ss.locals[i]); + for (i = 0; i < 6; i++) + err |= __put_user(rp->ins[i], &sf->ss.ins[i]); + err |= __put_user(rp->ins[6], &sf->ss.fp); + err |= __put_user(rp->ins[7], &sf->ss.callers_pc); + } if (err) - goto sigsegv; + return err; /* 3. signal handler back-trampoline and parameters */ regs->u_regs[UREG_FP] = (unsigned long) sf; - regs->u_regs[UREG_I0] = signr; + regs->u_regs[UREG_I0] = ksig->sig; regs->u_regs[UREG_I1] = (unsigned long) &sf->info; regs->u_regs[UREG_I2] = (unsigned long) &sf->regs; /* 4. signal handler */ - regs->tpc = (unsigned long) ka->sa.sa_handler; + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; regs->tnpc = (regs->tpc + 4); if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; @@ -724,8 +647,8 @@ static int setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs, } /* 5. return to kernel instructions */ - if (ka->ka_restorer) - regs->u_regs[UREG_I7] = (unsigned long)ka->ka_restorer; + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; else { unsigned long address = ((unsigned long)&(sf->insns[0])); @@ -737,45 +660,25 @@ static int setup_rt_frame32(struct k_sigaction *ka, struct pt_regs *regs, /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); if (err) - goto sigsegv; + return err; flush_signal_insns(address); } return 0; - -sigill: - do_exit(SIGILL); - return -EINVAL; - -sigsegv: - force_sigsegv(signr, current); - return -EFAULT; } -static inline int handle_signal32(unsigned long signr, struct k_sigaction *ka, - siginfo_t *info, - sigset_t *oldset, struct pt_regs *regs) +static inline void handle_signal32(struct ksignal *ksig, + struct pt_regs *regs) { + sigset_t *oldset = sigmask_to_save(); int err; - if (ka->sa.sa_flags & SA_SIGINFO) - err = setup_rt_frame32(ka, regs, signr, oldset, info); + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err = setup_rt_frame32(ksig, regs, oldset); else - err = setup_frame32(ka, regs, signr, oldset); - - if (err) - return err; + err = setup_frame32(ksig, regs, oldset); - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NOMASK)) - sigaddset(¤t->blocked,signr); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - tracehook_signal_handler(signr, info, ka, regs, 0); - - return 0; + signal_setup_done(err, ksig, 0); } static inline void syscall_restart32(unsigned long orig_i0, struct pt_regs *regs, @@ -803,59 +706,42 @@ static inline void syscall_restart32(unsigned long orig_i0, struct pt_regs *regs * want to handle. Thus you cannot kill init even with a SIGKILL even by * mistake. */ -void do_signal32(sigset_t *oldset, struct pt_regs * regs, - int restart_syscall, unsigned long orig_i0) +void do_signal32(struct pt_regs * regs) { - struct k_sigaction ka; - siginfo_t info; - int signr; - - signr = get_signal_to_deliver(&info, &ka, regs, NULL); - - /* If the debugger messes with the program counter, it clears - * the "in syscall" bit, directing us to not perform a syscall - * restart. - */ - if (restart_syscall && !pt_regs_is_syscall(regs)) - restart_syscall = 0; + struct ksignal ksig; + unsigned long orig_i0 = 0; + int restart_syscall = 0; + bool has_handler = get_signal(&ksig); + + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } - if (signr > 0) { + if (has_handler) { if (restart_syscall) - syscall_restart32(orig_i0, regs, &ka.sa); - if (handle_signal32(signr, &ka, &info, oldset, regs) == 0) { - /* A signal was successfully delivered; the saved - * sigmask will have been stored in the signal frame, - * and will be restored by sigreturn, so we can simply - * clear the TS_RESTORE_SIGMASK flag. - */ - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; + syscall_restart32(orig_i0, regs, &ksig.ka.sa); + handle_signal32(&ksig, regs); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + } } - return; - } - if (restart_syscall && - (regs->u_regs[UREG_I0] == ERESTARTNOHAND || - regs->u_regs[UREG_I0] == ERESTARTSYS || - regs->u_regs[UREG_I0] == ERESTARTNOINTR)) { - /* replay the system call when we are done */ - regs->u_regs[UREG_I0] = orig_i0; - regs->tpc -= 4; - regs->tnpc -= 4; - pt_regs_clear_syscall(regs); - } - if (restart_syscall && - regs->u_regs[UREG_I0] == ERESTART_RESTARTBLOCK) { - regs->u_regs[UREG_G1] = __NR_restart_syscall; - regs->tpc -= 4; - regs->tnpc -= 4; - pt_regs_clear_syscall(regs); - } - - /* If there's no signal to deliver, we just put the saved sigmask - * back - */ - if (current_thread_info()->status & TS_RESTORE_SIGMASK) { - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; - sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); + restore_saved_sigmask(); } } @@ -906,29 +792,3 @@ asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp) out: return ret; } - -asmlinkage long do_sys32_sigaltstack(u32 ussa, u32 uossa, unsigned long sp) -{ - stack_t uss, uoss; - u32 u_ss_sp = 0; - int ret; - mm_segment_t old_fs; - stack_t32 __user *uss32 = compat_ptr(ussa); - stack_t32 __user *uoss32 = compat_ptr(uossa); - - if (ussa && (get_user(u_ss_sp, &uss32->ss_sp) || - __get_user(uss.ss_flags, &uss32->ss_flags) || - __get_user(uss.ss_size, &uss32->ss_size))) - return -EFAULT; - uss.ss_sp = compat_ptr(u_ss_sp); - old_fs = get_fs(); - set_fs(KERNEL_DS); - ret = do_sigaltstack(ussa ? (stack_t __user *) &uss : NULL, - uossa ? (stack_t __user *) &uoss : NULL, sp); - set_fs(old_fs); - if (!ret && uossa && (put_user(ptr_to_compat(uoss.ss_sp), &uoss32->ss_sp) || - __put_user(uoss.ss_flags, &uoss32->ss_flags) || - __put_user(uoss.ss_size, &uoss32->ss_size))) - return -EFAULT; - return ret; -} diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index 5e5c5fd0378..9ee72fc8e0e 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -25,8 +25,10 @@ #include <asm/pgalloc.h> #include <asm/pgtable.h> #include <asm/cacheflush.h> /* flush_sig_insns */ +#include <asm/switch_to.h> -#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) +#include "sigutil.h" +#include "kernel.h" extern void fpsave(unsigned long *fpregs, unsigned long *fsr, void *fpqueue, unsigned long *fpqdepth); @@ -39,8 +41,8 @@ struct signal_frame { unsigned long insns[2] __attribute__ ((aligned (8))); unsigned int extramask[_NSIG_WORDS - 1]; unsigned int extra_size; /* Should be 0 */ - __siginfo_fpu_t fpu_state; -}; + __siginfo_rwin_t __user *rwin_save; +} __attribute__((aligned(8))); struct rt_signal_frame { struct sparc_stackf ss; @@ -51,71 +53,20 @@ struct rt_signal_frame { unsigned int insns[2]; stack_t stack; unsigned int extra_size; /* Should be 0 */ - __siginfo_fpu_t fpu_state; -}; + __siginfo_rwin_t __user *rwin_save; +} __attribute__((aligned(8))); /* Align macros */ #define SF_ALIGNEDSZ (((sizeof(struct signal_frame) + 7) & (~7))) #define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) -static int _sigpause_common(old_sigset_t set) -{ - set &= _BLOCKABLE; - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - siginitset(¤t->blocked, set); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - set_thread_flag(TIF_RESTORE_SIGMASK); - - return -ERESTARTNOHAND; -} - -asmlinkage int sys_sigsuspend(old_sigset_t set) -{ - return _sigpause_common(set); -} - -static inline int -restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - int err; -#ifdef CONFIG_SMP - if (test_tsk_thread_flag(current, TIF_USEDFPU)) - regs->psr &= ~PSR_EF; -#else - if (current == last_task_used_math) { - last_task_used_math = NULL; - regs->psr &= ~PSR_EF; - } -#endif - set_used_math(); - clear_tsk_thread_flag(current, TIF_USEDFPU); - - if (!access_ok(VERIFY_READ, fpu, sizeof(*fpu))) - return -EFAULT; - - err = __copy_from_user(¤t->thread.float_regs[0], &fpu->si_float_regs[0], - (sizeof(unsigned long) * 32)); - err |= __get_user(current->thread.fsr, &fpu->si_fsr); - err |= __get_user(current->thread.fpqdepth, &fpu->si_fpqdepth); - if (current->thread.fpqdepth != 0) - err |= __copy_from_user(¤t->thread.fpqueue[0], - &fpu->si_fpqueue[0], - ((sizeof(unsigned long) + - (sizeof(unsigned long *)))*16)); - return err; -} - asmlinkage void do_sigreturn(struct pt_regs *regs) { struct signal_frame __user *sf; unsigned long up_psr, pc, npc; sigset_t set; __siginfo_fpu_t __user *fpu_save; + __siginfo_rwin_t __user *rwin_save; int err; /* Always make any pending restarted system calls return -EINTR */ @@ -150,9 +101,11 @@ asmlinkage void do_sigreturn(struct pt_regs *regs) pt_regs_clear_syscall(regs); err |= __get_user(fpu_save, &sf->fpu_save); - if (fpu_save) err |= restore_fpu_state(regs, fpu_save); + err |= __get_user(rwin_save, &sf->rwin_save); + if (rwin_save) + err |= restore_rwin_state(rwin_save); /* This is pretty much atomic, no amount locking would prevent * the races which exist anyways. @@ -164,11 +117,7 @@ asmlinkage void do_sigreturn(struct pt_regs *regs) if (err) goto segv_and_exit; - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); return; segv_and_exit: @@ -180,9 +129,8 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) struct rt_signal_frame __user *sf; unsigned int psr, pc, npc; __siginfo_fpu_t __user *fpu_save; - mm_segment_t old_fs; + __siginfo_rwin_t __user *rwin_save; sigset_t set; - stack_t st; int err; synchronize_user_stack(); @@ -207,12 +155,10 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) pt_regs_clear_syscall(regs); err |= __get_user(fpu_save, &sf->fpu_save); - - if (fpu_save) + if (!err && fpu_save) err |= restore_fpu_state(regs, fpu_save); err |= __copy_from_user(&set, &sf->mask, sizeof(sigset_t)); - - err |= __copy_from_user(&st, &sf->stack, sizeof(stack_t)); + err |= restore_altstack(&sf->stack); if (err) goto segv; @@ -220,19 +166,13 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) regs->pc = pc; regs->npc = npc; - /* It is more difficult to avoid calling this function than to - * call it and ignore errors. - */ - old_fs = get_fs(); - set_fs(KERNEL_DS); - do_sigaltstack((const stack_t __user *) &st, NULL, (unsigned long)sf); - set_fs(old_fs); - - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(rwin_save)) + goto segv; + } + + set_current_blocked(&set); return; segv: force_sig(SIGSEGV, current); @@ -241,16 +181,13 @@ segv: /* Checks if the fp is valid */ static inline int invalid_frame_pointer(void __user *fp, int fplen) { - if ((((unsigned long) fp) & 7) || - !__access_ok((unsigned long)fp, fplen) || - ((sparc_cpu_model == sun4 || sparc_cpu_model == sun4c) && - ((unsigned long) fp < 0xe0000000 && (unsigned long) fp >= 0x20000000))) + if ((((unsigned long) fp) & 7) || !__access_ok((unsigned long)fp, fplen)) return 1; - + return 0; } -static inline void __user *get_sigframe(struct sigaction *sa, struct pt_regs *regs, unsigned long framesize) +static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP]; @@ -262,12 +199,7 @@ static inline void __user *get_sigframe(struct sigaction *sa, struct pt_regs *re return (void __user *) -1L; /* This is the X/Open sanctioned signal stack switching. */ - if (sa->sa_flags & SA_ONSTACK) { - if (sas_ss_flags(sp) == 0) - sp = current->sas_ss_sp + current->sas_ss_size; - } - - sp -= framesize; + sp = sigsp(sp, ksig) - framesize; /* Always align the stack frame. This handles two cases. First, * sigaltstack need not be mindful of platform specific stack @@ -280,62 +212,33 @@ static inline void __user *get_sigframe(struct sigaction *sa, struct pt_regs *re return (void __user *) sp; } -static inline int -save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - int err = 0; -#ifdef CONFIG_SMP - if (test_tsk_thread_flag(current, TIF_USEDFPU)) { - put_psr(get_psr() | PSR_EF); - fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, - ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); - regs->psr &= ~(PSR_EF); - clear_tsk_thread_flag(current, TIF_USEDFPU); - } -#else - if (current == last_task_used_math) { - put_psr(get_psr() | PSR_EF); - fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, - ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); - last_task_used_math = NULL; - regs->psr &= ~(PSR_EF); - } -#endif - err |= __copy_to_user(&fpu->si_float_regs[0], - ¤t->thread.float_regs[0], - (sizeof(unsigned long) * 32)); - err |= __put_user(current->thread.fsr, &fpu->si_fsr); - err |= __put_user(current->thread.fpqdepth, &fpu->si_fpqdepth); - if (current->thread.fpqdepth != 0) - err |= __copy_to_user(&fpu->si_fpqueue[0], - ¤t->thread.fpqueue[0], - ((sizeof(unsigned long) + - (sizeof(unsigned long *)))*16)); - clear_used_math(); - return err; -} - -static int setup_frame(struct k_sigaction *ka, struct pt_regs *regs, - int signo, sigset_t *oldset) +static int setup_frame(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) { struct signal_frame __user *sf; - int sigframe_size, err; + int sigframe_size, err, wsaved; + void __user *tail; /* 1. Make sure everything is clean */ synchronize_user_stack(); - sigframe_size = SF_ALIGNEDSZ; - if (!used_math()) - sigframe_size -= sizeof(__siginfo_fpu_t); + wsaved = current_thread_info()->w_saved; + + sigframe_size = sizeof(*sf); + if (used_math()) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); sf = (struct signal_frame __user *) - get_sigframe(&ka->sa, regs, sigframe_size); + get_sigframe(ksig, regs, sigframe_size); - if (invalid_frame_pointer(sf, sigframe_size)) - goto sigill_and_return; + if (invalid_frame_pointer(sf, sigframe_size)) { + do_exit(SIGILL); + return -EINVAL; + } - if (current_thread_info()->w_saved != 0) - goto sigill_and_return; + tail = sf + 1; /* 2. Save the current process state */ err = __copy_to_user(&sf->info.si_regs, regs, sizeof(struct pt_regs)); @@ -343,33 +246,50 @@ static int setup_frame(struct k_sigaction *ka, struct pt_regs *regs, err |= __put_user(0, &sf->extra_size); if (used_math()) { - err |= save_fpu_state(regs, &sf->fpu_state); - err |= __put_user(&sf->fpu_state, &sf->fpu_save); + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user(fp, &sf->fpu_save); } else { err |= __put_user(0, &sf->fpu_save); } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user(rwp, &sf->rwin_save); + } else { + err |= __put_user(0, &sf->rwin_save); + } err |= __put_user(oldset->sig[0], &sf->info.si_mask); err |= __copy_to_user(sf->extramask, &oldset->sig[1], (_NSIG_WORDS - 1) * sizeof(unsigned int)); - err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], - sizeof(struct reg_window32)); + if (!wsaved) { + err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], + sizeof(struct reg_window32)); + } else { + struct reg_window32 *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= __copy_to_user(sf, rp, sizeof(struct reg_window32)); + } if (err) - goto sigsegv; + return err; /* 3. signal handler back-trampoline and parameters */ regs->u_regs[UREG_FP] = (unsigned long) sf; - regs->u_regs[UREG_I0] = signo; + regs->u_regs[UREG_I0] = ksig->sig; regs->u_regs[UREG_I1] = (unsigned long) &sf->info; regs->u_regs[UREG_I2] = (unsigned long) &sf->info; /* 4. signal handler */ - regs->pc = (unsigned long) ka->sa.sa_handler; + regs->pc = (unsigned long) ksig->ka.sa.sa_handler; regs->npc = (regs->pc + 4); /* 5. return to kernel instructions */ - if (ka->ka_restorer) - regs->u_regs[UREG_I7] = (unsigned long)ka->ka_restorer; + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; else { regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); @@ -379,41 +299,38 @@ static int setup_frame(struct k_sigaction *ka, struct pt_regs *regs, /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); if (err) - goto sigsegv; + return err; /* Flush instruction space. */ flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0])); } return 0; - -sigill_and_return: - do_exit(SIGILL); - return -EINVAL; - -sigsegv: - force_sigsegv(signo, current); - return -EFAULT; } -static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, - int signo, sigset_t *oldset, siginfo_t *info) +static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) { struct rt_signal_frame __user *sf; - int sigframe_size; + int sigframe_size, wsaved; + void __user *tail; unsigned int psr; int err; synchronize_user_stack(); - sigframe_size = RT_ALIGNEDSZ; - if (!used_math()) - sigframe_size -= sizeof(__siginfo_fpu_t); + wsaved = current_thread_info()->w_saved; + sigframe_size = sizeof(*sf); + if (used_math()) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); sf = (struct rt_signal_frame __user *) - get_sigframe(&ka->sa, regs, sigframe_size); - if (invalid_frame_pointer(sf, sigframe_size)) - goto sigill; - if (current_thread_info()->w_saved != 0) - goto sigill; + get_sigframe(ksig, regs, sigframe_size); + if (invalid_frame_pointer(sf, sigframe_size)) { + do_exit(SIGILL); + return -EINVAL; + } + tail = sf + 1; err = __put_user(regs->pc, &sf->regs.pc); err |= __put_user(regs->npc, &sf->regs.npc); err |= __put_user(regs->y, &sf->regs.y); @@ -425,36 +342,51 @@ static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, err |= __put_user(0, &sf->extra_size); if (psr & PSR_EF) { - err |= save_fpu_state(regs, &sf->fpu_state); - err |= __put_user(&sf->fpu_state, &sf->fpu_save); + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user(fp, &sf->fpu_save); } else { err |= __put_user(0, &sf->fpu_save); } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user(rwp, &sf->rwin_save); + } else { + err |= __put_user(0, &sf->rwin_save); + } err |= __copy_to_user(&sf->mask, &oldset->sig[0], sizeof(sigset_t)); /* Setup sigaltstack */ - err |= __put_user(current->sas_ss_sp, &sf->stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->u_regs[UREG_FP]), &sf->stack.ss_flags); - err |= __put_user(current->sas_ss_size, &sf->stack.ss_size); + err |= __save_altstack(&sf->stack, regs->u_regs[UREG_FP]); - err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], - sizeof(struct reg_window32)); + if (!wsaved) { + err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], + sizeof(struct reg_window32)); + } else { + struct reg_window32 *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= __copy_to_user(sf, rp, sizeof(struct reg_window32)); + } - err |= copy_siginfo_to_user(&sf->info, info); + err |= copy_siginfo_to_user(&sf->info, &ksig->info); if (err) - goto sigsegv; + return err; regs->u_regs[UREG_FP] = (unsigned long) sf; - regs->u_regs[UREG_I0] = signo; + regs->u_regs[UREG_I0] = ksig->sig; regs->u_regs[UREG_I1] = (unsigned long) &sf->info; regs->u_regs[UREG_I2] = (unsigned long) &sf->regs; - regs->pc = (unsigned long) ka->sa.sa_handler; + regs->pc = (unsigned long) ksig->ka.sa.sa_handler; regs->npc = (regs->pc + 4); - if (ka->ka_restorer) - regs->u_regs[UREG_I7] = (unsigned long)ka->ka_restorer; + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; else { regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); @@ -464,46 +396,25 @@ static int setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, /* t 0x10 */ err |= __put_user(0x91d02010, &sf->insns[1]); if (err) - goto sigsegv; + return err; /* Flush instruction space. */ flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0])); } return 0; - -sigill: - do_exit(SIGILL); - return -EINVAL; - -sigsegv: - force_sigsegv(signo, current); - return -EFAULT; } -static inline int -handle_signal(unsigned long signr, struct k_sigaction *ka, - siginfo_t *info, sigset_t *oldset, struct pt_regs *regs) +static inline void +handle_signal(struct ksignal *ksig, struct pt_regs *regs) { + sigset_t *oldset = sigmask_to_save(); int err; - if (ka->sa.sa_flags & SA_SIGINFO) - err = setup_rt_frame(ka, regs, signr, oldset, info); + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err = setup_rt_frame(ksig, regs, oldset); else - err = setup_frame(ka, regs, signr, oldset); - - if (err) - return err; - - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NOMASK)) - sigaddset(¤t->blocked, signr); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - tracehook_signal_handler(signr, info, ka, regs, 0); - - return 0; + err = setup_frame(ksig, regs, oldset); + signal_setup_done(err, ksig, 0); } static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, @@ -533,88 +444,83 @@ static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, */ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) { - struct k_sigaction ka; + struct ksignal ksig; int restart_syscall; - sigset_t *oldset; - siginfo_t info; - int signr; - + bool has_handler; + + /* It's a lot of work and synchronization to add a new ptrace + * register for GDB to save and restore in order to get + * orig_i0 correct for syscall restarts when debugging. + * + * Although it should be the case that most of the global + * registers are volatile across a system call, glibc already + * depends upon that fact that we preserve them. So we can't + * just use any global register to save away the orig_i0 value. + * + * In particular %g2, %g3, %g4, and %g5 are all assumed to be + * preserved across a system call trap by various pieces of + * code in glibc. + * + * %g7 is used as the "thread register". %g6 is not used in + * any fixed manner. %g6 is used as a scratch register and + * a compiler temporary, but it's value is never used across + * a system call. Therefore %g6 is usable for orig_i0 storage. + */ if (pt_regs_is_syscall(regs) && (regs->psr & PSR_C)) - restart_syscall = 1; - else - restart_syscall = 0; - - if (test_thread_flag(TIF_RESTORE_SIGMASK)) - oldset = ¤t->saved_sigmask; - else - oldset = ¤t->blocked; + regs->u_regs[UREG_G6] = orig_i0; - signr = get_signal_to_deliver(&info, &ka, regs, NULL); + has_handler = get_signal(&ksig); /* If the debugger messes with the program counter, it clears * the software "in syscall" bit, directing us to not perform * a syscall restart. */ - if (restart_syscall && !pt_regs_is_syscall(regs)) - restart_syscall = 0; + restart_syscall = 0; + if (pt_regs_is_syscall(regs) && (regs->psr & PSR_C)) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } - if (signr > 0) { + if (has_handler) { if (restart_syscall) - syscall_restart(orig_i0, regs, &ka.sa); - if (handle_signal(signr, &ka, &info, oldset, regs) == 0) { - /* a signal was successfully delivered; the saved - * sigmask will have been stored in the signal frame, - * and will be restored by sigreturn, so we can simply - * clear the TIF_RESTORE_SIGMASK flag. - */ - if (test_thread_flag(TIF_RESTORE_SIGMASK)) - clear_thread_flag(TIF_RESTORE_SIGMASK); + syscall_restart(orig_i0, regs, &ksig.ka.sa); + handle_signal(&ksig, regs); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->pc -= 4; + regs->npc -= 4; + pt_regs_clear_syscall(regs); + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->pc -= 4; + regs->npc -= 4; + pt_regs_clear_syscall(regs); + } } - return; - } - if (restart_syscall && - (regs->u_regs[UREG_I0] == ERESTARTNOHAND || - regs->u_regs[UREG_I0] == ERESTARTSYS || - regs->u_regs[UREG_I0] == ERESTARTNOINTR)) { - /* replay the system call when we are done */ - regs->u_regs[UREG_I0] = orig_i0; - regs->pc -= 4; - regs->npc -= 4; - pt_regs_clear_syscall(regs); - } - if (restart_syscall && - regs->u_regs[UREG_I0] == ERESTART_RESTARTBLOCK) { - regs->u_regs[UREG_G1] = __NR_restart_syscall; - regs->pc -= 4; - regs->npc -= 4; - pt_regs_clear_syscall(regs); - } - - /* if there's no signal to deliver, we just put the saved sigmask - * back - */ - if (test_thread_flag(TIF_RESTORE_SIGMASK)) { - clear_thread_flag(TIF_RESTORE_SIGMASK); - sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); + restore_saved_sigmask(); } } void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { - if (thread_info_flags & (_TIF_SIGPENDING | _TIF_RESTORE_SIGMASK)) + if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, orig_i0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); - if (current->replacement_session_keyring) - key_replace_session_keyring(); } } -asmlinkage int -do_sys_sigstack(struct sigstack __user *ssptr, struct sigstack __user *ossptr, - unsigned long sp) +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp) { int ret = -EFAULT; diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c index 006fe451588..1a699986803 100644 --- a/arch/sparc/kernel/signal_64.c +++ b/arch/sparc/kernel/signal_64.c @@ -23,6 +23,7 @@ #include <linux/tty.h> #include <linux/binfmts.h> #include <linux/bitops.h> +#include <linux/context_tracking.h> #include <asm/uaccess.h> #include <asm/ptrace.h> @@ -31,17 +32,20 @@ #include <asm/uctx.h> #include <asm/siginfo.h> #include <asm/visasm.h> +#include <asm/switch_to.h> +#include <asm/cacheflush.h> -#include "entry.h" +#include "sigutil.h" #include "systbls.h" - -#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) +#include "kernel.h" +#include "entry.h" /* {set, get}context() needed for 64-bit SparcLinux userland. */ asmlinkage void sparc64_set_context(struct pt_regs *regs) { struct ucontext __user *ucp = (struct ucontext __user *) regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); mc_gregset_t __user *grp; unsigned long pc, npc, tstate; unsigned long fp, i7; @@ -68,11 +72,7 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) if (__copy_from_user(&set, &ucp->uc_sigmask, sizeof(sigset_t))) goto do_sigsegv; } - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); } if (test_thread_flag(TIF_32BIT)) { pc &= 0xffffffff; @@ -132,16 +132,19 @@ asmlinkage void sparc64_set_context(struct pt_regs *regs) } if (err) goto do_sigsegv; - +out: + exception_exit(prev_state); return; do_sigsegv: force_sig(SIGSEGV, current); + goto out; } asmlinkage void sparc64_get_context(struct pt_regs *regs) { struct ucontext __user *ucp = (struct ucontext __user *) regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); mc_gregset_t __user *grp; mcontext_t __user *mcp; unsigned long fp, i7; @@ -223,10 +226,12 @@ asmlinkage void sparc64_get_context(struct pt_regs *regs) } if (err) goto do_sigsegv; - +out: + exception_exit(prev_state); return; do_sigsegv: force_sig(SIGSEGV, current); + goto out; } struct rt_signal_frame { @@ -236,63 +241,15 @@ struct rt_signal_frame { __siginfo_fpu_t __user *fpu_save; stack_t stack; sigset_t mask; - __siginfo_fpu_t fpu_state; + __siginfo_rwin_t *rwin_save; }; -static long _sigpause_common(old_sigset_t set) -{ - set &= _BLOCKABLE; - spin_lock_irq(¤t->sighand->siglock); - current->saved_sigmask = current->blocked; - siginitset(¤t->blocked, set); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - current->state = TASK_INTERRUPTIBLE; - schedule(); - - set_restore_sigmask(); - - return -ERESTARTNOHAND; -} - -asmlinkage long sys_sigpause(unsigned int set) -{ - return _sigpause_common(set); -} - -asmlinkage long sys_sigsuspend(old_sigset_t set) -{ - return _sigpause_common(set); -} - -static inline int -restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - unsigned long *fpregs = current_thread_info()->fpregs; - unsigned long fprs; - int err; - - err = __get_user(fprs, &fpu->si_fprs); - fprs_write(0); - regs->tstate &= ~TSTATE_PEF; - if (fprs & FPRS_DL) - err |= copy_from_user(fpregs, &fpu->si_float_regs[0], - (sizeof(unsigned int) * 32)); - if (fprs & FPRS_DU) - err |= copy_from_user(fpregs+16, &fpu->si_float_regs[32], - (sizeof(unsigned int) * 32)); - err |= __get_user(current_thread_info()->xfsr[0], &fpu->si_fsr); - err |= __get_user(current_thread_info()->gsr[0], &fpu->si_gsr); - current_thread_info()->fpsaved[0] |= fprs; - return err; -} - void do_rt_sigreturn(struct pt_regs *regs) { struct rt_signal_frame __user *sf; unsigned long tpc, tnpc, tstate; __siginfo_fpu_t __user *fpu_save; + __siginfo_rwin_t __user *rwin_save; sigset_t set; int err; @@ -325,61 +282,41 @@ void do_rt_sigreturn(struct pt_regs *regs) regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC)); err |= __get_user(fpu_save, &sf->fpu_save); - if (fpu_save) - err |= restore_fpu_state(regs, &sf->fpu_state); + if (!err && fpu_save) + err |= restore_fpu_state(regs, fpu_save); err |= __copy_from_user(&set, &sf->mask, sizeof(sigset_t)); - err |= do_sigaltstack(&sf->stack, NULL, (unsigned long)sf); - + err |= restore_altstack(&sf->stack); if (err) goto segv; + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(rwin_save)) + goto segv; + } + regs->tpc = tpc; regs->tnpc = tnpc; /* Prevent syscall restart. */ pt_regs_clear_syscall(regs); - sigdelsetmask(&set, ~_BLOCKABLE); - spin_lock_irq(¤t->sighand->siglock); - current->blocked = set; - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); + set_current_blocked(&set); return; segv: force_sig(SIGSEGV, current); } /* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp, int fplen) +static int invalid_frame_pointer(void __user *fp) { if (((unsigned long) fp) & 15) return 1; return 0; } -static inline int -save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) -{ - unsigned long *fpregs = current_thread_info()->fpregs; - unsigned long fprs; - int err = 0; - - fprs = current_thread_info()->fpsaved[0]; - if (fprs & FPRS_DL) - err |= copy_to_user(&fpu->si_float_regs[0], fpregs, - (sizeof(unsigned int) * 32)); - if (fprs & FPRS_DU) - err |= copy_to_user(&fpu->si_float_regs[32], fpregs+16, - (sizeof(unsigned int) * 32)); - err |= __put_user(current_thread_info()->xfsr[0], &fpu->si_fsr); - err |= __put_user(current_thread_info()->gsr[0], &fpu->si_gsr); - err |= __put_user(fprs, &fpu->si_fprs); - - return err; -} - -static inline void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, unsigned long framesize) +static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP] + STACK_BIAS; @@ -391,12 +328,7 @@ static inline void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs * return (void __user *) -1L; /* This is the X/Open sanctioned signal stack switching. */ - if (ka->sa.sa_flags & SA_ONSTACK) { - if (sas_ss_flags(sp) == 0) - sp = current->sas_ss_sp + current->sas_ss_size; - } - - sp -= framesize; + sp = sigsp(sp, ksig) - framesize; /* Always align the stack frame. This handles two cases. First, * sigaltstack need not be mindful of platform specific stack @@ -410,62 +342,82 @@ static inline void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs * } static inline int -setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, - int signo, sigset_t *oldset, siginfo_t *info) +setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) { struct rt_signal_frame __user *sf; - int sigframe_size, err; + int wsaved, err, sf_size; + void __user *tail; /* 1. Make sure everything is clean */ synchronize_user_stack(); save_and_clear_fpu(); - sigframe_size = sizeof(struct rt_signal_frame); - if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) - sigframe_size -= sizeof(__siginfo_fpu_t); + wsaved = get_thread_wsaved(); + sf_size = sizeof(struct rt_signal_frame); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sf_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sf_size += sizeof(__siginfo_rwin_t); sf = (struct rt_signal_frame __user *) - get_sigframe(ka, regs, sigframe_size); - - if (invalid_frame_pointer (sf, sigframe_size)) - goto sigill; + get_sigframe(ksig, regs, sf_size); - if (get_thread_wsaved() != 0) - goto sigill; + if (invalid_frame_pointer (sf)) { + do_exit(SIGILL); /* won't return, actually */ + return -EINVAL; + } + + tail = (sf + 1); /* 2. Save the current process state */ err = copy_to_user(&sf->regs, regs, sizeof (*regs)); if (current_thread_info()->fpsaved[0] & FPRS_FEF) { - err |= save_fpu_state(regs, &sf->fpu_state); - err |= __put_user((u64)&sf->fpu_state, &sf->fpu_save); + __siginfo_fpu_t __user *fpu_save = tail; + tail += sizeof(__siginfo_fpu_t); + err |= save_fpu_state(regs, fpu_save); + err |= __put_user((u64)fpu_save, &sf->fpu_save); } else { err |= __put_user(0, &sf->fpu_save); } + if (wsaved) { + __siginfo_rwin_t __user *rwin_save = tail; + tail += sizeof(__siginfo_rwin_t); + err |= save_rwin_state(wsaved, rwin_save); + err |= __put_user((u64)rwin_save, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); + } /* Setup sigaltstack */ - err |= __put_user(current->sas_ss_sp, &sf->stack.ss_sp); - err |= __put_user(sas_ss_flags(regs->u_regs[UREG_FP]), &sf->stack.ss_flags); - err |= __put_user(current->sas_ss_size, &sf->stack.ss_size); + err |= __save_altstack(&sf->stack, regs->u_regs[UREG_FP]); - err |= copy_to_user(&sf->mask, oldset, sizeof(sigset_t)); + err |= copy_to_user(&sf->mask, sigmask_to_save(), sizeof(sigset_t)); - err |= copy_in_user((u64 __user *)sf, - (u64 __user *)(regs->u_regs[UREG_FP]+STACK_BIAS), - sizeof(struct reg_window)); + if (!wsaved) { + err |= copy_in_user((u64 __user *)sf, + (u64 __user *)(regs->u_regs[UREG_FP] + + STACK_BIAS), + sizeof(struct reg_window)); + } else { + struct reg_window *rp; - if (info) - err |= copy_siginfo_to_user(&sf->info, info); + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= copy_to_user(sf, rp, sizeof(struct reg_window)); + } + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err |= copy_siginfo_to_user(&sf->info, &ksig->info); else { - err |= __put_user(signo, &sf->info.si_signo); + err |= __put_user(ksig->sig, &sf->info.si_signo); err |= __put_user(SI_NOINFO, &sf->info.si_code); } if (err) - goto sigsegv; + return err; /* 3. signal handler back-trampoline and parameters */ regs->u_regs[UREG_FP] = ((unsigned long) sf) - STACK_BIAS; - regs->u_regs[UREG_I0] = signo; + regs->u_regs[UREG_I0] = ksig->sig; regs->u_regs[UREG_I1] = (unsigned long) &sf->info; /* The sigcontext is passed in this way because of how it @@ -475,44 +427,14 @@ setup_rt_frame(struct k_sigaction *ka, struct pt_regs *regs, regs->u_regs[UREG_I2] = (unsigned long) &sf->info; /* 5. signal handler */ - regs->tpc = (unsigned long) ka->sa.sa_handler; + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; regs->tnpc = (regs->tpc + 4); if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; regs->tnpc &= 0xffffffff; } /* 4. return to kernel instructions */ - regs->u_regs[UREG_I7] = (unsigned long)ka->ka_restorer; - return 0; - -sigill: - do_exit(SIGILL); - return -EINVAL; - -sigsegv: - force_sigsegv(signo, current); - return -EFAULT; -} - -static inline int handle_signal(unsigned long signr, struct k_sigaction *ka, - siginfo_t *info, - sigset_t *oldset, struct pt_regs *regs) -{ - int err; - - err = setup_rt_frame(ka, regs, signr, oldset, - (ka->sa.sa_flags & SA_SIGINFO) ? info : NULL); - if (err) - return err; - spin_lock_irq(¤t->sighand->siglock); - sigorsets(¤t->blocked,¤t->blocked,&ka->sa.sa_mask); - if (!(ka->sa.sa_flags & SA_NOMASK)) - sigaddset(¤t->blocked,signr); - recalc_sigpending(); - spin_unlock_irq(¤t->sighand->siglock); - - tracehook_signal_handler(signr, info, ka, regs, 0); - + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; return 0; } @@ -543,91 +465,83 @@ static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, */ static void do_signal(struct pt_regs *regs, unsigned long orig_i0) { - struct k_sigaction ka; + struct ksignal ksig; int restart_syscall; - sigset_t *oldset; - siginfo_t info; - int signr; + bool has_handler; + /* It's a lot of work and synchronization to add a new ptrace + * register for GDB to save and restore in order to get + * orig_i0 correct for syscall restarts when debugging. + * + * Although it should be the case that most of the global + * registers are volatile across a system call, glibc already + * depends upon that fact that we preserve them. So we can't + * just use any global register to save away the orig_i0 value. + * + * In particular %g2, %g3, %g4, and %g5 are all assumed to be + * preserved across a system call trap by various pieces of + * code in glibc. + * + * %g7 is used as the "thread register". %g6 is not used in + * any fixed manner. %g6 is used as a scratch register and + * a compiler temporary, but it's value is never used across + * a system call. Therefore %g6 is usable for orig_i0 storage. + */ if (pt_regs_is_syscall(regs) && - (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { - restart_syscall = 1; - } else - restart_syscall = 0; - - if (current_thread_info()->status & TS_RESTORE_SIGMASK) - oldset = ¤t->saved_sigmask; - else - oldset = ¤t->blocked; + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) + regs->u_regs[UREG_G6] = orig_i0; #ifdef CONFIG_COMPAT if (test_thread_flag(TIF_32BIT)) { - extern void do_signal32(sigset_t *, struct pt_regs *, - int restart_syscall, - unsigned long orig_i0); - do_signal32(oldset, regs, restart_syscall, orig_i0); + do_signal32(regs); return; } #endif - signr = get_signal_to_deliver(&info, &ka, regs, NULL); + has_handler = get_signal(&ksig); - /* If the debugger messes with the program counter, it clears - * the software "in syscall" bit, directing us to not perform - * a syscall restart. - */ - if (restart_syscall && !pt_regs_is_syscall(regs)) - restart_syscall = 0; + restart_syscall = 0; + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } - if (signr > 0) { + if (has_handler) { if (restart_syscall) - syscall_restart(orig_i0, regs, &ka.sa); - if (handle_signal(signr, &ka, &info, oldset, regs) == 0) { - /* A signal was successfully delivered; the saved - * sigmask will have been stored in the signal frame, - * and will be restored by sigreturn, so we can simply - * clear the TS_RESTORE_SIGMASK flag. - */ - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; + syscall_restart(orig_i0, regs, &ksig.ka.sa); + signal_setup_done(setup_rt_frame(&ksig, regs), &ksig, 0); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + } } - return; - } - if (restart_syscall && - (regs->u_regs[UREG_I0] == ERESTARTNOHAND || - regs->u_regs[UREG_I0] == ERESTARTSYS || - regs->u_regs[UREG_I0] == ERESTARTNOINTR)) { - /* replay the system call when we are done */ - regs->u_regs[UREG_I0] = orig_i0; - regs->tpc -= 4; - regs->tnpc -= 4; - pt_regs_clear_syscall(regs); - } - if (restart_syscall && - regs->u_regs[UREG_I0] == ERESTART_RESTARTBLOCK) { - regs->u_regs[UREG_G1] = __NR_restart_syscall; - regs->tpc -= 4; - regs->tnpc -= 4; - pt_regs_clear_syscall(regs); - } - - /* If there's no signal to deliver, we just put the saved sigmask - * back - */ - if (current_thread_info()->status & TS_RESTORE_SIGMASK) { - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; - sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); + restore_saved_sigmask(); } } void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) { + user_exit(); if (thread_info_flags & _TIF_SIGPENDING) do_signal(regs, orig_i0); if (thread_info_flags & _TIF_NOTIFY_RESUME) { clear_thread_flag(TIF_NOTIFY_RESUME); tracehook_notify_resume(regs); - if (current->replacement_session_keyring) - key_replace_session_keyring(); } + user_enter(); } diff --git a/arch/sparc/kernel/sigutil.h b/arch/sparc/kernel/sigutil.h new file mode 100644 index 00000000000..d223aa432bb --- /dev/null +++ b/arch/sparc/kernel/sigutil.h @@ -0,0 +1,9 @@ +#ifndef _SIGUTIL_H +#define _SIGUTIL_H + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu); +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu); +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin); +int restore_rwin_state(__siginfo_rwin_t __user *rp); + +#endif /* _SIGUTIL_H */ diff --git a/arch/sparc/kernel/sigutil_32.c b/arch/sparc/kernel/sigutil_32.c new file mode 100644 index 00000000000..0f6eebe71e6 --- /dev/null +++ b/arch/sparc/kernel/sigutil_32.c @@ -0,0 +1,121 @@ +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/thread_info.h> +#include <linux/uaccess.h> +#include <linux/sched.h> + +#include <asm/sigcontext.h> +#include <asm/fpumacro.h> +#include <asm/ptrace.h> +#include <asm/switch_to.h> + +#include "sigutil.h" + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + int err = 0; +#ifdef CONFIG_SMP + if (test_tsk_thread_flag(current, TIF_USEDFPU)) { + put_psr(get_psr() | PSR_EF); + fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, + ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); + regs->psr &= ~(PSR_EF); + clear_tsk_thread_flag(current, TIF_USEDFPU); + } +#else + if (current == last_task_used_math) { + put_psr(get_psr() | PSR_EF); + fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, + ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); + last_task_used_math = NULL; + regs->psr &= ~(PSR_EF); + } +#endif + err |= __copy_to_user(&fpu->si_float_regs[0], + ¤t->thread.float_regs[0], + (sizeof(unsigned long) * 32)); + err |= __put_user(current->thread.fsr, &fpu->si_fsr); + err |= __put_user(current->thread.fpqdepth, &fpu->si_fpqdepth); + if (current->thread.fpqdepth != 0) + err |= __copy_to_user(&fpu->si_fpqueue[0], + ¤t->thread.fpqueue[0], + ((sizeof(unsigned long) + + (sizeof(unsigned long *)))*16)); + clear_used_math(); + return err; +} + +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + int err; +#ifdef CONFIG_SMP + if (test_tsk_thread_flag(current, TIF_USEDFPU)) + regs->psr &= ~PSR_EF; +#else + if (current == last_task_used_math) { + last_task_used_math = NULL; + regs->psr &= ~PSR_EF; + } +#endif + set_used_math(); + clear_tsk_thread_flag(current, TIF_USEDFPU); + + if (!access_ok(VERIFY_READ, fpu, sizeof(*fpu))) + return -EFAULT; + + err = __copy_from_user(¤t->thread.float_regs[0], &fpu->si_float_regs[0], + (sizeof(unsigned long) * 32)); + err |= __get_user(current->thread.fsr, &fpu->si_fsr); + err |= __get_user(current->thread.fpqdepth, &fpu->si_fpqdepth); + if (current->thread.fpqdepth != 0) + err |= __copy_from_user(¤t->thread.fpqueue[0], + &fpu->si_fpqueue[0], + ((sizeof(unsigned long) + + (sizeof(unsigned long *)))*16)); + return err; +} + +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin) +{ + int i, err = __put_user(wsaved, &rwin->wsaved); + + for (i = 0; i < wsaved; i++) { + struct reg_window32 *rp; + unsigned long fp; + + rp = ¤t_thread_info()->reg_window[i]; + fp = current_thread_info()->rwbuf_stkptrs[i]; + err |= copy_to_user(&rwin->reg_window[i], rp, + sizeof(struct reg_window32)); + err |= __put_user(fp, &rwin->rwbuf_stkptrs[i]); + } + return err; +} + +int restore_rwin_state(__siginfo_rwin_t __user *rp) +{ + struct thread_info *t = current_thread_info(); + int i, wsaved, err; + + __get_user(wsaved, &rp->wsaved); + if (wsaved > NSWINS) + return -EFAULT; + + err = 0; + for (i = 0; i < wsaved; i++) { + err |= copy_from_user(&t->reg_window[i], + &rp->reg_window[i], + sizeof(struct reg_window32)); + err |= __get_user(t->rwbuf_stkptrs[i], + &rp->rwbuf_stkptrs[i]); + } + if (err) + return err; + + t->w_saved = wsaved; + synchronize_user_stack(); + if (t->w_saved) + return -EFAULT; + return 0; + +} diff --git a/arch/sparc/kernel/sigutil_64.c b/arch/sparc/kernel/sigutil_64.c new file mode 100644 index 00000000000..387834a9c56 --- /dev/null +++ b/arch/sparc/kernel/sigutil_64.c @@ -0,0 +1,95 @@ +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/thread_info.h> +#include <linux/uaccess.h> +#include <linux/errno.h> + +#include <asm/sigcontext.h> +#include <asm/fpumacro.h> +#include <asm/ptrace.h> +#include <asm/switch_to.h> + +#include "sigutil.h" + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + int err = 0; + + fprs = current_thread_info()->fpsaved[0]; + if (fprs & FPRS_DL) + err |= copy_to_user(&fpu->si_float_regs[0], fpregs, + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_to_user(&fpu->si_float_regs[32], fpregs+16, + (sizeof(unsigned int) * 32)); + err |= __put_user(current_thread_info()->xfsr[0], &fpu->si_fsr); + err |= __put_user(current_thread_info()->gsr[0], &fpu->si_gsr); + err |= __put_user(fprs, &fpu->si_fprs); + + return err; +} + +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + int err; + + err = __get_user(fprs, &fpu->si_fprs); + fprs_write(0); + regs->tstate &= ~TSTATE_PEF; + if (fprs & FPRS_DL) + err |= copy_from_user(fpregs, &fpu->si_float_regs[0], + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_from_user(fpregs+16, &fpu->si_float_regs[32], + (sizeof(unsigned int) * 32)); + err |= __get_user(current_thread_info()->xfsr[0], &fpu->si_fsr); + err |= __get_user(current_thread_info()->gsr[0], &fpu->si_gsr); + current_thread_info()->fpsaved[0] |= fprs; + return err; +} + +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin) +{ + int i, err = __put_user(wsaved, &rwin->wsaved); + + for (i = 0; i < wsaved; i++) { + struct reg_window *rp = ¤t_thread_info()->reg_window[i]; + unsigned long fp = current_thread_info()->rwbuf_stkptrs[i]; + + err |= copy_to_user(&rwin->reg_window[i], rp, + sizeof(struct reg_window)); + err |= __put_user(fp, &rwin->rwbuf_stkptrs[i]); + } + return err; +} + +int restore_rwin_state(__siginfo_rwin_t __user *rp) +{ + struct thread_info *t = current_thread_info(); + int i, wsaved, err; + + __get_user(wsaved, &rp->wsaved); + if (wsaved > NSWINS) + return -EFAULT; + + err = 0; + for (i = 0; i < wsaved; i++) { + err |= copy_from_user(&t->reg_window[i], + &rp->reg_window[i], + sizeof(struct reg_window)); + err |= __get_user(t->rwbuf_stkptrs[i], + &rp->rwbuf_stkptrs[i]); + } + if (err) + return err; + + set_thread_wsaved(wsaved); + synchronize_user_stack(); + if (get_thread_wsaved()) + return -EFAULT; + return 0; +} diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index 91c10fb7085..7958242d63c 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -20,9 +20,11 @@ #include <linux/seq_file.h> #include <linux/cache.h> #include <linux/delay.h> +#include <linux/profile.h> +#include <linux/cpu.h> #include <asm/ptrace.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/irq.h> #include <asm/page.h> @@ -32,16 +34,18 @@ #include <asm/cacheflush.h> #include <asm/tlbflush.h> #include <asm/cpudata.h> +#include <asm/timer.h> #include <asm/leon.h> +#include "kernel.h" #include "irq.h" -volatile unsigned long cpu_callin_map[NR_CPUS] __cpuinitdata = {0,}; -unsigned char boot_cpu_id = 0; -unsigned char boot_cpu_id4 = 0; /* boot_cpu_id << 2 */ +volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; cpumask_t smp_commenced_mask = CPU_MASK_NONE; +const struct sparc32_ipi_ops *sparc32_ipi_ops; + /* The only guaranteed locking primitive available on all Sparc * processors is 'ldstub [%reg + immediate], %dest_reg' which atomically * places the current byte at the effective address into dest_reg and @@ -50,9 +54,10 @@ cpumask_t smp_commenced_mask = CPU_MASK_NONE; * instruction which is much better... */ -void __cpuinit smp_store_cpu_info(int id) +void smp_store_cpu_info(int id) { int cpu_node; + int mid; cpu_data(id).udelay_val = loops_per_jiffy; @@ -60,16 +65,17 @@ void __cpuinit smp_store_cpu_info(int id) cpu_data(id).clock_tick = prom_getintdefault(cpu_node, "clock-frequency", 0); cpu_data(id).prom_node = cpu_node; - cpu_data(id).mid = cpu_get_hwmid(cpu_node); + mid = cpu_get_hwmid(cpu_node); - if (cpu_data(id).mid < 0) - panic("No MID found for CPU%d at node 0x%08d", id, cpu_node); + if (mid < 0) { + printk(KERN_NOTICE "No MID found for CPU%d at node 0x%08d", id, cpu_node); + mid = 0; + } + cpu_data(id).mid = mid; } void __init smp_cpus_done(unsigned int max_cpus) { - extern void smp4m_smp_done(void); - extern void smp4d_smp_done(void); unsigned long bogosum = 0; int cpu, num = 0; @@ -83,14 +89,6 @@ void __init smp_cpus_done(unsigned int max_cpus) (bogosum/(5000/HZ))%100); switch(sparc_cpu_model) { - case sun4: - printk("SUN4\n"); - BUG(); - break; - case sun4c: - printk("SUN4C\n"); - BUG(); - break; case sun4m: smp4m_smp_done(); break; @@ -112,7 +110,7 @@ void __init smp_cpus_done(unsigned int max_cpus) printk("UNKNOWN!\n"); BUG(); break; - }; + } } void cpu_panic(void) @@ -121,165 +119,69 @@ void cpu_panic(void) panic("SMP bolixed\n"); } -struct linux_prom_registers smp_penguin_ctable __cpuinitdata = { 0 }; +struct linux_prom_registers smp_penguin_ctable = { 0 }; void smp_send_reschedule(int cpu) { - /* See sparc64 */ + /* + * CPU model dependent way of implementing IPI generation targeting + * a single CPU. The trap handler needs only to do trap entry/return + * to call schedule. + */ + sparc32_ipi_ops->resched(cpu); } void smp_send_stop(void) { } -void smp_flush_cache_all(void) -{ - xc0((smpfunc_t) BTFIXUP_CALL(local_flush_cache_all)); - local_flush_cache_all(); -} - -void smp_flush_tlb_all(void) -{ - xc0((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_all)); - local_flush_tlb_all(); -} - -void smp_flush_cache_mm(struct mm_struct *mm) -{ - if(mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc1((smpfunc_t) BTFIXUP_CALL(local_flush_cache_mm), (unsigned long) mm); - local_flush_cache_mm(mm); - } -} - -void smp_flush_tlb_mm(struct mm_struct *mm) -{ - if(mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) { - xc1((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_mm), (unsigned long) mm); - if(atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) - cpumask_copy(mm_cpumask(mm), - cpumask_of(smp_processor_id())); - } - local_flush_tlb_mm(mm); - } -} - -void smp_flush_cache_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - - if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc3((smpfunc_t) BTFIXUP_CALL(local_flush_cache_range), (unsigned long) vma, start, end); - local_flush_cache_range(vma, start, end); - } -} - -void smp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end) +void arch_send_call_function_single_ipi(int cpu) { - struct mm_struct *mm = vma->vm_mm; - - if (mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc3((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_range), (unsigned long) vma, start, end); - local_flush_tlb_range(vma, start, end); - } + /* trigger one IPI single call on one CPU */ + sparc32_ipi_ops->single(cpu); } -void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +void arch_send_call_function_ipi_mask(const struct cpumask *mask) { - struct mm_struct *mm = vma->vm_mm; - - if(mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc2((smpfunc_t) BTFIXUP_CALL(local_flush_cache_page), (unsigned long) vma, page); - local_flush_cache_page(vma, page); - } -} + int cpu; -void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - - if(mm->context != NO_CONTEXT) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc2((smpfunc_t) BTFIXUP_CALL(local_flush_tlb_page), (unsigned long) vma, page); - local_flush_tlb_page(vma, page); - } + /* trigger IPI mask call on each CPU */ + for_each_cpu(cpu, mask) + sparc32_ipi_ops->mask_one(cpu); } -void smp_reschedule_irq(void) +void smp_resched_interrupt(void) { - set_need_resched(); + irq_enter(); + scheduler_ipi(); + local_cpu_data().irq_resched_count++; + irq_exit(); + /* re-schedule routine called by interrupt return code. */ } -void smp_flush_page_to_ram(unsigned long page) +void smp_call_function_single_interrupt(void) { - /* Current theory is that those who call this are the one's - * who have just dirtied their cache with the pages contents - * in kernel space, therefore we only run this on local cpu. - * - * XXX This experiment failed, research further... -DaveM - */ -#if 1 - xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_to_ram), page); -#endif - local_flush_page_to_ram(page); + irq_enter(); + generic_smp_call_function_single_interrupt(); + local_cpu_data().irq_call_count++; + irq_exit(); } -void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) +void smp_call_function_interrupt(void) { - cpumask_t cpu_mask = *mm_cpumask(mm); - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) - xc2((smpfunc_t) BTFIXUP_CALL(local_flush_sig_insns), (unsigned long) mm, insn_addr); - local_flush_sig_insns(mm, insn_addr); + irq_enter(); + generic_smp_call_function_interrupt(); + local_cpu_data().irq_call_count++; + irq_exit(); } -extern unsigned int lvl14_resolution; - -/* /proc/profile writes can call this, don't __init it please. */ -static DEFINE_SPINLOCK(prof_setup_lock); - int setup_profiling_timer(unsigned int multiplier) { - int i; - unsigned long flags; - - /* Prevent level14 ticker IRQ flooding. */ - if((!multiplier) || (lvl14_resolution / multiplier) < 500) - return -EINVAL; - - spin_lock_irqsave(&prof_setup_lock, flags); - for_each_possible_cpu(i) { - load_profile_irq(i, lvl14_resolution / multiplier); - prof_multiplier(i) = multiplier; - } - spin_unlock_irqrestore(&prof_setup_lock, flags); - - return 0; + return -EINVAL; } void __init smp_prepare_cpus(unsigned int max_cpus) { - extern void __init smp4m_boot_cpus(void); - extern void __init smp4d_boot_cpus(void); int i, cpuid, extra; printk("Entering SMP Mode...\n"); @@ -296,14 +198,6 @@ void __init smp_prepare_cpus(unsigned int max_cpus) smp_store_cpu_info(boot_cpu_id); switch(sparc_cpu_model) { - case sun4: - printk("SUN4\n"); - BUG(); - break; - case sun4c: - printk("SUN4C\n"); - BUG(); - break; case sun4m: smp4m_boot_cpus(); break; @@ -325,7 +219,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) printk("UNKNOWN!\n"); BUG(); break; - }; + } } /* Set this up early so that things like the scheduler can init @@ -362,29 +256,19 @@ void __init smp_prepare_boot_cpu(void) set_cpu_possible(cpuid, true); } -int __cpuinit __cpu_up(unsigned int cpu) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - extern int __cpuinit smp4m_boot_one_cpu(int); - extern int __cpuinit smp4d_boot_one_cpu(int); int ret=0; switch(sparc_cpu_model) { - case sun4: - printk("SUN4\n"); - BUG(); - break; - case sun4c: - printk("SUN4C\n"); - BUG(); - break; case sun4m: - ret = smp4m_boot_one_cpu(cpu); + ret = smp4m_boot_one_cpu(cpu, tidle); break; case sun4d: - ret = smp4d_boot_one_cpu(cpu); + ret = smp4d_boot_one_cpu(cpu, tidle); break; case sparc_leon: - ret = leon_boot_one_cpu(cpu); + ret = leon_boot_one_cpu(cpu, tidle); break; case sun4e: printk("SUN4E\n"); @@ -398,16 +282,99 @@ int __cpuinit __cpu_up(unsigned int cpu) printk("UNKNOWN!\n"); BUG(); break; - }; + } if (!ret) { - cpu_set(cpu, smp_commenced_mask); + cpumask_set_cpu(cpu, &smp_commenced_mask); while (!cpu_online(cpu)) mb(); } return ret; } +static void arch_cpu_pre_starting(void *arg) +{ + local_ops->cache_all(); + local_ops->tlb_all(); + + switch(sparc_cpu_model) { + case sun4m: + sun4m_cpu_pre_starting(arg); + break; + case sun4d: + sun4d_cpu_pre_starting(arg); + break; + case sparc_leon: + leon_cpu_pre_starting(arg); + break; + default: + BUG(); + } +} + +static void arch_cpu_pre_online(void *arg) +{ + unsigned int cpuid = hard_smp_processor_id(); + + register_percpu_ce(cpuid); + + calibrate_delay(); + smp_store_cpu_info(cpuid); + + local_ops->cache_all(); + local_ops->tlb_all(); + + switch(sparc_cpu_model) { + case sun4m: + sun4m_cpu_pre_online(arg); + break; + case sun4d: + sun4d_cpu_pre_online(arg); + break; + case sparc_leon: + leon_cpu_pre_online(arg); + break; + default: + BUG(); + } +} + +static void sparc_start_secondary(void *arg) +{ + unsigned int cpu; + + /* + * SMP booting is extremely fragile in some architectures. So run + * the cpu initialization code first before anything else. + */ + arch_cpu_pre_starting(arg); + + preempt_disable(); + cpu = smp_processor_id(); + + /* Invoke the CPU_STARTING notifier callbacks */ + notify_cpu_starting(cpu); + + arch_cpu_pre_online(arg); + + /* Set the CPU in the cpu_online_mask */ + set_cpu_online(cpu, true); + + /* Enable local interrupts now */ + local_irq_enable(); + + wmb(); + cpu_startup_entry(CPUHP_ONLINE); + + /* We should never reach here! */ + BUG(); +} + +void smp_callin(void) +{ + sparc_start_secondary(NULL); +} + void smp_bogo(struct seq_file *m) { int i; diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 555a76d1f4a..41aa2478f3c 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -3,7 +3,7 @@ * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/mm.h> @@ -25,16 +25,18 @@ #include <linux/ftrace.h> #include <linux/cpu.h> #include <linux/slab.h> +#include <linux/kgdb.h> #include <asm/head.h> #include <asm/ptrace.h> -#include <asm/atomic.h> +#include <linux/atomic.h> #include <asm/tlbflush.h> #include <asm/mmu_context.h> #include <asm/cpudata.h> #include <asm/hvtramp.h> #include <asm/io.h> #include <asm/timer.h> +#include <asm/setup.h> #include <asm/irq.h> #include <asm/irq_regs.h> @@ -52,8 +54,7 @@ #include <asm/pcr.h> #include "cpumap.h" - -int sparc64_multi_core __read_mostly; +#include "kernel.h" DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; cpumask_t cpu_core_map[NR_CPUS] __read_mostly = @@ -87,7 +88,7 @@ extern void setup_sparc64_timer(void); static volatile unsigned long callin_flag = 0; -void __cpuinit smp_callin(void) +void smp_callin(void) { int cpuid = hard_smp_processor_id(); @@ -103,8 +104,6 @@ void __cpuinit smp_callin(void) if (cheetah_pcache_forced_on) cheetah_enable_pcache(); - local_irq_enable(); - callin_flag = 1; __asm__ __volatile__("membar #Sync\n\t" "flush %%g6" : : : "memory"); @@ -121,15 +120,17 @@ void __cpuinit smp_callin(void) /* inform the notifiers about the new cpu */ notify_cpu_starting(cpuid); - while (!cpu_isset(cpuid, smp_commenced_mask)) + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) rmb(); - ipi_call_lock_irq(); - cpu_set(cpuid, cpu_online_map); - ipi_call_unlock_irq(); + set_cpu_online(cpuid, true); /* idle thread is expected to have preempt disabled */ preempt_disable(); + + local_irq_enable(); + + cpu_startup_entry(CPUHP_ONLINE); } void cpu_panic(void) @@ -151,7 +152,7 @@ void cpu_panic(void) #define NUM_ROUNDS 64 /* magic value */ #define NUM_ITERS 5 /* likewise */ -static DEFINE_SPINLOCK(itc_sync_lock); +static DEFINE_RAW_SPINLOCK(itc_sync_lock); static unsigned long go[SLAVE + 1]; #define DEBUG_TICK_SYNC 0 @@ -189,7 +190,7 @@ static inline long get_delta (long *rt, long *master) void smp_synchronize_tick_client(void) { long i, delta, adj, adjust_latency = 0, done = 0; - unsigned long flags, rt, master_time_stamp, bound; + unsigned long flags, rt, master_time_stamp; #if DEBUG_TICK_SYNC struct { long rt; /* roundtrip time */ @@ -208,10 +209,8 @@ void smp_synchronize_tick_client(void) { for (i = 0; i < NUM_ROUNDS; i++) { delta = get_delta(&rt, &master_time_stamp); - if (delta == 0) { + if (delta == 0) done = 1; /* let's lock on to this... */ - bound = rt; - } if (!done) { if (i > 0) { @@ -261,7 +260,7 @@ static void smp_synchronize_one_tick(int cpu) go[MASTER] = 0; membar_safe("#StoreLoad"); - spin_lock_irqsave(&itc_sync_lock, flags); + raw_spin_lock_irqsave(&itc_sync_lock, flags); { for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { while (!go[MASTER]) @@ -272,19 +271,12 @@ static void smp_synchronize_one_tick(int cpu) membar_safe("#StoreLoad"); } } - spin_unlock_irqrestore(&itc_sync_lock, flags); + raw_spin_unlock_irqrestore(&itc_sync_lock, flags); } #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) -/* XXX Put this in some common place. XXX */ -static unsigned long kimage_addr_to_ra(void *p) -{ - unsigned long val = (unsigned long) p; - - return kern_base + (val - KERNBASE); -} - -static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp) +static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, + void **descrp) { extern unsigned long sparc64_ttable_tl0; extern unsigned long kern_locked_tte_data; @@ -345,21 +337,17 @@ extern unsigned long sparc64_cpu_startup; */ static struct thread_info *cpu_new_thread = NULL; -static int __cpuinit smp_boot_one_cpu(unsigned int cpu) +static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) { unsigned long entry = (unsigned long)(&sparc64_cpu_startup); unsigned long cookie = (unsigned long)(&cpu_new_thread); - struct task_struct *p; void *descr = NULL; int timeout, ret; - p = fork_idle(cpu); - if (IS_ERR(p)) - return PTR_ERR(p); callin_flag = 0; - cpu_new_thread = task_thread_info(p); + cpu_new_thread = task_thread_info(idle); if (tlb_type == hypervisor) { #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) @@ -787,7 +775,7 @@ static void xcall_deliver(u64 data0, u64 data1, u64 data2, const cpumask_t *mask /* Send cross call to all processors mentioned in MASK_P * except self. Really, there are only two cases currently, - * "&cpu_online_map" and "&mm->cpu_vm_mask". + * "cpu_online_mask" and "mm_cpumask(mm)". */ static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask) { @@ -799,7 +787,7 @@ static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 d /* Send cross call to all processors except self. */ static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2) { - smp_cross_call_masked(func, ctx, data1, data2, &cpu_online_map); + smp_cross_call_masked(func, ctx, data1, data2, cpu_online_mask); } extern unsigned long xcall_sync_tick; @@ -807,7 +795,7 @@ extern unsigned long xcall_sync_tick; static void smp_start_sync_tick_client(int cpu) { xcall_deliver((u64) &xcall_sync_tick, 0, 0, - &cpumask_of_cpu(cpu)); + cpumask_of(cpu)); } extern unsigned long xcall_call_function; @@ -822,7 +810,7 @@ extern unsigned long xcall_call_function_single; void arch_send_call_function_single_ipi(int cpu) { xcall_deliver((u64) &xcall_call_function_single, 0, 0, - &cpumask_of_cpu(cpu)); + cpumask_of(cpu)); } void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs) @@ -842,7 +830,7 @@ static void tsb_sync(void *info) struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()]; struct mm_struct *mm = info; - /* It is not valid to test "currrent->active_mm == mm" here. + /* It is not valid to test "current->active_mm == mm" here. * * The value of "current" is not changed atomically with * switch_mm(). But that's OK, we just need to check the @@ -858,9 +846,11 @@ void smp_tsb_sync(struct mm_struct *mm) } extern unsigned long xcall_flush_tlb_mm; -extern unsigned long xcall_flush_tlb_pending; +extern unsigned long xcall_flush_tlb_page; extern unsigned long xcall_flush_tlb_kernel_range; extern unsigned long xcall_fetch_glob_regs; +extern unsigned long xcall_fetch_glob_pmu; +extern unsigned long xcall_fetch_glob_pmu_n4; extern unsigned long xcall_receive_signal; extern unsigned long xcall_new_mmu_context_version; #ifdef CONFIG_KGDB @@ -872,11 +862,6 @@ extern unsigned long xcall_flush_dcache_page_cheetah; #endif extern unsigned long xcall_flush_dcache_page_spitfire; -#ifdef CONFIG_DEBUG_DCFLUSH -extern atomic_t dcpage_flushes; -extern atomic_t dcpage_flushes_xcall; -#endif - static inline void __local_flush_dcache_page(struct page *page) { #ifdef DCACHE_ALIASING_POSSIBLE @@ -920,7 +905,7 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) } if (data0) { xcall_deliver(data0, __pa(pg_addr), - (u64) pg_addr, &cpumask_of_cpu(cpu)); + (u64) pg_addr, cpumask_of(cpu)); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes_xcall); #endif @@ -933,13 +918,12 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu) void flush_dcache_page_all(struct mm_struct *mm, struct page *page) { void *pg_addr; - int this_cpu; u64 data0; if (tlb_type == hypervisor) return; - this_cpu = get_cpu(); + preempt_disable(); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes); @@ -957,14 +941,14 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page) } if (data0) { xcall_deliver(data0, __pa(pg_addr), - (u64) pg_addr, &cpu_online_map); + (u64) pg_addr, cpu_online_mask); #ifdef CONFIG_DEBUG_DCFLUSH atomic_inc(&dcpage_flushes_xcall); #endif } __local_flush_dcache_page(page); - put_cpu(); + preempt_enable(); } void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs) @@ -1010,6 +994,15 @@ void smp_fetch_global_regs(void) smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0); } +void smp_fetch_global_pmu(void) +{ + if (tlb_type == hypervisor && + sun4v_chip_type >= SUN4V_CHIP_NIAGARA4) + smp_cross_call(&xcall_fetch_glob_pmu_n4, 0, 0, 0); + else + smp_cross_call(&xcall_fetch_glob_pmu, 0, 0, 0); +} + /* We know that the window frames of the user have been flushed * to the stack before we get here because all callers of us * are flush_tlb_*() routines, and these run after flush_cache_*() @@ -1073,23 +1066,56 @@ local_flush_and_out: put_cpu(); } +struct tlb_pending_info { + unsigned long ctx; + unsigned long nr; + unsigned long *vaddrs; +}; + +static void tlb_pending_func(void *info) +{ + struct tlb_pending_info *t = info; + + __flush_tlb_pending(t->ctx, t->nr, t->vaddrs); +} + void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs) { u32 ctx = CTX_HWBITS(mm->context); + struct tlb_pending_info info; int cpu = get_cpu(); + info.ctx = ctx; + info.nr = nr; + info.vaddrs = vaddrs; + if (mm == current->mm && atomic_read(&mm->mm_users) == 1) cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); else - smp_cross_call_masked(&xcall_flush_tlb_pending, - ctx, nr, (unsigned long) vaddrs, - mm_cpumask(mm)); + smp_call_function_many(mm_cpumask(mm), tlb_pending_func, + &info, 1); __flush_tlb_pending(ctx, nr, vaddrs); put_cpu(); } +void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) +{ + unsigned long context = CTX_HWBITS(mm->context); + int cpu = get_cpu(); + + if (mm == current->mm && atomic_read(&mm->mm_users) == 1) + cpumask_copy(mm_cpumask(mm), cpumask_of(cpu)); + else + smp_cross_call_masked(&xcall_flush_tlb_page, + context, vaddr, 0, + mm_cpumask(mm)); + __flush_tlb_page(context, vaddr); + + put_cpu(); +} + void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end) { start &= PAGE_MASK; @@ -1179,7 +1205,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) { } -void __devinit smp_prepare_boot_cpu(void) +void smp_prepare_boot_cpu(void) { } @@ -1193,52 +1219,52 @@ void __init smp_setup_processor_id(void) xcall_deliver_impl = hypervisor_xcall_deliver; } -void __devinit smp_fill_in_sib_core_maps(void) +void smp_fill_in_sib_core_maps(void) { unsigned int i; for_each_present_cpu(i) { unsigned int j; - cpus_clear(cpu_core_map[i]); + cpumask_clear(&cpu_core_map[i]); if (cpu_data(i).core_id == 0) { - cpu_set(i, cpu_core_map[i]); + cpumask_set_cpu(i, &cpu_core_map[i]); continue; } for_each_present_cpu(j) { if (cpu_data(i).core_id == cpu_data(j).core_id) - cpu_set(j, cpu_core_map[i]); + cpumask_set_cpu(j, &cpu_core_map[i]); } } for_each_present_cpu(i) { unsigned int j; - cpus_clear(per_cpu(cpu_sibling_map, i)); + cpumask_clear(&per_cpu(cpu_sibling_map, i)); if (cpu_data(i).proc_id == -1) { - cpu_set(i, per_cpu(cpu_sibling_map, i)); + cpumask_set_cpu(i, &per_cpu(cpu_sibling_map, i)); continue; } for_each_present_cpu(j) { if (cpu_data(i).proc_id == cpu_data(j).proc_id) - cpu_set(j, per_cpu(cpu_sibling_map, i)); + cpumask_set_cpu(j, &per_cpu(cpu_sibling_map, i)); } } } -int __cpuinit __cpu_up(unsigned int cpu) +int __cpu_up(unsigned int cpu, struct task_struct *tidle) { - int ret = smp_boot_one_cpu(cpu); + int ret = smp_boot_one_cpu(cpu, tidle); if (!ret) { - cpu_set(cpu, smp_commenced_mask); - while (!cpu_isset(cpu, cpu_online_map)) + cpumask_set_cpu(cpu, &smp_commenced_mask); + while (!cpu_online(cpu)) mb(); - if (!cpu_isset(cpu, cpu_online_map)) { + if (!cpu_online(cpu)) { ret = -ENODEV; } else { /* On SUN4V, writes to %tick and %stick are @@ -1272,7 +1298,7 @@ void cpu_play_dead(void) tb->nonresum_mondo_pa, 0); } - cpu_clear(cpu, smp_commenced_mask); + cpumask_clear_cpu(cpu, &smp_commenced_mask); membar_safe("#Sync"); local_irq_disable(); @@ -1293,13 +1319,13 @@ int __cpu_disable(void) cpuinfo_sparc *c; int i; - for_each_cpu_mask(i, cpu_core_map[cpu]) - cpu_clear(cpu, cpu_core_map[i]); - cpus_clear(cpu_core_map[cpu]); + for_each_cpu(i, &cpu_core_map[cpu]) + cpumask_clear_cpu(cpu, &cpu_core_map[i]); + cpumask_clear(&cpu_core_map[cpu]); - for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu)) - cpu_clear(cpu, per_cpu(cpu_sibling_map, i)); - cpus_clear(per_cpu(cpu_sibling_map, cpu)); + for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu)) + cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i)); + cpumask_clear(&per_cpu(cpu_sibling_map, cpu)); c = &cpu_data(cpu); @@ -1315,9 +1341,7 @@ int __cpu_disable(void) mdelay(1); local_irq_disable(); - ipi_call_lock(); - cpu_clear(cpu, cpu_online_map); - ipi_call_unlock(); + set_cpu_online(cpu, false); cpu_map_rebuild(); @@ -1330,11 +1354,11 @@ void __cpu_die(unsigned int cpu) for (i = 0; i < 100; i++) { smp_rmb(); - if (!cpu_isset(cpu, smp_commenced_mask)) + if (!cpumask_test_cpu(cpu, &smp_commenced_mask)) break; msleep(100); } - if (cpu_isset(cpu, smp_commenced_mask)) { + if (cpumask_test_cpu(cpu, &smp_commenced_mask)) { printk(KERN_ERR "CPU %u didn't die...\n", cpu); } else { #if defined(CONFIG_SUN_LDOMS) @@ -1344,7 +1368,7 @@ void __cpu_die(unsigned int cpu) do { hv_err = sun4v_cpu_stop(cpu); if (hv_err == HV_EOK) { - cpu_clear(cpu, cpu_present_map); + set_cpu_present(cpu, false); break; } } while (--limit > 0); @@ -1364,13 +1388,19 @@ void __init smp_cpus_done(unsigned int max_cpus) void smp_send_reschedule(int cpu) { - xcall_deliver((u64) &xcall_receive_signal, 0, 0, - &cpumask_of_cpu(cpu)); + if (cpu == smp_processor_id()) { + WARN_ON_ONCE(preemptible()); + set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); + } else { + xcall_deliver((u64) &xcall_receive_signal, + 0, 0, cpumask_of(cpu)); + } } void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) { clear_softint(1 << irq); + scheduler_ipi(); } /* This is a nop because we capture all other cpus diff --git a/arch/sparc/kernel/sparc_ksyms_32.c b/arch/sparc/kernel/sparc_ksyms_32.c index baeab872023..bf4ccb10a78 100644 --- a/arch/sparc/kernel/sparc_ksyms_32.c +++ b/arch/sparc/kernel/sparc_ksyms_32.c @@ -6,7 +6,6 @@ */ #include <linux/module.h> -#include <linux/init.h> #include <asm/pgtable.h> #include <asm/uaccess.h> @@ -28,19 +27,5 @@ EXPORT_SYMBOL(__ndelay); EXPORT_SYMBOL(__ret_efault); EXPORT_SYMBOL(empty_zero_page); -/* Defined using magic */ -#ifndef CONFIG_SMP -EXPORT_SYMBOL(BTFIXUP_CALL(___xchg32)); -#else -EXPORT_SYMBOL(BTFIXUP_CALL(__hard_smp_processor_id)); -#endif -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_unlockarea)); -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_lockarea)); -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_get_scsi_sgl)); -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_get_scsi_one)); -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_release_scsi_sgl)); -EXPORT_SYMBOL(BTFIXUP_CALL(mmu_release_scsi_one)); -EXPORT_SYMBOL(BTFIXUP_CALL(pgprot_noncached)); - /* Exporting a symbol from /init/main.c */ EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c index 372ad59c4cb..a92d5d2c46a 100644 --- a/arch/sparc/kernel/sparc_ksyms_64.c +++ b/arch/sparc/kernel/sparc_ksyms_64.c @@ -5,16 +5,16 @@ * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) */ -#include <linux/module.h> +#include <linux/export.h> #include <linux/pci.h> -#include <linux/init.h> +#include <linux/bitops.h> -#include <asm/system.h> #include <asm/cpudata.h> #include <asm/uaccess.h> #include <asm/spitfire.h> #include <asm/oplib.h> #include <asm/hypervisor.h> +#include <asm/cacheflush.h> struct poll { int fd; @@ -38,5 +38,15 @@ EXPORT_SYMBOL(sun4v_niagara_setperf); EXPORT_SYMBOL(sun4v_niagara2_getperf); EXPORT_SYMBOL(sun4v_niagara2_setperf); +/* from hweight.S */ +EXPORT_SYMBOL(__arch_hweight8); +EXPORT_SYMBOL(__arch_hweight16); +EXPORT_SYMBOL(__arch_hweight32); +EXPORT_SYMBOL(__arch_hweight64); + +/* from ffs_ffz.S */ +EXPORT_SYMBOL(ffs); +EXPORT_SYMBOL(__ffs); + /* Exporting a symbol from /init/main.c */ EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sstate.c b/arch/sparc/kernel/sstate.c index 8cdbe5946b4..c59af546f52 100644 --- a/arch/sparc/kernel/sstate.c +++ b/arch/sparc/kernel/sstate.c @@ -14,14 +14,9 @@ #include <asm/head.h> #include <asm/io.h> -static int hv_supports_soft_state; - -static unsigned long kimage_addr_to_ra(const char *p) -{ - unsigned long val = (unsigned long) p; +#include "kernel.h" - return kern_base + (val - KERNBASE); -} +static int hv_supports_soft_state; static void do_set_sstate(unsigned long state, const char *msg) { diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c index 3e081534963..e78386a0029 100644 --- a/arch/sparc/kernel/stacktrace.c +++ b/arch/sparc/kernel/stacktrace.c @@ -2,7 +2,7 @@ #include <linux/stacktrace.h> #include <linux/thread_info.h> #include <linux/ftrace.h> -#include <linux/module.h> +#include <linux/export.h> #include <asm/ptrace.h> #include <asm/stacktrace.h> diff --git a/arch/sparc/kernel/sun4c_irq.c b/arch/sparc/kernel/sun4c_irq.c deleted file mode 100644 index 892fb884910..00000000000 --- a/arch/sparc/kernel/sun4c_irq.c +++ /dev/null @@ -1,225 +0,0 @@ -/* sun4c_irq.c - * arch/sparc/kernel/sun4c_irq.c: - * - * djhr: Hacked out of irq.c into a CPU dependent version. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1995 Miguel de Icaza (miguel@nuclecu.unam.mx) - * Copyright (C) 1995 Pete A. Zaitcev (zaitcev@yahoo.com) - * Copyright (C) 1996 Dave Redman (djhr@tadpole.co.uk) - */ - -#include <linux/errno.h> -#include <linux/linkage.h> -#include <linux/kernel_stat.h> -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/ptrace.h> -#include <linux/interrupt.h> -#include <linux/init.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include "irq.h" - -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/system.h> -#include <asm/psr.h> -#include <asm/vaddrs.h> -#include <asm/timer.h> -#include <asm/openprom.h> -#include <asm/oplib.h> -#include <asm/traps.h> -#include <asm/irq.h> -#include <asm/io.h> -#include <asm/idprom.h> -#include <asm/machines.h> - -/* - * Bit field defines for the interrupt registers on various - * Sparc machines. - */ - -/* The sun4c interrupt register. */ -#define SUN4C_INT_ENABLE 0x01 /* Allow interrupts. */ -#define SUN4C_INT_E14 0x80 /* Enable level 14 IRQ. */ -#define SUN4C_INT_E10 0x20 /* Enable level 10 IRQ. */ -#define SUN4C_INT_E8 0x10 /* Enable level 8 IRQ. */ -#define SUN4C_INT_E6 0x08 /* Enable level 6 IRQ. */ -#define SUN4C_INT_E4 0x04 /* Enable level 4 IRQ. */ -#define SUN4C_INT_E1 0x02 /* Enable level 1 IRQ. */ - -/* Pointer to the interrupt enable byte - * - * Dave Redman (djhr@tadpole.co.uk) - * What you may not be aware of is that entry.S requires this variable. - * - * --- linux_trap_nmi_sun4c -- - * - * so don't go making it static, like I tried. sigh. - */ -unsigned char __iomem *interrupt_enable = NULL; - -static void sun4c_disable_irq(unsigned int irq_nr) -{ - unsigned long flags; - unsigned char current_mask, new_mask; - - local_irq_save(flags); - irq_nr &= (NR_IRQS - 1); - current_mask = sbus_readb(interrupt_enable); - switch(irq_nr) { - case 1: - new_mask = ((current_mask) & (~(SUN4C_INT_E1))); - break; - case 8: - new_mask = ((current_mask) & (~(SUN4C_INT_E8))); - break; - case 10: - new_mask = ((current_mask) & (~(SUN4C_INT_E10))); - break; - case 14: - new_mask = ((current_mask) & (~(SUN4C_INT_E14))); - break; - default: - local_irq_restore(flags); - return; - } - sbus_writeb(new_mask, interrupt_enable); - local_irq_restore(flags); -} - -static void sun4c_enable_irq(unsigned int irq_nr) -{ - unsigned long flags; - unsigned char current_mask, new_mask; - - local_irq_save(flags); - irq_nr &= (NR_IRQS - 1); - current_mask = sbus_readb(interrupt_enable); - switch(irq_nr) { - case 1: - new_mask = ((current_mask) | SUN4C_INT_E1); - break; - case 8: - new_mask = ((current_mask) | SUN4C_INT_E8); - break; - case 10: - new_mask = ((current_mask) | SUN4C_INT_E10); - break; - case 14: - new_mask = ((current_mask) | SUN4C_INT_E14); - break; - default: - local_irq_restore(flags); - return; - } - sbus_writeb(new_mask, interrupt_enable); - local_irq_restore(flags); -} - -struct sun4c_timer_info { - u32 l10_count; - u32 l10_limit; - u32 l14_count; - u32 l14_limit; -}; - -static struct sun4c_timer_info __iomem *sun4c_timers; - -static void sun4c_clear_clock_irq(void) -{ - sbus_readl(&sun4c_timers->l10_limit); -} - -static void sun4c_load_profile_irq(int cpu, unsigned int limit) -{ - /* Errm.. not sure how to do this.. */ -} - -static void __init sun4c_init_timers(irq_handler_t counter_fn) -{ - const struct linux_prom_irqs *irq; - struct device_node *dp; - const u32 *addr; - int err; - - dp = of_find_node_by_name(NULL, "counter-timer"); - if (!dp) { - prom_printf("sun4c_init_timers: Unable to find counter-timer\n"); - prom_halt(); - } - - addr = of_get_property(dp, "address", NULL); - if (!addr) { - prom_printf("sun4c_init_timers: No address property\n"); - prom_halt(); - } - - sun4c_timers = (void __iomem *) (unsigned long) addr[0]; - - irq = of_get_property(dp, "intr", NULL); - of_node_put(dp); - if (!irq) { - prom_printf("sun4c_init_timers: No intr property\n"); - prom_halt(); - } - - /* Have the level 10 timer tick at 100HZ. We don't touch the - * level 14 timer limit since we are letting the prom handle - * them until we have a real console driver so L1-A works. - */ - sbus_writel((((1000000/HZ) + 1) << 10), &sun4c_timers->l10_limit); - - master_l10_counter = &sun4c_timers->l10_count; - - err = request_irq(irq[0].pri, counter_fn, - (IRQF_DISABLED | SA_STATIC_ALLOC), - "timer", NULL); - if (err) { - prom_printf("sun4c_init_timers: request_irq() fails with %d\n", err); - prom_halt(); - } - - sun4c_disable_irq(irq[1].pri); -} - -#ifdef CONFIG_SMP -static void sun4c_nop(void) {} -#endif - -void __init sun4c_init_IRQ(void) -{ - struct device_node *dp; - const u32 *addr; - - dp = of_find_node_by_name(NULL, "interrupt-enable"); - if (!dp) { - prom_printf("sun4c_init_IRQ: Unable to find interrupt-enable\n"); - prom_halt(); - } - - addr = of_get_property(dp, "address", NULL); - of_node_put(dp); - if (!addr) { - prom_printf("sun4c_init_IRQ: No address property\n"); - prom_halt(); - } - - interrupt_enable = (void __iomem *) (unsigned long) addr[0]; - - BTFIXUPSET_CALL(enable_irq, sun4c_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_irq, sun4c_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(enable_pil_irq, sun4c_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_pil_irq, sun4c_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_clock_irq, sun4c_clear_clock_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(load_profile_irq, sun4c_load_profile_irq, BTFIXUPCALL_NOP); - sparc_init_timers = sun4c_init_timers; -#ifdef CONFIG_SMP - BTFIXUPSET_CALL(set_cpu_int, sun4c_nop, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(clear_cpu_int, sun4c_nop, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(set_irq_udt, sun4c_nop, BTFIXUPCALL_NOP); -#endif - sbus_writeb(SUN4C_INT_ENABLE, interrupt_enable); - /* Cannot enable interrupts until OBP ticker is disabled. */ -} diff --git a/arch/sparc/kernel/sun4d_irq.c b/arch/sparc/kernel/sun4d_irq.c index e11b4612dab..a1bb2675b28 100644 --- a/arch/sparc/kernel/sun4d_irq.c +++ b/arch/sparc/kernel/sun4d_irq.c @@ -1,50 +1,43 @@ /* - * arch/sparc/kernel/sun4d_irq.c: - * SS1000/SC2000 interrupt handling. + * SS1000/SC2000 interrupt handling. * * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) * Heavily based on arch/sparc/kernel/irq.c. */ -#include <linux/errno.h> -#include <linux/linkage.h> #include <linux/kernel_stat.h> -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/ptrace.h> -#include <linux/interrupt.h> #include <linux/slab.h> -#include <linux/random.h> -#include <linux/init.h> -#include <linux/smp.h> -#include <linux/spinlock.h> #include <linux/seq_file.h> -#include <linux/of.h> -#include <linux/of_device.h> - -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/system.h> -#include <asm/psr.h> -#include <asm/smp.h> -#include <asm/vaddrs.h> + #include <asm/timer.h> -#include <asm/openprom.h> -#include <asm/oplib.h> #include <asm/traps.h> #include <asm/irq.h> #include <asm/io.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> #include <asm/sbi.h> #include <asm/cacheflush.h> -#include <asm/irq_regs.h> +#include <asm/setup.h> +#include <asm/oplib.h> #include "kernel.h" #include "irq.h" -/* If you trust current SCSI layer to handle different SCSI IRQs, enable this. I don't trust it... -jj */ -/* #define DISTRIBUTE_IRQS */ +/* Sun4d interrupts fall roughly into two categories. SBUS and + * cpu local. CPU local interrupts cover the timer interrupts + * and whatnot, and we encode those as normal PILs between + * 0 and 15. + * SBUS interrupts are encodes as a combination of board, level and slot. + */ + +struct sun4d_handler_data { + unsigned int cpuid; /* target cpu */ + unsigned int real_irq; /* interrupt level */ +}; + + +static unsigned int sun4d_encode_irq(int board, int lvl, int slot) +{ + return (board + 1) << 5 | (lvl << 2) | slot; +} struct sun4d_timer_regs { u32 l10_timer_limit; @@ -56,377 +49,207 @@ struct sun4d_timer_regs { static struct sun4d_timer_regs __iomem *sun4d_timers; -#define TIMER_IRQ 10 - -#define MAX_STATIC_ALLOC 4 -extern int static_irq_count; -static unsigned char sbus_tid[32]; +#define SUN4D_TIMER_IRQ 10 -static struct irqaction *irq_action[NR_IRQS]; -extern spinlock_t irq_action_lock; - -static struct sbus_action { - struct irqaction *action; - /* For SMP this needs to be extended */ -} *sbus_actions; +/* Specify which cpu handle interrupts from which board. + * Index is board - value is cpu. + */ +static unsigned char board_to_cpu[32]; static int pil_to_sbus[] = { - 0, 0, 1, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 0, + 0, + 0, + 1, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 0, }; -static int sbus_to_pil[] = { - 0, 2, 3, 5, 7, 9, 11, 13, -}; - -static int nsbi; - /* Exported for sun4d_smp.c */ DEFINE_SPINLOCK(sun4d_imsk_lock); -int show_sun4d_interrupts(struct seq_file *p, void *v) -{ - int i = *(loff_t *) v, j = 0, k = 0, sbusl; - struct irqaction * action; - unsigned long flags; -#ifdef CONFIG_SMP - int x; -#endif - - spin_lock_irqsave(&irq_action_lock, flags); - if (i < NR_IRQS) { - sbusl = pil_to_sbus[i]; - if (!sbusl) { - action = *(i + irq_action); - if (!action) - goto out_unlock; - } else { - for (j = 0; j < nsbi; j++) { - for (k = 0; k < 4; k++) - if ((action = sbus_actions [(j << 5) + (sbusl << 2) + k].action)) - goto found_it; - } - goto out_unlock; - } -found_it: seq_printf(p, "%3d: ", i); -#ifndef CONFIG_SMP - seq_printf(p, "%10u ", kstat_irqs(i)); -#else - for_each_online_cpu(x) - seq_printf(p, "%10u ", - kstat_cpu(cpu_logical_map(x)).irqs[i]); -#endif - seq_printf(p, "%c %s", - (action->flags & IRQF_DISABLED) ? '+' : ' ', - action->name); - action = action->next; - for (;;) { - for (; action; action = action->next) { - seq_printf(p, ",%s %s", - (action->flags & IRQF_DISABLED) ? " +" : "", - action->name); - } - if (!sbusl) break; - k++; - if (k < 4) - action = sbus_actions [(j << 5) + (sbusl << 2) + k].action; - else { - j++; - if (j == nsbi) break; - k = 0; - action = sbus_actions [(j << 5) + (sbusl << 2)].action; - } - } - seq_putc(p, '\n'); - } -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); - return 0; -} - -void sun4d_free_irq(unsigned int irq, void *dev_id) +/* SBUS interrupts are encoded integers including the board number + * (plus one), the SBUS level, and the SBUS slot number. Sun4D + * IRQ dispatch is done by: + * + * 1) Reading the BW local interrupt table in order to get the bus + * interrupt mask. + * + * This table is indexed by SBUS interrupt level which can be + * derived from the PIL we got interrupted on. + * + * 2) For each bus showing interrupt pending from #1, read the + * SBI interrupt state register. This will indicate which slots + * have interrupts pending for that SBUS interrupt level. + * + * 3) Call the genreric IRQ support. + */ +static void sun4d_sbus_handler_irq(int sbusl) { - struct irqaction *action, **actionp; - struct irqaction *tmp = NULL; - unsigned long flags; - - spin_lock_irqsave(&irq_action_lock, flags); - if (irq < 15) - actionp = irq + irq_action; - else - actionp = &(sbus_actions[irq - (1 << 5)].action); - action = *actionp; - if (!action) { - printk("Trying to free free IRQ%d\n",irq); - goto out_unlock; - } - if (dev_id) { - for (; action; action = action->next) { - if (action->dev_id == dev_id) - break; - tmp = action; - } - if (!action) { - printk("Trying to free free shared IRQ%d\n",irq); - goto out_unlock; - } - } else if (action->flags & IRQF_SHARED) { - printk("Trying to free shared IRQ%d with NULL device ID\n", irq); - goto out_unlock; - } - if (action->flags & SA_STATIC_ALLOC) - { - /* This interrupt is marked as specially allocated - * so it is a bad idea to free it. + unsigned int bus_mask; + unsigned int sbino, slot; + unsigned int sbil; + + bus_mask = bw_get_intr_mask(sbusl) & 0x3ffff; + bw_clear_intr_mask(sbusl, bus_mask); + + sbil = (sbusl << 2); + /* Loop for each pending SBI */ + for (sbino = 0; bus_mask; sbino++, bus_mask >>= 1) { + unsigned int idx, mask; + + if (!(bus_mask & 1)) + continue; + /* XXX This seems to ACK the irq twice. acquire_sbi() + * XXX uses swap, therefore this writes 0xf << sbil, + * XXX then later release_sbi() will write the individual + * XXX bits which were set again. */ - printk("Attempt to free statically allocated IRQ%d (%s)\n", - irq, action->name); - goto out_unlock; - } - - if (tmp) - tmp->next = action->next; - else - *actionp = action->next; - - spin_unlock_irqrestore(&irq_action_lock, flags); + mask = acquire_sbi(SBI2DEVID(sbino), 0xf << sbil); + mask &= (0xf << sbil); - synchronize_irq(irq); + /* Loop for each pending SBI slot */ + slot = (1 << sbil); + for (idx = 0; mask != 0; idx++, slot <<= 1) { + unsigned int pil; + struct irq_bucket *p; - spin_lock_irqsave(&irq_action_lock, flags); + if (!(mask & slot)) + continue; - kfree(action); + mask &= ~slot; + pil = sun4d_encode_irq(sbino, sbusl, idx); - if (!(*actionp)) - __disable_irq(irq); + p = irq_map[pil]; + while (p) { + struct irq_bucket *next; -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); + next = p->next; + generic_handle_irq(p->irq); + p = next; + } + release_sbi(SBI2DEVID(sbino), slot); + } + } } -extern void unexpected_irq(int, void *, struct pt_regs *); - -void sun4d_handler_irq(int irq, struct pt_regs * regs) +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs) { struct pt_regs *old_regs; - struct irqaction * action; - int cpu = smp_processor_id(); /* SBUS IRQ level (1 - 7) */ - int sbusl = pil_to_sbus[irq]; - + int sbusl = pil_to_sbus[pil]; + /* FIXME: Is this necessary?? */ cc_get_ipen(); - - cc_set_iclr(1 << irq); - + + cc_set_iclr(1 << pil); + +#ifdef CONFIG_SMP + /* + * Check IPI data structures after IRQ has been cleared. Hard and Soft + * IRQ can happen at the same time, so both cases are always handled. + */ + if (pil == SUN4D_IPI_IRQ) + sun4d_ipi_interrupt(); +#endif + old_regs = set_irq_regs(regs); irq_enter(); - kstat_cpu(cpu).irqs[irq]++; - if (!sbusl) { - action = *(irq + irq_action); - if (!action) - unexpected_irq(irq, NULL, regs); - do { - action->handler(irq, action->dev_id); - action = action->next; - } while (action); + if (sbusl == 0) { + /* cpu interrupt */ + struct irq_bucket *p; + + p = irq_map[pil]; + while (p) { + struct irq_bucket *next; + + next = p->next; + generic_handle_irq(p->irq); + p = next; + } } else { - int bus_mask = bw_get_intr_mask(sbusl) & 0x3ffff; - int sbino; - struct sbus_action *actionp; - unsigned mask, slot; - int sbil = (sbusl << 2); - - bw_clear_intr_mask(sbusl, bus_mask); - - /* Loop for each pending SBI */ - for (sbino = 0; bus_mask; sbino++, bus_mask >>= 1) - if (bus_mask & 1) { - mask = acquire_sbi(SBI2DEVID(sbino), 0xf << sbil); - mask &= (0xf << sbil); - actionp = sbus_actions + (sbino << 5) + (sbil); - /* Loop for each pending SBI slot */ - for (slot = (1 << sbil); mask; slot <<= 1, actionp++) - if (mask & slot) { - mask &= ~slot; - action = actionp->action; - - if (!action) - unexpected_irq(irq, NULL, regs); - do { - action->handler(irq, action->dev_id); - action = action->next; - } while (action); - release_sbi(SBI2DEVID(sbino), slot); - } - } + /* SBUS interrupt */ + sun4d_sbus_handler_irq(sbusl); } irq_exit(); set_irq_regs(old_regs); } -int sun4d_request_irq(unsigned int irq, - irq_handler_t handler, - unsigned long irqflags, const char * devname, void *dev_id) -{ - struct irqaction *action, *tmp = NULL, **actionp; - unsigned long flags; - int ret; - - if(irq > 14 && irq < (1 << 5)) { - ret = -EINVAL; - goto out; - } - - if (!handler) { - ret = -EINVAL; - goto out; - } - - spin_lock_irqsave(&irq_action_lock, flags); - - if (irq >= (1 << 5)) - actionp = &(sbus_actions[irq - (1 << 5)].action); - else - actionp = irq + irq_action; - action = *actionp; - - if (action) { - if ((action->flags & IRQF_SHARED) && (irqflags & IRQF_SHARED)) { - for (tmp = action; tmp->next; tmp = tmp->next); - } else { - ret = -EBUSY; - goto out_unlock; - } - if ((action->flags & IRQF_DISABLED) ^ (irqflags & IRQF_DISABLED)) { - printk("Attempt to mix fast and slow interrupts on IRQ%d denied\n", irq); - ret = -EBUSY; - goto out_unlock; - } - action = NULL; /* Or else! */ - } - - /* If this is flagged as statically allocated then we use our - * private struct which is never freed. - */ - if (irqflags & SA_STATIC_ALLOC) { - if (static_irq_count < MAX_STATIC_ALLOC) - action = &static_irqaction[static_irq_count++]; - else - printk("Request for IRQ%d (%s) SA_STATIC_ALLOC failed using kmalloc\n", irq, devname); - } - - if (action == NULL) - action = kmalloc(sizeof(struct irqaction), - GFP_ATOMIC); - - if (!action) { - ret = -ENOMEM; - goto out_unlock; - } - action->handler = handler; - action->flags = irqflags; - action->name = devname; - action->next = NULL; - action->dev_id = dev_id; - - if (tmp) - tmp->next = action; - else - *actionp = action; - - __enable_irq(irq); - - ret = 0; -out_unlock: - spin_unlock_irqrestore(&irq_action_lock, flags); -out: - return ret; -} - -static void sun4d_disable_irq(unsigned int irq) +static void sun4d_mask_irq(struct irq_data *data) { - int tid = sbus_tid[(irq >> 5) - 1]; + struct sun4d_handler_data *handler_data = data->handler_data; + unsigned int real_irq; +#ifdef CONFIG_SMP + int cpuid = handler_data->cpuid; unsigned long flags; - - if (irq < NR_IRQS) - return; - +#endif + real_irq = handler_data->real_irq; +#ifdef CONFIG_SMP spin_lock_irqsave(&sun4d_imsk_lock, flags); - cc_set_imsk_other(tid, cc_get_imsk_other(tid) | (1 << sbus_to_pil[(irq >> 2) & 7])); + cc_set_imsk_other(cpuid, cc_get_imsk_other(cpuid) | (1 << real_irq)); spin_unlock_irqrestore(&sun4d_imsk_lock, flags); +#else + cc_set_imsk(cc_get_imsk() | (1 << real_irq)); +#endif } -static void sun4d_enable_irq(unsigned int irq) +static void sun4d_unmask_irq(struct irq_data *data) { - int tid = sbus_tid[(irq >> 5) - 1]; + struct sun4d_handler_data *handler_data = data->handler_data; + unsigned int real_irq; +#ifdef CONFIG_SMP + int cpuid = handler_data->cpuid; unsigned long flags; - - if (irq < NR_IRQS) - return; +#endif + real_irq = handler_data->real_irq; +#ifdef CONFIG_SMP spin_lock_irqsave(&sun4d_imsk_lock, flags); - cc_set_imsk_other(tid, cc_get_imsk_other(tid) & ~(1 << sbus_to_pil[(irq >> 2) & 7])); + cc_set_imsk_other(cpuid, cc_get_imsk_other(cpuid) & ~(1 << real_irq)); spin_unlock_irqrestore(&sun4d_imsk_lock, flags); +#else + cc_set_imsk(cc_get_imsk() & ~(1 << real_irq)); +#endif } -#ifdef CONFIG_SMP -static void sun4d_set_cpu_int(int cpu, int level) +static unsigned int sun4d_startup_irq(struct irq_data *data) { - sun4d_send_ipi(cpu, level); + irq_link(data->irq); + sun4d_unmask_irq(data); + return 0; } -static void sun4d_clear_ipi(int cpu, int level) +static void sun4d_shutdown_irq(struct irq_data *data) { + sun4d_mask_irq(data); + irq_unlink(data->irq); } -static void sun4d_set_udt(int cpu) -{ -} +static struct irq_chip sun4d_irq = { + .name = "sun4d", + .irq_startup = sun4d_startup_irq, + .irq_shutdown = sun4d_shutdown_irq, + .irq_unmask = sun4d_unmask_irq, + .irq_mask = sun4d_mask_irq, +}; +#ifdef CONFIG_SMP /* Setup IRQ distribution scheme. */ void __init sun4d_distribute_irqs(void) { struct device_node *dp; -#ifdef DISTRIBUTE_IRQS - cpumask_t sbus_serving_map; - - sbus_serving_map = cpu_present_map; - for_each_node_by_name(dp, "sbi") { - int board = of_getintprop_default(dp, "board#", 0); - - if ((board * 2) == boot_cpu_id && cpu_isset(board * 2 + 1, cpu_present_map)) - sbus_tid[board] = (board * 2 + 1); - else if (cpu_isset(board * 2, cpu_present_map)) - sbus_tid[board] = (board * 2); - else if (cpu_isset(board * 2 + 1, cpu_present_map)) - sbus_tid[board] = (board * 2 + 1); - else - sbus_tid[board] = 0xff; - if (sbus_tid[board] != 0xff) - cpu_clear(sbus_tid[board], sbus_serving_map); - } - for_each_node_by_name(dp, "sbi") { - int board = of_getintprop_default(dp, "board#", 0); - if (sbus_tid[board] == 0xff) { - int i = 31; - - if (cpus_empty(sbus_serving_map)) - sbus_serving_map = cpu_present_map; - while (cpu_isset(i, sbus_serving_map)) - i--; - sbus_tid[board] = i; - cpu_clear(i, sbus_serving_map); - } - } - for_each_node_by_name(dp, "sbi") { - int devid = of_getintprop_default(dp, "device-id", 0); - int board = of_getintprop_default(dp, "board#", 0); - printk("sbus%d IRQs directed to CPU%d\n", board, sbus_tid[board]); - set_sbi_tid(devid, sbus_tid[board] << 3); - } -#else int cpuid = cpu_logical_map(1); if (cpuid == -1) @@ -434,14 +257,13 @@ void __init sun4d_distribute_irqs(void) for_each_node_by_name(dp, "sbi") { int devid = of_getintprop_default(dp, "device-id", 0); int board = of_getintprop_default(dp, "board#", 0); - sbus_tid[board] = cpuid; + board_to_cpu[board] = cpuid; set_sbi_tid(devid, cpuid << 3); } - printk("All sbus IRQs directed to CPU%d\n", cpuid); -#endif + printk(KERN_ERR "All sbus IRQs directed to CPU%d\n", cpuid); } #endif - + static void sun4d_clear_clock_irq(void) { sbus_readl(&sun4d_timers->l10_timer_limit); @@ -449,7 +271,8 @@ static void sun4d_clear_clock_irq(void) static void sun4d_load_profile_irq(int cpu, unsigned int limit) { - bw_set_prof_limit(cpu, limit); + unsigned int value = limit ? timer_value(limit) : 0; + bw_set_prof_limit(cpu, value); } static void __init sun4d_load_profile_irqs(void) @@ -462,14 +285,116 @@ static void __init sun4d_load_profile_irqs(void) } } +static unsigned int _sun4d_build_device_irq(unsigned int real_irq, + unsigned int pil, + unsigned int board) +{ + struct sun4d_handler_data *handler_data; + unsigned int irq; + + irq = irq_alloc(real_irq, pil); + if (irq == 0) { + prom_printf("IRQ: allocate for %d %d %d failed\n", + real_irq, pil, board); + goto err_out; + } + + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) + goto err_out; + + handler_data = kzalloc(sizeof(struct sun4d_handler_data), GFP_ATOMIC); + if (unlikely(!handler_data)) { + prom_printf("IRQ: kzalloc(sun4d_handler_data) failed.\n"); + prom_halt(); + } + handler_data->cpuid = board_to_cpu[board]; + handler_data->real_irq = real_irq; + irq_set_chip_and_handler_name(irq, &sun4d_irq, + handle_level_irq, "level"); + irq_set_handler_data(irq, handler_data); + +err_out: + return irq; +} + + + +static unsigned int sun4d_build_device_irq(struct platform_device *op, + unsigned int real_irq) +{ + struct device_node *dp = op->dev.of_node; + struct device_node *board_parent, *bus = dp->parent; + char *bus_connection; + const struct linux_prom_registers *regs; + unsigned int pil; + unsigned int irq; + int board, slot; + int sbusl; + + irq = real_irq; + while (bus) { + if (!strcmp(bus->name, "sbi")) { + bus_connection = "io-unit"; + break; + } + + if (!strcmp(bus->name, "bootbus")) { + bus_connection = "cpu-unit"; + break; + } + + bus = bus->parent; + } + if (!bus) + goto err_out; + + regs = of_get_property(dp, "reg", NULL); + if (!regs) + goto err_out; + + slot = regs->which_io; + + /* + * If Bus nodes parent is not io-unit/cpu-unit or the io-unit/cpu-unit + * lacks a "board#" property, something is very wrong. + */ + if (!bus->parent || strcmp(bus->parent->name, bus_connection)) { + printk(KERN_ERR "%s: Error, parent is not %s.\n", + bus->full_name, bus_connection); + goto err_out; + } + board_parent = bus->parent; + board = of_getintprop_default(board_parent, "board#", -1); + if (board == -1) { + printk(KERN_ERR "%s: Error, lacks board# property.\n", + board_parent->full_name); + goto err_out; + } + + sbusl = pil_to_sbus[real_irq]; + if (sbusl) + pil = sun4d_encode_irq(board, sbusl, slot); + else + pil = real_irq; + + irq = _sun4d_build_device_irq(real_irq, pil, board); +err_out: + return irq; +} + +static unsigned int sun4d_build_timer_irq(unsigned int board, + unsigned int real_irq) +{ + return _sun4d_build_device_irq(real_irq, real_irq, board); +} + + static void __init sun4d_fixup_trap_table(void) { #ifdef CONFIG_SMP unsigned long flags; - extern unsigned long lvl14_save[4]; struct tt_entry *trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (14 - 1)]; - extern unsigned int real_irq_entry[], smp4d_ticker[]; - extern unsigned int patchme_maybe_smp_msg[]; /* Adjust so that we jump directly to smp4d_ticker */ lvl14_save[2] += smp4d_ticker - real_irq_entry; @@ -484,17 +409,19 @@ static void __init sun4d_fixup_trap_table(void) trap_table->inst_two = lvl14_save[1]; trap_table->inst_three = lvl14_save[2]; trap_table->inst_four = lvl14_save[3]; - local_flush_cache_all(); + local_ops->cache_all(); local_irq_restore(flags); #endif } -static void __init sun4d_init_timers(irq_handler_t counter_fn) +static void __init sun4d_init_timers(void) { struct device_node *dp; struct resource res; + unsigned int irq; const u32 *reg; int err; + int board; dp = of_find_node_by_name(NULL, "cpu-unit"); if (!dp) { @@ -507,12 +434,19 @@ static void __init sun4d_init_timers(irq_handler_t counter_fn) * bootbus. */ reg = of_get_property(dp, "reg", NULL); - of_node_put(dp); if (!reg) { prom_printf("sun4d_init_timers: No reg property\n"); prom_halt(); } + board = of_getintprop_default(dp, "board#", -1); + if (board == -1) { + prom_printf("sun4d_init_timers: No board# property on cpu-unit\n"); + prom_halt(); + } + + of_node_put(dp); + res.start = reg[1]; res.end = reg[2] - 1; res.flags = reg[0] & 0xff; @@ -523,15 +457,23 @@ static void __init sun4d_init_timers(irq_handler_t counter_fn) prom_halt(); } - sbus_writel((((1000000/HZ) + 1) << 10), &sun4d_timers->l10_timer_limit); +#ifdef CONFIG_SMP + sparc_config.cs_period = SBUS_CLOCK_RATE * 2; /* 2 seconds */ +#else + sparc_config.cs_period = SBUS_CLOCK_RATE / HZ; /* 1/HZ sec */ + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sbus_writel(timer_value(sparc_config.cs_period), + &sun4d_timers->l10_timer_limit); master_l10_counter = &sun4d_timers->l10_cur_count; - err = request_irq(TIMER_IRQ, counter_fn, - (IRQF_DISABLED | SA_STATIC_ALLOC), - "timer", NULL); + irq = sun4d_build_timer_irq(board, SUN4D_TIMER_IRQ); + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); if (err) { - prom_printf("sun4d_init_timers: request_irq() failed with %d\n", err); + prom_printf("sun4d_init_timers: request_irq() failed with %d\n", + err); prom_halt(); } sun4d_load_profile_irqs(); @@ -541,32 +483,22 @@ static void __init sun4d_init_timers(irq_handler_t counter_fn) void __init sun4d_init_sbi_irq(void) { struct device_node *dp; - int target_cpu = 0; + int target_cpu; -#ifdef CONFIG_SMP target_cpu = boot_cpu_id; -#endif - - nsbi = 0; - for_each_node_by_name(dp, "sbi") - nsbi++; - sbus_actions = kzalloc (nsbi * 8 * 4 * sizeof(struct sbus_action), GFP_ATOMIC); - if (!sbus_actions) { - prom_printf("SUN4D: Cannot allocate sbus_actions, halting.\n"); - prom_halt(); - } for_each_node_by_name(dp, "sbi") { int devid = of_getintprop_default(dp, "device-id", 0); int board = of_getintprop_default(dp, "board#", 0); unsigned int mask; set_sbi_tid(devid, target_cpu << 3); - sbus_tid[board] = target_cpu; + board_to_cpu[board] = target_cpu; /* Get rid of pending irqs from PROM */ mask = acquire_sbi(devid, 0xffffffff); if (mask) { - printk ("Clearing pending IRQs %08x on SBI %d\n", mask, board); + printk(KERN_ERR "Clearing pending IRQs %08x on SBI %d\n", + mask, board); release_sbi(devid, mask); } } @@ -576,15 +508,11 @@ void __init sun4d_init_IRQ(void) { local_irq_disable(); - BTFIXUPSET_CALL(enable_irq, sun4d_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_irq, sun4d_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_clock_irq, sun4d_clear_clock_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(load_profile_irq, sun4d_load_profile_irq, BTFIXUPCALL_NORM); - sparc_init_timers = sun4d_init_timers; -#ifdef CONFIG_SMP - BTFIXUPSET_CALL(set_cpu_int, sun4d_set_cpu_int, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_cpu_int, sun4d_clear_ipi, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(set_irq_udt, sun4d_set_udt, BTFIXUPCALL_NOP); -#endif + sparc_config.init_timers = sun4d_init_timers; + sparc_config.build_device_irq = sun4d_build_device_irq; + sparc_config.clock_rate = SBUS_CLOCK_RATE; + sparc_config.clear_clock_irq = sun4d_clear_clock_irq; + sparc_config.load_profile_irq = sun4d_load_profile_irq; + /* Cannot enable interrupts until OBP ticker is disabled. */ } diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c index 482f2ab9269..d5c319553fd 100644 --- a/arch/sparc/kernel/sun4d_smp.c +++ b/arch/sparc/kernel/sun4d_smp.c @@ -1,4 +1,4 @@ -/* sun4d_smp.c: Sparc SS1000/SC2000 SMP support. +/* Sparc SS1000/SC2000 SMP support. * * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) * @@ -6,59 +6,28 @@ * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ -#include <asm/head.h> - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/threads.h> -#include <linux/smp.h> +#include <linux/clockchips.h> #include <linux/interrupt.h> -#include <linux/kernel_stat.h> -#include <linux/init.h> -#include <linux/spinlock.h> -#include <linux/mm.h> -#include <linux/swap.h> #include <linux/profile.h> #include <linux/delay.h> +#include <linux/sched.h> #include <linux/cpu.h> -#include <asm/ptrace.h> -#include <asm/atomic.h> -#include <asm/irq_regs.h> - -#include <asm/irq.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> +#include <asm/cacheflush.h> +#include <asm/switch_to.h> +#include <asm/tlbflush.h> +#include <asm/timer.h> #include <asm/oplib.h> #include <asm/sbi.h> -#include <asm/tlbflush.h> -#include <asm/cacheflush.h> -#include <asm/cpudata.h> +#include <asm/mmu.h> +#include "kernel.h" #include "irq.h" -#define IRQ_CROSS_CALL 15 -extern ctxd_t *srmmu_ctx_table_phys; +#define IRQ_CROSS_CALL 15 -static volatile int smp_processors_ready = 0; +static volatile int smp_processors_ready; static int smp_highest_cpu; -extern volatile unsigned long cpu_callin_map[NR_CPUS]; -extern cpuinfo_sparc cpu_data[NR_CPUS]; -extern unsigned char boot_cpu_id; -extern volatile int smp_process_available; - -extern cpumask_t smp_commenced_mask; - -extern int __smp4d_processor_id(void); - -/* #define SMP_DEBUG */ - -#ifdef SMP_DEBUG -#define SMP_PRINTK(x) printk x -#else -#define SMP_PRINTK(x) -#endif static inline unsigned long sun4d_swap(volatile unsigned long *ptr, unsigned long val) { @@ -68,9 +37,7 @@ static inline unsigned long sun4d_swap(volatile unsigned long *ptr, unsigned lon return val; } -static void smp_setup_percpu_timer(void); -extern void cpu_probe(void); -extern void sun4d_distribute_irqs(void); +static void smp4d_ipi_init(void); static unsigned char cpu_leds[32]; @@ -83,50 +50,40 @@ static inline void show_leds(int cpuid) "i" (ASI_M_CTL)); } -void __cpuinit smp4d_callin(void) +void sun4d_cpu_pre_starting(void *arg) { - int cpuid = hard_smp4d_processor_id(); - extern spinlock_t sun4d_imsk_lock; - unsigned long flags; - + int cpuid = hard_smp_processor_id(); + /* Show we are alive */ cpu_leds[cpuid] = 0x6; show_leds(cpuid); /* Enable level15 interrupt, disable level14 interrupt for now */ cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000); +} - local_flush_cache_all(); - local_flush_tlb_all(); +void sun4d_cpu_pre_online(void *arg) +{ + unsigned long flags; + int cpuid; - notify_cpu_starting(cpuid); - /* - * Unblock the master CPU _only_ when the scheduler state + cpuid = hard_smp_processor_id(); + + /* Unblock the master CPU _only_ when the scheduler state * of all secondary CPUs will be up-to-date, so after * the SMP initialization the master will be just allowed * to call the scheduler code. */ - /* Get our local ticker going. */ - smp_setup_percpu_timer(); - - calibrate_delay(); - smp_store_cpu_info(cpuid); - local_flush_cache_all(); - local_flush_tlb_all(); - - /* Allow master to continue. */ sun4d_swap((unsigned long *)&cpu_callin_map[cpuid], 1); - local_flush_cache_all(); - local_flush_tlb_all(); - - cpu_probe(); + local_ops->cache_all(); + local_ops->tlb_all(); - while((unsigned long)current_set[cpuid] < PAGE_OFFSET) + while ((unsigned long)current_set[cpuid] < PAGE_OFFSET) barrier(); - - while(current_set[cpuid]->cpu != cpuid) + + while (current_set[cpuid]->cpu != cpuid) barrier(); - + /* Fix idle thread fields. */ __asm__ __volatile__("ld [%0], %%g6\n\t" : : "r" (¤t_set[cpuid]) @@ -134,86 +91,71 @@ void __cpuinit smp4d_callin(void) cpu_leds[cpuid] = 0x9; show_leds(cpuid); - + /* Attach to the address space of init_task. */ atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; - local_flush_cache_all(); - local_flush_tlb_all(); - - local_irq_enable(); /* We don't allow PIL 14 yet */ - - while (!cpu_isset(cpuid, smp_commenced_mask)) + local_ops->cache_all(); + local_ops->tlb_all(); + + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) barrier(); spin_lock_irqsave(&sun4d_imsk_lock, flags); cc_set_imsk(cc_get_imsk() & ~0x4000); /* Allow PIL 14 as well */ spin_unlock_irqrestore(&sun4d_imsk_lock, flags); - set_cpu_online(cpuid, true); - } -extern void init_IRQ(void); -extern void cpu_panic(void); - /* * Cycle through the processors asking the PROM to start each one. */ - -extern struct linux_prom_registers smp_penguin_ctable; - void __init smp4d_boot_cpus(void) { + smp4d_ipi_init(); if (boot_cpu_id) current_set[0] = NULL; - smp_setup_percpu_timer(); - local_flush_cache_all(); + local_ops->cache_all(); } -int __cpuinit smp4d_boot_one_cpu(int i) +int smp4d_boot_one_cpu(int i, struct task_struct *idle) { - extern unsigned long sun4d_cpu_startup; - unsigned long *entry = &sun4d_cpu_startup; - struct task_struct *p; - int timeout; - int cpu_node; + unsigned long *entry = &sun4d_cpu_startup; + int timeout; + int cpu_node; - cpu_find_by_instance(i, &cpu_node,NULL); - /* Cook up an idler for this guy. */ - p = fork_idle(i); - current_set[i] = task_thread_info(p); + cpu_find_by_instance(i, &cpu_node, NULL); + current_set[i] = task_thread_info(idle); + /* + * Initialize the contexts table + * Since the call to prom_startcpu() trashes the structure, + * we need to re-initialize it for each cpu + */ + smp_penguin_ctable.which_io = 0; + smp_penguin_ctable.phys_addr = (unsigned int) srmmu_ctx_table_phys; + smp_penguin_ctable.reg_size = 0; + + /* whirrr, whirrr, whirrrrrrrrr... */ + printk(KERN_INFO "Starting CPU %d at %p\n", i, entry); + local_ops->cache_all(); + prom_startcpu(cpu_node, + &smp_penguin_ctable, 0, (char *)entry); + + printk(KERN_INFO "prom_startcpu returned :)\n"); + + /* wheee... it's going... */ + for (timeout = 0; timeout < 10000; timeout++) { + if (cpu_callin_map[i]) + break; + udelay(200); + } - /* - * Initialize the contexts table - * Since the call to prom_startcpu() trashes the structure, - * we need to re-initialize it for each cpu - */ - smp_penguin_ctable.which_io = 0; - smp_penguin_ctable.phys_addr = (unsigned int) srmmu_ctx_table_phys; - smp_penguin_ctable.reg_size = 0; - - /* whirrr, whirrr, whirrrrrrrrr... */ - SMP_PRINTK(("Starting CPU %d at %p\n", i, entry)); - local_flush_cache_all(); - prom_startcpu(cpu_node, - &smp_penguin_ctable, 0, (char *)entry); - - SMP_PRINTK(("prom_startcpu returned :)\n")); - - /* wheee... it's going... */ - for(timeout = 0; timeout < 10000; timeout++) { - if(cpu_callin_map[i]) - break; - udelay(200); - } - if (!(cpu_callin_map[i])) { - printk("Processor %d is stuck.\n", i); + printk(KERN_ERR "Processor %d is stuck.\n", i); return -ENODEV; } - local_flush_cache_all(); + local_ops->cache_all(); return 0; } @@ -230,13 +172,100 @@ void __init smp4d_smp_done(void) prev = &cpu_data(i).next; } *prev = first; - local_flush_cache_all(); + local_ops->cache_all(); /* Ok, they are spinning and ready to go. */ smp_processors_ready = 1; sun4d_distribute_irqs(); } +/* Memory structure giving interrupt handler information about IPI generated */ +struct sun4d_ipi_work { + int single; + int msk; + int resched; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct sun4d_ipi_work, sun4d_ipi_work); + +/* Initialize IPIs on the SUN4D SMP machine */ +static void __init smp4d_ipi_init(void) +{ + int cpu; + struct sun4d_ipi_work *work; + + printk(KERN_INFO "smp4d: setup IPI at IRQ %d\n", SUN4D_IPI_IRQ); + + for_each_possible_cpu(cpu) { + work = &per_cpu(sun4d_ipi_work, cpu); + work->single = work->msk = work->resched = 0; + } +} + +void sun4d_ipi_interrupt(void) +{ + struct sun4d_ipi_work *work = &__get_cpu_var(sun4d_ipi_work); + + if (work->single) { + work->single = 0; + smp_call_function_single_interrupt(); + } + if (work->msk) { + work->msk = 0; + smp_call_function_interrupt(); + } + if (work->resched) { + work->resched = 0; + smp_resched_interrupt(); + } +} + +/* +-------+-------------+-----------+------------------------------------+ + * | bcast | devid | sid | levels mask | + * +-------+-------------+-----------+------------------------------------+ + * 31 30 23 22 15 14 0 + */ +#define IGEN_MESSAGE(bcast, devid, sid, levels) \ + (((bcast) << 31) | ((devid) << 23) | ((sid) << 15) | (levels)) + +static void sun4d_send_ipi(int cpu, int level) +{ + cc_set_igen(IGEN_MESSAGE(0, cpu << 3, 6 + ((level >> 1) & 7), 1 << (level - 1))); +} + +static void sun4d_ipi_single(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->single = 1; + + /* Generate IRQ on the CPU */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + +static void sun4d_ipi_mask_one(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->msk = 1; + + /* Generate IRQ on the CPU */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + +static void sun4d_ipi_resched(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->resched = 1; + + /* Generate IRQ on the CPU (any IRQ will cause resched) */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + static struct smp_funcall { smpfunc_t func; unsigned long arg1; @@ -251,18 +280,21 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void smp4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { - if(smp_processors_ready) { + if (smp_processors_ready) { register int high = smp_highest_cpu; unsigned long flags; spin_lock_irqsave(&cross_call_lock, flags); { - /* If you make changes here, make sure gcc generates proper code... */ + /* + * If you make changes here, make sure + * gcc generates proper code... + */ register smpfunc_t f asm("i0") = func; register unsigned long a1 asm("i1") = arg1; register unsigned long a2 asm("i2") = arg2; @@ -282,10 +314,10 @@ static void smp4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, { register int i; - cpu_clear(smp_processor_id(), mask); - cpus_and(mask, cpu_online_map, mask); - for(i = 0; i <= high; i++) { - if (cpu_isset(i, mask)) { + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); + for (i = 0; i <= high; i++) { + if (cpumask_test_cpu(i, &mask)) { ccall_info.processors_in[i] = 0; ccall_info.processors_out[i] = 0; sun4d_send_ipi(i, IRQ_CROSS_CALL); @@ -298,19 +330,19 @@ static void smp4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; - while(!ccall_info.processors_in[i]) + while (!ccall_info.processors_in[i]) barrier(); - } while(++i <= high); + } while (++i <= high); i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; - while(!ccall_info.processors_out[i]) + while (!ccall_info.processors_out[i]) barrier(); - } while(++i <= high); + } while (++i <= high); } spin_unlock_irqrestore(&cross_call_lock, flags); @@ -320,7 +352,7 @@ static void smp4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, /* Running cross calls. */ void smp4d_cross_call_irq(void) { - int i = hard_smp4d_processor_id(); + int i = hard_smp_processor_id(); ccall_info.processors_in[i] = 1; ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, @@ -331,12 +363,13 @@ void smp4d_cross_call_irq(void) void smp4d_percpu_timer_interrupt(struct pt_regs *regs) { struct pt_regs *old_regs; - int cpu = hard_smp4d_processor_id(); + int cpu = hard_smp_processor_id(); + struct clock_event_device *ce; static int cpu_tick[NR_CPUS]; static char led_mask[] = { 0xe, 0xd, 0xb, 0x7, 0xb, 0xd }; old_regs = set_irq_regs(regs); - bw_get_prof_limit(cpu); + bw_get_prof_limit(cpu); bw_clear_intr_mask(0, 1); /* INTR_TABLE[0] & 1 is Profile IRQ */ cpu_tick[cpu]++; @@ -347,62 +380,31 @@ void smp4d_percpu_timer_interrupt(struct pt_regs *regs) show_leds(cpu); } - profile_tick(CPU_PROFILING); - - if(!--prof_counter(cpu)) { - int user = user_mode(regs); + ce = &per_cpu(sparc32_clockevent, cpu); - irq_enter(); - update_process_times(user); - irq_exit(); + irq_enter(); + ce->event_handler(ce); + irq_exit(); - prof_counter(cpu) = prof_multiplier(cpu); - } set_irq_regs(old_regs); } -extern unsigned int lvl14_resolution; - -static void __cpuinit smp_setup_percpu_timer(void) -{ - int cpu = hard_smp4d_processor_id(); - - prof_counter(cpu) = prof_multiplier(cpu) = 1; - load_profile_irq(cpu, lvl14_resolution); -} - -void __init smp4d_blackbox_id(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - - addr[0] = 0xc0800800 | rd; /* lda [%g0] ASI_M_VIKING_TMP1, reg */ - addr[1] = 0x01000000; /* nop */ - addr[2] = 0x01000000; /* nop */ -} - -void __init smp4d_blackbox_current(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - - addr[0] = 0xc0800800 | rd; /* lda [%g0] ASI_M_VIKING_TMP1, reg */ - addr[2] = 0x81282002 | rd | (rd >> 11); /* sll reg, 2, reg */ - addr[4] = 0x01000000; /* nop */ -} +static const struct sparc32_ipi_ops sun4d_ipi_ops = { + .cross_call = sun4d_cross_call, + .resched = sun4d_ipi_resched, + .single = sun4d_ipi_single, + .mask_one = sun4d_ipi_mask_one, +}; void __init sun4d_init_smp(void) { int i; - extern unsigned int t_nmi[], linux_trap_ipi15_sun4d[], linux_trap_ipi15_sun4m[]; /* Patch ipi15 trap table */ t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_sun4d - linux_trap_ipi15_sun4m); - - /* And set btfixup... */ - BTFIXUPSET_BLACKBOX(hard_smp_processor_id, smp4d_blackbox_id); - BTFIXUPSET_BLACKBOX(load_current, smp4d_blackbox_current); - BTFIXUPSET_CALL(smp_cross_call, smp4d_cross_call, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__hard_smp_processor_id, __smp4d_processor_id, BTFIXUPCALL_NORM); - + + sparc32_ipi_ops = &sun4d_ipi_ops; + for (i = 0; i < NR_CPUS; i++) { ccall_info.processors_in[i] = 1; ccall_info.processors_out[i] = 1; diff --git a/arch/sparc/kernel/sun4m_irq.c b/arch/sparc/kernel/sun4m_irq.c index 7f3b97ff62c..8bb3b3fddea 100644 --- a/arch/sparc/kernel/sun4m_irq.c +++ b/arch/sparc/kernel/sun4m_irq.c @@ -1,5 +1,5 @@ -/* sun4m_irq.c - * arch/sparc/kernel/sun4m_irq.c: +/* + * sun4m irq support * * djhr: Hacked out of irq.c into a CPU dependent version. * @@ -9,101 +9,46 @@ * Copyright (C) 1996 Dave Redman (djhr@tadpole.co.uk) */ -#include <linux/errno.h> -#include <linux/linkage.h> -#include <linux/kernel_stat.h> -#include <linux/signal.h> -#include <linux/sched.h> -#include <linux/ptrace.h> -#include <linux/smp.h> -#include <linux/interrupt.h> -#include <linux/init.h> -#include <linux/ioport.h> -#include <linux/of.h> -#include <linux/of_device.h> - -#include <asm/ptrace.h> -#include <asm/processor.h> -#include <asm/system.h> -#include <asm/psr.h> -#include <asm/vaddrs.h> +#include <linux/slab.h> + #include <asm/timer.h> -#include <asm/openprom.h> -#include <asm/oplib.h> #include <asm/traps.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> -#include <asm/smp.h> #include <asm/irq.h> #include <asm/io.h> #include <asm/cacheflush.h> #include "irq.h" +#include "kernel.h" -struct sun4m_irq_percpu { - u32 pending; - u32 clear; - u32 set; -}; - -struct sun4m_irq_global { - u32 pending; - u32 mask; - u32 mask_clear; - u32 mask_set; - u32 interrupt_target; -}; - -/* Code in entry.S needs to get at these register mappings. */ -struct sun4m_irq_percpu __iomem *sun4m_irq_percpu[SUN4M_NCPUS]; -struct sun4m_irq_global __iomem *sun4m_irq_global; - -/* Dave Redman (djhr@tadpole.co.uk) - * The sun4m interrupt registers. - */ -#define SUN4M_INT_ENABLE 0x80000000 -#define SUN4M_INT_E14 0x00000080 -#define SUN4M_INT_E10 0x00080000 - -#define SUN4M_HARD_INT(x) (0x000000001 << (x)) -#define SUN4M_SOFT_INT(x) (0x000010000 << (x)) - -#define SUN4M_INT_MASKALL 0x80000000 /* mask all interrupts */ -#define SUN4M_INT_MODULE_ERR 0x40000000 /* module error */ -#define SUN4M_INT_M2S_WRITE_ERR 0x20000000 /* write buffer error */ -#define SUN4M_INT_ECC_ERR 0x10000000 /* ecc memory error */ -#define SUN4M_INT_VME_ERR 0x08000000 /* vme async error */ -#define SUN4M_INT_FLOPPY 0x00400000 /* floppy disk */ -#define SUN4M_INT_MODULE 0x00200000 /* module interrupt */ -#define SUN4M_INT_VIDEO 0x00100000 /* onboard video */ -#define SUN4M_INT_REALTIME 0x00080000 /* system timer */ -#define SUN4M_INT_SCSI 0x00040000 /* onboard scsi */ -#define SUN4M_INT_AUDIO 0x00020000 /* audio/isdn */ -#define SUN4M_INT_ETHERNET 0x00010000 /* onboard ethernet */ -#define SUN4M_INT_SERIAL 0x00008000 /* serial ports */ -#define SUN4M_INT_KBDMS 0x00004000 /* keyboard/mouse */ -#define SUN4M_INT_SBUSBITS 0x00003F80 /* sbus int bits */ -#define SUN4M_INT_VMEBITS 0x0000007F /* vme int bits */ - -#define SUN4M_INT_ERROR (SUN4M_INT_MODULE_ERR | \ - SUN4M_INT_M2S_WRITE_ERR | \ - SUN4M_INT_ECC_ERR | \ - SUN4M_INT_VME_ERR) - -#define SUN4M_INT_SBUS(x) (1 << (x+7)) -#define SUN4M_INT_VME(x) (1 << (x)) - -/* Interrupt levels used by OBP */ -#define OBP_INT_LEVEL_SOFT 0x10 -#define OBP_INT_LEVEL_ONBOARD 0x20 -#define OBP_INT_LEVEL_SBUS 0x30 -#define OBP_INT_LEVEL_VME 0x40 - -/* Interrupt level assignment on sun4m: +/* Sample sun4m IRQ layout: + * + * 0x22 - Power + * 0x24 - ESP SCSI + * 0x26 - Lance ethernet + * 0x2b - Floppy + * 0x2c - Zilog uart + * 0x32 - SBUS level 0 + * 0x33 - Parallel port, SBUS level 1 + * 0x35 - SBUS level 2 + * 0x37 - SBUS level 3 + * 0x39 - Audio, Graphics card, SBUS level 4 + * 0x3b - SBUS level 5 + * 0x3d - SBUS level 6 + * + * Each interrupt source has a mask bit in the interrupt registers. + * When the mask bit is set, this blocks interrupt deliver. So you + * clear the bit to enable the interrupt. + * + * Interrupts numbered less than 0x10 are software triggered interrupts + * and unused by Linux. + * + * Interrupt level assignment on sun4m: * * level source * ------------------------------------------------------------ - * 1 softint-1 + * 1 softint-1 * 2 softint-2, VME/SBUS level 1 * 3 softint-3, VME/SBUS level 2 * 4 softint-4, onboard SCSI @@ -138,10 +83,10 @@ struct sun4m_irq_global __iomem *sun4m_irq_global; * 'intr' property IRQ priority values from ss4, ss5, ss10, ss20, and * Tadpole S3 GX systems. * - * esp: 0x24 onboard ESP SCSI - * le: 0x26 onboard Lance ETHERNET + * esp: 0x24 onboard ESP SCSI + * le: 0x26 onboard Lance ETHERNET * p9100: 0x32 SBUS level 1 P9100 video - * bpp: 0x33 SBUS level 2 BPP parallel port device + * bpp: 0x33 SBUS level 2 BPP parallel port device * DBRI: 0x39 SBUS level 5 DBRI ISDN audio * SUNW,leo: 0x39 SBUS level 5 LEO video * pcmcia: 0x3b SBUS level 6 PCMCIA controller @@ -152,8 +97,59 @@ struct sun4m_irq_global __iomem *sun4m_irq_global; * power: 0x22 onboard power device (XXX unknown mask bit XXX) */ -static unsigned long irq_mask[0x50] = { - /* SMP */ + +/* Code in entry.S needs to get at these register mappings. */ +struct sun4m_irq_percpu __iomem *sun4m_irq_percpu[SUN4M_NCPUS]; +struct sun4m_irq_global __iomem *sun4m_irq_global; + +struct sun4m_handler_data { + bool percpu; + long mask; +}; + +/* Dave Redman (djhr@tadpole.co.uk) + * The sun4m interrupt registers. + */ +#define SUN4M_INT_ENABLE 0x80000000 +#define SUN4M_INT_E14 0x00000080 +#define SUN4M_INT_E10 0x00080000 + +#define SUN4M_INT_MASKALL 0x80000000 /* mask all interrupts */ +#define SUN4M_INT_MODULE_ERR 0x40000000 /* module error */ +#define SUN4M_INT_M2S_WRITE_ERR 0x20000000 /* write buffer error */ +#define SUN4M_INT_ECC_ERR 0x10000000 /* ecc memory error */ +#define SUN4M_INT_VME_ERR 0x08000000 /* vme async error */ +#define SUN4M_INT_FLOPPY 0x00400000 /* floppy disk */ +#define SUN4M_INT_MODULE 0x00200000 /* module interrupt */ +#define SUN4M_INT_VIDEO 0x00100000 /* onboard video */ +#define SUN4M_INT_REALTIME 0x00080000 /* system timer */ +#define SUN4M_INT_SCSI 0x00040000 /* onboard scsi */ +#define SUN4M_INT_AUDIO 0x00020000 /* audio/isdn */ +#define SUN4M_INT_ETHERNET 0x00010000 /* onboard ethernet */ +#define SUN4M_INT_SERIAL 0x00008000 /* serial ports */ +#define SUN4M_INT_KBDMS 0x00004000 /* keyboard/mouse */ +#define SUN4M_INT_SBUSBITS 0x00003F80 /* sbus int bits */ +#define SUN4M_INT_VMEBITS 0x0000007F /* vme int bits */ + +#define SUN4M_INT_ERROR (SUN4M_INT_MODULE_ERR | \ + SUN4M_INT_M2S_WRITE_ERR | \ + SUN4M_INT_ECC_ERR | \ + SUN4M_INT_VME_ERR) + +#define SUN4M_INT_SBUS(x) (1 << (x+7)) +#define SUN4M_INT_VME(x) (1 << (x)) + +/* Interrupt levels used by OBP */ +#define OBP_INT_LEVEL_SOFT 0x10 +#define OBP_INT_LEVEL_ONBOARD 0x20 +#define OBP_INT_LEVEL_SBUS 0x30 +#define OBP_INT_LEVEL_VME 0x40 + +#define SUN4M_TIMER_IRQ (OBP_INT_LEVEL_ONBOARD | 10) +#define SUN4M_PROFILE_IRQ (OBP_INT_LEVEL_ONBOARD | 14) + +static unsigned long sun4m_imask[0x50] = { + /* 0x00 - SMP */ 0, SUN4M_SOFT_INT(1), SUN4M_SOFT_INT(2), SUN4M_SOFT_INT(3), SUN4M_SOFT_INT(4), SUN4M_SOFT_INT(5), @@ -162,7 +158,7 @@ static unsigned long irq_mask[0x50] = { SUN4M_SOFT_INT(10), SUN4M_SOFT_INT(11), SUN4M_SOFT_INT(12), SUN4M_SOFT_INT(13), SUN4M_SOFT_INT(14), SUN4M_SOFT_INT(15), - /* soft */ + /* 0x10 - soft */ 0, SUN4M_SOFT_INT(1), SUN4M_SOFT_INT(2), SUN4M_SOFT_INT(3), SUN4M_SOFT_INT(4), SUN4M_SOFT_INT(5), @@ -171,129 +167,119 @@ static unsigned long irq_mask[0x50] = { SUN4M_SOFT_INT(10), SUN4M_SOFT_INT(11), SUN4M_SOFT_INT(12), SUN4M_SOFT_INT(13), SUN4M_SOFT_INT(14), SUN4M_SOFT_INT(15), - /* onboard */ + /* 0x20 - onboard */ 0, 0, 0, 0, SUN4M_INT_SCSI, 0, SUN4M_INT_ETHERNET, 0, SUN4M_INT_VIDEO, SUN4M_INT_MODULE, SUN4M_INT_REALTIME, SUN4M_INT_FLOPPY, (SUN4M_INT_SERIAL | SUN4M_INT_KBDMS), - SUN4M_INT_AUDIO, 0, SUN4M_INT_MODULE_ERR, - /* sbus */ + SUN4M_INT_AUDIO, SUN4M_INT_E14, SUN4M_INT_MODULE_ERR, + /* 0x30 - sbus */ 0, 0, SUN4M_INT_SBUS(0), SUN4M_INT_SBUS(1), 0, SUN4M_INT_SBUS(2), 0, SUN4M_INT_SBUS(3), 0, SUN4M_INT_SBUS(4), 0, SUN4M_INT_SBUS(5), 0, SUN4M_INT_SBUS(6), 0, 0, - /* vme */ + /* 0x40 - vme */ 0, 0, SUN4M_INT_VME(0), SUN4M_INT_VME(1), 0, SUN4M_INT_VME(2), 0, SUN4M_INT_VME(3), 0, SUN4M_INT_VME(4), 0, SUN4M_INT_VME(5), 0, SUN4M_INT_VME(6), 0, 0 }; -static unsigned long sun4m_get_irqmask(unsigned int irq) +static void sun4m_mask_irq(struct irq_data *data) { - unsigned long mask; - - if (irq < 0x50) - mask = irq_mask[irq]; - else - mask = 0; - - if (!mask) - printk(KERN_ERR "sun4m_get_irqmask: IRQ%d has no valid mask!\n", - irq); - - return mask; -} - -static void sun4m_disable_irq(unsigned int irq_nr) -{ - unsigned long mask, flags; + struct sun4m_handler_data *handler_data = data->handler_data; int cpu = smp_processor_id(); - mask = sun4m_get_irqmask(irq_nr); - local_irq_save(flags); - if (irq_nr > 15) - sbus_writel(mask, &sun4m_irq_global->mask_set); - else - sbus_writel(mask, &sun4m_irq_percpu[cpu]->set); - local_irq_restore(flags); + if (handler_data->mask) { + unsigned long flags; + + local_irq_save(flags); + if (handler_data->percpu) { + sbus_writel(handler_data->mask, &sun4m_irq_percpu[cpu]->set); + } else { + sbus_writel(handler_data->mask, &sun4m_irq_global->mask_set); + } + local_irq_restore(flags); + } } -static void sun4m_enable_irq(unsigned int irq_nr) +static void sun4m_unmask_irq(struct irq_data *data) { - unsigned long mask, flags; + struct sun4m_handler_data *handler_data = data->handler_data; int cpu = smp_processor_id(); - /* Dreadful floppy hack. When we use 0x2b instead of - * 0x0b the system blows (it starts to whistle!). - * So we continue to use 0x0b. Fixme ASAP. --P3 - */ - if (irq_nr != 0x0b) { - mask = sun4m_get_irqmask(irq_nr); - local_irq_save(flags); - if (irq_nr > 15) - sbus_writel(mask, &sun4m_irq_global->mask_clear); - else - sbus_writel(mask, &sun4m_irq_percpu[cpu]->clear); - local_irq_restore(flags); - } else { + if (handler_data->mask) { + unsigned long flags; + local_irq_save(flags); - sbus_writel(SUN4M_INT_FLOPPY, &sun4m_irq_global->mask_clear); + if (handler_data->percpu) { + sbus_writel(handler_data->mask, &sun4m_irq_percpu[cpu]->clear); + } else { + sbus_writel(handler_data->mask, &sun4m_irq_global->mask_clear); + } local_irq_restore(flags); } } -static unsigned long cpu_pil_to_imask[16] = { -/*0*/ 0x00000000, -/*1*/ 0x00000000, -/*2*/ SUN4M_INT_SBUS(0) | SUN4M_INT_VME(0), -/*3*/ SUN4M_INT_SBUS(1) | SUN4M_INT_VME(1), -/*4*/ SUN4M_INT_SCSI, -/*5*/ SUN4M_INT_SBUS(2) | SUN4M_INT_VME(2), -/*6*/ SUN4M_INT_ETHERNET, -/*7*/ SUN4M_INT_SBUS(3) | SUN4M_INT_VME(3), -/*8*/ SUN4M_INT_VIDEO, -/*9*/ SUN4M_INT_SBUS(4) | SUN4M_INT_VME(4) | SUN4M_INT_MODULE_ERR, -/*10*/ SUN4M_INT_REALTIME, -/*11*/ SUN4M_INT_SBUS(5) | SUN4M_INT_VME(5) | SUN4M_INT_FLOPPY, -/*12*/ SUN4M_INT_SERIAL | SUN4M_INT_KBDMS, -/*13*/ SUN4M_INT_SBUS(6) | SUN4M_INT_VME(6) | SUN4M_INT_AUDIO, -/*14*/ SUN4M_INT_E14, -/*15*/ SUN4M_INT_ERROR -}; - -/* We assume the caller has disabled local interrupts when these are called, - * or else very bizarre behavior will result. - */ -static void sun4m_disable_pil_irq(unsigned int pil) +static unsigned int sun4m_startup_irq(struct irq_data *data) { - sbus_writel(cpu_pil_to_imask[pil], &sun4m_irq_global->mask_set); + irq_link(data->irq); + sun4m_unmask_irq(data); + return 0; } -static void sun4m_enable_pil_irq(unsigned int pil) +static void sun4m_shutdown_irq(struct irq_data *data) { - sbus_writel(cpu_pil_to_imask[pil], &sun4m_irq_global->mask_clear); + sun4m_mask_irq(data); + irq_unlink(data->irq); } -#ifdef CONFIG_SMP -static void sun4m_send_ipi(int cpu, int level) -{ - unsigned long mask = sun4m_get_irqmask(level); - sbus_writel(mask, &sun4m_irq_percpu[cpu]->set); -} +static struct irq_chip sun4m_irq = { + .name = "sun4m", + .irq_startup = sun4m_startup_irq, + .irq_shutdown = sun4m_shutdown_irq, + .irq_mask = sun4m_mask_irq, + .irq_unmask = sun4m_unmask_irq, +}; -static void sun4m_clear_ipi(int cpu, int level) -{ - unsigned long mask = sun4m_get_irqmask(level); - sbus_writel(mask, &sun4m_irq_percpu[cpu]->clear); -} -static void sun4m_set_udt(int cpu) +static unsigned int sun4m_build_device_irq(struct platform_device *op, + unsigned int real_irq) { - sbus_writel(cpu, &sun4m_irq_global->interrupt_target); + struct sun4m_handler_data *handler_data; + unsigned int irq; + unsigned int pil; + + if (real_irq >= OBP_INT_LEVEL_VME) { + prom_printf("Bogus sun4m IRQ %u\n", real_irq); + prom_halt(); + } + pil = (real_irq & 0xf); + irq = irq_alloc(real_irq, pil); + + if (irq == 0) + goto out; + + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) + goto out; + + handler_data = kzalloc(sizeof(struct sun4m_handler_data), GFP_ATOMIC); + if (unlikely(!handler_data)) { + prom_printf("IRQ: kzalloc(sun4m_handler_data) failed.\n"); + prom_halt(); + } + + handler_data->mask = sun4m_imask[real_irq]; + handler_data->percpu = real_irq < OBP_INT_LEVEL_ONBOARD; + irq_set_chip_and_handler_name(irq, &sun4m_irq, + handle_level_irq, "level"); + irq_set_handler_data(irq, handler_data); + +out: + return irq; } -#endif struct sun4m_timer_percpu { u32 l14_limit; @@ -314,10 +300,6 @@ struct sun4m_timer_global { static struct sun4m_timer_global __iomem *timers_global; -#define TIMER_IRQ (OBP_INT_LEVEL_ONBOARD | 10) - -unsigned int lvl14_resolution = (((1000000/HZ) + 1) << 10); - static void sun4m_clear_clock_irq(void) { sbus_readl(&timers_global->l10_limit); @@ -350,7 +332,15 @@ void sun4m_nmi(struct pt_regs *regs) prom_halt(); } -/* Exported for sun4m_smp.c */ +void sun4m_unmask_profile_irq(void) +{ + unsigned long flags; + + local_irq_save(flags); + sbus_writel(sun4m_imask[SUN4M_PROFILE_IRQ], &sun4m_irq_global->mask_clear); + local_irq_restore(flags); +} + void sun4m_clear_profile_irq(int cpu) { sbus_readl(&timers_percpu[cpu]->l14_limit); @@ -358,13 +348,15 @@ void sun4m_clear_profile_irq(int cpu) static void sun4m_load_profile_irq(int cpu, unsigned int limit) { - sbus_writel(limit, &timers_percpu[cpu]->l14_limit); + unsigned int value = limit ? timer_value(limit) : 0; + sbus_writel(value, &timers_percpu[cpu]->l14_limit); } -static void __init sun4m_init_timers(irq_handler_t counter_fn) +static void __init sun4m_init_timers(void) { struct device_node *dp = of_find_node_by_name(NULL, "counter"); int i, err, len, num_cpu_timers; + unsigned int irq; const u32 *addr; if (!dp) { @@ -387,12 +379,25 @@ static void __init sun4m_init_timers(irq_handler_t counter_fn) timers_global = (void __iomem *) (unsigned long) addr[num_cpu_timers]; - sbus_writel((((1000000/HZ) + 1) << 10), &timers_global->l10_limit); + /* Every per-cpu timer works in timer mode */ + sbus_writel(0x00000000, &timers_global->timer_config); + +#ifdef CONFIG_SMP + sparc_config.cs_period = SBUS_CLOCK_RATE * 2; /* 2 seconds */ + sparc_config.features |= FEAT_L14_ONESHOT; +#else + sparc_config.cs_period = SBUS_CLOCK_RATE / HZ; /* 1/HZ sec */ + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sbus_writel(timer_value(sparc_config.cs_period), + &timers_global->l10_limit); master_l10_counter = &timers_global->l10_count; - err = request_irq(TIMER_IRQ, counter_fn, - (IRQF_DISABLED | SA_STATIC_ALLOC), "timer", NULL); + irq = sun4m_build_device_irq(NULL, SUN4M_TIMER_IRQ); + + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); if (err) { printk(KERN_ERR "sun4m_init_timers: Register IRQ error %d.\n", err); @@ -407,7 +412,6 @@ static void __init sun4m_init_timers(irq_handler_t counter_fn) #ifdef CONFIG_SMP { unsigned long flags; - extern unsigned long lvl14_save[4]; struct tt_entry *trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (14 - 1)]; /* For SMP we use the level 14 ticker, however the bootup code @@ -419,7 +423,7 @@ static void __init sun4m_init_timers(irq_handler_t counter_fn) trap_table->inst_two = lvl14_save[1]; trap_table->inst_three = lvl14_save[2]; trap_table->inst_four = lvl14_save[3]; - local_flush_cache_all(); + local_ops->cache_all(); local_irq_restore(flags); } #endif @@ -460,18 +464,12 @@ void __init sun4m_init_IRQ(void) if (num_cpu_iregs == 4) sbus_writel(0, &sun4m_irq_global->interrupt_target); - BTFIXUPSET_CALL(enable_irq, sun4m_enable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_irq, sun4m_disable_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(enable_pil_irq, sun4m_enable_pil_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(disable_pil_irq, sun4m_disable_pil_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_clock_irq, sun4m_clear_clock_irq, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(load_profile_irq, sun4m_load_profile_irq, BTFIXUPCALL_NORM); - sparc_init_timers = sun4m_init_timers; -#ifdef CONFIG_SMP - BTFIXUPSET_CALL(set_cpu_int, sun4m_send_ipi, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(clear_cpu_int, sun4m_clear_ipi, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(set_irq_udt, sun4m_set_udt, BTFIXUPCALL_NORM); -#endif + sparc_config.init_timers = sun4m_init_timers; + sparc_config.build_device_irq = sun4m_build_device_irq; + sparc_config.clock_rate = SBUS_CLOCK_RATE; + sparc_config.clear_clock_irq = sun4m_clear_clock_irq; + sparc_config.load_profile_irq = sun4m_load_profile_irq; + /* Cannot enable interrupts until OBP ticker is disabled. */ } diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c index 762d6eedd94..d3408e72d20 100644 --- a/arch/sparc/kernel/sun4m_smp.c +++ b/arch/sparc/kernel/sun4m_smp.c @@ -1,59 +1,30 @@ -/* sun4m_smp.c: Sparc SUN4M SMP support. +/* + * sun4m SMP support. * * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ -#include <asm/head.h> - -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/threads.h> -#include <linux/smp.h> +#include <linux/clockchips.h> #include <linux/interrupt.h> -#include <linux/kernel_stat.h> -#include <linux/init.h> -#include <linux/spinlock.h> -#include <linux/mm.h> -#include <linux/swap.h> #include <linux/profile.h> #include <linux/delay.h> +#include <linux/sched.h> #include <linux/cpu.h> #include <asm/cacheflush.h> +#include <asm/switch_to.h> #include <asm/tlbflush.h> -#include <asm/irq_regs.h> - -#include <asm/ptrace.h> -#include <asm/atomic.h> - -#include <asm/irq.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> +#include <asm/timer.h> #include <asm/oplib.h> -#include <asm/cpudata.h> #include "irq.h" +#include "kernel.h" +#define IRQ_IPI_SINGLE 12 +#define IRQ_IPI_MASK 13 +#define IRQ_IPI_RESCHED 14 #define IRQ_CROSS_CALL 15 -extern ctxd_t *srmmu_ctx_table_phys; - -extern volatile unsigned long cpu_callin_map[NR_CPUS]; -extern unsigned char boot_cpu_id; - -extern cpumask_t smp_commenced_mask; - -extern int __smp4m_processor_id(void); - -/*#define SMP_DEBUG*/ - -#ifdef SMP_DEBUG -#define SMP_PRINTK(x) printk x -#else -#define SMP_PRINTK(x) -#endif - static inline unsigned long swap_ulong(volatile unsigned long *ptr, unsigned long val) { @@ -63,41 +34,24 @@ swap_ulong(volatile unsigned long *ptr, unsigned long val) return val; } -static void smp_setup_percpu_timer(void); -extern void cpu_probe(void); +void sun4m_cpu_pre_starting(void *arg) +{ +} -void __cpuinit smp4m_callin(void) +void sun4m_cpu_pre_online(void *arg) { int cpuid = hard_smp_processor_id(); - local_flush_cache_all(); - local_flush_tlb_all(); - - notify_cpu_starting(cpuid); - - /* Get our local ticker going. */ - smp_setup_percpu_timer(); - - calibrate_delay(); - smp_store_cpu_info(cpuid); - - local_flush_cache_all(); - local_flush_tlb_all(); - - /* - * Unblock the master CPU _only_ when the scheduler state - * of all secondary CPUs will be up-to-date, so after - * the SMP initialization the master will be just allowed - * to call the scheduler code. + /* Allow master to continue. The master will then give us the + * go-ahead by setting the smp_commenced_mask and will wait without + * timeouts until our setup is completed fully (signified by + * our bit being set in the cpu_online_mask). */ - /* Allow master to continue. */ swap_ulong(&cpu_callin_map[cpuid], 1); /* XXX: What's up with all the flushes? */ - local_flush_cache_all(); - local_flush_tlb_all(); - - cpu_probe(); + local_ops->cache_all(); + local_ops->tlb_all(); /* Fix idle thread fields. */ __asm__ __volatile__("ld [%0], %%g6\n\t" @@ -108,41 +62,30 @@ void __cpuinit smp4m_callin(void) atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; - while (!cpu_isset(cpuid, smp_commenced_mask)) + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) mb(); - - local_irq_enable(); - - set_cpu_online(cpuid, true); } /* * Cycle through the processors asking the PROM to start each one. */ - -extern struct linux_prom_registers smp_penguin_ctable; - void __init smp4m_boot_cpus(void) { - smp_setup_percpu_timer(); - local_flush_cache_all(); + sun4m_unmask_profile_irq(); + local_ops->cache_all(); } -int __cpuinit smp4m_boot_one_cpu(int i) +int smp4m_boot_one_cpu(int i, struct task_struct *idle) { - extern unsigned long sun4m_cpu_startup; unsigned long *entry = &sun4m_cpu_startup; - struct task_struct *p; int timeout; int cpu_node; cpu_find_by_mid(i, &cpu_node); + current_set[i] = task_thread_info(idle); - /* Cook up an idler for this guy. */ - p = fork_idle(i); - current_set[i] = task_thread_info(p); /* See trampoline.S for details... */ - entry += ((i-1) * 3); + entry += ((i - 1) * 3); /* * Initialize the contexts table @@ -154,24 +97,23 @@ int __cpuinit smp4m_boot_one_cpu(int i) smp_penguin_ctable.reg_size = 0; /* whirrr, whirrr, whirrrrrrrrr... */ - printk("Starting CPU %d at %p\n", i, entry); - local_flush_cache_all(); - prom_startcpu(cpu_node, - &smp_penguin_ctable, 0, (char *)entry); + printk(KERN_INFO "Starting CPU %d at %p\n", i, entry); + local_ops->cache_all(); + prom_startcpu(cpu_node, &smp_penguin_ctable, 0, (char *)entry); /* wheee... it's going... */ - for(timeout = 0; timeout < 10000; timeout++) { - if(cpu_callin_map[i]) + for (timeout = 0; timeout < 10000; timeout++) { + if (cpu_callin_map[i]) break; udelay(200); } if (!(cpu_callin_map[i])) { - printk("Processor %d is stuck.\n", i); + printk(KERN_ERR "Processor %d is stuck.\n", i); return -ENODEV; } - local_flush_cache_all(); + local_ops->cache_all(); return 0; } @@ -188,22 +130,29 @@ void __init smp4m_smp_done(void) prev = &cpu_data(i).next; } *prev = first; - local_flush_cache_all(); + local_ops->cache_all(); /* Ok, they are spinning and ready to go. */ } -/* At each hardware IRQ, we get this called to forward IRQ reception - * to the next processor. The caller must disable the IRQ level being - * serviced globally so that there are no double interrupts received. - * - * XXX See sparc64 irq.c. - */ -void smp4m_irq_rotate(int cpu) +static void sun4m_send_ipi(int cpu, int level) { - int next = cpu_data(cpu).next; - if (next != cpu) - set_irq_udt(next); + sbus_writel(SUN4M_SOFT_INT(level), &sun4m_irq_percpu[cpu]->set); +} + +static void sun4m_ipi_resched(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_RESCHED); +} + +static void sun4m_ipi_single(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_SINGLE); +} + +static void sun4m_ipi_mask_one(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_MASK); } static struct smp_funcall { @@ -220,7 +169,7 @@ static struct smp_funcall { static DEFINE_SPINLOCK(cross_call_lock); /* Cross calls must be serialized, at least currently. */ -static void smp4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, +static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, unsigned long arg2, unsigned long arg3, unsigned long arg4) { @@ -241,13 +190,13 @@ static void smp4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, { register int i; - cpu_clear(smp_processor_id(), mask); - cpus_and(mask, cpu_online_map, mask); - for(i = 0; i < ncpus; i++) { - if (cpu_isset(i, mask)) { + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); + for (i = 0; i < ncpus; i++) { + if (cpumask_test_cpu(i, &mask)) { ccall_info.processors_in[i] = 0; ccall_info.processors_out[i] = 0; - set_cpu_int(i, IRQ_CROSS_CALL); + sun4m_send_ipi(i, IRQ_CROSS_CALL); } else { ccall_info.processors_in[i] = 1; ccall_info.processors_out[i] = 1; @@ -260,21 +209,20 @@ static void smp4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1, i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; - while(!ccall_info.processors_in[i]) + while (!ccall_info.processors_in[i]) barrier(); - } while(++i < ncpus); + } while (++i < ncpus); i = 0; do { - if (!cpu_isset(i, mask)) + if (!cpumask_test_cpu(i, &mask)) continue; - while(!ccall_info.processors_out[i]) + while (!ccall_info.processors_out[i]) barrier(); - } while(++i < ncpus); + } while (++i < ncpus); } - spin_unlock_irqrestore(&cross_call_lock, flags); } @@ -289,68 +237,36 @@ void smp4m_cross_call_irq(void) ccall_info.processors_out[i] = 1; } -extern void sun4m_clear_profile_irq(int cpu); - void smp4m_percpu_timer_interrupt(struct pt_regs *regs) { struct pt_regs *old_regs; + struct clock_event_device *ce; int cpu = smp_processor_id(); old_regs = set_irq_regs(regs); - sun4m_clear_profile_irq(cpu); - - profile_tick(CPU_PROFILING); + ce = &per_cpu(sparc32_clockevent, cpu); - if(!--prof_counter(cpu)) { - int user = user_mode(regs); + if (ce->mode & CLOCK_EVT_MODE_PERIODIC) + sun4m_clear_profile_irq(cpu); + else + sparc_config.load_profile_irq(cpu, 0); /* Is this needless? */ - irq_enter(); - update_process_times(user); - irq_exit(); + irq_enter(); + ce->event_handler(ce); + irq_exit(); - prof_counter(cpu) = prof_multiplier(cpu); - } set_irq_regs(old_regs); } -extern unsigned int lvl14_resolution; - -static void __cpuinit smp_setup_percpu_timer(void) -{ - int cpu = smp_processor_id(); - - prof_counter(cpu) = prof_multiplier(cpu) = 1; - load_profile_irq(cpu, lvl14_resolution); - - if(cpu == boot_cpu_id) - enable_pil_irq(14); -} - -static void __init smp4m_blackbox_id(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - int rs1 = rd >> 11; - - addr[0] = 0x81580000 | rd; /* rd %tbr, reg */ - addr[1] = 0x8130200c | rd | rs1; /* srl reg, 0xc, reg */ - addr[2] = 0x80082003 | rd | rs1; /* and reg, 3, reg */ -} - -static void __init smp4m_blackbox_current(unsigned *addr) -{ - int rd = *addr & 0x3e000000; - int rs1 = rd >> 11; - - addr[0] = 0x81580000 | rd; /* rd %tbr, reg */ - addr[2] = 0x8130200a | rd | rs1; /* srl reg, 0xa, reg */ - addr[4] = 0x8008200c | rd | rs1; /* and reg, 0xc, reg */ -} +static const struct sparc32_ipi_ops sun4m_ipi_ops = { + .cross_call = sun4m_cross_call, + .resched = sun4m_ipi_resched, + .single = sun4m_ipi_single, + .mask_one = sun4m_ipi_mask_one, +}; void __init sun4m_init_smp(void) { - BTFIXUPSET_BLACKBOX(hard_smp_processor_id, smp4m_blackbox_id); - BTFIXUPSET_BLACKBOX(load_current, smp4m_blackbox_current); - BTFIXUPSET_CALL(smp_cross_call, smp4m_cross_call, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__hard_smp_processor_id, __smp4m_processor_id, BTFIXUPCALL_NORM); + sparc32_ipi_ops = &sun4m_ipi_ops; } diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S index e1fbf8c7578..e0c09bf8561 100644 --- a/arch/sparc/kernel/sun4v_tlb_miss.S +++ b/arch/sparc/kernel/sun4v_tlb_miss.S @@ -176,13 +176,13 @@ sun4v_tsb_miss_common: sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mov SCRATCHPAD_UTSBREG2, %g5 ldxa [%g5] ASI_SCRATCHPAD, %g5 cmp %g5, -1 be,pt %xcc, 80f nop - COMPUTE_TSB_PTR(%g5, %g4, HPAGE_SHIFT, %g2, %g7) + COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) /* That clobbered %g2, reload it. */ ldxa [%g0] ASI_SCRATCHPAD, %g2 diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S index 44e5faf1ad5..f834224208e 100644 --- a/arch/sparc/kernel/sys32.S +++ b/arch/sparc/kernel/sys32.S @@ -36,109 +36,19 @@ STUB: sra REG1, 0, REG1; \ jmpl %g1 + %lo(SYSCALL), %g0; \ sra REG3, 0, REG3 -#define SIGN4(STUB,SYSCALL,REG1,REG2,REG3,REG4) \ - .align 32; \ - .globl STUB; \ -STUB: sra REG1, 0, REG1; \ - sethi %hi(SYSCALL), %g1; \ - sra REG2, 0, REG2; \ - sra REG3, 0, REG3; \ - jmpl %g1 + %lo(SYSCALL), %g0; \ - sra REG4, 0, REG4 - -SIGN1(sys32_exit, sparc_exit, %o0) -SIGN1(sys32_exit_group, sys_exit_group, %o0) -SIGN1(sys32_wait4, compat_sys_wait4, %o2) -SIGN1(sys32_creat, sys_creat, %o1) -SIGN1(sys32_mknod, sys_mknod, %o1) -SIGN1(sys32_umount, sys_umount, %o1) -SIGN1(sys32_signal, sys_signal, %o0) -SIGN1(sys32_access, sys_access, %o1) -SIGN1(sys32_msync, sys_msync, %o2) -SIGN2(sys32_reboot, sys_reboot, %o0, %o1) -SIGN1(sys32_setitimer, compat_sys_setitimer, %o0) -SIGN1(sys32_getitimer, compat_sys_getitimer, %o0) -SIGN1(sys32_sethostname, sys_sethostname, %o1) -SIGN1(sys32_swapon, sys_swapon, %o1) -SIGN1(sys32_sigaction, compat_sys_sigaction, %o0) -SIGN1(sys32_rt_sigaction, compat_sys_rt_sigaction, %o0) -SIGN1(sys32_sigprocmask, compat_sys_sigprocmask, %o0) -SIGN1(sys32_rt_sigprocmask, compat_sys_rt_sigprocmask, %o0) -SIGN2(sys32_rt_sigqueueinfo, compat_sys_rt_sigqueueinfo, %o0, %o1) -SIGN1(sys32_getrusage, compat_sys_getrusage, %o0) -SIGN1(sys32_setxattr, sys_setxattr, %o4) -SIGN1(sys32_lsetxattr, sys_lsetxattr, %o4) -SIGN1(sys32_fsetxattr, sys_fsetxattr, %o4) -SIGN1(sys32_fgetxattr, sys_fgetxattr, %o0) -SIGN1(sys32_flistxattr, sys_flistxattr, %o0) -SIGN1(sys32_fremovexattr, sys_fremovexattr, %o0) -SIGN2(sys32_tkill, sys_tkill, %o0, %o1) -SIGN1(sys32_epoll_create, sys_epoll_create, %o0) -SIGN3(sys32_epoll_ctl, sys_epoll_ctl, %o0, %o1, %o2) -SIGN3(sys32_epoll_wait, sys_epoll_wait, %o0, %o2, %o3) SIGN1(sys32_readahead, compat_sys_readahead, %o0) SIGN2(sys32_fadvise64, compat_sys_fadvise64, %o0, %o4) SIGN2(sys32_fadvise64_64, compat_sys_fadvise64_64, %o0, %o5) -SIGN2(sys32_bdflush, sys_bdflush, %o0, %o1) -SIGN1(sys32_mlockall, sys_mlockall, %o0) -SIGN1(sys32_nfsservctl, compat_sys_nfsservctl, %o0) SIGN1(sys32_clock_nanosleep, compat_sys_clock_nanosleep, %o1) SIGN1(sys32_timer_settime, compat_sys_timer_settime, %o1) SIGN1(sys32_io_submit, compat_sys_io_submit, %o1) SIGN1(sys32_mq_open, compat_sys_mq_open, %o1) SIGN1(sys32_select, compat_sys_select, %o0) -SIGN1(sys32_mkdir, sys_mkdir, %o1) -SIGN3(sys32_futex, compat_sys_futex, %o1, %o2, %o5) -SIGN1(sys32_sysfs, compat_sys_sysfs, %o0) -SIGN2(sys32_sendfile, compat_sys_sendfile, %o0, %o1) -SIGN2(sys32_sendfile64, compat_sys_sendfile64, %o0, %o1) -SIGN1(sys32_prctl, sys_prctl, %o0) -SIGN1(sys32_sched_rr_get_interval, compat_sys_sched_rr_get_interval, %o0) -SIGN2(sys32_waitpid, sys_waitpid, %o0, %o2) -SIGN1(sys32_getgroups, sys_getgroups, %o0) -SIGN1(sys32_getpgid, sys_getpgid, %o0) -SIGN2(sys32_getpriority, sys_getpriority, %o0, %o1) -SIGN1(sys32_getsid, sys_getsid, %o0) -SIGN2(sys32_kill, sys_kill, %o0, %o1) -SIGN1(sys32_nice, sys_nice, %o0) -SIGN1(sys32_lseek, sys_lseek, %o1) -SIGN2(sys32_open, sparc32_open, %o1, %o2) -SIGN1(sys32_readlink, sys_readlink, %o2) -SIGN1(sys32_sched_get_priority_max, sys_sched_get_priority_max, %o0) -SIGN1(sys32_sched_get_priority_min, sys_sched_get_priority_min, %o0) -SIGN1(sys32_sched_getparam, sys_sched_getparam, %o0) -SIGN1(sys32_sched_getscheduler, sys_sched_getscheduler, %o0) -SIGN1(sys32_sched_setparam, sys_sched_setparam, %o0) -SIGN2(sys32_sched_setscheduler, sys_sched_setscheduler, %o0, %o1) -SIGN1(sys32_getdomainname, sys_getdomainname, %o1) -SIGN1(sys32_setdomainname, sys_setdomainname, %o1) -SIGN1(sys32_setgroups, sys_setgroups, %o0) -SIGN2(sys32_setpgid, sys_setpgid, %o0, %o1) -SIGN3(sys32_setpriority, sys_setpriority, %o0, %o1, %o2) -SIGN1(sys32_ssetmask, sys_ssetmask, %o0) -SIGN2(sys32_syslog, sys_syslog, %o0, %o2) -SIGN1(sys32_umask, sys_umask, %o0) -SIGN3(sys32_tgkill, sys_tgkill, %o0, %o1, %o2) -SIGN1(sys32_sendto, sys_sendto, %o0) +SIGN1(sys32_futex, compat_sys_futex, %o1) SIGN1(sys32_recvfrom, compat_sys_recvfrom, %o0) -SIGN3(sys32_socket, sys_socket, %o0, %o1, %o2) -SIGN2(sys32_connect, sys_connect, %o0, %o2) -SIGN2(sys32_bind, sys_bind, %o0, %o2) -SIGN2(sys32_listen, sys_listen, %o0, %o1) SIGN1(sys32_recvmsg, compat_sys_recvmsg, %o0) SIGN1(sys32_sendmsg, compat_sys_sendmsg, %o0) -SIGN2(sys32_shutdown, sys_shutdown, %o0, %o1) -SIGN3(sys32_socketpair, sys_socketpair, %o0, %o1, %o2) -SIGN1(sys32_getpeername, sys_getpeername, %o0) -SIGN1(sys32_getsockname, sys_getsockname, %o0) -SIGN2(sys32_ioprio_get, sys_ioprio_get, %o0, %o1) -SIGN3(sys32_ioprio_set, sys_ioprio_set, %o0, %o1, %o2) -SIGN2(sys32_splice, sys_splice, %o0, %o2) -SIGN2(sys32_sync_file_range, compat_sync_file_range, %o0, %o5) -SIGN2(sys32_tee, sys_tee, %o0, %o1) -SIGN1(sys32_vmsplice, compat_sys_vmsplice, %o0) -SIGN1(sys32_truncate, sys_truncate, %o1) -SIGN1(sys32_ftruncate, sys_ftruncate, %o1) +SIGN2(sys32_renameat2, sys_renameat2, %o0, %o2) .globl sys32_mmap2 sys32_mmap2: @@ -330,15 +240,6 @@ do_sys_accept4: /* sys_accept4(int, struct sockaddr *, int *, int) */ nop nop - .globl sys32_fanotify_mark -sys32_fanotify_mark: - sethi %hi(sys_fanotify_mark), %g1 - sllx %o2, 32, %o2 - or %o2, %o3, %o2 - mov %o4, %o3 - jmpl %g1 + %lo(sys_fanotify_mark), %g0 - mov %o5, %o4 - .section __ex_table,"a" .align 4 .word 1b, __retl_efault, 2b, __retl_efault diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c index 6db18c6927f..022c30c72eb 100644 --- a/arch/sparc/kernel/sys_sparc32.c +++ b/arch/sparc/kernel/sys_sparc32.c @@ -23,7 +23,6 @@ #include <linux/uio.h> #include <linux/nfs_fs.h> #include <linux/quota.h> -#include <linux/module.h> #include <linux/poll.h> #include <linux/personality.h> #include <linux/stat.h> @@ -50,70 +49,7 @@ #include <asm/mmu_context.h> #include <asm/compat_signal.h> -#ifdef CONFIG_SYSVIPC -asmlinkage long compat_sys_ipc(u32 call, u32 first, u32 second, u32 third, compat_uptr_t ptr, u32 fifth) -{ - int version; - - version = call >> 16; /* hack for backward compatibility */ - call &= 0xffff; - - switch (call) { - case SEMTIMEDOP: - if (fifth) - /* sign extend semid */ - return compat_sys_semtimedop((int)first, - compat_ptr(ptr), second, - compat_ptr(fifth)); - /* else fall through for normal semop() */ - case SEMOP: - /* struct sembuf is the same on 32 and 64bit :)) */ - /* sign extend semid */ - return sys_semtimedop((int)first, compat_ptr(ptr), second, - NULL); - case SEMGET: - /* sign extend key, nsems */ - return sys_semget((int)first, (int)second, third); - case SEMCTL: - /* sign extend semid, semnum */ - return compat_sys_semctl((int)first, (int)second, third, - compat_ptr(ptr)); - - case MSGSND: - /* sign extend msqid */ - return compat_sys_msgsnd((int)first, (int)second, third, - compat_ptr(ptr)); - case MSGRCV: - /* sign extend msqid, msgtyp */ - return compat_sys_msgrcv((int)first, second, (int)fifth, - third, version, compat_ptr(ptr)); - case MSGGET: - /* sign extend key */ - return sys_msgget((int)first, second); - case MSGCTL: - /* sign extend msqid */ - return compat_sys_msgctl((int)first, second, compat_ptr(ptr)); - - case SHMAT: - /* sign extend shmid */ - return compat_sys_shmat((int)first, second, third, version, - compat_ptr(ptr)); - case SHMDT: - return sys_shmdt(compat_ptr(ptr)); - case SHMGET: - /* sign extend key_t */ - return sys_shmget((int)first, second, third); - case SHMCTL: - /* sign extend shmid */ - return compat_sys_shmctl((int)first, second, compat_ptr(ptr)); - - default: - return -ENOSYS; - }; - - return -ENOSYS; -} -#endif +#include "systbls.h" asmlinkage long sys32_truncate64(const char __user * path, unsigned long high, unsigned long low) { @@ -140,8 +76,8 @@ static int cp_compat_stat64(struct kstat *stat, err |= put_user(stat->ino, &statbuf->st_ino); err |= put_user(stat->mode, &statbuf->st_mode); err |= put_user(stat->nlink, &statbuf->st_nlink); - err |= put_user(stat->uid, &statbuf->st_uid); - err |= put_user(stat->gid, &statbuf->st_gid); + err |= put_user(from_kuid_munged(current_user_ns(), stat->uid), &statbuf->st_uid); + err |= put_user(from_kgid_munged(current_user_ns(), stat->gid), &statbuf->st_gid); err |= put_user(huge_encode_dev(stat->rdev), &statbuf->st_rdev); err |= put_user(0, (unsigned long __user *) &statbuf->__pad3[0]); err |= put_user(stat->size, &statbuf->st_size); @@ -207,147 +143,19 @@ asmlinkage long compat_sys_fstatat64(unsigned int dfd, return cp_compat_stat64(&stat, statbuf); } -asmlinkage long compat_sys_sysfs(int option, u32 arg1, u32 arg2) -{ - return sys_sysfs(option, arg1, arg2); -} - -asmlinkage long compat_sys_sched_rr_get_interval(compat_pid_t pid, struct compat_timespec __user *interval) +COMPAT_SYSCALL_DEFINE3(sparc_sigaction, int, sig, + struct compat_old_sigaction __user *,act, + struct compat_old_sigaction __user *,oact) { - struct timespec t; - int ret; - mm_segment_t old_fs = get_fs (); - - set_fs (KERNEL_DS); - ret = sys_sched_rr_get_interval(pid, (struct timespec __user *) &t); - set_fs (old_fs); - if (put_compat_timespec(&t, interval)) - return -EFAULT; - return ret; -} - -asmlinkage long compat_sys_rt_sigprocmask(int how, - compat_sigset_t __user *set, - compat_sigset_t __user *oset, - compat_size_t sigsetsize) -{ - sigset_t s; - compat_sigset_t s32; - int ret; - mm_segment_t old_fs = get_fs(); - - if (set) { - if (copy_from_user (&s32, set, sizeof(compat_sigset_t))) - return -EFAULT; - switch (_NSIG_WORDS) { - case 4: s.sig[3] = s32.sig[6] | (((long)s32.sig[7]) << 32); - case 3: s.sig[2] = s32.sig[4] | (((long)s32.sig[5]) << 32); - case 2: s.sig[1] = s32.sig[2] | (((long)s32.sig[3]) << 32); - case 1: s.sig[0] = s32.sig[0] | (((long)s32.sig[1]) << 32); - } - } - set_fs (KERNEL_DS); - ret = sys_rt_sigprocmask(how, - set ? (sigset_t __user *) &s : NULL, - oset ? (sigset_t __user *) &s : NULL, - sigsetsize); - set_fs (old_fs); - if (ret) return ret; - if (oset) { - switch (_NSIG_WORDS) { - case 4: s32.sig[7] = (s.sig[3] >> 32); s32.sig[6] = s.sig[3]; - case 3: s32.sig[5] = (s.sig[2] >> 32); s32.sig[4] = s.sig[2]; - case 2: s32.sig[3] = (s.sig[1] >> 32); s32.sig[2] = s.sig[1]; - case 1: s32.sig[1] = (s.sig[0] >> 32); s32.sig[0] = s.sig[0]; - } - if (copy_to_user (oset, &s32, sizeof(compat_sigset_t))) - return -EFAULT; - } - return 0; -} - -asmlinkage long sys32_rt_sigpending(compat_sigset_t __user *set, - compat_size_t sigsetsize) -{ - sigset_t s; - compat_sigset_t s32; - int ret; - mm_segment_t old_fs = get_fs(); - - set_fs (KERNEL_DS); - ret = sys_rt_sigpending((sigset_t __user *) &s, sigsetsize); - set_fs (old_fs); - if (!ret) { - switch (_NSIG_WORDS) { - case 4: s32.sig[7] = (s.sig[3] >> 32); s32.sig[6] = s.sig[3]; - case 3: s32.sig[5] = (s.sig[2] >> 32); s32.sig[4] = s.sig[2]; - case 2: s32.sig[3] = (s.sig[1] >> 32); s32.sig[2] = s.sig[1]; - case 1: s32.sig[1] = (s.sig[0] >> 32); s32.sig[0] = s.sig[0]; - } - if (copy_to_user (set, &s32, sizeof(compat_sigset_t))) - return -EFAULT; - } - return ret; -} - -asmlinkage long compat_sys_rt_sigqueueinfo(int pid, int sig, - struct compat_siginfo __user *uinfo) -{ - siginfo_t info; - int ret; - mm_segment_t old_fs = get_fs(); - - if (copy_siginfo_from_user32(&info, uinfo)) - return -EFAULT; - - set_fs (KERNEL_DS); - ret = sys_rt_sigqueueinfo(pid, sig, (siginfo_t __user *) &info); - set_fs (old_fs); - return ret; -} - -asmlinkage long compat_sys_sigaction(int sig, struct old_sigaction32 __user *act, - struct old_sigaction32 __user *oact) -{ - struct k_sigaction new_ka, old_ka; - int ret; - WARN_ON_ONCE(sig >= 0); - sig = -sig; - - if (act) { - compat_old_sigset_t mask; - u32 u_handler, u_restorer; - - ret = get_user(u_handler, &act->sa_handler); - new_ka.sa.sa_handler = compat_ptr(u_handler); - ret |= __get_user(u_restorer, &act->sa_restorer); - new_ka.sa.sa_restorer = compat_ptr(u_restorer); - ret |= __get_user(new_ka.sa.sa_flags, &act->sa_flags); - ret |= __get_user(mask, &act->sa_mask); - if (ret) - return ret; - new_ka.ka_restorer = NULL; - siginitset(&new_ka.sa.sa_mask, mask); - } - - ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); - - if (!ret && oact) { - ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler); - ret |= __put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); - ret |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); - ret |= __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask); - } - - return ret; + return compat_sys_sigaction(-sig, act, oact); } -asmlinkage long compat_sys_rt_sigaction(int sig, - struct sigaction32 __user *act, - struct sigaction32 __user *oact, - void __user *restorer, - compat_size_t sigsetsize) +COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, + struct compat_sigaction __user *,act, + struct compat_sigaction __user *,oact, + void __user *,restorer, + compat_size_t,sigsetsize) { struct k_sigaction new_ka, old_ka; int ret; @@ -363,15 +171,10 @@ asmlinkage long compat_sys_rt_sigaction(int sig, new_ka.ka_restorer = restorer; ret = get_user(u_handler, &act->sa_handler); new_ka.sa.sa_handler = compat_ptr(u_handler); - ret |= __copy_from_user(&set32, &act->sa_mask, sizeof(compat_sigset_t)); - switch (_NSIG_WORDS) { - case 4: new_ka.sa.sa_mask.sig[3] = set32.sig[6] | (((long)set32.sig[7]) << 32); - case 3: new_ka.sa.sa_mask.sig[2] = set32.sig[4] | (((long)set32.sig[5]) << 32); - case 2: new_ka.sa.sa_mask.sig[1] = set32.sig[2] | (((long)set32.sig[3]) << 32); - case 1: new_ka.sa.sa_mask.sig[0] = set32.sig[0] | (((long)set32.sig[1]) << 32); - } - ret |= __get_user(new_ka.sa.sa_flags, &act->sa_flags); - ret |= __get_user(u_restorer, &act->sa_restorer); + ret |= copy_from_user(&set32, &act->sa_mask, sizeof(compat_sigset_t)); + sigset_from_compat(&new_ka.sa.sa_mask, &set32); + ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags); + ret |= get_user(u_restorer, &act->sa_restorer); new_ka.sa.sa_restorer = compat_ptr(u_restorer); if (ret) return -EFAULT; @@ -380,16 +183,11 @@ asmlinkage long compat_sys_rt_sigaction(int sig, ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); if (!ret && oact) { - switch (_NSIG_WORDS) { - case 4: set32.sig[7] = (old_ka.sa.sa_mask.sig[3] >> 32); set32.sig[6] = old_ka.sa.sa_mask.sig[3]; - case 3: set32.sig[5] = (old_ka.sa.sa_mask.sig[2] >> 32); set32.sig[4] = old_ka.sa.sa_mask.sig[2]; - case 2: set32.sig[3] = (old_ka.sa.sa_mask.sig[1] >> 32); set32.sig[2] = old_ka.sa.sa_mask.sig[1]; - case 1: set32.sig[1] = (old_ka.sa.sa_mask.sig[0] >> 32); set32.sig[0] = old_ka.sa.sa_mask.sig[0]; - } + sigset_to_compat(&set32, &old_ka.sa.sa_mask); ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler); - ret |= __copy_to_user(&oact->sa_mask, &set32, sizeof(compat_sigset_t)); - ret |= __put_user(old_ka.sa.sa_flags, &oact->sa_flags); - ret |= __put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); + ret |= copy_to_user(&oact->sa_mask, &set32, sizeof(compat_sigset_t)); + ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags); + ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); if (ret) ret = -EFAULT; } @@ -397,71 +195,6 @@ asmlinkage long compat_sys_rt_sigaction(int sig, return ret; } -/* - * sparc32_execve() executes a new program after the asm stub has set - * things up for us. This should basically do what I want it to. - */ -asmlinkage long sparc32_execve(struct pt_regs *regs) -{ - int error, base = 0; - char *filename; - - /* User register window flush is done by entry.S */ - - /* Check for indirect call. */ - if ((u32)regs->u_regs[UREG_G1] == 0) - base = 1; - - filename = getname(compat_ptr(regs->u_regs[base + UREG_I0])); - error = PTR_ERR(filename); - if (IS_ERR(filename)) - goto out; - - error = compat_do_execve(filename, - compat_ptr(regs->u_regs[base + UREG_I1]), - compat_ptr(regs->u_regs[base + UREG_I2]), regs); - - putname(filename); - - if (!error) { - fprs_write(0); - current_thread_info()->xfsr[0] = 0; - current_thread_info()->fpsaved[0] = 0; - regs->tstate &= ~TSTATE_PEF; - } -out: - return error; -} - -#ifdef CONFIG_MODULES - -asmlinkage long sys32_init_module(void __user *umod, u32 len, - const char __user *uargs) -{ - return sys_init_module(umod, len, uargs); -} - -asmlinkage long sys32_delete_module(const char __user *name_user, - unsigned int flags) -{ - return sys_delete_module(name_user, flags); -} - -#else /* CONFIG_MODULES */ - -asmlinkage long sys32_init_module(const char __user *name_user, - struct module __user *mod_user) -{ - return -ENOSYS; -} - -asmlinkage long sys32_delete_module(const char __user *name_user) -{ - return -ENOSYS; -} - -#endif /* CONFIG_MODULES */ - asmlinkage compat_ssize_t sys32_pread64(unsigned int fd, char __user *ubuf, compat_size_t count, @@ -507,71 +240,7 @@ long compat_sys_fadvise64_64(int fd, advice); } -asmlinkage long compat_sys_sendfile(int out_fd, int in_fd, - compat_off_t __user *offset, - compat_size_t count) -{ - mm_segment_t old_fs = get_fs(); - int ret; - off_t of; - - if (offset && get_user(of, offset)) - return -EFAULT; - - set_fs(KERNEL_DS); - ret = sys_sendfile(out_fd, in_fd, - offset ? (off_t __user *) &of : NULL, - count); - set_fs(old_fs); - - if (offset && put_user(of, offset)) - return -EFAULT; - - return ret; -} - -asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd, - compat_loff_t __user *offset, - compat_size_t count) -{ - mm_segment_t old_fs = get_fs(); - int ret; - loff_t lof; - - if (offset && get_user(lof, offset)) - return -EFAULT; - - set_fs(KERNEL_DS); - ret = sys_sendfile64(out_fd, in_fd, - offset ? (loff_t __user *) &lof : NULL, - count); - set_fs(old_fs); - - if (offset && put_user(lof, offset)) - return -EFAULT; - - return ret; -} - -/* This is just a version for 32-bit applications which does - * not force O_LARGEFILE on. - */ - -asmlinkage long sparc32_open(const char __user *filename, - int flags, int mode) -{ - return do_sys_open(AT_FDCWD, filename, flags, mode); -} - -long sys32_lookup_dcookie(unsigned long cookie_high, - unsigned long cookie_low, - char __user *buf, size_t len) -{ - return sys_lookup_dcookie((cookie_high << 32) | cookie_low, - buf, len); -} - -long compat_sync_file_range(int fd, unsigned long off_high, unsigned long off_low, unsigned long nb_high, unsigned long nb_low, int flags) +long sys32_sync_file_range(unsigned int fd, unsigned long off_high, unsigned long off_low, unsigned long nb_high, unsigned long nb_low, unsigned int flags) { return sys_sync_file_range(fd, (off_high << 32) | off_low, diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c index 42b282fa611..646988d4c1a 100644 --- a/arch/sparc/kernel/sys_sparc_32.c +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -24,6 +24,8 @@ #include <asm/uaccess.h> #include <asm/unistd.h> +#include "systbls.h" + /* #define DEBUG_UNIMP_SYSCALL */ /* XXX Make this per-binary type, this way we can detect the type of @@ -34,11 +36,9 @@ asmlinkage unsigned long sys_getpagesize(void) return PAGE_SIZE; /* Possibly older binaries want 8192 on sun4's? */ } -#define COLOUR_ALIGN(addr) (((addr)+SHMLBA-1)&~(SHMLBA-1)) - unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - struct vm_area_struct * vmm; + struct vm_unmapped_area_info info; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -53,37 +53,24 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi /* See asm-sparc/uaccess.h */ if (len > TASK_SIZE - PAGE_SIZE) return -ENOMEM; - if (ARCH_SUN4C && len > 0x20000000) - return -ENOMEM; if (!addr) addr = TASK_UNMAPPED_BASE; - if (flags & MAP_SHARED) - addr = COLOUR_ALIGN(addr); - else - addr = PAGE_ALIGN(addr); - - for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) { - /* At this point: (!vmm || addr < vmm->vm_end). */ - if (ARCH_SUN4C && addr < 0xe0000000 && 0x20000000 - len < addr) { - addr = PAGE_OFFSET; - vmm = find_vma(current->mm, PAGE_OFFSET); - } - if (TASK_SIZE - PAGE_SIZE - len < addr) - return -ENOMEM; - if (!vmm || addr + len <= vmm->vm_start) - return addr; - addr = vmm->vm_end; - if (flags & MAP_SHARED) - addr = COLOUR_ALIGN(addr); - } + info.flags = 0; + info.length = len; + info.low_limit = addr; + info.high_limit = TASK_SIZE; + info.align_mask = (flags & MAP_SHARED) ? + (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + return vm_unmapped_area(&info); } /* * sys_pipe() is the normal C calling standard for creating * a pipe. It's not the way unix traditionally does this, though. */ -asmlinkage int sparc_pipe(struct pt_regs *regs) +asmlinkage long sparc_pipe(struct pt_regs *regs) { int fd[2]; int error; @@ -99,11 +86,6 @@ out: int sparc_mmap_check(unsigned long addr, unsigned long len) { - if (ARCH_SUN4C && - (len > 0x20000000 || - (addr < 0xe0000000 && addr + len > 0x20000000))) - return -EINVAL; - /* See asm-sparc/uaccess.h */ if (len > TASK_SIZE - PAGE_SIZE || addr + len > TASK_SIZE - PAGE_SIZE) return -EINVAL; @@ -113,7 +95,7 @@ int sparc_mmap_check(unsigned long addr, unsigned long len) /* Linux version of mmap */ -asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len, +asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long pgoff) { @@ -123,7 +105,7 @@ asmlinkage unsigned long sys_mmap2(unsigned long addr, unsigned long len, pgoff >> (PAGE_SHIFT - 12)); } -asmlinkage unsigned long sys_mmap(unsigned long addr, unsigned long len, +asmlinkage long sys_mmap(unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, unsigned long fd, unsigned long off) { @@ -180,54 +162,19 @@ sparc_breakpoint (struct pt_regs *regs) #endif } -asmlinkage int -sparc_sigaction (int sig, const struct old_sigaction __user *act, - struct old_sigaction __user *oact) +SYSCALL_DEFINE3(sparc_sigaction, int, sig, + struct old_sigaction __user *,act, + struct old_sigaction __user *,oact) { - struct k_sigaction new_ka, old_ka; - int ret; - WARN_ON_ONCE(sig >= 0); - sig = -sig; - - if (act) { - unsigned long mask; - - if (!access_ok(VERIFY_READ, act, sizeof(*act)) || - __get_user(new_ka.sa.sa_handler, &act->sa_handler) || - __get_user(new_ka.sa.sa_restorer, &act->sa_restorer)) - return -EFAULT; - __get_user(new_ka.sa.sa_flags, &act->sa_flags); - __get_user(mask, &act->sa_mask); - siginitset(&new_ka.sa.sa_mask, mask); - new_ka.ka_restorer = NULL; - } - - ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); - - if (!ret && oact) { - /* In the clone() case we could copy half consistent - * state to the user, however this could sleep and - * deadlock us if we held the signal lock on SMP. So for - * now I take the easy way out and do no locking. - */ - if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) || - __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || - __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer)) - return -EFAULT; - __put_user(old_ka.sa.sa_flags, &oact->sa_flags); - __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask); - } - - return ret; + return sys_sigaction(-sig, act, oact); } -asmlinkage long -sys_rt_sigaction(int sig, - const struct sigaction __user *act, - struct sigaction __user *oact, - void __user *restorer, - size_t sigsetsize) +SYSCALL_DEFINE5(rt_sigaction, int, sig, + const struct sigaction __user *, act, + struct sigaction __user *, oact, + void __user *, restorer, + size_t, sigsetsize) { struct k_sigaction new_ka, old_ka; int ret; @@ -252,7 +199,7 @@ sys_rt_sigaction(int sig, return ret; } -asmlinkage int sys_getdomainname(char __user *name, int len) +asmlinkage long sys_getdomainname(char __user *name, int len) { int nlen, err; @@ -274,27 +221,3 @@ out: up_read(&uts_sem); return err; } - -/* - * Do a system call from kernel instead of calling sys_execve so we - * end up with proper pt_regs. - */ -int kernel_execve(const char *filename, - const char *const argv[], - const char *const envp[]) -{ - long __res; - register long __g1 __asm__ ("g1") = __NR_execve; - register long __o0 __asm__ ("o0") = (long)(filename); - register long __o1 __asm__ ("o1") = (long)(argv); - register long __o2 __asm__ ("o2") = (long)(envp); - asm volatile ("t 0x10\n\t" - "bcc 1f\n\t" - "mov %%o0, %0\n\t" - "sub %%g0, %%o0, %0\n\t" - "1:\n\t" - : "=r" (__res), "=&r" (__o0) - : "1" (__o0), "r" (__o1), "r" (__o2), "r" (__g1) - : "cc"); - return __res; -} diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index f836f4e93af..c85403d0496 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -23,13 +23,15 @@ #include <linux/ipc.h> #include <linux/personality.h> #include <linux/random.h> -#include <linux/module.h> +#include <linux/export.h> +#include <linux/context_tracking.h> #include <asm/uaccess.h> #include <asm/utrap.h> #include <asm/unistd.h> #include "entry.h" +#include "kernel.h" #include "systbls.h" /* #define DEBUG_UNIMP_SYSCALL */ @@ -39,9 +41,6 @@ asmlinkage unsigned long sys_getpagesize(void) return PAGE_SIZE; } -#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) -#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) - /* Does addr --> addr+len fall within 4GB of the VA-space hole or * overflow past the end of the 64-bit address space? */ @@ -66,23 +65,6 @@ static inline int invalid_64bit_range(unsigned long addr, unsigned long len) return 0; } -/* Does start,end straddle the VA-space hole? */ -static inline int straddles_64bit_va_hole(unsigned long start, unsigned long end) -{ - unsigned long va_exclude_start, va_exclude_end; - - va_exclude_start = VA_EXCLUDE_START; - va_exclude_end = VA_EXCLUDE_END; - - if (likely(start < va_exclude_start && end < va_exclude_start)) - return 0; - - if (likely(start >= va_exclude_end && end >= va_exclude_end)) - return 0; - - return 1; -} - /* These functions differ from the default implementations in * mm/mmap.c in two ways: * @@ -92,7 +74,7 @@ static inline int straddles_64bit_va_hole(unsigned long start, unsigned long end * the spitfire/niagara VA-hole. */ -static inline unsigned long COLOUR_ALIGN(unsigned long addr, +static inline unsigned long COLOR_ALIGN(unsigned long addr, unsigned long pgoff) { unsigned long base = (addr+SHMLBA-1)&~(SHMLBA-1); @@ -101,24 +83,13 @@ static inline unsigned long COLOUR_ALIGN(unsigned long addr, return base + off; } -static inline unsigned long COLOUR_ALIGN_DOWN(unsigned long addr, - unsigned long pgoff) -{ - unsigned long base = addr & ~(SHMLBA-1); - unsigned long off = (pgoff<<PAGE_SHIFT) & (SHMLBA-1); - - if (base + off <= addr) - return base + off; - return base - off; -} - unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { struct mm_struct *mm = current->mm; struct vm_area_struct * vma; unsigned long task_size = TASK_SIZE; - unsigned long start_addr; int do_color_align; + struct vm_unmapped_area_info info; if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate @@ -141,7 +112,7 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi if (addr) { if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); + addr = COLOR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); @@ -151,50 +122,22 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi return addr; } - if (len > mm->cached_hole_size) { - start_addr = addr = mm->free_area_cache; - } else { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; + info.flags = 0; + info.length = len; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = min(task_size, VA_EXCLUDE_START); + info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); + + if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { + VM_BUG_ON(addr != -ENOMEM); + info.low_limit = VA_EXCLUDE_END; + info.high_limit = task_size; + addr = vm_unmapped_area(&info); } - task_size -= len; - -full_search: - if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); - else - addr = PAGE_ALIGN(addr); - - for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { - /* At this point: (!vma || addr < vma->vm_end). */ - if (addr < VA_EXCLUDE_START && - (addr + len) >= VA_EXCLUDE_START) { - addr = VA_EXCLUDE_END; - vma = find_vma(mm, VA_EXCLUDE_END); - } - if (unlikely(task_size < addr)) { - if (start_addr != TASK_UNMAPPED_BASE) { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; - goto full_search; - } - return -ENOMEM; - } - if (likely(!vma || addr + len <= vma->vm_start)) { - /* - * Remember the place where we stopped the search: - */ - mm->free_area_cache = addr + len; - return addr; - } - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - addr = vma->vm_end; - if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); - } + return addr; } unsigned long @@ -207,6 +150,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, unsigned long task_size = STACK_TOP32; unsigned long addr = addr0; int do_color_align; + struct vm_unmapped_area_info info; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); @@ -231,7 +175,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, /* requesting a specific address */ if (addr) { if (do_color_align) - addr = COLOUR_ALIGN(addr, pgoff); + addr = COLOR_ALIGN(addr, pgoff); else addr = PAGE_ALIGN(addr); @@ -241,73 +185,27 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, return addr; } - /* check if free_area_cache is useful for us */ - if (len <= mm->cached_hole_size) { - mm->cached_hole_size = 0; - mm->free_area_cache = mm->mmap_base; - } - - /* either no address requested or can't fit in requested address hole */ - addr = mm->free_area_cache; - if (do_color_align) { - unsigned long base = COLOUR_ALIGN_DOWN(addr-len, pgoff); - - addr = base + len; - } - - /* make sure it can fit in the remaining address space */ - if (likely(addr > len)) { - vma = find_vma(mm, addr-len); - if (!vma || addr <= vma->vm_start) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr-len); - } - } - - if (unlikely(mm->mmap_base < len)) - goto bottomup; - - addr = mm->mmap_base-len; - if (do_color_align) - addr = COLOUR_ALIGN_DOWN(addr, pgoff); - - do { - /* - * Lookup failure means no vma is above this address, - * else if new region fits below vma->vm_start, - * return with success: - */ - vma = find_vma(mm, addr); - if (likely(!vma || addr+len <= vma->vm_start)) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr); - } + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = mm->mmap_base; + info.align_mask = do_color_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); - /* remember the largest hole we saw so far */ - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - /* try just below the current vma->vm_start */ - addr = vma->vm_start-len; - if (do_color_align) - addr = COLOUR_ALIGN_DOWN(addr, pgoff); - } while (likely(len < vma->vm_start)); - -bottomup: /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ - mm->cached_hole_size = ~0UL; - mm->free_area_cache = TASK_UNMAPPED_BASE; - addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags); - /* - * Restore the topdown base: - */ - mm->free_area_cache = mm->mmap_base; - mm->cached_hole_size = ~0UL; + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = STACK_TOP32; + addr = vm_unmapped_area(&info); + } return addr; } @@ -360,20 +258,25 @@ unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, u } EXPORT_SYMBOL(get_fb_unmapped_area); -/* Essentially the same as PowerPC... */ -void arch_pick_mmap_layout(struct mm_struct *mm) +/* Essentially the same as PowerPC. */ +static unsigned long mmap_rnd(void) { - unsigned long random_factor = 0UL; - unsigned long gap; + unsigned long rnd = 0UL; if (current->flags & PF_RANDOMIZE) { - random_factor = get_random_int(); + unsigned long val = get_random_int(); if (test_thread_flag(TIF_32BIT)) - random_factor &= ((1 * 1024 * 1024) - 1); + rnd = (val % (1UL << (23UL-PAGE_SHIFT))); else - random_factor = ((random_factor << PAGE_SHIFT) & - 0xffffffffUL); + rnd = (val % (1UL << (30UL-PAGE_SHIFT))); } + return rnd << PAGE_SHIFT; +} + +void arch_pick_mmap_layout(struct mm_struct *mm) +{ + unsigned long random_factor = mmap_rnd(); + unsigned long gap; /* * Fall back to the standard layout if the personality @@ -386,7 +289,6 @@ void arch_pick_mmap_layout(struct mm_struct *mm) sysctl_legacy_va_layout) { mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; mm->get_unmapped_area = arch_get_unmapped_area; - mm->unmap_area = arch_unmap_area; } else { /* We know it's 32-bit */ unsigned long task_size = STACK_TOP32; @@ -398,7 +300,6 @@ void arch_pick_mmap_layout(struct mm_struct *mm) mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor); mm->get_unmapped_area = arch_get_unmapped_area_topdown; - mm->unmap_area = arch_unmap_area_topdown; } } @@ -449,13 +350,13 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second case SEMCTL: { err = sys_semctl(first, second, (int)third | IPC_64, - (union semun) ptr); + (unsigned long) ptr); goto out; } default: err = -ENOSYS; goto out; - }; + } } if (call <= MSGCTL) { switch (call) { @@ -476,13 +377,13 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second default: err = -ENOSYS; goto out; - }; + } } if (call <= SHMCTL) { switch (call) { case SHMAT: { ulong raddr; - err = do_shmat(first, ptr, (int)second, &raddr); + err = do_shmat(first, ptr, (int)second, &raddr, SHMLBA); if (!err) { if (put_user(raddr, (ulong __user *) third)) @@ -502,7 +403,7 @@ SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second default: err = -ENOSYS; goto out; - }; + } } else { err = -ENOSYS; } @@ -514,12 +415,12 @@ SYSCALL_DEFINE1(sparc64_personality, unsigned long, personality) { int ret; - if (current->personality == PER_LINUX32 && - personality == PER_LINUX) - personality = PER_LINUX32; + if (personality(current->personality) == PER_LINUX32 && + personality(personality) == PER_LINUX) + personality |= PER_LINUX32; ret = sys_personality(personality); - if (ret == PER_LINUX32) - ret = PER_LINUX; + if (personality(ret) == PER_LINUX32) + ret &= ~PER_LINUX32; return ret; } @@ -561,35 +462,19 @@ out: SYSCALL_DEFINE2(64_munmap, unsigned long, addr, size_t, len) { - long ret; - if (invalid_64bit_range(addr, len)) return -EINVAL; - down_write(¤t->mm->mmap_sem); - ret = do_munmap(current->mm, addr, len); - up_write(¤t->mm->mmap_sem); - return ret; + return vm_munmap(addr, len); } - -extern unsigned long do_mremap(unsigned long addr, - unsigned long old_len, unsigned long new_len, - unsigned long flags, unsigned long new_addr); SYSCALL_DEFINE5(64_mremap, unsigned long, addr, unsigned long, old_len, unsigned long, new_len, unsigned long, flags, unsigned long, new_addr) { - unsigned long ret = -EINVAL; - if (test_thread_flag(TIF_32BIT)) - goto out; - - down_write(¤t->mm->mmap_sem); - ret = do_mremap(addr, old_len, new_len, flags, new_addr); - up_write(¤t->mm->mmap_sem); -out: - return ret; + return -EINVAL; + return sys_mremap(addr, old_len, new_len, flags, new_addr); } /* we come to here via sys_nis_syscall so it can setup the regs argument */ @@ -613,6 +498,7 @@ asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs) asmlinkage void sparc_breakpoint(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (test_thread_flag(TIF_32BIT)) { @@ -631,6 +517,7 @@ asmlinkage void sparc_breakpoint(struct pt_regs *regs) #ifdef DEBUG_SPARC_BREAKPOINT printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); #endif + exception_exit(prev_state); } extern void check_pending(int signum); @@ -754,24 +641,7 @@ SYSCALL_DEFINE5(rt_sigaction, int, sig, const struct sigaction __user *, act, return ret; } -/* - * Do a system call from kernel instead of calling sys_execve so we - * end up with proper pt_regs. - */ -int kernel_execve(const char *filename, - const char *const argv[], - const char *const envp[]) +asmlinkage long sys_kern_features(void) { - long __res; - register long __g1 __asm__ ("g1") = __NR_execve; - register long __o0 __asm__ ("o0") = (long)(filename); - register long __o1 __asm__ ("o1") = (long)(argv); - register long __o2 __asm__ ("o2") = (long)(envp); - asm volatile ("t 0x6d\n\t" - "sub %%g0, %%o0, %0\n\t" - "movcc %%xcc, %%o0, %0\n\t" - : "=r" (__res), "=&r" (__o0) - : "1" (__o0), "r" (__o1), "r" (__o2), "r" (__g1) - : "cc"); - return __res; + return KERN_FEATURE_MIXED_MODE_STACK; } diff --git a/arch/sparc/kernel/syscalls.S b/arch/sparc/kernel/syscalls.S index 1d7e274f3f2..33a17e7b3cc 100644 --- a/arch/sparc/kernel/syscalls.S +++ b/arch/sparc/kernel/syscalls.S @@ -1,23 +1,19 @@ /* SunOS's execv() call only specifies the argv argument, the * environment settings are the same as the calling processes. */ -sys_execve: - sethi %hi(sparc_execve), %g1 - ba,pt %xcc, execve_merge - or %g1, %lo(sparc_execve), %g1 +sys64_execve: + set sys_execve, %g1 + jmpl %g1, %g0 + flushw #ifdef CONFIG_COMPAT sunos_execv: - stx %g0, [%sp + PTREGS_OFF + PT_V9_I2] + mov %g0, %o2 sys32_execve: - sethi %hi(sparc32_execve), %g1 - or %g1, %lo(sparc32_execve), %g1 -#endif - -execve_merge: - flushw + set compat_sys_execve, %g1 jmpl %g1, %g0 - add %sp, PTREGS_OFF, %o0 + flushw +#endif .align 32 sys_sparc_pipe: @@ -29,16 +25,10 @@ sys_nis_syscall: sys_memory_ordering: ba,pt %xcc, sparc_memory_ordering add %sp, PTREGS_OFF, %o1 -sys_sigaltstack: - ba,pt %xcc, do_sigaltstack - add %i6, STACK_BIAS, %o2 #ifdef CONFIG_COMPAT sys32_sigstack: ba,pt %xcc, do_sys32_sigstack mov %i6, %o2 -sys32_sigaltstack: - ba,pt %xcc, do_sys32_sigaltstack - mov %i6, %o2 #endif .align 32 #ifdef CONFIG_COMPAT @@ -62,7 +52,7 @@ sys32_rt_sigreturn: #endif .align 32 1: ldx [%g6 + TI_FLAGS], %l5 - andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 be,pt %icc, rtrap nop call syscall_trace_leave @@ -108,20 +98,35 @@ sys_clone: ba,pt %xcc, sparc_do_fork add %sp, PTREGS_OFF, %o2 - .globl ret_from_syscall -ret_from_syscall: + .globl ret_from_fork +ret_from_fork: /* Clear current_thread_info()->new_child. */ stb %g0, [%g6 + TI_NEW_CHILD] - ldx [%g6 + TI_FLAGS], %l0 call schedule_tail mov %g7, %o0 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %o0 + brnz,pt %o0, ret_sys_call + ldx [%g6 + TI_FLAGS], %l0 + ldx [%sp + PTREGS_OFF + PT_V9_G1], %l1 + call %l1 + ldx [%sp + PTREGS_OFF + PT_V9_G2], %o0 ba,pt %xcc, ret_sys_call - ldx [%sp + PTREGS_OFF + PT_V9_I0], %o0 + mov 0, %o0 + + .globl sparc_exit_group + .type sparc_exit_group,#function +sparc_exit_group: + sethi %hi(sys_exit_group), %g7 + ba,pt %xcc, 1f + or %g7, %lo(sys_exit_group), %g7 + .size sparc_exit_group,.-sparc_exit_group .globl sparc_exit .type sparc_exit,#function sparc_exit: - rdpr %pstate, %g2 + sethi %hi(sys_exit), %g7 + or %g7, %lo(sys_exit), %g7 +1: rdpr %pstate, %g2 wrpr %g2, PSTATE_IE, %pstate rdpr %otherwin, %g1 rdpr %cansave, %g3 @@ -129,7 +134,7 @@ sparc_exit: wrpr %g3, 0x0, %cansave wrpr %g0, 0x0, %otherwin wrpr %g2, 0x0, %pstate - ba,pt %xcc, sys_exit + jmpl %g7, %g0 stb %g0, [%g6 + TI_WSAVED] .size sparc_exit,.-sparc_exit @@ -147,7 +152,7 @@ linux_syscall_trace32: srl %i4, 0, %o4 srl %i1, 0, %o1 srl %i2, 0, %o2 - ba,pt %xcc, 2f + ba,pt %xcc, 5f srl %i3, 0, %o3 linux_syscall_trace: @@ -177,14 +182,15 @@ linux_sparc_syscall32: srl %i1, 0, %o1 ! IEU0 Group ldx [%g6 + TI_FLAGS], %l0 ! Load - srl %i5, 0, %o5 ! IEU1 + srl %i3, 0, %o3 ! IEU0 srl %i2, 0, %o2 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 bne,pn %icc, linux_syscall_trace32 ! CTI mov %i0, %l5 ! IEU1 - call %l7 ! CTI Group brk forced - srl %i3, 0, %o3 ! IEU0 - ba,a,pt %xcc, 3f +5: call %l7 ! CTI Group brk forced + srl %i5, 0, %o5 ! IEU1 + ba,pt %xcc, 3f + sra %o0, 0, %o0 /* Linux native system calls enter here... */ .align 32 @@ -202,7 +208,7 @@ linux_sparc_syscall: mov %i3, %o3 ! IEU1 mov %i4, %o4 ! IEU0 Group - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %g0 + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 bne,pn %icc, linux_syscall_trace ! CTI Group mov %i0, %l5 ! IEU0 2: call %l7 ! CTI Group brk forced @@ -212,24 +218,18 @@ linux_sparc_syscall: 3: stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] ret_sys_call: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %g3 - ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc - sra %o0, 0, %o0 mov %ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2 sllx %g2, 32, %g2 - /* Check if force_successful_syscall_return() - * was invoked. - */ - ldub [%g6 + TI_SYS_NOERROR], %l2 - brnz,a,pn %l2, 80f - stb %g0, [%g6 + TI_SYS_NOERROR] - cmp %o0, -ERESTART_RESTARTBLOCK bgeu,pn %xcc, 1f - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %l6 -80: + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc + +2: /* System call success, clear Carry condition code. */ andn %g3, %g2, %g3 +3: stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] bne,pn %icc, linux_syscall_trace2 add %l1, 0x4, %l2 ! npc = npc+4 @@ -238,20 +238,20 @@ ret_sys_call: stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] 1: + /* Check if force_successful_syscall_return() + * was invoked. + */ + ldub [%g6 + TI_SYS_NOERROR], %l2 + brnz,pn %l2, 2b + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc /* System call failure, set Carry condition code. * Also, get abs(errno) to return to the process. */ - andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT), %l6 sub %g0, %o0, %o0 - or %g3, %g2, %g3 stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] - stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] - bne,pn %icc, linux_syscall_trace2 - add %l1, 0x4, %l2 ! npc = npc+4 - stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + ba,pt %xcc, 3b + or %g3, %g2, %g3 - b,pt %xcc, rtrap - stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] linux_syscall_trace2: call syscall_trace_leave add %sp, PTREGS_OFF, %o0 diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index 1eb8b00aed7..7f41d40b7e6 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -3,7 +3,7 @@ * Copyright (C) 2007 David S. Miller <davem@davemloft.net> */ #include <linux/sched.h> -#include <linux/sysdev.h> +#include <linux/device.h> #include <linux/cpu.h> #include <linux/smp.h> #include <linux/percpu.h> @@ -16,13 +16,13 @@ static DEFINE_PER_CPU(struct hv_mmu_statistics, mmu_stats) __attribute__((aligned(64))); #define SHOW_MMUSTAT_ULONG(NAME) \ -static ssize_t show_##NAME(struct sys_device *dev, \ - struct sysdev_attribute *attr, char *buf) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ { \ struct hv_mmu_statistics *p = &per_cpu(mmu_stats, dev->id); \ return sprintf(buf, "%lu\n", p->NAME); \ } \ -static SYSDEV_ATTR(NAME, 0444, show_##NAME, NULL) +static DEVICE_ATTR(NAME, 0444, show_##NAME, NULL) SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctx0_8k_tte); SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctx0_8k_tte); @@ -58,38 +58,38 @@ SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctxnon0_256mb_tte); SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctxnon0_256mb_tte); static struct attribute *mmu_stat_attrs[] = { - &attr_immu_tsb_hits_ctx0_8k_tte.attr, - &attr_immu_tsb_ticks_ctx0_8k_tte.attr, - &attr_immu_tsb_hits_ctx0_64k_tte.attr, - &attr_immu_tsb_ticks_ctx0_64k_tte.attr, - &attr_immu_tsb_hits_ctx0_4mb_tte.attr, - &attr_immu_tsb_ticks_ctx0_4mb_tte.attr, - &attr_immu_tsb_hits_ctx0_256mb_tte.attr, - &attr_immu_tsb_ticks_ctx0_256mb_tte.attr, - &attr_immu_tsb_hits_ctxnon0_8k_tte.attr, - &attr_immu_tsb_ticks_ctxnon0_8k_tte.attr, - &attr_immu_tsb_hits_ctxnon0_64k_tte.attr, - &attr_immu_tsb_ticks_ctxnon0_64k_tte.attr, - &attr_immu_tsb_hits_ctxnon0_4mb_tte.attr, - &attr_immu_tsb_ticks_ctxnon0_4mb_tte.attr, - &attr_immu_tsb_hits_ctxnon0_256mb_tte.attr, - &attr_immu_tsb_ticks_ctxnon0_256mb_tte.attr, - &attr_dmmu_tsb_hits_ctx0_8k_tte.attr, - &attr_dmmu_tsb_ticks_ctx0_8k_tte.attr, - &attr_dmmu_tsb_hits_ctx0_64k_tte.attr, - &attr_dmmu_tsb_ticks_ctx0_64k_tte.attr, - &attr_dmmu_tsb_hits_ctx0_4mb_tte.attr, - &attr_dmmu_tsb_ticks_ctx0_4mb_tte.attr, - &attr_dmmu_tsb_hits_ctx0_256mb_tte.attr, - &attr_dmmu_tsb_ticks_ctx0_256mb_tte.attr, - &attr_dmmu_tsb_hits_ctxnon0_8k_tte.attr, - &attr_dmmu_tsb_ticks_ctxnon0_8k_tte.attr, - &attr_dmmu_tsb_hits_ctxnon0_64k_tte.attr, - &attr_dmmu_tsb_ticks_ctxnon0_64k_tte.attr, - &attr_dmmu_tsb_hits_ctxnon0_4mb_tte.attr, - &attr_dmmu_tsb_ticks_ctxnon0_4mb_tte.attr, - &attr_dmmu_tsb_hits_ctxnon0_256mb_tte.attr, - &attr_dmmu_tsb_ticks_ctxnon0_256mb_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_8k_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_8k_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_64k_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_64k_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_4mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_4mb_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_256mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_256mb_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_8k_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_8k_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_64k_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_64k_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_4mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_4mb_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_256mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_256mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_8k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_8k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_64k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_64k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_4mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_4mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_256mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_256mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_8k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_8k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_64k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_64k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_4mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_4mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_256mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_256mb_tte.attr, NULL, }; @@ -103,9 +103,10 @@ static unsigned long run_on_cpu(unsigned long cpu, unsigned long (*func)(unsigned long), unsigned long arg) { - cpumask_t old_affinity = current->cpus_allowed; + cpumask_t old_affinity; unsigned long ret; + cpumask_copy(&old_affinity, tsk_cpus_allowed(current)); /* should return -EINVAL to userspace */ if (set_cpus_allowed_ptr(current, cpumask_of(cpu))) return 0; @@ -138,19 +139,19 @@ static unsigned long write_mmustat_enable(unsigned long val) return sun4v_mmustat_conf(ra, &orig_ra); } -static ssize_t show_mmustat_enable(struct sys_device *s, - struct sysdev_attribute *attr, char *buf) +static ssize_t show_mmustat_enable(struct device *s, + struct device_attribute *attr, char *buf) { unsigned long val = run_on_cpu(s->id, read_mmustat_enable, 0); return sprintf(buf, "%lx\n", val); } -static ssize_t store_mmustat_enable(struct sys_device *s, - struct sysdev_attribute *attr, const char *buf, +static ssize_t store_mmustat_enable(struct device *s, + struct device_attribute *attr, const char *buf, size_t count) { unsigned long val, err; - int ret = sscanf(buf, "%ld", &val); + int ret = sscanf(buf, "%lu", &val); if (ret != 1) return -EINVAL; @@ -162,39 +163,39 @@ static ssize_t store_mmustat_enable(struct sys_device *s, return count; } -static SYSDEV_ATTR(mmustat_enable, 0644, show_mmustat_enable, store_mmustat_enable); +static DEVICE_ATTR(mmustat_enable, 0644, show_mmustat_enable, store_mmustat_enable); static int mmu_stats_supported; -static int register_mmu_stats(struct sys_device *s) +static int register_mmu_stats(struct device *s) { if (!mmu_stats_supported) return 0; - sysdev_create_file(s, &attr_mmustat_enable); + device_create_file(s, &dev_attr_mmustat_enable); return sysfs_create_group(&s->kobj, &mmu_stat_group); } #ifdef CONFIG_HOTPLUG_CPU -static void unregister_mmu_stats(struct sys_device *s) +static void unregister_mmu_stats(struct device *s) { if (!mmu_stats_supported) return; sysfs_remove_group(&s->kobj, &mmu_stat_group); - sysdev_remove_file(s, &attr_mmustat_enable); + device_remove_file(s, &dev_attr_mmustat_enable); } #endif #define SHOW_CPUDATA_ULONG_NAME(NAME, MEMBER) \ -static ssize_t show_##NAME(struct sys_device *dev, \ - struct sysdev_attribute *attr, char *buf) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ { \ cpuinfo_sparc *c = &cpu_data(dev->id); \ return sprintf(buf, "%lu\n", c->MEMBER); \ } #define SHOW_CPUDATA_UINT_NAME(NAME, MEMBER) \ -static ssize_t show_##NAME(struct sys_device *dev, \ - struct sysdev_attribute *attr, char *buf) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ { \ cpuinfo_sparc *c = &cpu_data(dev->id); \ return sprintf(buf, "%u\n", c->MEMBER); \ @@ -208,14 +209,14 @@ SHOW_CPUDATA_UINT_NAME(l1_icache_line_size, icache_line_size); SHOW_CPUDATA_UINT_NAME(l2_cache_size, ecache_size); SHOW_CPUDATA_UINT_NAME(l2_cache_line_size, ecache_line_size); -static struct sysdev_attribute cpu_core_attrs[] = { - _SYSDEV_ATTR(clock_tick, 0444, show_clock_tick, NULL), - _SYSDEV_ATTR(l1_dcache_size, 0444, show_l1_dcache_size, NULL), - _SYSDEV_ATTR(l1_dcache_line_size, 0444, show_l1_dcache_line_size, NULL), - _SYSDEV_ATTR(l1_icache_size, 0444, show_l1_icache_size, NULL), - _SYSDEV_ATTR(l1_icache_line_size, 0444, show_l1_icache_line_size, NULL), - _SYSDEV_ATTR(l2_cache_size, 0444, show_l2_cache_size, NULL), - _SYSDEV_ATTR(l2_cache_line_size, 0444, show_l2_cache_line_size, NULL), +static struct device_attribute cpu_core_attrs[] = { + __ATTR(clock_tick, 0444, show_clock_tick, NULL), + __ATTR(l1_dcache_size, 0444, show_l1_dcache_size, NULL), + __ATTR(l1_dcache_line_size, 0444, show_l1_dcache_line_size, NULL), + __ATTR(l1_icache_size, 0444, show_l1_icache_size, NULL), + __ATTR(l1_icache_line_size, 0444, show_l1_icache_line_size, NULL), + __ATTR(l2_cache_size, 0444, show_l2_cache_size, NULL), + __ATTR(l2_cache_line_size, 0444, show_l2_cache_line_size, NULL), }; static DEFINE_PER_CPU(struct cpu, cpu_devices); @@ -223,11 +224,11 @@ static DEFINE_PER_CPU(struct cpu, cpu_devices); static void register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); - struct sys_device *s = &c->sysdev; + struct device *s = &c->dev; int i; for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) - sysdev_create_file(s, &cpu_core_attrs[i]); + device_create_file(s, &cpu_core_attrs[i]); register_mmu_stats(s); } @@ -236,16 +237,16 @@ static void register_cpu_online(unsigned int cpu) static void unregister_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); - struct sys_device *s = &c->sysdev; + struct device *s = &c->dev; int i; unregister_mmu_stats(s); for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) - sysdev_remove_file(s, &cpu_core_attrs[i]); + device_remove_file(s, &cpu_core_attrs[i]); } #endif -static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, +static int sysfs_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned int)(long)hcpu; @@ -265,7 +266,7 @@ static int __cpuinit sysfs_cpu_notify(struct notifier_block *self, return NOTIFY_OK; } -static struct notifier_block __cpuinitdata sysfs_cpu_nb = { +static struct notifier_block sysfs_cpu_nb = { .notifier_call = sysfs_cpu_notify, }; @@ -299,7 +300,7 @@ static int __init topology_init(void) check_mmu_stats(); - register_cpu_notifier(&sysfs_cpu_nb); + cpu_notifier_register_begin(); for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); @@ -309,6 +310,10 @@ static int __init topology_init(void) register_cpu_online(cpu); } + __register_cpu_notifier(&sysfs_cpu_nb); + + cpu_notifier_register_done(); + return 0; } diff --git a/arch/sparc/kernel/systbls.h b/arch/sparc/kernel/systbls.h index 118759cd734..2dab8236d49 100644 --- a/arch/sparc/kernel/systbls.h +++ b/arch/sparc/kernel/systbls.h @@ -1,43 +1,103 @@ #ifndef _SYSTBLS_H #define _SYSTBLS_H +#include <linux/signal.h> #include <linux/kernel.h> +#include <linux/compat.h> #include <linux/types.h> + #include <asm/utrap.h> -#include <asm/signal.h> -extern asmlinkage unsigned long sys_getpagesize(void); -extern asmlinkage long sparc_pipe(struct pt_regs *regs); -extern asmlinkage long sys_sparc_ipc(unsigned int call, int first, - unsigned long second, - unsigned long third, - void __user *ptr, long fifth); -extern asmlinkage long sparc64_personality(unsigned long personality); -extern asmlinkage long sys64_munmap(unsigned long addr, size_t len); -extern asmlinkage unsigned long sys64_mremap(unsigned long addr, - unsigned long old_len, - unsigned long new_len, - unsigned long flags, - unsigned long new_addr); -extern asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs); -extern asmlinkage long sys_getdomainname(char __user *name, int len); -extern asmlinkage long sys_utrap_install(utrap_entry_t type, - utrap_handler_t new_p, - utrap_handler_t new_d, - utrap_handler_t __user *old_p, - utrap_handler_t __user *old_d); -extern asmlinkage long sparc_memory_ordering(unsigned long model, - struct pt_regs *regs); -extern asmlinkage long sys_rt_sigaction(int sig, - const struct sigaction __user *act, - struct sigaction __user *oact, - void __user *restorer, - size_t sigsetsize); +asmlinkage unsigned long sys_getpagesize(void); +asmlinkage long sparc_pipe(struct pt_regs *regs); +asmlinkage unsigned long c_sys_nis_syscall(struct pt_regs *regs); +asmlinkage long sys_getdomainname(char __user *name, int len); +void do_rt_sigreturn(struct pt_regs *regs); +asmlinkage long sys_mmap(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long off); +asmlinkage void sparc_breakpoint(struct pt_regs *regs); + +#ifdef CONFIG_SPARC32 +asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long sparc_remap_file_pages(unsigned long start, unsigned long size, + unsigned long prot, unsigned long pgoff, + unsigned long flags); -extern asmlinkage void sparc64_set_context(struct pt_regs *regs); -extern asmlinkage void sparc64_get_context(struct pt_regs *regs); -extern asmlinkage long sys_sigpause(unsigned int set); -extern asmlinkage long sys_sigsuspend(old_sigset_t set); -extern void do_rt_sigreturn(struct pt_regs *regs); +#endif /* CONFIG_SPARC32 */ +#ifdef CONFIG_SPARC64 +asmlinkage long sys_sparc_ipc(unsigned int call, int first, + unsigned long second, + unsigned long third, + void __user *ptr, long fifth); +asmlinkage long sparc64_personality(unsigned long personality); +asmlinkage long sys64_munmap(unsigned long addr, size_t len); +asmlinkage unsigned long sys64_mremap(unsigned long addr, + unsigned long old_len, + unsigned long new_len, + unsigned long flags, + unsigned long new_addr); +asmlinkage long sys_utrap_install(utrap_entry_t type, + utrap_handler_t new_p, + utrap_handler_t new_d, + utrap_handler_t __user *old_p, + utrap_handler_t __user *old_d); +asmlinkage long sparc_memory_ordering(unsigned long model, + struct pt_regs *regs); +asmlinkage void sparc64_set_context(struct pt_regs *regs); +asmlinkage void sparc64_get_context(struct pt_regs *regs); +asmlinkage long sys32_truncate64(const char __user * path, + unsigned long high, + unsigned long low); +asmlinkage long sys32_ftruncate64(unsigned int fd, + unsigned long high, + unsigned long low); +struct compat_stat64; +asmlinkage long compat_sys_stat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_lstat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, int flag); +asmlinkage compat_ssize_t sys32_pread64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + unsigned long poshi, + unsigned long poslo); +asmlinkage compat_ssize_t sys32_pwrite64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + unsigned long poshi, + unsigned long poslo); +asmlinkage long compat_sys_readahead(int fd, + unsigned long offhi, + unsigned long offlo, + compat_size_t count); +long compat_sys_fadvise64(int fd, + unsigned long offhi, + unsigned long offlo, + compat_size_t len, int advice); +long compat_sys_fadvise64_64(int fd, + unsigned long offhi, unsigned long offlo, + unsigned long lenhi, unsigned long lenlo, + int advice); +long sys32_sync_file_range(unsigned int fd, + unsigned long off_high, unsigned long off_low, + unsigned long nb_high, unsigned long nb_low, + unsigned int flags); +asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, + u32 lenhi, u32 lenlo); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, + int flag); +#endif /* CONFIG_SPARC64 */ #endif /* _SYSTBLS_H */ diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S index ec396e1916b..85fe9b1087c 100644 --- a/arch/sparc/kernel/systbls_32.S +++ b/arch/sparc/kernel/systbls_32.S @@ -55,7 +55,7 @@ sys_call_table: /*180*/ .long sys_flistxattr, sys_removexattr, sys_lremovexattr, sys_sigpending, sys_ni_syscall /*185*/ .long sys_setpgid, sys_fremovexattr, sys_tkill, sys_exit_group, sys_newuname /*190*/ .long sys_init_module, sys_personality, sparc_remap_file_pages, sys_epoll_create, sys_epoll_ctl -/*195*/ .long sys_epoll_wait, sys_ioprio_set, sys_getppid, sparc_sigaction, sys_sgetmask +/*195*/ .long sys_epoll_wait, sys_ioprio_set, sys_getppid, sys_sparc_sigaction, sys_sgetmask /*200*/ .long sys_ssetmask, sys_sigsuspend, sys_newlstat, sys_uselib, sys_old_readdir /*205*/ .long sys_readahead, sys_socketcall, sys_syslog, sys_lookup_dcookie, sys_fadvise64 /*210*/ .long sys_fadvise64_64, sys_tgkill, sys_waitpid, sys_swapoff, sys_sysinfo @@ -67,7 +67,7 @@ sys_call_table: /*235*/ .long sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall /*240*/ .long sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler /*245*/ .long sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, sys_sched_rr_get_interval, sys_nanosleep -/*250*/ .long sys_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_nfsservctl +/*250*/ .long sys_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_ni_syscall /*255*/ .long sys_sync_file_range, sys_clock_settime, sys_clock_gettime, sys_clock_getres, sys_clock_nanosleep /*260*/ .long sys_sched_getaffinity, sys_sched_setaffinity, sys_timer_settime, sys_timer_gettime, sys_timer_getoverrun /*265*/ .long sys_timer_delete, sys_timer_create, sys_nis_syscall, sys_io_setup, sys_io_destroy @@ -83,5 +83,7 @@ sys_call_table: /*315*/ .long sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1 /*320*/ .long sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv /*325*/ .long sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init -/*330*/ .long sys_fanotify_mark, sys_prlimit64 - +/*330*/ .long sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime +/*335*/ .long sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev +/*340*/ .long sys_ni_syscall, sys_kcmp, sys_finit_module, sys_sched_setattr, sys_sched_getattr +/*345*/ .long sys_renameat2 diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S index 8cfcaa54958..33ecba2826e 100644 --- a/arch/sparc/kernel/systbls_64.S +++ b/arch/sparc/kernel/systbls_64.S @@ -18,63 +18,63 @@ .globl sys_call_table32 sys_call_table32: -/*0*/ .word sys_restart_syscall, sys32_exit, sys_fork, sys_read, sys_write -/*5*/ .word sys32_open, sys_close, sys32_wait4, sys32_creat, sys_link -/*10*/ .word sys_unlink, sunos_execv, sys_chdir, sys_chown16, sys32_mknod -/*15*/ .word sys_chmod, sys_lchown16, sys_brk, sys_nis_syscall, sys32_lseek +/*0*/ .word sys_restart_syscall, sparc_exit, sys_fork, sys_read, sys_write +/*5*/ .word compat_sys_open, sys_close, compat_sys_wait4, sys_creat, sys_link +/*10*/ .word sys_unlink, sunos_execv, sys_chdir, sys_chown16, sys_mknod +/*15*/ .word sys_chmod, sys_lchown16, sys_brk, sys_nis_syscall, compat_sys_lseek /*20*/ .word sys_getpid, sys_capget, sys_capset, sys_setuid16, sys_getuid16 -/*25*/ .word sys32_vmsplice, compat_sys_ptrace, sys_alarm, sys32_sigaltstack, sys_pause -/*30*/ .word compat_sys_utime, sys_lchown, sys_fchown, sys32_access, sys32_nice - .word sys_chown, sys_sync, sys32_kill, compat_sys_newstat, sys32_sendfile +/*25*/ .word compat_sys_vmsplice, compat_sys_ptrace, sys_alarm, compat_sys_sigaltstack, sys_pause +/*30*/ .word compat_sys_utime, sys_lchown, sys_fchown, sys_access, sys_nice + .word sys_chown, sys_sync, sys_kill, compat_sys_newstat, compat_sys_sendfile /*40*/ .word compat_sys_newlstat, sys_dup, sys_sparc_pipe, compat_sys_times, sys_getuid - .word sys32_umount, sys_setgid16, sys_getgid16, sys32_signal, sys_geteuid16 + .word sys_umount, sys_setgid16, sys_getgid16, sys_signal, sys_geteuid16 /*50*/ .word sys_getegid16, sys_acct, sys_nis_syscall, sys_getgid, compat_sys_ioctl - .word sys32_reboot, sys32_mmap2, sys_symlink, sys32_readlink, sys32_execve -/*60*/ .word sys32_umask, sys_chroot, compat_sys_newfstat, compat_sys_fstat64, sys_getpagesize - .word sys32_msync, sys_vfork, sys32_pread64, sys32_pwrite64, sys_geteuid + .word sys_reboot, sys32_mmap2, sys_symlink, sys_readlink, sys32_execve +/*60*/ .word sys_umask, sys_chroot, compat_sys_newfstat, compat_sys_fstat64, sys_getpagesize + .word sys_msync, sys_vfork, sys32_pread64, sys32_pwrite64, sys_geteuid /*70*/ .word sys_getegid, sys_mmap, sys_setreuid, sys_munmap, sys_mprotect .word sys_madvise, sys_vhangup, sys32_truncate64, sys_mincore, sys_getgroups16 -/*80*/ .word sys_setgroups16, sys_getpgrp, sys32_setgroups, sys32_setitimer, sys32_ftruncate64 - .word sys32_swapon, sys32_getitimer, sys_setuid, sys32_sethostname, sys_setgid +/*80*/ .word sys_setgroups16, sys_getpgrp, sys_setgroups, compat_sys_setitimer, sys32_ftruncate64 + .word sys_swapon, compat_sys_getitimer, sys_setuid, sys_sethostname, sys_setgid /*90*/ .word sys_dup2, sys_setfsuid, compat_sys_fcntl, sys32_select, sys_setfsgid - .word sys_fsync, sys32_setpriority, sys_nis_syscall, sys_nis_syscall, sys_nis_syscall -/*100*/ .word sys32_getpriority, sys32_rt_sigreturn, sys32_rt_sigaction, sys32_rt_sigprocmask, sys32_rt_sigpending - .word compat_sys_rt_sigtimedwait, sys32_rt_sigqueueinfo, compat_sys_rt_sigsuspend, sys_setresuid, sys_getresuid + .word sys_fsync, sys_setpriority, sys_nis_syscall, sys_nis_syscall, sys_nis_syscall +/*100*/ .word sys_getpriority, sys32_rt_sigreturn, compat_sys_rt_sigaction, compat_sys_rt_sigprocmask, compat_sys_rt_sigpending + .word compat_sys_rt_sigtimedwait, compat_sys_rt_sigqueueinfo, compat_sys_rt_sigsuspend, sys_setresuid, sys_getresuid /*110*/ .word sys_setresgid, sys_getresgid, sys_setregid, sys_nis_syscall, sys_nis_syscall - .word sys32_getgroups, compat_sys_gettimeofday, sys32_getrusage, sys_nis_syscall, sys_getcwd + .word sys_getgroups, compat_sys_gettimeofday, compat_sys_getrusage, sys_nis_syscall, sys_getcwd /*120*/ .word compat_sys_readv, compat_sys_writev, compat_sys_settimeofday, sys_fchown16, sys_fchmod - .word sys_nis_syscall, sys_setreuid16, sys_setregid16, sys_rename, sys32_truncate -/*130*/ .word sys32_ftruncate, sys_flock, compat_sys_lstat64, sys_nis_syscall, sys_nis_syscall - .word sys_nis_syscall, sys32_mkdir, sys_rmdir, compat_sys_utimes, compat_sys_stat64 -/*140*/ .word sys32_sendfile64, sys_nis_syscall, sys32_futex, sys_gettid, compat_sys_getrlimit - .word compat_sys_setrlimit, sys_pivot_root, sys32_prctl, sys_pciconfig_read, sys_pciconfig_write + .word sys_nis_syscall, sys_setreuid16, sys_setregid16, sys_rename, compat_sys_truncate +/*130*/ .word compat_sys_ftruncate, sys_flock, compat_sys_lstat64, sys_nis_syscall, sys_nis_syscall + .word sys_nis_syscall, sys_mkdir, sys_rmdir, compat_sys_utimes, compat_sys_stat64 +/*140*/ .word sys_sendfile64, sys_nis_syscall, sys32_futex, sys_gettid, compat_sys_getrlimit + .word compat_sys_setrlimit, sys_pivot_root, sys_prctl, sys_pciconfig_read, sys_pciconfig_write /*150*/ .word sys_nis_syscall, sys_inotify_init, sys_inotify_add_watch, sys_poll, sys_getdents64 .word compat_sys_fcntl64, sys_inotify_rm_watch, compat_sys_statfs, compat_sys_fstatfs, sys_oldumount -/*160*/ .word compat_sys_sched_setaffinity, compat_sys_sched_getaffinity, sys32_getdomainname, sys32_setdomainname, sys_nis_syscall - .word sys_quotactl, sys_set_tid_address, compat_sys_mount, compat_sys_ustat, sys32_setxattr -/*170*/ .word sys32_lsetxattr, sys32_fsetxattr, sys_getxattr, sys_lgetxattr, compat_sys_getdents - .word sys_setsid, sys_fchdir, sys32_fgetxattr, sys_listxattr, sys_llistxattr -/*180*/ .word sys32_flistxattr, sys_removexattr, sys_lremovexattr, compat_sys_sigpending, sys_ni_syscall - .word sys32_setpgid, sys32_fremovexattr, sys32_tkill, sys32_exit_group, sys_newuname -/*190*/ .word sys32_init_module, sys_sparc64_personality, sys_remap_file_pages, sys32_epoll_create, sys32_epoll_ctl - .word sys32_epoll_wait, sys32_ioprio_set, sys_getppid, sys32_sigaction, sys_sgetmask -/*200*/ .word sys32_ssetmask, sys_sigsuspend, compat_sys_newlstat, sys_uselib, compat_sys_old_readdir - .word sys32_readahead, sys32_socketcall, sys32_syslog, sys32_lookup_dcookie, sys32_fadvise64 -/*210*/ .word sys32_fadvise64_64, sys32_tgkill, sys32_waitpid, sys_swapoff, compat_sys_sysinfo - .word compat_sys_ipc, sys32_sigreturn, sys_clone, sys32_ioprio_get, compat_sys_adjtimex -/*220*/ .word sys32_sigprocmask, sys_ni_syscall, sys32_delete_module, sys_ni_syscall, sys32_getpgid - .word sys32_bdflush, sys32_sysfs, sys_nis_syscall, sys_setfsuid16, sys_setfsgid16 -/*230*/ .word sys32_select, compat_sys_time, sys32_splice, compat_sys_stime, compat_sys_statfs64 - .word compat_sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys32_mlockall -/*240*/ .word sys_munlockall, sys32_sched_setparam, sys32_sched_getparam, sys32_sched_setscheduler, sys32_sched_getscheduler - .word sys_sched_yield, sys32_sched_get_priority_max, sys32_sched_get_priority_min, sys32_sched_rr_get_interval, compat_sys_nanosleep -/*250*/ .word sys_mremap, compat_sys_sysctl, sys32_getsid, sys_fdatasync, sys32_nfsservctl +/*160*/ .word compat_sys_sched_setaffinity, compat_sys_sched_getaffinity, sys_getdomainname, sys_setdomainname, sys_nis_syscall + .word sys_quotactl, sys_set_tid_address, compat_sys_mount, compat_sys_ustat, sys_setxattr +/*170*/ .word sys_lsetxattr, sys_fsetxattr, sys_getxattr, sys_lgetxattr, compat_sys_getdents + .word sys_setsid, sys_fchdir, sys_fgetxattr, sys_listxattr, sys_llistxattr +/*180*/ .word sys_flistxattr, sys_removexattr, sys_lremovexattr, compat_sys_sigpending, sys_ni_syscall + .word sys_setpgid, sys_fremovexattr, sys_tkill, sparc_exit_group, sys_newuname +/*190*/ .word sys_init_module, sys_sparc64_personality, sys_remap_file_pages, sys_epoll_create, sys_epoll_ctl + .word sys_epoll_wait, sys_ioprio_set, sys_getppid, compat_sys_sparc_sigaction, sys_sgetmask +/*200*/ .word sys_ssetmask, sys_sigsuspend, compat_sys_newlstat, sys_uselib, compat_sys_old_readdir + .word sys32_readahead, sys32_socketcall, sys_syslog, compat_sys_lookup_dcookie, sys32_fadvise64 +/*210*/ .word sys32_fadvise64_64, sys_tgkill, sys_waitpid, sys_swapoff, compat_sys_sysinfo + .word compat_sys_ipc, sys32_sigreturn, sys_clone, sys_ioprio_get, compat_sys_adjtimex +/*220*/ .word compat_sys_sigprocmask, sys_ni_syscall, sys_delete_module, sys_ni_syscall, sys_getpgid + .word sys_bdflush, sys_sysfs, sys_nis_syscall, sys_setfsuid16, sys_setfsgid16 +/*230*/ .word sys32_select, compat_sys_time, sys_splice, compat_sys_stime, compat_sys_statfs64 + .word compat_sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall +/*240*/ .word sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler + .word sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, compat_sys_sched_rr_get_interval, compat_sys_nanosleep +/*250*/ .word sys_mremap, compat_sys_sysctl, sys_getsid, sys_fdatasync, sys_nis_syscall .word sys32_sync_file_range, compat_sys_clock_settime, compat_sys_clock_gettime, compat_sys_clock_getres, sys32_clock_nanosleep /*260*/ .word compat_sys_sched_getaffinity, compat_sys_sched_setaffinity, sys32_timer_settime, compat_sys_timer_gettime, sys_timer_getoverrun .word sys_timer_delete, compat_sys_timer_create, sys_ni_syscall, compat_sys_io_setup, sys_io_destroy /*270*/ .word sys32_io_submit, sys_io_cancel, compat_sys_io_getevents, sys32_mq_open, sys_mq_unlink .word compat_sys_mq_timedsend, compat_sys_mq_timedreceive, compat_sys_mq_notify, compat_sys_mq_getsetattr, compat_sys_waitid -/*280*/ .word sys32_tee, sys_add_key, sys_request_key, sys_keyctl, compat_sys_openat +/*280*/ .word sys_tee, sys_add_key, sys_request_key, compat_sys_keyctl, compat_sys_openat .word sys_mkdirat, sys_mknodat, sys_fchownat, compat_sys_futimesat, compat_sys_fstatat64 /*290*/ .word sys_unlinkat, sys_renameat, sys_linkat, sys_symlinkat, sys_readlinkat .word sys_fchmodat, sys_faccessat, compat_sys_pselect6, compat_sys_ppoll, sys_unshare @@ -84,7 +84,10 @@ sys_call_table32: .word compat_sys_timerfd_settime, compat_sys_timerfd_gettime, compat_sys_signalfd4, sys_eventfd2, sys_epoll_create1 /*320*/ .word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, compat_sys_preadv .word compat_sys_pwritev, compat_sys_rt_tgsigqueueinfo, sys_perf_event_open, compat_sys_recvmmsg, sys_fanotify_init -/*330*/ .word sys32_fanotify_mark, sys_prlimit64 +/*330*/ .word compat_sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, compat_sys_open_by_handle_at, compat_sys_clock_adjtime + .word sys_syncfs, compat_sys_sendmmsg, sys_setns, compat_sys_process_vm_readv, compat_sys_process_vm_writev +/*340*/ .word sys_kern_features, sys_kcmp, sys_finit_module, sys_sched_setattr, sys_sched_getattr + .word sys32_renameat2 #endif /* CONFIG_COMPAT */ @@ -105,7 +108,7 @@ sys_call_table: /*40*/ .word sys_newlstat, sys_dup, sys_sparc_pipe, sys_times, sys_nis_syscall .word sys_umount, sys_setgid, sys_getgid, sys_signal, sys_geteuid /*50*/ .word sys_getegid, sys_acct, sys_memory_ordering, sys_nis_syscall, sys_ioctl - .word sys_reboot, sys_nis_syscall, sys_symlink, sys_readlink, sys_execve + .word sys_reboot, sys_nis_syscall, sys_symlink, sys_readlink, sys64_execve /*60*/ .word sys_umask, sys_chroot, sys_newfstat, sys_fstat64, sys_getpagesize .word sys_msync, sys_vfork, sys_pread64, sys_pwrite64, sys_nis_syscall /*70*/ .word sys_nis_syscall, sys_mmap, sys_nis_syscall, sys_64_munmap, sys_mprotect @@ -131,7 +134,7 @@ sys_call_table: /*170*/ .word sys_lsetxattr, sys_fsetxattr, sys_getxattr, sys_lgetxattr, sys_getdents .word sys_setsid, sys_fchdir, sys_fgetxattr, sys_listxattr, sys_llistxattr /*180*/ .word sys_flistxattr, sys_removexattr, sys_lremovexattr, sys_nis_syscall, sys_ni_syscall - .word sys_setpgid, sys_fremovexattr, sys_tkill, sys_exit_group, sys_newuname + .word sys_setpgid, sys_fremovexattr, sys_tkill, sparc_exit_group, sys_newuname /*190*/ .word sys_init_module, sys_sparc64_personality, sys_remap_file_pages, sys_epoll_create, sys_epoll_ctl .word sys_epoll_wait, sys_ioprio_set, sys_getppid, sys_nis_syscall, sys_sgetmask /*200*/ .word sys_ssetmask, sys_nis_syscall, sys_newlstat, sys_uselib, sys_nis_syscall @@ -144,7 +147,7 @@ sys_call_table: .word sys_fstatfs64, sys_llseek, sys_mlock, sys_munlock, sys_mlockall /*240*/ .word sys_munlockall, sys_sched_setparam, sys_sched_getparam, sys_sched_setscheduler, sys_sched_getscheduler .word sys_sched_yield, sys_sched_get_priority_max, sys_sched_get_priority_min, sys_sched_rr_get_interval, sys_nanosleep -/*250*/ .word sys_64_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_nfsservctl +/*250*/ .word sys_64_mremap, sys_sysctl, sys_getsid, sys_fdatasync, sys_nis_syscall .word sys_sync_file_range, sys_clock_settime, sys_clock_gettime, sys_clock_getres, sys_clock_nanosleep /*260*/ .word sys_sched_getaffinity, sys_sched_setaffinity, sys_timer_settime, sys_timer_gettime, sys_timer_getoverrun .word sys_timer_delete, sys_timer_create, sys_ni_syscall, sys_io_setup, sys_io_destroy @@ -160,4 +163,7 @@ sys_call_table: .word sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1 /*320*/ .word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv .word sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init -/*330*/ .word sys_fanotify_mark, sys_prlimit64 +/*330*/ .word sys_fanotify_mark, sys_prlimit64, sys_name_to_handle_at, sys_open_by_handle_at, sys_clock_adjtime + .word sys_syncfs, sys_sendmmsg, sys_setns, sys_process_vm_readv, sys_process_vm_writev +/*340*/ .word sys_kern_features, sys_kcmp, sys_finit_module, sys_sched_setattr, sys_sched_getattr + .word sys_renameat2 diff --git a/arch/sparc/kernel/tadpole.c b/arch/sparc/kernel/tadpole.c deleted file mode 100644 index 9aba8bd5a78..00000000000 --- a/arch/sparc/kernel/tadpole.c +++ /dev/null @@ -1,126 +0,0 @@ -/* tadpole.c: Probing for the tadpole clock stopping h/w at boot time. - * - * Copyright (C) 1996 David Redman (djhr@tadpole.co.uk) - */ - -#include <linux/string.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <linux/init.h> - -#include <asm/asi.h> -#include <asm/oplib.h> -#include <asm/io.h> - -#define MACIO_SCSI_CSR_ADDR 0x78400000 -#define MACIO_EN_DMA 0x00000200 -#define CLOCK_INIT_DONE 1 - -static int clk_state; -static volatile unsigned char *clk_ctrl; -void (*cpu_pwr_save)(void); - -static inline unsigned int ldphys(unsigned int addr) -{ - unsigned long data; - - __asm__ __volatile__("\n\tlda [%1] %2, %0\n\t" : - "=r" (data) : - "r" (addr), "i" (ASI_M_BYPASS)); - return data; -} - -static void clk_init(void) -{ - __asm__ __volatile__("mov 0x6c, %%g1\n\t" - "mov 0x4c, %%g2\n\t" - "mov 0xdf, %%g3\n\t" - "stb %%g1, [%0+3]\n\t" - "stb %%g2, [%0+3]\n\t" - "stb %%g3, [%0+3]\n\t" : : - "r" (clk_ctrl) : - "g1", "g2", "g3"); -} - -static void clk_slow(void) -{ - __asm__ __volatile__("mov 0xcc, %%g2\n\t" - "mov 0x4c, %%g3\n\t" - "mov 0xcf, %%g4\n\t" - "mov 0xdf, %%g5\n\t" - "stb %%g2, [%0+3]\n\t" - "stb %%g3, [%0+3]\n\t" - "stb %%g4, [%0+3]\n\t" - "stb %%g5, [%0+3]\n\t" : : - "r" (clk_ctrl) : - "g2", "g3", "g4", "g5"); -} - -/* - * Tadpole is guaranteed to be UP, using local_irq_save. - */ -static void tsu_clockstop(void) -{ - unsigned int mcsr; - unsigned long flags; - - if (!clk_ctrl) - return; - if (!(clk_state & CLOCK_INIT_DONE)) { - local_irq_save(flags); - clk_init(); - clk_state |= CLOCK_INIT_DONE; /* all done */ - local_irq_restore(flags); - return; - } - if (!(clk_ctrl[2] & 1)) - return; /* no speed up yet */ - - local_irq_save(flags); - - /* if SCSI DMA in progress, don't slow clock */ - mcsr = ldphys(MACIO_SCSI_CSR_ADDR); - if ((mcsr&MACIO_EN_DMA) != 0) { - local_irq_restore(flags); - return; - } - /* TODO... the minimum clock setting ought to increase the - * memory refresh interval.. - */ - clk_slow(); - local_irq_restore(flags); -} - -static void swift_clockstop(void) -{ - if (!clk_ctrl) - return; - clk_ctrl[0] = 0; -} - -void __init clock_stop_probe(void) -{ - phandle node, clk_nd; - char name[20]; - - prom_getstring(prom_root_node, "name", name, sizeof(name)); - if (strncmp(name, "Tadpole", 7)) - return; - node = prom_getchild(prom_root_node); - node = prom_searchsiblings(node, "obio"); - node = prom_getchild(node); - clk_nd = prom_searchsiblings(node, "clk-ctrl"); - if (!clk_nd) - return; - printk("Clock Stopping h/w detected... "); - clk_ctrl = (char *) prom_getint(clk_nd, "address"); - clk_state = 0; - if (name[10] == '\0') { - cpu_pwr_save = tsu_clockstop; - printk("enabled (S3)\n"); - } else if ((name[10] == 'X') || (name[10] == 'G')) { - cpu_pwr_save = swift_clockstop; - printk("enabled (%s)\n",name+7); - } else - printk("disabled %s\n",name+7); -} diff --git a/arch/sparc/kernel/tick14.c b/arch/sparc/kernel/tick14.c deleted file mode 100644 index 138bbf5f872..00000000000 --- a/arch/sparc/kernel/tick14.c +++ /dev/null @@ -1,39 +0,0 @@ -/* tick14.c - * - * Copyright (C) 1996 David Redman (djhr@tadpole.co.uk) - * - * This file handles the Sparc specific level14 ticker - * This is really useful for profiling OBP uses it for keyboard - * aborts and other stuff. - */ -#include <linux/kernel.h> - -extern unsigned long lvl14_save[5]; -static unsigned long *linux_lvl14 = NULL; -static unsigned long obp_lvl14[4]; - -/* - * Call with timer IRQ closed. - * First time we do it with disable_irq, later prom code uses spin_lock_irq(). - */ -void install_linux_ticker(void) -{ - - if (!linux_lvl14) - return; - linux_lvl14[0] = lvl14_save[0]; - linux_lvl14[1] = lvl14_save[1]; - linux_lvl14[2] = lvl14_save[2]; - linux_lvl14[3] = lvl14_save[3]; -} - -void install_obp_ticker(void) -{ - - if (!linux_lvl14) - return; - linux_lvl14[0] = obp_lvl14[0]; - linux_lvl14[1] = obp_lvl14[1]; - linux_lvl14[2] = obp_lvl14[2]; - linux_lvl14[3] = obp_lvl14[3]; -} diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index 9c743b1886f..5923d1e4e7c 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -26,6 +26,8 @@ #include <linux/rtc.h> #include <linux/rtc/m48t59.h> #include <linux/timex.h> +#include <linux/clocksource.h> +#include <linux/clockchips.h> #include <linux/init.h> #include <linux/pci.h> #include <linux/ioport.h> @@ -34,20 +36,32 @@ #include <linux/of_device.h> #include <linux/platform_device.h> +#include <asm/mc146818rtc.h> #include <asm/oplib.h> #include <asm/timex.h> #include <asm/timer.h> -#include <asm/system.h> #include <asm/irq.h> #include <asm/io.h> #include <asm/idprom.h> -#include <asm/machines.h> #include <asm/page.h> #include <asm/pcic.h> #include <asm/irq_regs.h> +#include <asm/setup.h> +#include "kernel.h" #include "irq.h" +static __cacheline_aligned_in_smp DEFINE_SEQLOCK(timer_cs_lock); +static __volatile__ u64 timer_cs_internal_counter = 0; +static char timer_cs_enabled = 0; + +static struct clock_event_device timer_ce; +static char timer_ce_enabled = 0; + +#ifdef CONFIG_SMP +DEFINE_PER_CPU(struct clock_event_device, sparc32_clockevent); +#endif + DEFINE_SPINLOCK(rtc_lock); EXPORT_SYMBOL(rtc_lock); @@ -56,7 +70,6 @@ static int set_rtc_mmss(unsigned long); unsigned long profile_pc(struct pt_regs *regs) { extern char __copy_user_begin[], __copy_user_end[]; - extern char __atomic_begin[], __atomic_end[]; extern char __bzero_begin[], __bzero_end[]; unsigned long pc = regs->pc; @@ -64,8 +77,6 @@ unsigned long profile_pc(struct pt_regs *regs) if (in_lock_functions(pc) || (pc >= (unsigned long) __copy_user_begin && pc < (unsigned long) __copy_user_end) || - (pc >= (unsigned long) __atomic_begin && - pc < (unsigned long) __atomic_end) || (pc >= (unsigned long) __bzero_begin && pc < (unsigned long) __bzero_end)) pc = regs->u_regs[UREG_RETPC]; @@ -74,43 +85,170 @@ unsigned long profile_pc(struct pt_regs *regs) EXPORT_SYMBOL(profile_pc); -__volatile__ unsigned int *master_l10_counter; - -u32 (*do_arch_gettimeoffset)(void); +volatile u32 __iomem *master_l10_counter; int update_persistent_clock(struct timespec now) { return set_rtc_mmss(now.tv_sec); } -/* - * timer_interrupt() needs to keep up the real-time clock, - * as well as call the "do_timer()" routine every clocktick - */ +irqreturn_t notrace timer_interrupt(int dummy, void *dev_id) +{ + if (timer_cs_enabled) { + write_seqlock(&timer_cs_lock); + timer_cs_internal_counter++; + sparc_config.clear_clock_irq(); + write_sequnlock(&timer_cs_lock); + } else { + sparc_config.clear_clock_irq(); + } -#define TICK_SIZE (tick_nsec / 1000) + if (timer_ce_enabled) + timer_ce.event_handler(&timer_ce); -static irqreturn_t timer_interrupt(int dummy, void *dev_id) + return IRQ_HANDLED; +} + +static void timer_ce_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) { -#ifndef CONFIG_SMP - profile_tick(CPU_PROFILING); -#endif + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + case CLOCK_EVT_MODE_RESUME: + timer_ce_enabled = 1; + break; + case CLOCK_EVT_MODE_SHUTDOWN: + timer_ce_enabled = 0; + break; + default: + break; + } + smp_mb(); +} + +static __init void setup_timer_ce(void) +{ + struct clock_event_device *ce = &timer_ce; + + BUG_ON(smp_processor_id() != boot_cpu_id); + + ce->name = "timer_ce"; + ce->rating = 100; + ce->features = CLOCK_EVT_FEAT_PERIODIC; + ce->set_mode = timer_ce_set_mode; + ce->cpumask = cpu_possible_mask; + ce->shift = 32; + ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, + ce->shift); + clockevents_register_device(ce); +} + +static unsigned int sbus_cycles_offset(void) +{ + u32 val, offset; - /* Protect counter clear so that do_gettimeoffset works */ - write_seqlock(&xtime_lock); + val = sbus_readl(master_l10_counter); + offset = (val >> TIMER_VALUE_SHIFT) & TIMER_VALUE_MASK; - clear_clock_irq(); + /* Limit hit? */ + if (val & TIMER_LIMIT_BIT) + offset += sparc_config.cs_period; - do_timer(1); + return offset; +} - write_sequnlock(&xtime_lock); +static cycle_t timer_cs_read(struct clocksource *cs) +{ + unsigned int seq, offset; + u64 cycles; -#ifndef CONFIG_SMP - update_process_times(user_mode(get_irq_regs())); -#endif - return IRQ_HANDLED; + do { + seq = read_seqbegin(&timer_cs_lock); + + cycles = timer_cs_internal_counter; + offset = sparc_config.get_cycles_offset(); + } while (read_seqretry(&timer_cs_lock, seq)); + + /* Count absolute cycles */ + cycles *= sparc_config.cs_period; + cycles += offset; + + return cycles; +} + +static struct clocksource timer_cs = { + .name = "timer_cs", + .rating = 100, + .read = timer_cs_read, + .mask = CLOCKSOURCE_MASK(64), + .shift = 2, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +static __init int setup_timer_cs(void) +{ + timer_cs_enabled = 1; + timer_cs.mult = clocksource_hz2mult(sparc_config.clock_rate, + timer_cs.shift); + + return clocksource_register(&timer_cs); +} + +#ifdef CONFIG_SMP +static void percpu_ce_setup(enum clock_event_mode mode, + struct clock_event_device *evt) +{ + int cpu = __first_cpu(evt->cpumask); + + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + sparc_config.load_profile_irq(cpu, + SBUS_CLOCK_RATE / HZ); + break; + case CLOCK_EVT_MODE_ONESHOT: + case CLOCK_EVT_MODE_SHUTDOWN: + case CLOCK_EVT_MODE_UNUSED: + sparc_config.load_profile_irq(cpu, 0); + break; + default: + break; + } } +static int percpu_ce_set_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + int cpu = __first_cpu(evt->cpumask); + unsigned int next = (unsigned int)delta; + + sparc_config.load_profile_irq(cpu, next); + return 0; +} + +void register_percpu_ce(int cpu) +{ + struct clock_event_device *ce = &per_cpu(sparc32_clockevent, cpu); + unsigned int features = CLOCK_EVT_FEAT_PERIODIC; + + if (sparc_config.features & FEAT_L14_ONESHOT) + features |= CLOCK_EVT_FEAT_ONESHOT; + + ce->name = "percpu_ce"; + ce->rating = 200; + ce->features = features; + ce->set_mode = percpu_ce_setup; + ce->set_next_event = percpu_ce_set_next_event; + ce->cpumask = cpumask_of(cpu); + ce->shift = 32; + ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, + ce->shift); + ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce); + ce->min_delta_ns = clockevent_delta2ns(100, ce); + + clockevents_register_device(ce); +} +#endif + static unsigned char mostek_read_byte(struct device *dev, u32 ofs) { struct platform_device *pdev = to_platform_device(dev); @@ -142,7 +280,7 @@ static struct platform_device m48t59_rtc = { }, }; -static int __devinit clock_probe(struct platform_device *op, const struct of_device_id *match) +static int clock_probe(struct platform_device *op) { struct device_node *dp = op->dev.of_node; const char *model = of_get_property(dp, "model", NULL); @@ -150,6 +288,10 @@ static int __devinit clock_probe(struct platform_device *op, const struct of_dev if (!model) return -ENODEV; + /* Only the primary RTC has an address property */ + if (!of_find_property(dp, "address", NULL)) + return -ENODEV; + m48t59_rtc.resource = &op->resource[0]; if (!strcmp(model, "mk48t02")) { /* Map the clock register io area read-only */ @@ -169,14 +311,14 @@ static int __devinit clock_probe(struct platform_device *op, const struct of_dev return 0; } -static struct of_device_id __initdata clock_match[] = { +static struct of_device_id clock_match[] = { { .name = "eeprom", }, {}, }; -static struct of_platform_driver clock_driver = { +static struct platform_driver clock_driver = { .probe = clock_probe, .driver = { .name = "rtc", @@ -189,7 +331,7 @@ static struct of_platform_driver clock_driver = { /* Probe for the mostek real time clock chip. */ static int __init clock_init(void) { - return of_register_platform_driver(&clock_driver); + return platform_driver_register(&clock_driver); } /* Must be after subsys_initcall() so that busses are probed. Must * be before device_initcall() because things like the RTC driver @@ -197,46 +339,32 @@ static int __init clock_init(void) */ fs_initcall(clock_init); - -u32 sbus_do_gettimeoffset(void) +static void __init sparc32_late_time_init(void) { - unsigned long val = *master_l10_counter; - unsigned long usec = (val >> 10) & 0x1fffff; - - /* Limit hit? */ - if (val & 0x80000000) - usec += 1000000 / HZ; - - return usec * 1000; -} - - -u32 arch_gettimeoffset(void) -{ - if (unlikely(!do_arch_gettimeoffset)) - return 0; - return do_arch_gettimeoffset(); + if (sparc_config.features & FEAT_L10_CLOCKEVENT) + setup_timer_ce(); + if (sparc_config.features & FEAT_L10_CLOCKSOURCE) + setup_timer_cs(); +#ifdef CONFIG_SMP + register_percpu_ce(smp_processor_id()); +#endif } static void __init sbus_time_init(void) { - do_arch_gettimeoffset = sbus_do_gettimeoffset; - - btfixup(); - - sparc_init_timers(timer_interrupt); + sparc_config.get_cycles_offset = sbus_cycles_offset; + sparc_config.init_timers(); } void __init time_init(void) { -#ifdef CONFIG_PCI - extern void pci_time_init(void); - if (pcic_present()) { + sparc_config.features = 0; + late_time_init = sparc32_late_time_init; + + if (pcic_present()) pci_time_init(); - return; - } -#endif - sbus_time_init(); + else + sbus_time_init(); } diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 3bc9c9979b9..3fddf64c7fc 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -9,7 +9,7 @@ */ #include <linux/errno.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/param.h> @@ -419,7 +419,7 @@ static struct platform_device rtc_cmos_device = { .num_resources = 1, }; -static int __devinit rtc_probe(struct platform_device *op, const struct of_device_id *match) +static int rtc_probe(struct platform_device *op) { struct resource *r; @@ -442,7 +442,7 @@ static int __devinit rtc_probe(struct platform_device *op, const struct of_devic return platform_device_register(&rtc_cmos_device); } -static struct of_device_id __initdata rtc_match[] = { +static const struct of_device_id rtc_match[] = { { .name = "rtc", .compatible = "m5819", @@ -462,7 +462,7 @@ static struct of_device_id __initdata rtc_match[] = { {}, }; -static struct of_platform_driver rtc_driver = { +static struct platform_driver rtc_driver = { .probe = rtc_probe, .driver = { .name = "rtc", @@ -477,7 +477,7 @@ static struct platform_device rtc_bq4802_device = { .num_resources = 1, }; -static int __devinit bq4802_probe(struct platform_device *op, const struct of_device_id *match) +static int bq4802_probe(struct platform_device *op) { printk(KERN_INFO "%s: BQ4802 regs at 0x%llx\n", @@ -487,7 +487,7 @@ static int __devinit bq4802_probe(struct platform_device *op, const struct of_de return platform_device_register(&rtc_bq4802_device); } -static struct of_device_id __initdata bq4802_match[] = { +static const struct of_device_id bq4802_match[] = { { .name = "rtc", .compatible = "bq4802", @@ -495,7 +495,7 @@ static struct of_device_id __initdata bq4802_match[] = { {}, }; -static struct of_platform_driver bq4802_driver = { +static struct platform_driver bq4802_driver = { .probe = bq4802_probe, .driver = { .name = "bq4802", @@ -534,7 +534,7 @@ static struct platform_device m48t59_rtc = { }, }; -static int __devinit mostek_probe(struct platform_device *op, const struct of_device_id *match) +static int mostek_probe(struct platform_device *op) { struct device_node *dp = op->dev.of_node; @@ -552,14 +552,14 @@ static int __devinit mostek_probe(struct platform_device *op, const struct of_de return platform_device_register(&m48t59_rtc); } -static struct of_device_id __initdata mostek_match[] = { +static const struct of_device_id mostek_match[] = { { .name = "eeprom", }, {}, }; -static struct of_platform_driver mostek_driver = { +static struct platform_driver mostek_driver = { .probe = mostek_probe, .driver = { .name = "mostek", @@ -586,9 +586,9 @@ static int __init clock_init(void) if (tlb_type == hypervisor) return platform_device_register(&rtc_sun4v_device); - (void) of_register_platform_driver(&rtc_driver); - (void) of_register_platform_driver(&mostek_driver); - (void) of_register_platform_driver(&bq4802_driver); + (void) platform_driver_register(&rtc_driver); + (void) platform_driver_register(&mostek_driver); + (void) platform_driver_register(&bq4802_driver); return 0; } @@ -659,8 +659,7 @@ static int sparc64_cpufreq_notifier(struct notifier_block *nb, unsigned long val ft->clock_tick_ref = cpu_data(cpu).clock_tick; } if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || - (val == CPUFREQ_POSTCHANGE && freq->old > freq->new) || - (val == CPUFREQ_RESUMECHANGE)) { + (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { cpu_data(cpu).clock_tick = cpufreq_scale(ft->clock_tick_ref, ft->ref_freq, @@ -708,7 +707,7 @@ static void sparc64_timer_setup(enum clock_event_mode mode, case CLOCK_EVT_MODE_UNUSED: WARN_ON(1); break; - }; + } } static struct clock_event_device sparc64_clockevent = { @@ -733,7 +732,7 @@ void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) irq_enter(); local_cpu_data().irq0_irqs++; - kstat_incr_irqs_this_cpu(0, irq_to_desc(0)); + kstat_incr_irq_this_cpu(0); if (unlikely(!evt->event_handler)) { printk(KERN_WARNING @@ -746,7 +745,7 @@ void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) set_irq_regs(old_regs); } -void __devinit setup_sparc64_timer(void) +void setup_sparc64_timer(void) { struct clock_event_device *sevt; unsigned long pstate; @@ -816,14 +815,12 @@ void __init time_init(void) clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); clocksource_tick.name = tick_ops->name; - clocksource_calc_mult_shift(&clocksource_tick, freq, 4); clocksource_tick.read = clocksource_tick_read; + clocksource_register_hz(&clocksource_tick, freq); printk("clocksource: mult[%x] shift[%d]\n", clocksource_tick.mult, clocksource_tick.shift); - clocksource_register(&clocksource_tick); - sparc64_clockevent.name = tick_ops->name; clockevents_calc_mult_shift(&sparc64_clockevent, freq, 4); @@ -846,7 +843,7 @@ unsigned long long sched_clock(void) >> SPARC64_NSEC_PER_CYC_SHIFT; } -int __devinit read_current_timer(unsigned long *timer_val) +int read_current_timer(unsigned long *timer_val) { *timer_val = tick_ops->get_tick(); return 0; diff --git a/arch/sparc/kernel/trampoline_32.S b/arch/sparc/kernel/trampoline_32.S index 691f484e03b..3eed99fc698 100644 --- a/arch/sparc/kernel/trampoline_32.S +++ b/arch/sparc/kernel/trampoline_32.S @@ -5,7 +5,6 @@ * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#include <linux/init.h> #include <asm/head.h> #include <asm/psr.h> #include <asm/page.h> @@ -15,10 +14,9 @@ #include <asm/contregs.h> #include <asm/thread_info.h> - .globl sun4m_cpu_startup, __smp4m_processor_id, __leon_processor_id - .globl sun4d_cpu_startup, __smp4d_processor_id + .globl sun4m_cpu_startup + .globl sun4d_cpu_startup - __CPUINIT .align 4 /* When we start up a cpu for the first time it enters this routine. @@ -79,43 +77,21 @@ cpu3_startup: nop /* Start this processor. */ - call smp4m_callin + call smp_callin nop - b,a smp_do_cpu_idle + b,a smp_panic .text .align 4 -smp_do_cpu_idle: - call cpu_idle - mov 0, %o0 - +smp_panic: call cpu_panic nop -__smp4m_processor_id: - rd %tbr, %g2 - srl %g2, 12, %g2 - and %g2, 3, %g2 - retl - mov %g1, %o7 - -__smp4d_processor_id: - lda [%g0] ASI_M_VIKING_TMP1, %g2 - retl - mov %g1, %o7 - -__leon_processor_id: - rd %asr17,%g2 - srl %g2,28,%g2 - retl - mov %g1, %o7 - /* CPUID in bootbus can be found at PA 0xff0140000 */ #define SUN4D_BOOTBUS_CPUID 0xf0140000 - __CPUINIT .align 4 sun4d_cpu_startup: @@ -162,14 +138,11 @@ sun4d_cpu_startup: nop /* Start this processor. */ - call smp4d_callin + call smp_callin nop - b,a smp_do_cpu_idle - -#ifdef CONFIG_SPARC_LEON + b,a smp_panic - __CPUINIT .align 4 .global leon_smp_cpu_startup, smp_penguin_ctable @@ -179,7 +152,7 @@ leon_smp_cpu_startup: ld [%g1+4],%g1 srl %g1,4,%g1 set 0x00000100,%g5 /* SRMMU_CTXTBL_PTR */ - sta %g1, [%g5] ASI_M_MMUREGS + sta %g1, [%g5] ASI_LEON_MMUREGS /* Set up a sane %psr -- PIL<0xf> S<0x1> PS<0x1> CWP<0x0> */ set (PSR_PIL | PSR_S | PSR_PS), %g1 @@ -221,9 +194,7 @@ leon_smp_cpu_startup: nop /* Start this processor. */ - call leon_callin + call smp_callin nop - b,a smp_do_cpu_idle - -#endif + b,a smp_panic diff --git a/arch/sparc/kernel/trampoline_64.S b/arch/sparc/kernel/trampoline_64.S index da1b781b5e6..737f8cbc7d5 100644 --- a/arch/sparc/kernel/trampoline_64.S +++ b/arch/sparc/kernel/trampoline_64.S @@ -4,7 +4,6 @@ * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) */ -#include <linux/init.h> #include <asm/head.h> #include <asm/asi.h> @@ -32,13 +31,11 @@ itlb_load: dtlb_load: .asciz "SUNW,dtlb-load" - /* XXX __cpuinit this thing XXX */ #define TRAMP_STACK_SIZE 1024 .align 16 tramp_stack: .skip TRAMP_STACK_SIZE - __CPUINIT .align 8 .globl sparc64_cpu_startup, sparc64_cpu_startup_end sparc64_cpu_startup: @@ -131,7 +128,6 @@ startup_continue: clr %l5 sethi %hi(num_kernel_image_mappings), %l6 lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 - add %l6, 1, %l6 mov 15, %l7 BRANCH_IF_ANY_CHEETAH(g1,g5,2f) @@ -224,7 +220,6 @@ niagara_lock_tlb: clr %l5 sethi %hi(num_kernel_image_mappings), %l6 lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 - add %l6, 1, %l6 1: mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 @@ -407,8 +402,7 @@ after_lock_tlb: call smp_callin nop - call cpu_idle - mov 0, %o0 + call cpu_panic nop 1: b,a,pt %xcc, 1b diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c index c0490c7bbde..6fd386c5232 100644 --- a/arch/sparc/kernel/traps_32.c +++ b/arch/sparc/kernel/traps_32.c @@ -14,9 +14,9 @@ #include <linux/signal.h> #include <linux/smp.h> #include <linux/kdebug.h> +#include <linux/export.h> #include <asm/delay.h> -#include <asm/system.h> #include <asm/ptrace.h> #include <asm/oplib.h> #include <asm/page.h> @@ -44,7 +44,7 @@ static void instruction_dump(unsigned long *pc) #define __SAVE __asm__ __volatile__("save %sp, -0x40, %sp\n\t") #define __RESTORE __asm__ __volatile__("restore %g0, %g0, %g0\n\t") -void die_if_kernel(char *str, struct pt_regs *regs) +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) { static int die_counter; int count = 0; @@ -58,7 +58,7 @@ void die_if_kernel(char *str, struct pt_regs *regs) printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter); show_regs(regs); - add_taint(TAINT_DIE); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); __SAVE; __SAVE; __SAVE; __SAVE; __SAVE; __SAVE; __SAVE; __SAVE; @@ -120,8 +120,6 @@ void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned lon printk("Ill instr. at pc=%08lx instruction is %08lx\n", regs->pc, *(unsigned long *)regs->pc); #endif - if (!do_user_muldiv (regs, pc)) - return; info.si_signo = SIGILL; info.si_errno = 0; @@ -221,8 +219,6 @@ static unsigned long fake_fsr; static unsigned long fake_queue[32] __attribute__ ((aligned (8))); static unsigned long fake_depth; -extern int do_mathemu(struct pt_regs *, struct task_struct *); - void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, unsigned long psr) { diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index 1e9770936c3..fb6640ec855 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -1,6 +1,6 @@ /* arch/sparc64/kernel/traps.c * - * Copyright (C) 1995,1997,2008,2009 David S. Miller (davem@davemloft.net) + * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net) * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com) */ @@ -18,11 +18,12 @@ #include <linux/init.h> #include <linux/kdebug.h> #include <linux/ftrace.h> +#include <linux/reboot.h> #include <linux/gfp.h> +#include <linux/context_tracking.h> #include <asm/smp.h> #include <asm/delay.h> -#include <asm/system.h> #include <asm/ptrace.h> #include <asm/oplib.h> #include <asm/page.h> @@ -41,8 +42,11 @@ #include <asm/head.h> #include <asm/prom.h> #include <asm/memctrl.h> +#include <asm/cacheflush.h> +#include <asm/setup.h> #include "entry.h" +#include "kernel.h" #include "kstack.h" /* When an irrecoverable trap occurs at tl > 0, the trap entry @@ -185,11 +189,12 @@ EXPORT_SYMBOL_GPL(unregister_dimm_printer); void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "instruction access exception", regs, 0, 0x8, SIGTRAP) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { printk("spitfire_insn_access_exception: SFSR[%016lx] " @@ -206,6 +211,8 @@ void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, un info.si_addr = (void __user *)regs->tpc; info.si_trapno = 0; force_sig_info(SIGSEGV, &info, current); +out: + exception_exit(prev_state); } void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) @@ -259,11 +266,12 @@ void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, u void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "data access exception", regs, 0, 0x30, SIGTRAP) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { /* Test if this comes from uaccess places. */ @@ -279,7 +287,7 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un #endif regs->tpc = entry->fixup; regs->tnpc = regs->tpc + 4; - return; + goto out; } /* Shit... */ printk("spitfire_data_access_exception: SFSR[%016lx] " @@ -293,6 +301,8 @@ void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, un info.si_addr = (void __user *)sfar; info.si_trapno = 0; force_sig_info(SIGSEGV, &info, current); +out: + exception_exit(prev_state); } void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) @@ -850,7 +860,7 @@ void __init cheetah_ecache_flush_init(void) ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size); if (ecache_flush_physbase == ~0UL) { - prom_printf("cheetah_ecache_flush_init: Cannot find %d byte " + prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte " "contiguous physical memory.\n", ecache_flush_size); prom_halt(); @@ -1760,85 +1770,223 @@ void cheetah_plus_parity_error(int type, struct pt_regs *regs) } struct sun4v_error_entry { - u64 err_handle; - u64 err_stick; + /* Unique error handle */ +/*0x00*/u64 err_handle; + + /* %stick value at the time of the error */ +/*0x08*/u64 err_stick; + +/*0x10*/u8 reserved_1[3]; - u32 err_type; + /* Error type */ +/*0x13*/u8 err_type; #define SUN4V_ERR_TYPE_UNDEFINED 0 #define SUN4V_ERR_TYPE_UNCORRECTED_RES 1 #define SUN4V_ERR_TYPE_PRECISE_NONRES 2 #define SUN4V_ERR_TYPE_DEFERRED_NONRES 3 -#define SUN4V_ERR_TYPE_WARNING_RES 4 +#define SUN4V_ERR_TYPE_SHUTDOWN_RQST 4 +#define SUN4V_ERR_TYPE_DUMP_CORE 5 +#define SUN4V_ERR_TYPE_SP_STATE_CHANGE 6 +#define SUN4V_ERR_TYPE_NUM 7 - u32 err_attrs; + /* Error attributes */ +/*0x14*/u32 err_attrs; #define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001 #define SUN4V_ERR_ATTRS_MEMORY 0x00000002 #define SUN4V_ERR_ATTRS_PIO 0x00000004 #define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008 #define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010 -#define SUN4V_ERR_ATTRS_USER_MODE 0x01000000 -#define SUN4V_ERR_ATTRS_PRIV_MODE 0x02000000 +#define SUN4V_ERR_ATTRS_SHUTDOWN_RQST 0x00000020 +#define SUN4V_ERR_ATTRS_ASR 0x00000040 +#define SUN4V_ERR_ATTRS_ASI 0x00000080 +#define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100 +#define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600 +#define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9 +#define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000 +#define SUN4V_ERR_ATTRS_MODE_SHFT 24 #define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000 - u64 err_raddr; - u32 err_size; - u16 err_cpu; - u16 err_pad; +#define SUN4V_ERR_SPSTATE_FAULTED 0 +#define SUN4V_ERR_SPSTATE_AVAILABLE 1 +#define SUN4V_ERR_SPSTATE_NOT_PRESENT 2 + +#define SUN4V_ERR_MODE_USER 1 +#define SUN4V_ERR_MODE_PRIV 2 + + /* Real address of the memory region or PIO transaction */ +/*0x18*/u64 err_raddr; + + /* Size of the operation triggering the error, in bytes */ +/*0x20*/u32 err_size; + + /* ID of the CPU */ +/*0x24*/u16 err_cpu; + + /* Grace periof for shutdown, in seconds */ +/*0x26*/u16 err_secs; + + /* Value of the %asi register */ +/*0x28*/u8 err_asi; + +/*0x29*/u8 reserved_2; + + /* Value of the ASR register number */ +/*0x2a*/u16 err_asr; +#define SUN4V_ERR_ASR_VALID 0x8000 + +/*0x2c*/u32 reserved_3; +/*0x30*/u64 reserved_4; +/*0x38*/u64 reserved_5; }; static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0); static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0); -static const char *sun4v_err_type_to_str(u32 type) -{ - switch (type) { - case SUN4V_ERR_TYPE_UNDEFINED: - return "undefined"; - case SUN4V_ERR_TYPE_UNCORRECTED_RES: - return "uncorrected resumable"; - case SUN4V_ERR_TYPE_PRECISE_NONRES: - return "precise nonresumable"; - case SUN4V_ERR_TYPE_DEFERRED_NONRES: - return "deferred nonresumable"; - case SUN4V_ERR_TYPE_WARNING_RES: - return "warning resumable"; - default: - return "unknown"; +static const char *sun4v_err_type_to_str(u8 type) +{ + static const char *types[SUN4V_ERR_TYPE_NUM] = { + "undefined", + "uncorrected resumable", + "precise nonresumable", + "deferred nonresumable", + "shutdown request", + "dump core", + "SP state change", }; + + if (type < SUN4V_ERR_TYPE_NUM) + return types[type]; + + return "unknown"; +} + +static void sun4v_emit_err_attr_strings(u32 attrs) +{ + static const char *attr_names[] = { + "processor", + "memory", + "PIO", + "int-registers", + "fpu-registers", + "shutdown-request", + "ASR", + "ASI", + "priv-reg", + }; + static const char *sp_states[] = { + "sp-faulted", + "sp-available", + "sp-not-present", + "sp-state-reserved", + }; + static const char *modes[] = { + "mode-reserved0", + "user", + "priv", + "mode-reserved1", + }; + u32 sp_state, mode; + int i; + + for (i = 0; i < ARRAY_SIZE(attr_names); i++) { + if (attrs & (1U << i)) { + const char *s = attr_names[i]; + + pr_cont("%s ", s); + } + } + + sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >> + SUN4V_ERR_ATTRS_SPSTATE_SHFT); + pr_cont("%s ", sp_states[sp_state]); + + mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >> + SUN4V_ERR_ATTRS_MODE_SHFT); + pr_cont("%s ", modes[mode]); + + if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) + pr_cont("res-queue-full "); +} + +/* When the report contains a real-address of "-1" it means that the + * hardware did not provide the address. So we compute the effective + * address of the load or store instruction at regs->tpc and report + * that. Usually when this happens it's a PIO and in such a case we + * are using physical addresses with bypass ASIs anyways, so what we + * report here is exactly what we want. + */ +static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs) +{ + unsigned int insn; + u64 addr; + + if (!(regs->tstate & TSTATE_PRIV)) + return; + + insn = *(unsigned int *) regs->tpc; + + addr = compute_effective_address(regs, insn, 0); + + printk("%s: insn effective address [0x%016llx]\n", + pfx, addr); } -static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, int cpu, const char *pfx, atomic_t *ocnt) +static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, + int cpu, const char *pfx, atomic_t *ocnt) { + u64 *raw_ptr = (u64 *) ent; + u32 attrs; int cnt; printk("%s: Reporting on cpu %d\n", pfx, cpu); - printk("%s: err_handle[%llx] err_stick[%llx] err_type[%08x:%s]\n", - pfx, - ent->err_handle, ent->err_stick, - ent->err_type, - sun4v_err_type_to_str(ent->err_type)); - printk("%s: err_attrs[%08x:%s %s %s %s %s %s %s %s]\n", - pfx, - ent->err_attrs, - ((ent->err_attrs & SUN4V_ERR_ATTRS_PROCESSOR) ? - "processor" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_MEMORY) ? - "memory" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_PIO) ? - "pio" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_INT_REGISTERS) ? - "integer-regs" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_FPU_REGISTERS) ? - "fpu-regs" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_USER_MODE) ? - "user" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_PRIV_MODE) ? - "privileged" : ""), - ((ent->err_attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) ? - "queue-full" : "")); - printk("%s: err_raddr[%016llx] err_size[%u] err_cpu[%u]\n", - pfx, - ent->err_raddr, ent->err_size, ent->err_cpu); + printk("%s: TPC [0x%016lx] <%pS>\n", + pfx, regs->tpc, (void *) regs->tpc); + + printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n", + pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]); + printk("%s: %016llx:%016llx:%016llx:%016llx]\n", + pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]); + + printk("%s: handle [0x%016llx] stick [0x%016llx]\n", + pfx, ent->err_handle, ent->err_stick); + + printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type)); + + attrs = ent->err_attrs; + printk("%s: attrs [0x%08x] < ", pfx, attrs); + sun4v_emit_err_attr_strings(attrs); + pr_cont(">\n"); + + /* Various fields in the error report are only valid if + * certain attribute bits are set. + */ + if (attrs & (SUN4V_ERR_ATTRS_MEMORY | + SUN4V_ERR_ATTRS_PIO | + SUN4V_ERR_ATTRS_ASI)) { + printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr); + + if (ent->err_raddr == ~(u64)0) + sun4v_report_real_raddr(pfx, regs); + } + + if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI)) + printk("%s: size [0x%x]\n", pfx, ent->err_size); + + if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR | + SUN4V_ERR_ATTRS_INT_REGISTERS | + SUN4V_ERR_ATTRS_FPU_REGISTERS | + SUN4V_ERR_ATTRS_PRIV_REG)) + printk("%s: cpu[%u]\n", pfx, ent->err_cpu); + + if (attrs & SUN4V_ERR_ATTRS_ASI) + printk("%s: asi [0x%02x]\n", pfx, ent->err_asi); + + if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS | + SUN4V_ERR_ATTRS_FPU_REGISTERS | + SUN4V_ERR_ATTRS_PRIV_REG)) && + (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0) + printk("%s: reg [0x%04x]\n", + pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID); show_regs(regs); @@ -1855,6 +2003,7 @@ static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, */ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) { + enum ctx_state prev_state = exception_enter(); struct sun4v_error_entry *ent, local_copy; struct trap_per_cpu *tb; unsigned long paddr; @@ -1874,19 +2023,23 @@ void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) put_cpu(); - if (ent->err_type == SUN4V_ERR_TYPE_WARNING_RES) { - /* If err_type is 0x4, it's a powerdown request. Do - * not do the usual resumable error log because that - * makes it look like some abnormal error. + if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) { + /* We should really take the seconds field of + * the error report and use it for the shutdown + * invocation, but for now do the same thing we + * do for a DS shutdown request. */ - printk(KERN_INFO "Power down request...\n"); - kill_cad_pid(SIGINT, 1); - return; + pr_info("Shutdown request, %u seconds...\n", + local_copy.err_secs); + orderly_poweroff(true); + goto out; } sun4v_log_error(regs, &local_copy, cpu, KERN_ERR "RESUMABLE ERROR", &sun4v_resum_oflow_cnt); +out: + exception_exit(prev_state); } /* If we try to printk() we'll probably make matters worse, by trying @@ -2011,7 +2164,7 @@ void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op) err, op); } -void do_fpe_common(struct pt_regs *regs) +static void do_fpe_common(struct pt_regs *regs) { if (regs->tstate & TSTATE_PRIV) { regs->tpc = regs->tnpc; @@ -2047,42 +2200,48 @@ void do_fpe_common(struct pt_regs *regs) void do_fpieee(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); + if (notify_die(DIE_TRAP, "fpu exception ieee", regs, 0, 0x24, SIGFPE) == NOTIFY_STOP) - return; + goto out; do_fpe_common(regs); +out: + exception_exit(prev_state); } -extern int do_mathemu(struct pt_regs *, struct fpustate *); - void do_fpother(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); struct fpustate *f = FPUSTATE; int ret = 0; if (notify_die(DIE_TRAP, "fpu exception other", regs, 0, 0x25, SIGFPE) == NOTIFY_STOP) - return; + goto out; switch ((current_thread_info()->xfsr[0] & 0x1c000)) { case (2 << 14): /* unfinished_FPop */ case (3 << 14): /* unimplemented_FPop */ - ret = do_mathemu(regs, f); + ret = do_mathemu(regs, f, false); break; } if (ret) - return; + goto out; do_fpe_common(regs); +out: + exception_exit(prev_state); } void do_tof(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, 0, 0x26, SIGEMT) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) die_if_kernel("Penguin overflow trap from kernel mode", regs); @@ -2096,15 +2255,18 @@ void do_tof(struct pt_regs *regs) info.si_addr = (void __user *)regs->tpc; info.si_trapno = 0; force_sig_info(SIGEMT, &info, current); +out: + exception_exit(prev_state); } void do_div0(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "integer division by zero", regs, 0, 0x28, SIGFPE) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) die_if_kernel("TL0: Kernel divide by zero.", regs); @@ -2118,6 +2280,8 @@ void do_div0(struct pt_regs *regs) info.si_addr = (void __user *)regs->tpc; info.si_trapno = 0; force_sig_info(SIGFPE, &info, current); +out: + exception_exit(prev_state); } static void instruction_dump(unsigned int *pc) @@ -2152,7 +2316,7 @@ static void user_instruction_dump(unsigned int __user *pc) void show_stack(struct task_struct *tsk, unsigned long *_ksp) { - unsigned long fp, thread_base, ksp; + unsigned long fp, ksp; struct thread_info *tp; int count = 0; #ifdef CONFIG_FUNCTION_GRAPH_TRACER @@ -2173,7 +2337,6 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) flushw_all(); fp = ksp + STACK_BIAS; - thread_base = (unsigned long) tp; printk("Call Trace:\n"); do { @@ -2210,13 +2373,6 @@ void show_stack(struct task_struct *tsk, unsigned long *_ksp) } while (++count < 16); } -void dump_stack(void) -{ - show_stack(current, NULL); -} - -EXPORT_SYMBOL(dump_stack); - static inline struct reg_window *kernel_stack_up(struct reg_window *rw) { unsigned long fp = rw->ins[6]; @@ -2227,7 +2383,7 @@ static inline struct reg_window *kernel_stack_up(struct reg_window *rw) return (struct reg_window *) (fp + STACK_BIAS); } -void die_if_kernel(char *str, struct pt_regs *regs) +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) { static int die_counter; int count = 0; @@ -2243,7 +2399,7 @@ void die_if_kernel(char *str, struct pt_regs *regs) notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV); __asm__ __volatile__("flushw"); show_regs(regs); - add_taint(TAINT_DIE); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); if (regs->tstate & TSTATE_PRIV) { struct thread_info *tp = current_thread_info(); struct reg_window *rw = (struct reg_window *) @@ -2277,11 +2433,9 @@ EXPORT_SYMBOL(die_if_kernel); #define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19)) #define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19)) -extern int handle_popc(u32 insn, struct pt_regs *regs); -extern int handle_ldf_stq(u32 insn, struct pt_regs *regs); - void do_illegal_instruction(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; @@ -2289,7 +2443,7 @@ void do_illegal_instruction(struct pt_regs *regs) if (notify_die(DIE_TRAP, "illegal instruction", regs, 0, 0x10, SIGILL) == NOTIFY_STOP) - return; + goto out; if (tstate & TSTATE_PRIV) die_if_kernel("Kernel illegal instruction", regs); @@ -2298,22 +2452,24 @@ void do_illegal_instruction(struct pt_regs *regs) if (get_user(insn, (u32 __user *) pc) != -EFAULT) { if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { if (handle_popc(insn, regs)) - return; + goto out; } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { if (handle_ldf_stq(insn, regs)) - return; + goto out; } else if (tlb_type == hypervisor) { if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { if (!vis_emul(regs, insn)) - return; + goto out; } else { struct fpustate *f = FPUSTATE; - /* XXX maybe verify XFSR bits like - * XXX do_fpother() does? + /* On UltraSPARC T2 and later, FPU insns which + * are not implemented in HW signal an illegal + * instruction trap and do not set the FP Trap + * Trap in the %fsr to unimplemented_FPop. */ - if (do_mathemu(regs, f)) - return; + if (do_mathemu(regs, f, true)) + goto out; } } } @@ -2323,21 +2479,22 @@ void do_illegal_instruction(struct pt_regs *regs) info.si_addr = (void __user *)pc; info.si_trapno = 0; force_sig_info(SIGILL, &info, current); +out: + exception_exit(prev_state); } -extern void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); - void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "memory address unaligned", regs, 0, 0x34, SIGSEGV) == NOTIFY_STOP) - return; + goto out; if (regs->tstate & TSTATE_PRIV) { kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); - return; + goto out; } info.si_signo = SIGBUS; info.si_errno = 0; @@ -2345,6 +2502,8 @@ void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned lo info.si_addr = (void __user *)sfar; info.si_trapno = 0; force_sig_info(SIGBUS, &info, current); +out: + exception_exit(prev_state); } void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) @@ -2369,11 +2528,12 @@ void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_c void do_privop(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); siginfo_t info; if (notify_die(DIE_TRAP, "privileged operation", regs, 0, 0x11, SIGILL) == NOTIFY_STOP) - return; + goto out; if (test_thread_flag(TIF_32BIT)) { regs->tpc &= 0xffffffff; @@ -2385,6 +2545,8 @@ void do_privop(struct pt_regs *regs) info.si_addr = (void __user *)regs->tpc; info.si_trapno = 0; force_sig_info(SIGILL, &info, current); +out: + exception_exit(prev_state); } void do_privact(struct pt_regs *regs) @@ -2395,99 +2557,116 @@ void do_privact(struct pt_regs *regs) /* Trap level 1 stuff or other traps we should never see... */ void do_cee(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Cache Error Exception", regs); } void do_cee_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Cache Error Exception", regs); } void do_dae_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Data Access Exception", regs); } void do_iae_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Instruction Access Exception", regs); } void do_div0_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: DIV0 Exception", regs); } void do_fpdis_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: FPU Disabled", regs); } void do_fpieee_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: FPU IEEE Exception", regs); } void do_fpother_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: FPU Other Exception", regs); } void do_ill_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Illegal Instruction Exception", regs); } void do_irq_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: IRQ Exception", regs); } void do_lddfmna_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: LDDF Exception", regs); } void do_stdfmna_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: STDF Exception", regs); } void do_paw(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Phys Watchpoint Exception", regs); } void do_paw_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Phys Watchpoint Exception", regs); } void do_vaw(struct pt_regs *regs) { + exception_enter(); die_if_kernel("TL0: Virt Watchpoint Exception", regs); } void do_vaw_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Virt Watchpoint Exception", regs); } void do_tof_tl1(struct pt_regs *regs) { + exception_enter(); dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); die_if_kernel("TL1: Tag Overflow Exception", regs); } @@ -2546,8 +2725,8 @@ void __init trap_init(void) TI_PRE_COUNT != offsetof(struct thread_info, preempt_count) || TI_NEW_CHILD != offsetof(struct thread_info, new_child) || - TI_SYS_NOERROR != offsetof(struct thread_info, - syscall_noerror) || + TI_CURRENT_DS != offsetof(struct thread_info, + current_ds) || TI_RESTART_BLOCK != offsetof(struct thread_info, restart_block) || TI_KUNA_REGS != offsetof(struct thread_info, diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index db15d123f05..14158d40ba7 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S @@ -49,7 +49,7 @@ tsb_miss_page_table_walk: /* Before committing to a full page table walk, * check the huge page TSB. */ -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 661: ldx [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5 nop @@ -75,7 +75,7 @@ tsb_miss_page_table_walk: mov 512, %g7 andn %g5, 0x7, %g5 sllx %g7, %g6, %g7 - srlx %g4, HPAGE_SHIFT, %g6 + srlx %g4, REAL_HPAGE_SHIFT, %g6 sub %g7, 1, %g7 and %g6, %g7, %g6 sllx %g6, 4, %g6 @@ -110,12 +110,9 @@ tsb_miss_page_table_walk: tsb_miss_page_table_walk_sun4v_fastpath: USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) - /* Load and check PTE. */ - ldxa [%g5] ASI_PHYS_USE_EC, %g5 - brgez,pn %g5, tsb_do_fault - nop + /* Valid PTE is now in %g5. */ -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) 661: sethi %uhi(_PAGE_SZALL_4U), %g7 sllx %g7, 32, %g7 .section .sun4v_2insn_patch, "ax" @@ -139,12 +136,43 @@ tsb_miss_page_table_walk_sun4v_fastpath: nop /* It is a huge page, use huge page TSB entry address we - * calculated above. + * calculated above. If the huge page TSB has not been + * allocated, setup a trap stack and call hugetlb_setup() + * to do so, then return from the trap to replay the TLB + * miss. + * + * This is necessary to handle the case of transparent huge + * pages where we don't really have a non-atomic context + * in which to allocate the hugepage TSB hash table. When + * the 'mm' faults in the hugepage for the first time, we + * thus handle it here. This also makes sure that we can + * allocate the TSB hash table on the correct NUMA node. */ TRAP_LOAD_TRAP_BLOCK(%g7, %g2) - ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g2 - cmp %g2, -1 - movne %xcc, %g2, %g1 + ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g1 + cmp %g1, -1 + bne,pt %xcc, 60f + nop + +661: rdpr %pstate, %g5 + wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + SET_GL(1) + nop + .previous + + rdpr %tl, %g3 + cmp %g3, 1 + bne,pn %xcc, winfix_trampoline + nop + ba,pt %xcc, etrap + rd %pc, %g7 + call hugetlb_setup + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + 60: #endif diff --git a/arch/sparc/kernel/ttable_32.S b/arch/sparc/kernel/ttable_32.S new file mode 100644 index 00000000000..8a7a96ca676 --- /dev/null +++ b/arch/sparc/kernel/ttable_32.S @@ -0,0 +1,417 @@ +/* The Sparc trap table, bootloader gives us control at _start. */ + __HEAD + + .globl _start +_start: + + .globl _stext +_stext: + + .globl trapbase +trapbase: + +#ifdef CONFIG_SMP +trapbase_cpu0: +#endif +/* We get control passed to us here at t_zero. */ +t_zero: b gokernel; nop; nop; nop; +t_tflt: SRMMU_TFAULT /* Inst. Access Exception */ +t_bins: TRAP_ENTRY(0x2, bad_instruction) /* Illegal Instruction */ +t_pins: TRAP_ENTRY(0x3, priv_instruction) /* Privileged Instruction */ +t_fpd: TRAP_ENTRY(0x4, fpd_trap_handler) /* Floating Point Disabled */ +t_wovf: WINDOW_SPILL /* Window Overflow */ +t_wunf: WINDOW_FILL /* Window Underflow */ +t_mna: TRAP_ENTRY(0x7, mna_handler) /* Memory Address Not Aligned */ +t_fpe: TRAP_ENTRY(0x8, fpe_trap_handler) /* Floating Point Exception */ +t_dflt: SRMMU_DFAULT /* Data Miss Exception */ +t_tio: TRAP_ENTRY(0xa, do_tag_overflow) /* Tagged Instruction Ovrflw */ +t_wpt: TRAP_ENTRY(0xb, do_watchpoint) /* Watchpoint Detected */ +t_badc: BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) +t_irq1: TRAP_ENTRY_INTERRUPT(1) /* IRQ Software/SBUS Level 1 */ +t_irq2: TRAP_ENTRY_INTERRUPT(2) /* IRQ SBUS Level 2 */ +t_irq3: TRAP_ENTRY_INTERRUPT(3) /* IRQ SCSI/DMA/SBUS Level 3 */ +t_irq4: TRAP_ENTRY_INTERRUPT(4) /* IRQ Software Level 4 */ +t_irq5: TRAP_ENTRY_INTERRUPT(5) /* IRQ SBUS/Ethernet Level 5 */ +t_irq6: TRAP_ENTRY_INTERRUPT(6) /* IRQ Software Level 6 */ +t_irq7: TRAP_ENTRY_INTERRUPT(7) /* IRQ Video/SBUS Level 5 */ +t_irq8: TRAP_ENTRY_INTERRUPT(8) /* IRQ SBUS Level 6 */ +t_irq9: TRAP_ENTRY_INTERRUPT(9) /* IRQ SBUS Level 7 */ +t_irq10:TRAP_ENTRY_INTERRUPT(10) /* IRQ Timer #1 (one we use) */ +t_irq11:TRAP_ENTRY_INTERRUPT(11) /* IRQ Floppy Intr. */ +t_irq12:TRAP_ENTRY_INTERRUPT(12) /* IRQ Zilog serial chip */ +t_irq13:TRAP_ENTRY_INTERRUPT(13) /* IRQ Audio Intr. */ +t_irq14:TRAP_ENTRY_INTERRUPT(14) /* IRQ Timer #2 */ + + .globl t_nmi +t_nmi: TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + +t_racc: TRAP_ENTRY(0x20, do_reg_access) /* General Register Access Error */ +t_iacce:BAD_TRAP(0x21) /* Instr Access Error */ +t_bad22:BAD_TRAP(0x22) + BAD_TRAP(0x23) +t_cpdis:TRAP_ENTRY(0x24, do_cp_disabled) /* Co-Processor Disabled */ +t_uflsh:SKIP_TRAP(0x25, unimp_flush) /* Unimplemented FLUSH inst. */ +t_bad26:BAD_TRAP(0x26) BAD_TRAP(0x27) +t_cpexc:TRAP_ENTRY(0x28, do_cp_exception) /* Co-Processor Exception */ +t_dacce:SRMMU_DFAULT /* Data Access Error */ +t_hwdz: TRAP_ENTRY(0x2a, do_hw_divzero) /* Division by zero, you lose... */ +t_dserr:BAD_TRAP(0x2b) /* Data Store Error */ +t_daccm:BAD_TRAP(0x2c) /* Data Access MMU-Miss */ +t_bad2d:BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) +t_bad32:BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) +t_bad37:BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) +t_iaccm:BAD_TRAP(0x3c) /* Instr Access MMU-Miss */ +t_bad3d:BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) BAD_TRAP(0x41) +t_bad42:BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) BAD_TRAP(0x46) +t_bad47:BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) BAD_TRAP(0x4b) +t_bad4c:BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) BAD_TRAP(0x50) +t_bad51:BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) +t_bad56:BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) +t_bad5b:BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) +t_bad60:BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) +t_bad65:BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) +t_bad6a:BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) +t_bad6f:BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) +t_bad74:BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) +t_bad79:BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) +t_bad7e:BAD_TRAP(0x7e) BAD_TRAP(0x7f) +t_bad80:BAD_TRAP(0x80) /* SunOS System Call */ +t_sbkpt:BREAKPOINT_TRAP /* Software Breakpoint/KGDB */ +t_divz: TRAP_ENTRY(0x82, do_hw_divzero) /* Divide by zero trap */ +t_flwin:TRAP_ENTRY(0x83, do_flush_windows) /* Flush Windows Trap */ +t_clwin:BAD_TRAP(0x84) /* Clean Windows Trap */ +t_rchk: BAD_TRAP(0x85) /* Range Check */ +t_funal:BAD_TRAP(0x86) /* Fix Unaligned Access Trap */ +t_iovf: BAD_TRAP(0x87) /* Integer Overflow Trap */ +t_bad88:BAD_TRAP(0x88) /* Slowaris System Call */ +t_bad89:BAD_TRAP(0x89) /* Net-B.S. System Call */ +t_bad8a:BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) BAD_TRAP(0x8d) BAD_TRAP(0x8e) +t_bad8f:BAD_TRAP(0x8f) +t_linux:LINUX_SYSCALL_TRAP /* Linux System Call */ +t_bad91:BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) BAD_TRAP(0x95) +t_bad96:BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) BAD_TRAP(0x9a) +t_bad9b:BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) BAD_TRAP(0x9f) +t_getcc:GETCC_TRAP /* Get Condition Codes */ +t_setcc:SETCC_TRAP /* Set Condition Codes */ +t_getpsr:GETPSR_TRAP /* Get PSR Register */ +t_bada3:BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) +t_bada7:BAD_TRAP(0xa7) +t_bada8:BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) +t_badac:BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) +t_badb1:BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) +t_badb6:BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) +t_badbb:BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) +t_badc0:BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) +t_badc5:BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) +t_badca:BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) +t_badcf:BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) +t_badd4:BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) +t_badd9:BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) +t_badde:BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) +t_bade3:BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) +t_bade8:BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) +t_baded:BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) +t_badf2:BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) +t_badf7:BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) +t_badfc:BAD_TRAP(0xfc) +t_kgdb: KGDB_TRAP(0xfd) +dbtrap: BAD_TRAP(0xfe) /* Debugger/PROM breakpoint #1 */ +dbtrap2:BAD_TRAP(0xff) /* Debugger/PROM breakpoint #2 */ + + .globl end_traptable +end_traptable: + +#ifdef CONFIG_SMP + /* Trap tables for the other cpus. */ + .globl trapbase_cpu1, trapbase_cpu2, trapbase_cpu3 +trapbase_cpu1: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) + BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) BAD_TRAP(0x85) BAD_TRAP(0x86) + BAD_TRAP(0x87) BAD_TRAP(0x88) BAD_TRAP(0x89) + BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP BAD_TRAP(0x91) + BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +trapbase_cpu2: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) + TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) + TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) + TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) + TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) + TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) + TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) + TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) + BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) + BAD_TRAP(0x85) + BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) + BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP BAD_TRAP(0x91) + BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +trapbase_cpu3: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) + TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) + TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) + TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) + TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) + TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) + TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) + TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) BAD_TRAP(0x85) + BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) + BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP + BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +#endif diff --git a/arch/sparc/kernel/ttable.S b/arch/sparc/kernel/ttable_64.S index c6dfdaa29e2..c6dfdaa29e2 100644 --- a/arch/sparc/kernel/ttable.S +++ b/arch/sparc/kernel/ttable_64.S diff --git a/arch/sparc/kernel/una_asm_64.S b/arch/sparc/kernel/una_asm_64.S index be183fe4144..1c8d33228b2 100644 --- a/arch/sparc/kernel/una_asm_64.S +++ b/arch/sparc/kernel/una_asm_64.S @@ -127,7 +127,7 @@ do_int_load: wr %o5, 0x0, %asi retl mov 0, %o0 - .size __do_int_load, .-__do_int_load + .size do_int_load, .-do_int_load .section __ex_table,"a" .word 4b, __retl_efault diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c index 4491f4cb269..c5c61b3c6b5 100644 --- a/arch/sparc/kernel/unaligned_32.c +++ b/arch/sparc/kernel/unaligned_32.c @@ -10,14 +10,16 @@ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/mm.h> -#include <linux/module.h> #include <asm/ptrace.h> #include <asm/processor.h> -#include <asm/system.h> #include <asm/uaccess.h> #include <linux/smp.h> #include <linux/perf_event.h> +#include <asm/setup.h> + +#include "kernel.h" + enum direction { load, /* ld, ldd, ldh, ldsh */ store, /* st, std, sth, stsh */ @@ -247,7 +249,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) unsigned long addr = compute_effective_address(regs, insn); int err; - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch (dir) { case load: err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), @@ -338,7 +340,7 @@ asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn) } addr = compute_effective_address(regs, insn); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch(dir) { case load: err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c index c752c4c479b..62098a89bbb 100644 --- a/arch/sparc/kernel/unaligned_64.c +++ b/arch/sparc/kernel/unaligned_64.c @@ -16,13 +16,18 @@ #include <asm/ptrace.h> #include <asm/pstate.h> #include <asm/processor.h> -#include <asm/system.h> #include <asm/uaccess.h> #include <linux/smp.h> #include <linux/bitops.h> #include <linux/perf_event.h> #include <linux/ratelimit.h> +#include <linux/context_tracking.h> #include <asm/fpumacro.h> +#include <asm/cacheflush.h> +#include <asm/setup.h> + +#include "entry.h" +#include "kernel.h" enum direction { load, /* ld, ldd, ldh, ldsh */ @@ -113,21 +118,24 @@ static inline long sign_extend_imm13(long imm) static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) { - unsigned long value; + unsigned long value, fp; if (reg < 16) return (!reg ? 0 : regs->u_regs[reg]); + + fp = regs->u_regs[UREG_FP]; + if (regs->tstate & TSTATE_PRIV) { struct reg_window *win; - win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window *)(fp + STACK_BIAS); value = win->locals[reg - 16]; - } else if (test_thread_flag(TIF_32BIT)) { + } else if (!test_thread_64bit_stack(fp)) { struct reg_window32 __user *win32; - win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); get_user(value, &win32->locals[reg - 16]); } else { struct reg_window __user *win; - win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window __user *)(fp + STACK_BIAS); get_user(value, &win->locals[reg - 16]); } return value; @@ -135,19 +143,24 @@ static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) { + unsigned long fp; + if (reg < 16) return ®s->u_regs[reg]; + + fp = regs->u_regs[UREG_FP]; + if (regs->tstate & TSTATE_PRIV) { struct reg_window *win; - win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window *)(fp + STACK_BIAS); return &win->locals[reg - 16]; - } else if (test_thread_flag(TIF_32BIT)) { + } else if (!test_thread_64bit_stack(fp)) { struct reg_window32 *win32; - win32 = (struct reg_window32 *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + win32 = (struct reg_window32 *)((unsigned long)((u32)fp)); return (unsigned long *)&win32->locals[reg - 16]; } else { struct reg_window *win; - win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window *)(fp + STACK_BIAS); return &win->locals[reg - 16]; } } @@ -155,17 +168,23 @@ static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) unsigned long compute_effective_address(struct pt_regs *regs, unsigned int insn, unsigned int rd) { + int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; unsigned int rs1 = (insn >> 14) & 0x1f; unsigned int rs2 = insn & 0x1f; - int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + unsigned long addr; if (insn & 0x2000) { maybe_flush_windows(rs1, 0, rd, from_kernel); - return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); + addr = (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); } else { maybe_flush_windows(rs1, rs2, rd, from_kernel); - return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); + addr = (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); } + + if (!from_kernel && test_thread_flag(TIF_32BIT)) + addr &= 0xffffffff; + + return addr; } /* This is just to make gcc think die_if_kernel does return... */ @@ -211,7 +230,7 @@ static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr, default: BUG(); break; - }; + } } return __do_int_store(dst_addr, size, src_val, asi); } @@ -317,7 +336,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) addr = compute_effective_address(regs, insn, ((insn >> 25) & 0x1f)); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, addr); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); switch (asi) { case ASI_NL: case ASI_AIUPL: @@ -328,7 +347,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) case ASI_SNFL: asi &= ~0x08; break; - }; + } switch (dir) { case load: reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs); @@ -351,7 +370,7 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) default: BUG(); break; - }; + } *reg_addr = val_in; } break; @@ -373,18 +392,13 @@ asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) } } -static char popc_helper[] = { -0, 1, 1, 2, 1, 2, 2, 3, -1, 2, 2, 3, 2, 3, 3, 4, -}; - int handle_popc(u32 insn, struct pt_regs *regs) { - u64 value; - int ret, i, rd = ((insn >> 25) & 0x1f); int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + int ret, rd = ((insn >> 25) & 0x1f); + u64 value; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (insn & 0x2000) { maybe_flush_windows(0, 0, rd, from_kernel); value = sign_extend_imm13(insn); @@ -392,21 +406,20 @@ int handle_popc(u32 insn, struct pt_regs *regs) maybe_flush_windows(0, insn & 0x1f, rd, from_kernel); value = fetch_reg(insn & 0x1f, regs); } - for (ret = 0, i = 0; i < 16; i++) { - ret += popc_helper[value & 0xf]; - value >>= 4; - } + ret = hweight64(value); if (rd < 16) { if (rd) regs->u_regs[rd] = ret; } else { - if (test_thread_flag(TIF_32BIT)) { + unsigned long fp = regs->u_regs[UREG_FP]; + + if (!test_thread_64bit_stack(fp)) { struct reg_window32 __user *win32; - win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); put_user(ret, &win32->locals[rd - 16]); } else { struct reg_window __user *win; - win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window __user *)(fp + STACK_BIAS); put_user(ret, &win->locals[rd - 16]); } } @@ -416,9 +429,6 @@ int handle_popc(u32 insn, struct pt_regs *regs) extern void do_fpother(struct pt_regs *regs); extern void do_privact(struct pt_regs *regs); -extern void spitfire_data_access_exception(struct pt_regs *regs, - unsigned long sfsr, - unsigned long sfar); extern void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx); @@ -431,7 +441,7 @@ int handle_ldf_stq(u32 insn, struct pt_regs *regs) int asi = decode_asi(insn, regs); int flag = (freg < 32) ? FPRS_DL : FPRS_DU; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); save_and_clear_fpu(); current_thread_info()->xfsr[0] &= ~0x1c000; @@ -554,7 +564,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; unsigned long *reg; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); maybe_flush_windows(0, 0, rd, from_kernel); reg = fetch_reg_addr(rd, regs); @@ -562,7 +572,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) reg[0] = 0; if ((insn & 0x780000) == 0x180000) reg[1] = 0; - } else if (test_thread_flag(TIF_32BIT)) { + } else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) { put_user(0, (int __user *) reg); if ((insn & 0x780000) == 0x180000) put_user(0, ((int __user *) reg) + 1); @@ -576,6 +586,7 @@ void handle_ld_nf(u32 insn, struct pt_regs *regs) void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; @@ -586,7 +597,7 @@ void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr if (tstate & TSTATE_PRIV) die_if_kernel("lddfmna from kernel", regs); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { @@ -630,13 +641,16 @@ daex: sun4v_data_access_exception(regs, sfar, sfsr); else spitfire_data_access_exception(regs, sfsr, sfar); - return; + goto out; } advance(regs); +out: + exception_exit(prev_state); } void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) { + enum ctx_state prev_state = exception_enter(); unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; u32 insn; @@ -647,7 +661,7 @@ void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr if (tstate & TSTATE_PRIV) die_if_kernel("stdfmna from kernel", regs); - perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, 0, regs, sfar); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { @@ -678,7 +692,9 @@ daex: sun4v_data_access_exception(regs, sfar, sfsr); else spitfire_data_access_exception(regs, sfsr, sfar); - return; + goto out; } advance(regs); +out: + exception_exit(prev_state); } diff --git a/arch/sparc/kernel/us2e_cpufreq.c b/arch/sparc/kernel/us2e_cpufreq.c deleted file mode 100644 index 8f982b76c71..00000000000 --- a/arch/sparc/kernel/us2e_cpufreq.c +++ /dev/null @@ -1,413 +0,0 @@ -/* us2e_cpufreq.c: UltraSPARC-IIe cpu frequency support - * - * Copyright (C) 2003 David S. Miller (davem@redhat.com) - * - * Many thanks to Dominik Brodowski for fixing up the cpufreq - * infrastructure in order to make this driver easier to implement. - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/cpufreq.h> -#include <linux/threads.h> -#include <linux/slab.h> -#include <linux/delay.h> -#include <linux/init.h> - -#include <asm/asi.h> -#include <asm/timer.h> - -static struct cpufreq_driver *cpufreq_us2e_driver; - -struct us2e_freq_percpu_info { - struct cpufreq_frequency_table table[6]; -}; - -/* Indexed by cpu number. */ -static struct us2e_freq_percpu_info *us2e_freq_table; - -#define HBIRD_MEM_CNTL0_ADDR 0x1fe0000f010UL -#define HBIRD_ESTAR_MODE_ADDR 0x1fe0000f080UL - -/* UltraSPARC-IIe has five dividers: 1, 2, 4, 6, and 8. These are controlled - * in the ESTAR mode control register. - */ -#define ESTAR_MODE_DIV_1 0x0000000000000000UL -#define ESTAR_MODE_DIV_2 0x0000000000000001UL -#define ESTAR_MODE_DIV_4 0x0000000000000003UL -#define ESTAR_MODE_DIV_6 0x0000000000000002UL -#define ESTAR_MODE_DIV_8 0x0000000000000004UL -#define ESTAR_MODE_DIV_MASK 0x0000000000000007UL - -#define MCTRL0_SREFRESH_ENAB 0x0000000000010000UL -#define MCTRL0_REFR_COUNT_MASK 0x0000000000007f00UL -#define MCTRL0_REFR_COUNT_SHIFT 8 -#define MCTRL0_REFR_INTERVAL 7800 -#define MCTRL0_REFR_CLKS_P_CNT 64 - -static unsigned long read_hbreg(unsigned long addr) -{ - unsigned long ret; - - __asm__ __volatile__("ldxa [%1] %2, %0" - : "=&r" (ret) - : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); - return ret; -} - -static void write_hbreg(unsigned long addr, unsigned long val) -{ - __asm__ __volatile__("stxa %0, [%1] %2\n\t" - "membar #Sync" - : /* no outputs */ - : "r" (val), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E) - : "memory"); - if (addr == HBIRD_ESTAR_MODE_ADDR) { - /* Need to wait 16 clock cycles for the PLL to lock. */ - udelay(1); - } -} - -static void self_refresh_ctl(int enable) -{ - unsigned long mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR); - - if (enable) - mctrl |= MCTRL0_SREFRESH_ENAB; - else - mctrl &= ~MCTRL0_SREFRESH_ENAB; - write_hbreg(HBIRD_MEM_CNTL0_ADDR, mctrl); - (void) read_hbreg(HBIRD_MEM_CNTL0_ADDR); -} - -static void frob_mem_refresh(int cpu_slowing_down, - unsigned long clock_tick, - unsigned long old_divisor, unsigned long divisor) -{ - unsigned long old_refr_count, refr_count, mctrl; - - refr_count = (clock_tick * MCTRL0_REFR_INTERVAL); - refr_count /= (MCTRL0_REFR_CLKS_P_CNT * divisor * 1000000000UL); - - mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR); - old_refr_count = (mctrl & MCTRL0_REFR_COUNT_MASK) - >> MCTRL0_REFR_COUNT_SHIFT; - - mctrl &= ~MCTRL0_REFR_COUNT_MASK; - mctrl |= refr_count << MCTRL0_REFR_COUNT_SHIFT; - write_hbreg(HBIRD_MEM_CNTL0_ADDR, mctrl); - mctrl = read_hbreg(HBIRD_MEM_CNTL0_ADDR); - - if (cpu_slowing_down && !(mctrl & MCTRL0_SREFRESH_ENAB)) { - unsigned long usecs; - - /* We have to wait for both refresh counts (old - * and new) to go to zero. - */ - usecs = (MCTRL0_REFR_CLKS_P_CNT * - (refr_count + old_refr_count) * - 1000000UL * - old_divisor) / clock_tick; - udelay(usecs + 1UL); - } -} - -static void us2e_transition(unsigned long estar, unsigned long new_bits, - unsigned long clock_tick, - unsigned long old_divisor, unsigned long divisor) -{ - unsigned long flags; - - local_irq_save(flags); - - estar &= ~ESTAR_MODE_DIV_MASK; - - /* This is based upon the state transition diagram in the IIe manual. */ - if (old_divisor == 2 && divisor == 1) { - self_refresh_ctl(0); - write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits); - frob_mem_refresh(0, clock_tick, old_divisor, divisor); - } else if (old_divisor == 1 && divisor == 2) { - frob_mem_refresh(1, clock_tick, old_divisor, divisor); - write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits); - self_refresh_ctl(1); - } else if (old_divisor == 1 && divisor > 2) { - us2e_transition(estar, ESTAR_MODE_DIV_2, clock_tick, - 1, 2); - us2e_transition(estar, new_bits, clock_tick, - 2, divisor); - } else if (old_divisor > 2 && divisor == 1) { - us2e_transition(estar, ESTAR_MODE_DIV_2, clock_tick, - old_divisor, 2); - us2e_transition(estar, new_bits, clock_tick, - 2, divisor); - } else if (old_divisor < divisor) { - frob_mem_refresh(0, clock_tick, old_divisor, divisor); - write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits); - } else if (old_divisor > divisor) { - write_hbreg(HBIRD_ESTAR_MODE_ADDR, estar | new_bits); - frob_mem_refresh(1, clock_tick, old_divisor, divisor); - } else { - BUG(); - } - - local_irq_restore(flags); -} - -static unsigned long index_to_estar_mode(unsigned int index) -{ - switch (index) { - case 0: - return ESTAR_MODE_DIV_1; - - case 1: - return ESTAR_MODE_DIV_2; - - case 2: - return ESTAR_MODE_DIV_4; - - case 3: - return ESTAR_MODE_DIV_6; - - case 4: - return ESTAR_MODE_DIV_8; - - default: - BUG(); - }; -} - -static unsigned long index_to_divisor(unsigned int index) -{ - switch (index) { - case 0: - return 1; - - case 1: - return 2; - - case 2: - return 4; - - case 3: - return 6; - - case 4: - return 8; - - default: - BUG(); - }; -} - -static unsigned long estar_to_divisor(unsigned long estar) -{ - unsigned long ret; - - switch (estar & ESTAR_MODE_DIV_MASK) { - case ESTAR_MODE_DIV_1: - ret = 1; - break; - case ESTAR_MODE_DIV_2: - ret = 2; - break; - case ESTAR_MODE_DIV_4: - ret = 4; - break; - case ESTAR_MODE_DIV_6: - ret = 6; - break; - case ESTAR_MODE_DIV_8: - ret = 8; - break; - default: - BUG(); - }; - - return ret; -} - -static unsigned int us2e_freq_get(unsigned int cpu) -{ - cpumask_t cpus_allowed; - unsigned long clock_tick, estar; - - if (!cpu_online(cpu)) - return 0; - - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - clock_tick = sparc64_get_clock_tick(cpu) / 1000; - estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR); - - set_cpus_allowed_ptr(current, &cpus_allowed); - - return clock_tick / estar_to_divisor(estar); -} - -static void us2e_set_cpu_divider_index(unsigned int cpu, unsigned int index) -{ - unsigned long new_bits, new_freq; - unsigned long clock_tick, divisor, old_divisor, estar; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; - - if (!cpu_online(cpu)) - return; - - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - new_freq = clock_tick = sparc64_get_clock_tick(cpu) / 1000; - new_bits = index_to_estar_mode(index); - divisor = index_to_divisor(index); - new_freq /= divisor; - - estar = read_hbreg(HBIRD_ESTAR_MODE_ADDR); - - old_divisor = estar_to_divisor(estar); - - freqs.old = clock_tick / old_divisor; - freqs.new = new_freq; - freqs.cpu = cpu; - cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); - - if (old_divisor != divisor) - us2e_transition(estar, new_bits, clock_tick * 1000, - old_divisor, divisor); - - cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); -} - -static int us2e_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int new_index = 0; - - if (cpufreq_frequency_table_target(policy, - &us2e_freq_table[policy->cpu].table[0], - target_freq, relation, &new_index)) - return -EINVAL; - - us2e_set_cpu_divider_index(policy->cpu, new_index); - - return 0; -} - -static int us2e_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &us2e_freq_table[policy->cpu].table[0]); -} - -static int __init us2e_freq_cpu_init(struct cpufreq_policy *policy) -{ - unsigned int cpu = policy->cpu; - unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000; - struct cpufreq_frequency_table *table = - &us2e_freq_table[cpu].table[0]; - - table[0].index = 0; - table[0].frequency = clock_tick / 1; - table[1].index = 1; - table[1].frequency = clock_tick / 2; - table[2].index = 2; - table[2].frequency = clock_tick / 4; - table[2].index = 3; - table[2].frequency = clock_tick / 6; - table[2].index = 4; - table[2].frequency = clock_tick / 8; - table[2].index = 5; - table[3].frequency = CPUFREQ_TABLE_END; - - policy->cpuinfo.transition_latency = 0; - policy->cur = clock_tick; - - return cpufreq_frequency_table_cpuinfo(policy, table); -} - -static int us2e_freq_cpu_exit(struct cpufreq_policy *policy) -{ - if (cpufreq_us2e_driver) - us2e_set_cpu_divider_index(policy->cpu, 0); - - return 0; -} - -static int __init us2e_freq_init(void) -{ - unsigned long manuf, impl, ver; - int ret; - - if (tlb_type != spitfire) - return -ENODEV; - - __asm__("rdpr %%ver, %0" : "=r" (ver)); - manuf = ((ver >> 48) & 0xffff); - impl = ((ver >> 32) & 0xffff); - - if (manuf == 0x17 && impl == 0x13) { - struct cpufreq_driver *driver; - - ret = -ENOMEM; - driver = kzalloc(sizeof(struct cpufreq_driver), GFP_KERNEL); - if (!driver) - goto err_out; - - us2e_freq_table = kzalloc( - (NR_CPUS * sizeof(struct us2e_freq_percpu_info)), - GFP_KERNEL); - if (!us2e_freq_table) - goto err_out; - - driver->init = us2e_freq_cpu_init; - driver->verify = us2e_freq_verify; - driver->target = us2e_freq_target; - driver->get = us2e_freq_get; - driver->exit = us2e_freq_cpu_exit; - driver->owner = THIS_MODULE, - strcpy(driver->name, "UltraSPARC-IIe"); - - cpufreq_us2e_driver = driver; - ret = cpufreq_register_driver(driver); - if (ret) - goto err_out; - - return 0; - -err_out: - if (driver) { - kfree(driver); - cpufreq_us2e_driver = NULL; - } - kfree(us2e_freq_table); - us2e_freq_table = NULL; - return ret; - } - - return -ENODEV; -} - -static void __exit us2e_freq_exit(void) -{ - if (cpufreq_us2e_driver) { - cpufreq_unregister_driver(cpufreq_us2e_driver); - kfree(cpufreq_us2e_driver); - cpufreq_us2e_driver = NULL; - kfree(us2e_freq_table); - us2e_freq_table = NULL; - } -} - -MODULE_AUTHOR("David S. Miller <davem@redhat.com>"); -MODULE_DESCRIPTION("cpufreq driver for UltraSPARC-IIe"); -MODULE_LICENSE("GPL"); - -module_init(us2e_freq_init); -module_exit(us2e_freq_exit); diff --git a/arch/sparc/kernel/us3_cpufreq.c b/arch/sparc/kernel/us3_cpufreq.c deleted file mode 100644 index f35d1e79454..00000000000 --- a/arch/sparc/kernel/us3_cpufreq.c +++ /dev/null @@ -1,274 +0,0 @@ -/* us3_cpufreq.c: UltraSPARC-III cpu frequency support - * - * Copyright (C) 2003 David S. Miller (davem@redhat.com) - * - * Many thanks to Dominik Brodowski for fixing up the cpufreq - * infrastructure in order to make this driver easier to implement. - */ - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/cpufreq.h> -#include <linux/threads.h> -#include <linux/slab.h> -#include <linux/init.h> - -#include <asm/head.h> -#include <asm/timer.h> - -static struct cpufreq_driver *cpufreq_us3_driver; - -struct us3_freq_percpu_info { - struct cpufreq_frequency_table table[4]; -}; - -/* Indexed by cpu number. */ -static struct us3_freq_percpu_info *us3_freq_table; - -/* UltraSPARC-III has three dividers: 1, 2, and 32. These are controlled - * in the Safari config register. - */ -#define SAFARI_CFG_DIV_1 0x0000000000000000UL -#define SAFARI_CFG_DIV_2 0x0000000040000000UL -#define SAFARI_CFG_DIV_32 0x0000000080000000UL -#define SAFARI_CFG_DIV_MASK 0x00000000C0000000UL - -static unsigned long read_safari_cfg(void) -{ - unsigned long ret; - - __asm__ __volatile__("ldxa [%%g0] %1, %0" - : "=&r" (ret) - : "i" (ASI_SAFARI_CONFIG)); - return ret; -} - -static void write_safari_cfg(unsigned long val) -{ - __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" - "membar #Sync" - : /* no outputs */ - : "r" (val), "i" (ASI_SAFARI_CONFIG) - : "memory"); -} - -static unsigned long get_current_freq(unsigned int cpu, unsigned long safari_cfg) -{ - unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000; - unsigned long ret; - - switch (safari_cfg & SAFARI_CFG_DIV_MASK) { - case SAFARI_CFG_DIV_1: - ret = clock_tick / 1; - break; - case SAFARI_CFG_DIV_2: - ret = clock_tick / 2; - break; - case SAFARI_CFG_DIV_32: - ret = clock_tick / 32; - break; - default: - BUG(); - }; - - return ret; -} - -static unsigned int us3_freq_get(unsigned int cpu) -{ - cpumask_t cpus_allowed; - unsigned long reg; - unsigned int ret; - - if (!cpu_online(cpu)) - return 0; - - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - reg = read_safari_cfg(); - ret = get_current_freq(cpu, reg); - - set_cpus_allowed_ptr(current, &cpus_allowed); - - return ret; -} - -static void us3_set_cpu_divider_index(unsigned int cpu, unsigned int index) -{ - unsigned long new_bits, new_freq, reg; - cpumask_t cpus_allowed; - struct cpufreq_freqs freqs; - - if (!cpu_online(cpu)) - return; - - cpus_allowed = current->cpus_allowed; - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - - new_freq = sparc64_get_clock_tick(cpu) / 1000; - switch (index) { - case 0: - new_bits = SAFARI_CFG_DIV_1; - new_freq /= 1; - break; - case 1: - new_bits = SAFARI_CFG_DIV_2; - new_freq /= 2; - break; - case 2: - new_bits = SAFARI_CFG_DIV_32; - new_freq /= 32; - break; - - default: - BUG(); - }; - - reg = read_safari_cfg(); - - freqs.old = get_current_freq(cpu, reg); - freqs.new = new_freq; - freqs.cpu = cpu; - cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); - - reg &= ~SAFARI_CFG_DIV_MASK; - reg |= new_bits; - write_safari_cfg(reg); - - cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); - - set_cpus_allowed_ptr(current, &cpus_allowed); -} - -static int us3_freq_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - unsigned int new_index = 0; - - if (cpufreq_frequency_table_target(policy, - &us3_freq_table[policy->cpu].table[0], - target_freq, - relation, - &new_index)) - return -EINVAL; - - us3_set_cpu_divider_index(policy->cpu, new_index); - - return 0; -} - -static int us3_freq_verify(struct cpufreq_policy *policy) -{ - return cpufreq_frequency_table_verify(policy, - &us3_freq_table[policy->cpu].table[0]); -} - -static int __init us3_freq_cpu_init(struct cpufreq_policy *policy) -{ - unsigned int cpu = policy->cpu; - unsigned long clock_tick = sparc64_get_clock_tick(cpu) / 1000; - struct cpufreq_frequency_table *table = - &us3_freq_table[cpu].table[0]; - - table[0].index = 0; - table[0].frequency = clock_tick / 1; - table[1].index = 1; - table[1].frequency = clock_tick / 2; - table[2].index = 2; - table[2].frequency = clock_tick / 32; - table[3].index = 0; - table[3].frequency = CPUFREQ_TABLE_END; - - policy->cpuinfo.transition_latency = 0; - policy->cur = clock_tick; - - return cpufreq_frequency_table_cpuinfo(policy, table); -} - -static int us3_freq_cpu_exit(struct cpufreq_policy *policy) -{ - if (cpufreq_us3_driver) - us3_set_cpu_divider_index(policy->cpu, 0); - - return 0; -} - -static int __init us3_freq_init(void) -{ - unsigned long manuf, impl, ver; - int ret; - - if (tlb_type != cheetah && tlb_type != cheetah_plus) - return -ENODEV; - - __asm__("rdpr %%ver, %0" : "=r" (ver)); - manuf = ((ver >> 48) & 0xffff); - impl = ((ver >> 32) & 0xffff); - - if (manuf == CHEETAH_MANUF && - (impl == CHEETAH_IMPL || - impl == CHEETAH_PLUS_IMPL || - impl == JAGUAR_IMPL || - impl == PANTHER_IMPL)) { - struct cpufreq_driver *driver; - - ret = -ENOMEM; - driver = kzalloc(sizeof(struct cpufreq_driver), GFP_KERNEL); - if (!driver) - goto err_out; - - us3_freq_table = kzalloc( - (NR_CPUS * sizeof(struct us3_freq_percpu_info)), - GFP_KERNEL); - if (!us3_freq_table) - goto err_out; - - driver->init = us3_freq_cpu_init; - driver->verify = us3_freq_verify; - driver->target = us3_freq_target; - driver->get = us3_freq_get; - driver->exit = us3_freq_cpu_exit; - driver->owner = THIS_MODULE, - strcpy(driver->name, "UltraSPARC-III"); - - cpufreq_us3_driver = driver; - ret = cpufreq_register_driver(driver); - if (ret) - goto err_out; - - return 0; - -err_out: - if (driver) { - kfree(driver); - cpufreq_us3_driver = NULL; - } - kfree(us3_freq_table); - us3_freq_table = NULL; - return ret; - } - - return -ENODEV; -} - -static void __exit us3_freq_exit(void) -{ - if (cpufreq_us3_driver) { - cpufreq_unregister_driver(cpufreq_us3_driver); - kfree(cpufreq_us3_driver); - cpufreq_us3_driver = NULL; - kfree(us3_freq_table); - us3_freq_table = NULL; - } -} - -MODULE_AUTHOR("David S. Miller <davem@redhat.com>"); -MODULE_DESCRIPTION("cpufreq driver for UltraSPARC-III"); -MODULE_LICENSE("GPL"); - -module_init(us3_freq_init); -module_exit(us3_freq_exit); diff --git a/arch/sparc/kernel/vio.c b/arch/sparc/kernel/vio.c index 3cb1def9806..8647fcc5ca6 100644 --- a/arch/sparc/kernel/vio.c +++ b/arch/sparc/kernel/vio.c @@ -12,6 +12,7 @@ #include <linux/kernel.h> #include <linux/slab.h> #include <linux/irq.h> +#include <linux/export.h> #include <linux/init.h> #include <asm/mdesc.h> @@ -118,13 +119,17 @@ static struct bus_type vio_bus_type = { .remove = vio_device_remove, }; -int vio_register_driver(struct vio_driver *viodrv) +int __vio_register_driver(struct vio_driver *viodrv, struct module *owner, + const char *mod_name) { viodrv->driver.bus = &vio_bus_type; + viodrv->driver.name = viodrv->name; + viodrv->driver.owner = owner; + viodrv->driver.mod_name = mod_name; return driver_register(&viodrv->driver); } -EXPORT_SYMBOL(vio_register_driver); +EXPORT_SYMBOL(__vio_register_driver); void vio_unregister_driver(struct vio_driver *viodrv) { @@ -337,6 +342,7 @@ static void vio_remove(struct mdesc_handle *hp, u64 node) printk(KERN_INFO "VIO: Removing device %s\n", dev_name(dev)); device_unregister(dev); + put_device(dev); } } @@ -438,7 +444,7 @@ static int __init vio_init(void) root_vdev = vio_create_one(hp, root, NULL); err = -ENODEV; if (!root_vdev) { - printk(KERN_ERR "VIO: Coult not create root device.\n"); + printk(KERN_ERR "VIO: Could not create root device.\n"); goto out_release; } diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c index aa6ac70d4fd..f8e7dd53e1c 100644 --- a/arch/sparc/kernel/viohs.c +++ b/arch/sparc/kernel/viohs.c @@ -4,7 +4,7 @@ */ #include <linux/kernel.h> -#include <linux/module.h> +#include <linux/export.h> #include <linux/string.h> #include <linux/delay.h> #include <linux/sched.h> @@ -363,7 +363,7 @@ static int process_ver(struct vio_driver_state *vio, struct vio_ver_info *pkt) default: return handshake_failure(vio); - }; + } } static int process_attr(struct vio_driver_state *vio, void *pkt) diff --git a/arch/sparc/kernel/visemul.c b/arch/sparc/kernel/visemul.c index 9dfd2ebcb15..c096c624ac4 100644 --- a/arch/sparc/kernel/visemul.c +++ b/arch/sparc/kernel/visemul.c @@ -9,9 +9,9 @@ #include <asm/ptrace.h> #include <asm/pstate.h> -#include <asm/system.h> #include <asm/fpumacro.h> #include <asm/uaccess.h> +#include <asm/cacheflush.h> /* OPF field of various VIS instructions. */ @@ -149,21 +149,24 @@ static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) { - unsigned long value; + unsigned long value, fp; if (reg < 16) return (!reg ? 0 : regs->u_regs[reg]); + + fp = regs->u_regs[UREG_FP]; + if (regs->tstate & TSTATE_PRIV) { struct reg_window *win; - win = (struct reg_window *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window *)(fp + STACK_BIAS); value = win->locals[reg - 16]; - } else if (test_thread_flag(TIF_32BIT)) { + } else if (!test_thread_64bit_stack(fp)) { struct reg_window32 __user *win32; - win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); get_user(value, &win32->locals[reg - 16]); } else { struct reg_window __user *win; - win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window __user *)(fp + STACK_BIAS); get_user(value, &win->locals[reg - 16]); } return value; @@ -172,16 +175,18 @@ static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) static inline unsigned long __user *__fetch_reg_addr_user(unsigned int reg, struct pt_regs *regs) { + unsigned long fp = regs->u_regs[UREG_FP]; + BUG_ON(reg < 16); BUG_ON(regs->tstate & TSTATE_PRIV); - if (test_thread_flag(TIF_32BIT)) { + if (!test_thread_64bit_stack(fp)) { struct reg_window32 __user *win32; - win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); return (unsigned long __user *)&win32->locals[reg - 16]; } else { struct reg_window __user *win; - win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); + win = (struct reg_window __user *)(fp + STACK_BIAS); return &win->locals[reg - 16]; } } @@ -204,7 +209,7 @@ static void store_reg(struct pt_regs *regs, unsigned long val, unsigned long rd) } else { unsigned long __user *rd_user = __fetch_reg_addr_user(rd, regs); - if (test_thread_flag(TIF_32BIT)) + if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) __put_user((u32)val, (u32 __user *)rd_user); else __put_user(val, rd_user); @@ -334,7 +339,7 @@ static void edge(struct pt_regs *regs, unsigned int insn, unsigned int opf) left = edge32_tab_l[(rs1 >> 2) & 0x1].left; right = edge32_tab_l[(rs2 >> 2) & 0x1].right; break; - }; + } if ((rs1 & ~0x7UL) == (rs2 & ~0x7UL)) rd_val = right & left; @@ -360,7 +365,7 @@ static void edge(struct pt_regs *regs, unsigned int insn, unsigned int opf) tstate = regs->tstate & ~(TSTATE_XCC | TSTATE_ICC); regs->tstate = tstate | (ccr << 32UL); } - }; + } } static void array(struct pt_regs *regs, unsigned int insn, unsigned int opf) @@ -392,7 +397,7 @@ static void array(struct pt_regs *regs, unsigned int insn, unsigned int opf) case ARRAY32_OPF: rd_val <<= 2; - }; + } store_reg(regs, rd_val, RD(insn)); } @@ -577,7 +582,7 @@ static void pformat(struct pt_regs *regs, unsigned int insn, unsigned int opf) *fpd_regaddr(f, RD(insn)) = rd_val; break; } - }; + } } static void pmul(struct pt_regs *regs, unsigned int insn, unsigned int opf) @@ -693,7 +698,7 @@ static void pmul(struct pt_regs *regs, unsigned int insn, unsigned int opf) *fpd_regaddr(f, RD(insn)) = rd_val; break; } - }; + } } static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) @@ -713,17 +718,17 @@ static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) s16 b = (rs2 >> (i * 16)) & 0xffff; if (a > b) - rd_val |= 1 << i; + rd_val |= 8 >> i; } break; case FCMPGT32_OPF: for (i = 0; i < 2; i++) { - s32 a = (rs1 >> (i * 32)) & 0xffff; - s32 b = (rs2 >> (i * 32)) & 0xffff; + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; if (a > b) - rd_val |= 1 << i; + rd_val |= 2 >> i; } break; @@ -733,17 +738,17 @@ static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) s16 b = (rs2 >> (i * 16)) & 0xffff; if (a <= b) - rd_val |= 1 << i; + rd_val |= 8 >> i; } break; case FCMPLE32_OPF: for (i = 0; i < 2; i++) { - s32 a = (rs1 >> (i * 32)) & 0xffff; - s32 b = (rs2 >> (i * 32)) & 0xffff; + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; if (a <= b) - rd_val |= 1 << i; + rd_val |= 2 >> i; } break; @@ -753,17 +758,17 @@ static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) s16 b = (rs2 >> (i * 16)) & 0xffff; if (a != b) - rd_val |= 1 << i; + rd_val |= 8 >> i; } break; case FCMPNE32_OPF: for (i = 0; i < 2; i++) { - s32 a = (rs1 >> (i * 32)) & 0xffff; - s32 b = (rs2 >> (i * 32)) & 0xffff; + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; if (a != b) - rd_val |= 1 << i; + rd_val |= 2 >> i; } break; @@ -773,20 +778,20 @@ static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) s16 b = (rs2 >> (i * 16)) & 0xffff; if (a == b) - rd_val |= 1 << i; + rd_val |= 8 >> i; } break; case FCMPEQ32_OPF: for (i = 0; i < 2; i++) { - s32 a = (rs1 >> (i * 32)) & 0xffff; - s32 b = (rs2 >> (i * 32)) & 0xffff; + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; if (a == b) - rd_val |= 1 << i; + rd_val |= 2 >> i; } break; - }; + } maybe_flush_windows(0, 0, RD(insn), 0); store_reg(regs, rd_val, RD(insn)); @@ -802,7 +807,7 @@ int vis_emul(struct pt_regs *regs, unsigned int insn) BUG_ON(regs->tstate & TSTATE_PRIV); - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; @@ -885,7 +890,7 @@ int vis_emul(struct pt_regs *regs, unsigned int insn) case BSHUFFLE_OPF: bshuffle(regs, insn); break; - }; + } regs->tpc = regs->tnpc; regs->tnpc += 4; diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 0c1e6783657..932ff90fd76 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -107,8 +107,42 @@ SECTIONS *(.sun4v_2insn_patch) __sun4v_2insn_patch_end = .; } - - PERCPU(PAGE_SIZE) + .leon_1insn_patch : { + __leon_1insn_patch = .; + *(.leon_1insn_patch) + __leon_1insn_patch_end = .; + } + .swapper_tsb_phys_patch : { + __swapper_tsb_phys_patch = .; + *(.swapper_tsb_phys_patch) + __swapper_tsb_phys_patch_end = .; + } + .swapper_4m_tsb_phys_patch : { + __swapper_4m_tsb_phys_patch = .; + *(.swapper_4m_tsb_phys_patch) + __swapper_4m_tsb_phys_patch_end = .; + } + .page_offset_shift_patch : { + __page_offset_shift_patch = .; + *(.page_offset_shift_patch) + __page_offset_shift_patch_end = .; + } + .popc_3insn_patch : { + __popc_3insn_patch = .; + *(.popc_3insn_patch) + __popc_3insn_patch_end = .; + } + .popc_6insn_patch : { + __popc_6insn_patch = .; + *(.popc_6insn_patch) + __popc_6insn_patch_end = .; + } + .pause_3insn_patch : { + __pause_3insn_patch = .; + *(.pause_3insn_patch) + __pause_3insn_patch_end = .; + } + PERCPU_SECTION(SMP_CACHE_BYTES) . = ALIGN(PAGE_SIZE); __init_end = .; diff --git a/arch/sparc/kernel/windows.c b/arch/sparc/kernel/windows.c index 3107381e576..87bab0a3857 100644 --- a/arch/sparc/kernel/windows.c +++ b/arch/sparc/kernel/windows.c @@ -10,8 +10,11 @@ #include <linux/mm.h> #include <linux/smp.h> +#include <asm/cacheflush.h> #include <asm/uaccess.h> +#include "kernel.h" + /* Do save's until all user register windows are out of the cpu. */ void flush_user_windows(void) { diff --git a/arch/sparc/kernel/winfixup.S b/arch/sparc/kernel/winfixup.S index a6b0863c27d..1e67ce95836 100644 --- a/arch/sparc/kernel/winfixup.S +++ b/arch/sparc/kernel/winfixup.S @@ -43,6 +43,8 @@ spill_fixup_mna: spill_fixup_dax: TRAP_LOAD_THREAD_REG(%g6, %g1) ldx [%g6 + TI_FLAGS], %g1 + andcc %sp, 0x1, %g0 + movne %icc, 0, %g1 andcc %g1, _TIF_32BIT, %g0 ldub [%g6 + TI_WSAVED], %g1 sll %g1, 3, %g3 diff --git a/arch/sparc/kernel/wof.S b/arch/sparc/kernel/wof.S index 3bbcd8dc9ab..28a7bc69f82 100644 --- a/arch/sparc/kernel/wof.S +++ b/arch/sparc/kernel/wof.S @@ -163,9 +163,8 @@ spwin_fromuser: * the label 'spwin_user_stack_is_bolixed' which will take * care of things at that point. */ - .globl spwin_mmu_patchme -spwin_mmu_patchme: b spwin_sun4c_stackchk - andcc %sp, 0x7, %g0 + b spwin_srmmu_stackchk + andcc %sp, 0x7, %g0 spwin_good_ustack: /* LOCATION: Window to be saved */ @@ -306,73 +305,6 @@ spwin_bad_ustack_from_kernel: * As noted above %curptr cannot be touched by this routine at all. */ -spwin_sun4c_stackchk: - /* LOCATION: Window to be saved on the stack */ - - /* See if the stack is in the address space hole but first, - * check results of callers andcc %sp, 0x7, %g0 - */ - be 1f - sra %sp, 29, %glob_tmp - - rd %psr, %glob_tmp - b spwin_user_stack_is_bolixed + 0x4 - nop - -1: - add %glob_tmp, 0x1, %glob_tmp - andncc %glob_tmp, 0x1, %g0 - be 1f - and %sp, 0xfff, %glob_tmp ! delay slot - - rd %psr, %glob_tmp - b spwin_user_stack_is_bolixed + 0x4 - nop - - /* See if our dump area will be on more than one - * page. - */ -1: - add %glob_tmp, 0x38, %glob_tmp - andncc %glob_tmp, 0xff8, %g0 - be spwin_sun4c_onepage ! only one page to check - lda [%sp] ASI_PTE, %glob_tmp ! have to check first page anyways - -spwin_sun4c_twopages: - /* Is first page ok permission wise? */ - srl %glob_tmp, 29, %glob_tmp - cmp %glob_tmp, 0x6 - be 1f - add %sp, 0x38, %glob_tmp /* Is second page in vma hole? */ - - rd %psr, %glob_tmp - b spwin_user_stack_is_bolixed + 0x4 - nop - -1: - sra %glob_tmp, 29, %glob_tmp - add %glob_tmp, 0x1, %glob_tmp - andncc %glob_tmp, 0x1, %g0 - be 1f - add %sp, 0x38, %glob_tmp - - rd %psr, %glob_tmp - b spwin_user_stack_is_bolixed + 0x4 - nop - -1: - lda [%glob_tmp] ASI_PTE, %glob_tmp - -spwin_sun4c_onepage: - srl %glob_tmp, 29, %glob_tmp - cmp %glob_tmp, 0x6 ! can user write to it? - be spwin_good_ustack ! success - nop - - rd %psr, %glob_tmp - b spwin_user_stack_is_bolixed + 0x4 - nop - /* This is a generic SRMMU routine. As far as I know this * works for all current v8/srmmu implementations, we'll * see... @@ -400,24 +332,30 @@ spwin_srmmu_stackchk: mov AC_M_SFSR, %glob_tmp /* Clear the fault status and turn on the no_fault bit. */ - lda [%glob_tmp] ASI_M_MMUREGS, %g0 ! eat SFSR +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) ! eat SFSR +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) ! eat SFSR - lda [%g0] ASI_M_MMUREGS, %glob_tmp ! read MMU control +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %glob_tmp) ! read MMU control +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %glob_tmp) ! read MMU control or %glob_tmp, 0x2, %glob_tmp ! or in no_fault bit - sta %glob_tmp, [%g0] ASI_M_MMUREGS ! set it +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) ! set it +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) ! set it /* Dump the registers and cross fingers. */ STORE_WINDOW(sp) /* Clear the no_fault bit and check the status. */ andn %glob_tmp, 0x2, %glob_tmp - sta %glob_tmp, [%g0] ASI_M_MMUREGS +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) mov AC_M_SFAR, %glob_tmp - lda [%glob_tmp] ASI_M_MMUREGS, %g0 +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) mov AC_M_SFSR, %glob_tmp - lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %glob_tmp) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp) andcc %glob_tmp, 0x2, %g0 ! did we fault? be,a spwin_finish_up + 0x4 ! cool beans, success restore %g0, %g0, %g0 diff --git a/arch/sparc/kernel/wuf.S b/arch/sparc/kernel/wuf.S index 779ff750603..2c21cc59683 100644 --- a/arch/sparc/kernel/wuf.S +++ b/arch/sparc/kernel/wuf.S @@ -131,12 +131,9 @@ fwin_from_user: /* LOCATION: Window 'W' */ - /* Branch to the architecture specific stack validation - * routine. They can be found below... - */ - .globl fwin_mmu_patchme -fwin_mmu_patchme: b sun4c_fwin_stackchk - andcc %sp, 0x7, %g0 + /* Branch to the stack validation routine */ + b srmmu_fwin_stackchk + andcc %sp, 0x7, %g0 #define STACK_OFFSET (THREAD_SIZE - TRACEREG_SZ - STACKFRAME_SZ) @@ -242,57 +239,6 @@ fwin_user_finish_up: * 'someone elses' window possibly. */ - .align 4 -sun4c_fwin_stackchk: - /* LOCATION: Window 'W' */ - - /* Caller did 'andcc %sp, 0x7, %g0' */ - be 1f - and %sp, 0xfff, %l0 ! delay slot - - b,a fwin_user_stack_is_bolixed - - /* See if we have to check the sanity of one page or two */ -1: - add %l0, 0x38, %l0 - sra %sp, 29, %l5 - add %l5, 0x1, %l5 - andncc %l5, 0x1, %g0 - be 1f - andncc %l0, 0xff8, %g0 - - b,a fwin_user_stack_is_bolixed /* %sp is in vma hole, yuck */ - -1: - be sun4c_fwin_onepage /* Only one page to check */ - lda [%sp] ASI_PTE, %l1 -sun4c_fwin_twopages: - add %sp, 0x38, %l0 - sra %l0, 29, %l5 - add %l5, 0x1, %l5 - andncc %l5, 0x1, %g0 - be 1f - lda [%l0] ASI_PTE, %l1 - - b,a fwin_user_stack_is_bolixed /* Second page in vma hole */ - -1: - srl %l1, 29, %l1 - andcc %l1, 0x4, %g0 - bne sun4c_fwin_onepage - lda [%sp] ASI_PTE, %l1 - - b,a fwin_user_stack_is_bolixed /* Second page has bad perms */ - -sun4c_fwin_onepage: - srl %l1, 29, %l1 - andcc %l1, 0x4, %g0 - bne fwin_user_stack_is_ok - nop - - /* A page had bad page permissions, losing... */ - b,a fwin_user_stack_is_bolixed - .globl srmmu_fwin_stackchk srmmu_fwin_stackchk: /* LOCATION: Window 'W' */ @@ -308,16 +254,19 @@ srmmu_fwin_stackchk: mov AC_M_SFSR, %l4 cmp %l5, %sp bleu fwin_user_stack_is_bolixed - lda [%l4] ASI_M_MMUREGS, %g0 ! clear fault status +LEON_PI( lda [%l4] ASI_LEON_MMUREGS, %g0) ! clear fault status +SUN_PI_( lda [%l4] ASI_M_MMUREGS, %g0) ! clear fault status /* The technique is, turn off faults on this processor, * just let the load rip, then check the sfsr to see if * a fault did occur. Then we turn on fault traps again * and branch conditionally based upon what happened. */ - lda [%g0] ASI_M_MMUREGS, %l5 ! read mmu-ctrl reg +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %l5) ! read mmu-ctrl reg +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %l5) ! read mmu-ctrl reg or %l5, 0x2, %l5 ! turn on no-fault bit - sta %l5, [%g0] ASI_M_MMUREGS ! store it +LEON_PI(sta %l5, [%g0] ASI_LEON_MMUREGS) ! store it +SUN_PI_(sta %l5, [%g0] ASI_M_MMUREGS) ! store it /* Cross fingers and go for it. */ LOAD_WINDOW(sp) @@ -329,18 +278,22 @@ srmmu_fwin_stackchk: /* LOCATION: Window 'T' */ - lda [%g0] ASI_M_MMUREGS, %twin_tmp1 ! load mmu-ctrl again - andn %twin_tmp1, 0x2, %twin_tmp1 ! clear no-fault bit - sta %twin_tmp1, [%g0] ASI_M_MMUREGS ! store it +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %twin_tmp1) ! load mmu-ctrl again +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %twin_tmp1) ! load mmu-ctrl again + andn %twin_tmp1, 0x2, %twin_tmp1 ! clear no-fault bit +LEON_PI(sta %twin_tmp1, [%g0] ASI_LEON_MMUREGS) ! store it +SUN_PI_(sta %twin_tmp1, [%g0] ASI_M_MMUREGS) ! store it mov AC_M_SFAR, %twin_tmp2 - lda [%twin_tmp2] ASI_M_MMUREGS, %g0 ! read fault address +LEON_PI(lda [%twin_tmp2] ASI_LEON_MMUREGS, %g0) ! read fault address +SUN_PI_(lda [%twin_tmp2] ASI_M_MMUREGS, %g0) ! read fault address mov AC_M_SFSR, %twin_tmp2 - lda [%twin_tmp2] ASI_M_MMUREGS, %twin_tmp2 ! read fault status - andcc %twin_tmp2, 0x2, %g0 ! did fault occur? +LEON_PI(lda [%twin_tmp2] ASI_LEON_MMUREGS, %twin_tmp2) ! read fault status +SUN_PI_(lda [%twin_tmp2] ASI_M_MMUREGS, %twin_tmp2) ! read fault status + andcc %twin_tmp2, 0x2, %g0 ! did fault occur? - bne 1f ! yep, cleanup + bne 1f ! yep, cleanup nop wr %t_psr, 0x0, %psr diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 846d1c4374e..3269b023409 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -4,18 +4,16 @@ asflags-y := -ansi -DST_DIV0=0x02 ccflags-y := -Werror -lib-$(CONFIG_SPARC32) += mul.o rem.o sdiv.o udiv.o umul.o urem.o ashrdi3.o +lib-$(CONFIG_SPARC32) += ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o lib-y += strlen.o lib-y += checksum_$(BITS).o lib-$(CONFIG_SPARC32) += blockops.o lib-y += memscan_$(BITS).o memcmp.o strncmp_$(BITS).o -lib-y += strncpy_from_user_$(BITS).o strlen_user_$(BITS).o lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o lib-$(CONFIG_SPARC32) += copy_user.o locks.o -lib-y += atomic_$(BITS).o +lib-$(CONFIG_SPARC64) += atomic_64.o lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o -lib-$(CONFIG_SPARC32) += rwsem_32.o lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o @@ -32,16 +30,18 @@ lib-$(CONFIG_SPARC64) += NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o -lib-$(CONFIG_SPARC64) += NG2patch.o NG2page.o +lib-$(CONFIG_SPARC64) += NG2patch.o + +lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o +lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o lib-$(CONFIG_SPARC64) += copy_in_user.o user_fixup.o memmove.o -lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o +lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o -obj-y += iomap.o -obj-$(CONFIG_SPARC32) += atomic32.o +obj-$(CONFIG_SPARC64) += iomap.o +obj-$(CONFIG_SPARC32) += atomic32.o ucmpdi2.o obj-y += ksyms.o obj-$(CONFIG_SPARC64) += PeeCeeI.o -obj-y += usercopy.o diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index 0aed75653b5..30eee6e8a81 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S @@ -14,7 +14,7 @@ #define FPRS_FEF 0x04 #ifdef MEMCPY_DEBUG #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ - clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; + clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; #define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs #else #define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs @@ -90,49 +90,49 @@ faligndata %x7, %x8, %f14; #define FREG_MOVE_1(x0) \ - fmovd %x0, %f0; + fsrc2 %x0, %f0; #define FREG_MOVE_2(x0, x1) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; #define FREG_MOVE_3(x0, x1, x2) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; #define FREG_MOVE_4(x0, x1, x2, x3) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; #define FREG_MOVE_5(x0, x1, x2, x3, x4) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; #define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; #define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; #define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; \ - fmovd %x7, %f14; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; \ + fsrc2 %x7, %f14; #define FREG_LOAD_1(base, x0) \ EX_LD(LOAD(ldd, base + 0x00, %x0)) #define FREG_LOAD_2(base, x0, x1) \ @@ -182,13 +182,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ cmp %g2, 0 tne %xcc, 5 PREAMBLE - mov %o0, GLOBAL_SPARE + mov %o0, %o3 cmp %o2, 0 be,pn %XCC, 85f - or %o0, %o1, %o3 + or %o0, %o1, GLOBAL_SPARE cmp %o2, 16 blu,a,pn %XCC, 80f - or %o3, %o2, %o3 + or GLOBAL_SPARE, %o2, GLOBAL_SPARE /* 2 blocks (128 bytes) is the minimum we can do the block * copy with. We need to ensure that we'll iterate at least @@ -202,7 +202,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ */ cmp %o2, (4 * 64) blu,pt %XCC, 75f - andcc %o3, 0x7, %g0 + andcc GLOBAL_SPARE, 0x7, %g0 /* %o0: dst * %o1: src @@ -236,6 +236,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ */ VISEntryHalf + membar #Sync alignaddr %o1, %g0, %g0 add %o1, (64 - 1), %o4 @@ -404,13 +405,13 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ * over. If anything is left, we copy it one byte at a time. */ brz,pt %o2, 85f - sub %o0, %o1, %o3 + sub %o0, %o1, GLOBAL_SPARE ba,a,pt %XCC, 90f .align 64 75: /* 16 < len <= 64 */ bne,pn %XCC, 75f - sub %o0, %o1, %o3 + sub %o0, %o1, GLOBAL_SPARE 72: andn %o2, 0xf, %o4 @@ -420,9 +421,9 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ add %o1, 0x08, %o1 EX_LD(LOAD(ldx, %o1, %g1)) sub %o1, 0x08, %o1 - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) add %o1, 0x8, %o1 - EX_ST(STORE(stx, %g1, %o1 + %o3)) + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE)) bgu,pt %XCC, 1b add %o1, 0x8, %o1 73: andcc %o2, 0x8, %g0 @@ -430,14 +431,14 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop sub %o2, 0x8, %o2 EX_LD(LOAD(ldx, %o1, %o5)) - EX_ST(STORE(stx, %o5, %o1 + %o3)) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE)) add %o1, 0x8, %o1 1: andcc %o2, 0x4, %g0 be,pt %XCC, 1f nop sub %o2, 0x4, %o2 EX_LD(LOAD(lduw, %o1, %o5)) - EX_ST(STORE(stw, %o5, %o1 + %o3)) + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE)) add %o1, 0x4, %o1 1: cmp %o2, 0 be,pt %XCC, 85f @@ -454,11 +455,11 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ 1: subcc %g1, 1, %g1 EX_LD(LOAD(ldub, %o1, %o5)) - EX_ST(STORE(stb, %o5, %o1 + %o3)) + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE)) bgu,pt %icc, 1b add %o1, 1, %o1 -2: add %o1, %o3, %o0 +2: add %o1, GLOBAL_SPARE, %o0 andcc %o1, 0x7, %g1 bne,pt %icc, 8f sll %g1, 3, %g1 @@ -468,16 +469,16 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ nop ba,a,pt %xcc, 73b -8: mov 64, %o3 +8: mov 64, GLOBAL_SPARE andn %o1, 0x7, %o1 EX_LD(LOAD(ldx, %o1, %g2)) - sub %o3, %g1, %o3 + sub GLOBAL_SPARE, %g1, GLOBAL_SPARE andn %o2, 0x7, %o4 sllx %g2, %g1, %g2 1: add %o1, 0x8, %o1 EX_LD(LOAD(ldx, %o1, %g3)) subcc %o4, 0x8, %o4 - srlx %g3, %o3, %o5 + srlx %g3, GLOBAL_SPARE, %o5 or %o5, %g2, %o5 EX_ST(STORE(stx, %o5, %o0)) add %o0, 0x8, %o0 @@ -489,32 +490,32 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ be,pn %icc, 85f add %o1, %g1, %o1 ba,pt %xcc, 90f - sub %o0, %o1, %o3 + sub %o0, %o1, GLOBAL_SPARE .align 64 80: /* 0 < len <= 16 */ - andcc %o3, 0x3, %g0 + andcc GLOBAL_SPARE, 0x3, %g0 bne,pn %XCC, 90f - sub %o0, %o1, %o3 + sub %o0, %o1, GLOBAL_SPARE 1: subcc %o2, 4, %o2 EX_LD(LOAD(lduw, %o1, %g1)) - EX_ST(STORE(stw, %g1, %o1 + %o3)) + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE)) bgu,pt %XCC, 1b add %o1, 4, %o1 85: retl - mov EX_RETVAL(GLOBAL_SPARE), %o0 + mov EX_RETVAL(%o3), %o0 .align 32 90: subcc %o2, 1, %o2 EX_LD(LOAD(ldub, %o1, %g1)) - EX_ST(STORE(stb, %g1, %o1 + %o3)) + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE)) bgu,pt %XCC, 90b add %o1, 1, %o1 retl - mov EX_RETVAL(GLOBAL_SPARE), %o0 + mov EX_RETVAL(%o3), %o0 .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG2page.S b/arch/sparc/lib/NG2page.S deleted file mode 100644 index 73b6b7c72cb..00000000000 --- a/arch/sparc/lib/NG2page.S +++ /dev/null @@ -1,61 +0,0 @@ -/* NG2page.S: Niagara-2 optimized clear and copy page. - * - * Copyright (C) 2007 (davem@davemloft.net) - */ - -#include <asm/asi.h> -#include <asm/page.h> -#include <asm/visasm.h> - - .text - .align 32 - - /* This is heavily simplified from the sun4u variants - * because Niagara-2 does not have any D-cache aliasing issues. - */ -NG2copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ - prefetch [%o1 + 0x00], #one_read - prefetch [%o1 + 0x40], #one_read - VISEntryHalf - set PAGE_SIZE, %g7 - sub %o0, %o1, %g3 -1: stxa %g0, [%o1 + %g3] ASI_BLK_INIT_QUAD_LDD_P - subcc %g7, 64, %g7 - ldda [%o1] ASI_BLK_P, %f0 - stda %f0, [%o1 + %g3] ASI_BLK_P - add %o1, 64, %o1 - bne,pt %xcc, 1b - prefetch [%o1 + 0x40], #one_read - membar #Sync - VISExitHalf - retl - nop - -#define BRANCH_ALWAYS 0x10680000 -#define NOP 0x01000000 -#define NG_DO_PATCH(OLD, NEW) \ - sethi %hi(NEW), %g1; \ - or %g1, %lo(NEW), %g1; \ - sethi %hi(OLD), %g2; \ - or %g2, %lo(OLD), %g2; \ - sub %g1, %g2, %g1; \ - sethi %hi(BRANCH_ALWAYS), %g3; \ - sll %g1, 11, %g1; \ - srl %g1, 11 + 2, %g1; \ - or %g3, %lo(BRANCH_ALWAYS), %g3; \ - or %g3, %g1, %g3; \ - stw %g3, [%g2]; \ - sethi %hi(NOP), %g3; \ - or %g3, %lo(NOP), %g3; \ - stw %g3, [%g2 + 0x4]; \ - flush %g2; - - .globl niagara2_patch_pageops - .type niagara2_patch_pageops,#function -niagara2_patch_pageops: - NG_DO_PATCH(copy_user_page, NG2copy_user_page) - NG_DO_PATCH(_clear_page, NGclear_page) - NG_DO_PATCH(clear_user_page, NGclear_user_page) - retl - nop - .size niagara2_patch_pageops,.-niagara2_patch_pageops diff --git a/arch/sparc/lib/NG4clear_page.S b/arch/sparc/lib/NG4clear_page.S new file mode 100644 index 00000000000..e16c88204a4 --- /dev/null +++ b/arch/sparc/lib/NG4clear_page.S @@ -0,0 +1,29 @@ +/* NG4copy_page.S: Niagara-4 optimized clear page. + * + * Copyright (C) 2012 (davem@davemloft.net) + */ + +#include <asm/asi.h> +#include <asm/page.h> + + .text + + .register %g3, #scratch + + .align 32 + .globl NG4clear_page + .globl NG4clear_user_page +NG4clear_page: /* %o0=dest */ +NG4clear_user_page: /* %o0=dest, %o1=vaddr */ + set PAGE_SIZE, %g7 + mov 0x20, %g3 +1: stxa %g0, [%o0 + %g0] ASI_ST_BLKINIT_MRU_P + subcc %g7, 0x40, %g7 + stxa %g0, [%o0 + %g3] ASI_ST_BLKINIT_MRU_P + bne,pt %xcc, 1b + add %o0, 0x40, %o0 + membar #StoreLoad|#StoreStore + retl + nop + .size NG4clear_page,.-NG4clear_page + .size NG4clear_user_page,.-NG4clear_user_page
\ No newline at end of file diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S new file mode 100644 index 00000000000..fd9f903ffa3 --- /dev/null +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -0,0 +1,30 @@ +/* NG4copy_from_user.S: Niagara-4 optimized copy from userspace. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, __retl_one_asi;\ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NG4copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, ___copy_in_user; \ + nop +#endif + +#include "NG4memcpy.S" diff --git a/arch/sparc/lib/NG4copy_page.S b/arch/sparc/lib/NG4copy_page.S new file mode 100644 index 00000000000..28504e88c53 --- /dev/null +++ b/arch/sparc/lib/NG4copy_page.S @@ -0,0 +1,57 @@ +/* NG4copy_page.S: Niagara-4 optimized copy page. + * + * Copyright (C) 2012 (davem@davemloft.net) + */ + +#include <asm/asi.h> +#include <asm/page.h> + + .text + .align 32 + + .register %g2, #scratch + .register %g3, #scratch + + .globl NG4copy_user_page +NG4copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + prefetch [%o1 + 0x000], #n_reads_strong + prefetch [%o1 + 0x040], #n_reads_strong + prefetch [%o1 + 0x080], #n_reads_strong + prefetch [%o1 + 0x0c0], #n_reads_strong + set PAGE_SIZE, %g7 + prefetch [%o1 + 0x100], #n_reads_strong + prefetch [%o1 + 0x140], #n_reads_strong + prefetch [%o1 + 0x180], #n_reads_strong + prefetch [%o1 + 0x1c0], #n_reads_strong +1: + ldx [%o1 + 0x00], %o2 + subcc %g7, 0x40, %g7 + ldx [%o1 + 0x08], %o3 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + ldx [%o1 + 0x20], %g1 + stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x28], %g2 + stxa %o3, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x30], %g3 + stxa %o4, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x38], %o2 + add %o1, 0x40, %o1 + stxa %o5, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g1, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g3, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + bne,pt %icc, 1b + prefetch [%o1 + 0x200], #n_reads_strong + retl + membar #StoreLoad | #StoreStore + .size NG4copy_user_page,.-NG4copy_user_page diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S new file mode 100644 index 00000000000..9744c4540a8 --- /dev/null +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -0,0 +1,39 @@ +/* NG4copy_to_user.S: Niagara-4 optimized copy to userspace. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, __retl_one_asi;\ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME NG4copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, ___copy_in_user; \ + nop +#endif + +#include "NG4memcpy.S" diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S new file mode 100644 index 00000000000..9cf2ee01cee --- /dev/null +++ b/arch/sparc/lib/NG4memcpy.S @@ -0,0 +1,360 @@ +/* NG4memcpy.S: Niagara-4 optimized memcpy. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include <asm/visasm.h> +#include <asm/asi.h> +#define GLOBAL_SPARE %g7 +#else +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define FPRS_FEF 0x04 + +/* On T4 it is very expensive to access ASRs like %fprs and + * %asi, avoiding a read or a write can save ~50 cycles. + */ +#define FPU_ENTER \ + rd %fprs, %o5; \ + andcc %o5, FPRS_FEF, %g0; \ + be,a,pn %icc, 999f; \ + wr %g0, FPRS_FEF, %fprs; \ + 999: + +#ifdef MEMCPY_DEBUG +#define VISEntryHalf FPU_ENTER; \ + clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf FPU_ENTER +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif + +#define GLOBAL_SPARE %g5 +#endif + +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#ifndef MEMCPY_DEBUG +#define STORE(type,src,addr) type src, [addr] +#else +#define STORE(type,src,addr) type##a src, [addr] %asi +#endif +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NG4memcpy +#endif +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ +#ifdef MEMCPY_DEBUG + wr %g0, 0x80, %asi +#endif + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %XCC, 5 + PREAMBLE + mov %o0, %o3 + brz,pn %o2, .Lexit + cmp %o2, 3 + ble,pn %icc, .Ltiny + cmp %o2, 19 + ble,pn %icc, .Lsmall + or %o0, %o1, %g2 + cmp %o2, 128 + bl,pn %icc, .Lmedium + nop + +.Llarge:/* len >= 0x80 */ + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, 51f + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) + add %o1, 1, %o1 + subcc %g1, 1, %g1 + add %o0, 1, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01)) + +51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) + LOAD(prefetch, %o1 + 0x080, #n_reads_strong) + LOAD(prefetch, %o1 + 0x0c0, #n_reads_strong) + LOAD(prefetch, %o1 + 0x100, #n_reads_strong) + LOAD(prefetch, %o1 + 0x140, #n_reads_strong) + LOAD(prefetch, %o1 + 0x180, #n_reads_strong) + LOAD(prefetch, %o1 + 0x1c0, #n_reads_strong) + LOAD(prefetch, %o1 + 0x200, #n_reads_strong) + + /* Check if we can use the straight fully aligned + * loop, or we require the alignaddr/faligndata variant. + */ + andcc %o1, 0x7, %o5 + bne,pn %icc, .Llarge_src_unaligned + sub %g0, %o0, %g1 + + /* Legitimize the use of initializing stores by getting dest + * to be 64-byte aligned. + */ + and %g1, 0x3f, %g1 + brz,pt %g1, .Llarge_aligned + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2)) + add %o1, 8, %o1 + subcc %g1, 8, %g1 + add %o0, 8, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stx, %g2, %o0 - 0x08)) + +.Llarge_aligned: + /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ + andn %o2, 0x3f, %o4 + sub %o2, %o4, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + add %o1, 0x40, %o1 + EX_LD(LOAD(ldx, %o1 - 0x38, %g2)) + subcc %o4, 0x40, %o4 + EX_LD(LOAD(ldx, %o1 - 0x30, %g3)) + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE)) + EX_LD(LOAD(ldx, %o1 - 0x20, %o5)) + EX_ST(STORE_INIT(%g1, %o0)) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g2, %o0)) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x18, %g2)) + EX_ST(STORE_INIT(%g3, %o0)) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x10, %g3)) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE)) + EX_ST(STORE_INIT(%o5, %o0)) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g2, %o0)) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g3, %o0)) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0)) + add %o0, 0x08, %o0 + bne,pt %icc, 1b + LOAD(prefetch, %o1 + 0x200, #n_reads_strong) + + membar #StoreLoad | #StoreStore + + brz,pn %o2, .Lexit + cmp %o2, 19 + ble,pn %icc, .Lsmall_unaligned + nop + ba,a,pt %icc, .Lmedium_noprefetch + +.Lexit: retl + mov EX_RETVAL(%o3), %o0 + +.Llarge_src_unaligned: + andn %o2, 0x3f, %o4 + sub %o2, %o4, %o2 + VISEntryHalf + alignaddr %o1, %g0, %g1 + add %o1, %o4, %o1 + EX_LD(LOAD(ldd, %g1 + 0x00, %f0)) +1: EX_LD(LOAD(ldd, %g1 + 0x08, %f2)) + subcc %o4, 0x40, %o4 + EX_LD(LOAD(ldd, %g1 + 0x10, %f4)) + EX_LD(LOAD(ldd, %g1 + 0x18, %f6)) + EX_LD(LOAD(ldd, %g1 + 0x20, %f8)) + EX_LD(LOAD(ldd, %g1 + 0x28, %f10)) + EX_LD(LOAD(ldd, %g1 + 0x30, %f12)) + EX_LD(LOAD(ldd, %g1 + 0x38, %f14)) + faligndata %f0, %f2, %f16 + EX_LD(LOAD(ldd, %g1 + 0x40, %f0)) + faligndata %f2, %f4, %f18 + add %g1, 0x40, %g1 + faligndata %f4, %f6, %f20 + faligndata %f6, %f8, %f22 + faligndata %f8, %f10, %f24 + faligndata %f10, %f12, %f26 + faligndata %f12, %f14, %f28 + faligndata %f14, %f0, %f30 + EX_ST(STORE(std, %f16, %o0 + 0x00)) + EX_ST(STORE(std, %f18, %o0 + 0x08)) + EX_ST(STORE(std, %f20, %o0 + 0x10)) + EX_ST(STORE(std, %f22, %o0 + 0x18)) + EX_ST(STORE(std, %f24, %o0 + 0x20)) + EX_ST(STORE(std, %f26, %o0 + 0x28)) + EX_ST(STORE(std, %f28, %o0 + 0x30)) + EX_ST(STORE(std, %f30, %o0 + 0x38)) + add %o0, 0x40, %o0 + bne,pt %icc, 1b + LOAD(prefetch, %g1 + 0x200, #n_reads_strong) + VISExitHalf + + brz,pn %o2, .Lexit + cmp %o2, 19 + ble,pn %icc, .Lsmall_unaligned + nop + ba,a,pt %icc, .Lmedium_unaligned + +.Lmedium: + LOAD(prefetch, %o1 + 0x40, #n_reads_strong) + andcc %g2, 0x7, %g0 + bne,pn %icc, .Lmedium_unaligned + nop +.Lmedium_noprefetch: + andncc %o2, 0x20 - 1, %o5 + be,pn %icc, 2f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2)) + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE)) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4)) + add %o1, 0x20, %o1 + subcc %o5, 0x20, %o5 + EX_ST(STORE(stx, %g1, %o0 + 0x00)) + EX_ST(STORE(stx, %g2, %o0 + 0x08)) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10)) + EX_ST(STORE(stx, %o4, %o0 + 0x18)) + bne,pt %icc, 1b + add %o0, 0x20, %o0 +2: andcc %o2, 0x18, %o5 + be,pt %icc, 3f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1)) + add %o1, 0x08, %o1 + add %o0, 0x08, %o0 + subcc %o5, 0x08, %o5 + bne,pt %icc, 1b + EX_ST(STORE(stx, %g1, %o0 - 0x08)) +3: brz,pt %o2, .Lexit + cmp %o2, 0x04 + bl,pn %icc, .Ltiny + nop + EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + add %o1, 0x04, %o1 + add %o0, 0x04, %o0 + subcc %o2, 0x04, %o2 + bne,pn %icc, .Ltiny + EX_ST(STORE(stw, %g1, %o0 - 0x04)) + ba,a,pt %icc, .Lexit +.Lmedium_unaligned: + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, 2f + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2)) + add %o1, 1, %o1 + subcc %g1, 1, %g1 + add %o0, 1, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01)) +2: + and %o1, 0x7, %g1 + brz,pn %g1, .Lmedium_noprefetch + sll %g1, 3, %g1 + mov 64, %g2 + sub %g2, %g1, %g2 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4)) + sllx %o4, %g1, %o4 + andn %o2, 0x08 - 1, %o5 + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3)) + add %o1, 0x08, %o1 + subcc %o5, 0x08, %o5 + srlx %g3, %g2, GLOBAL_SPARE + or GLOBAL_SPARE, %o4, GLOBAL_SPARE + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00)) + add %o0, 0x08, %o0 + bne,pt %icc, 1b + sllx %g3, %g1, %o4 + srl %g1, 3, %g1 + add %o1, %g1, %o1 + brz,pn %o2, .Lexit + nop + ba,pt %icc, .Lsmall_unaligned + +.Ltiny: + EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) + subcc %o2, 1, %o2 + be,pn %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x00)) + EX_LD(LOAD(ldub, %o1 + 0x01, %g1)) + subcc %o2, 1, %o2 + be,pn %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x01)) + EX_LD(LOAD(ldub, %o1 + 0x02, %g1)) + ba,pt %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x02)) + +.Lsmall: + andcc %g2, 0x3, %g0 + bne,pn %icc, .Lsmall_unaligned + andn %o2, 0x4 - 1, %o5 + sub %o2, %o5, %o2 +1: + EX_LD(LOAD(lduw, %o1 + 0x00, %g1)) + add %o1, 0x04, %o1 + subcc %o5, 0x04, %o5 + add %o0, 0x04, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stw, %g1, %o0 - 0x04)) + brz,pt %o2, .Lexit + nop + ba,a,pt %icc, .Ltiny + +.Lsmall_unaligned: +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1)) + add %o1, 1, %o1 + add %o0, 1, %o0 + subcc %o2, 1, %o2 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g1, %o0 - 0x01)) + ba,a,pt %icc, .Lexit + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S new file mode 100644 index 00000000000..41da4bdd95c --- /dev/null +++ b/arch/sparc/lib/NG4memset.S @@ -0,0 +1,105 @@ +/* NG4memset.S: Niagara-4 optimized memset/bzero. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#include <asm/asi.h> + + .register %g2, #scratch + .register %g3, #scratch + + .text + .align 32 + .globl NG4memset +NG4memset: + andcc %o1, 0xff, %o4 + be,pt %icc, 1f + mov %o2, %o1 + sllx %o4, 8, %g1 + or %g1, %o4, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %icc, 1f + or %g1, %o2, %o4 + .size NG4memset,.-NG4memset + + .align 32 + .globl NG4bzero +NG4bzero: + clr %o4 +1: cmp %o1, 16 + ble %icc, .Ltiny + mov %o0, %o3 + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, .Laligned8 + sub %o1, %g1, %o1 +1: stb %o4, [%o0 + 0x00] + subcc %g1, 1, %g1 + bne,pt %icc, 1b + add %o0, 1, %o0 +.Laligned8: + cmp %o1, 64 + (64 - 8) + ble .Lmedium + sub %g0, %o0, %g1 + andcc %g1, (64 - 1), %g1 + brz,pn %g1, .Laligned64 + sub %o1, %g1, %o1 +1: stx %o4, [%o0 + 0x00] + subcc %g1, 8, %g1 + bne,pt %icc, 1b + add %o0, 0x8, %o0 +.Laligned64: + andn %o1, 64 - 1, %g1 + sub %o1, %g1, %o1 + brnz,pn %o4, .Lnon_bzero_loop + mov 0x20, %g2 +1: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + subcc %g1, 0x40, %g1 + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + bne,pt %icc, 1b + add %o0, 0x40, %o0 +.Lpostloop: + cmp %o1, 8 + bl,pn %icc, .Ltiny + membar #StoreStore|#StoreLoad +.Lmedium: + andn %o1, 0x7, %g1 + sub %o1, %g1, %o1 +1: stx %o4, [%o0 + 0x00] + subcc %g1, 0x8, %g1 + bne,pt %icc, 1b + add %o0, 0x08, %o0 + andcc %o1, 0x4, %g1 + be,pt %icc, .Ltiny + sub %o1, %g1, %o1 + stw %o4, [%o0 + 0x00] + add %o0, 0x4, %o0 +.Ltiny: + cmp %o1, 0 + be,pn %icc, .Lexit +1: subcc %o1, 1, %o1 + stb %o4, [%o0 + 0x00] + bne,pt %icc, 1b + add %o0, 1, %o0 +.Lexit: + retl + mov %o3, %o0 +.Lnon_bzero_loop: + mov 0x08, %g3 + mov 0x28, %o5 +1: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + subcc %g1, 0x40, %g1 + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P + add %o0, 0x10, %o0 + stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P + bne,pt %icc, 1b + add %o0, 0x30, %o0 + ba,a,pt %icc, .Lpostloop + .size NG4bzero,.-NG4bzero diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S new file mode 100644 index 00000000000..a114cbcf2a4 --- /dev/null +++ b/arch/sparc/lib/NG4patch.S @@ -0,0 +1,54 @@ +/* NG4patch.S: Patch Ultra-I routines with Niagara-4 variant. + * + * Copyright (C) 2012 David S. Miller <davem@davemloft.net> + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara4_patch_copyops + .type niagara4_patch_copyops,#function +niagara4_patch_copyops: + NG_DO_PATCH(memcpy, NG4memcpy) + NG_DO_PATCH(___copy_from_user, NG4copy_from_user) + NG_DO_PATCH(___copy_to_user, NG4copy_to_user) + retl + nop + .size niagara4_patch_copyops,.-niagara4_patch_copyops + + .globl niagara4_patch_bzero + .type niagara4_patch_bzero,#function +niagara4_patch_bzero: + NG_DO_PATCH(memset, NG4memset) + NG_DO_PATCH(__bzero, NG4bzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop + .size niagara4_patch_bzero,.-niagara4_patch_bzero + + .globl niagara4_patch_pageops + .type niagara4_patch_pageops,#function +niagara4_patch_pageops: + NG_DO_PATCH(copy_user_page, NG4copy_user_page) + NG_DO_PATCH(_clear_page, NG4clear_page) + NG_DO_PATCH(clear_user_page, NG4clear_user_page) + retl + nop + .size niagara4_patch_pageops,.-niagara4_patch_pageops diff --git a/arch/sparc/lib/NGpage.S b/arch/sparc/lib/NGpage.S index 428920de05b..423d46e2258 100644 --- a/arch/sparc/lib/NGpage.S +++ b/arch/sparc/lib/NGpage.S @@ -16,55 +16,93 @@ */ NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ - prefetch [%o1 + 0x00], #one_read - mov 8, %g1 - mov 16, %g2 - mov 24, %g3 + save %sp, -192, %sp + rd %asi, %g3 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi set PAGE_SIZE, %g7 + prefetch [%i1 + 0x00], #one_read + prefetch [%i1 + 0x40], #one_read -1: ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 - ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 - prefetch [%o1 + 0x40], #one_read - add %o1, 32, %o1 - stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P - stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P - ldda [%o1 + %g0] ASI_BLK_INIT_QUAD_LDD_P, %o2 - stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P - stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P - ldda [%o1 + %g2] ASI_BLK_INIT_QUAD_LDD_P, %o4 - add %o1, 32, %o1 - add %o0, 32, %o0 - stxa %o2, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P - stxa %o3, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P - stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P - stxa %o5, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P - subcc %g7, 64, %g7 +1: prefetch [%i1 + 0x80], #one_read + prefetch [%i1 + 0xc0], #one_read + ldda [%i1 + 0x00] %asi, %o2 + ldda [%i1 + 0x10] %asi, %o4 + ldda [%i1 + 0x20] %asi, %l2 + ldda [%i1 + 0x30] %asi, %l4 + stxa %o2, [%i0 + 0x00] %asi + stxa %o3, [%i0 + 0x08] %asi + stxa %o4, [%i0 + 0x10] %asi + stxa %o5, [%i0 + 0x18] %asi + stxa %l2, [%i0 + 0x20] %asi + stxa %l3, [%i0 + 0x28] %asi + stxa %l4, [%i0 + 0x30] %asi + stxa %l5, [%i0 + 0x38] %asi + ldda [%i1 + 0x40] %asi, %o2 + ldda [%i1 + 0x50] %asi, %o4 + ldda [%i1 + 0x60] %asi, %l2 + ldda [%i1 + 0x70] %asi, %l4 + stxa %o2, [%i0 + 0x40] %asi + stxa %o3, [%i0 + 0x48] %asi + stxa %o4, [%i0 + 0x50] %asi + stxa %o5, [%i0 + 0x58] %asi + stxa %l2, [%i0 + 0x60] %asi + stxa %l3, [%i0 + 0x68] %asi + stxa %l4, [%i0 + 0x70] %asi + stxa %l5, [%i0 + 0x78] %asi + add %i1, 128, %i1 + subcc %g7, 128, %g7 bne,pt %xcc, 1b - add %o0, 32, %o0 + add %i0, 128, %i0 + wr %g3, 0x0, %asi membar #Sync - retl - nop + ret + restore - .globl NGclear_page, NGclear_user_page + .align 32 + .globl NGclear_page + .globl NGclear_user_page NGclear_page: /* %o0=dest */ NGclear_user_page: /* %o0=dest, %o1=vaddr */ - mov 8, %g1 - mov 16, %g2 - mov 24, %g3 + rd %asi, %g3 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi set PAGE_SIZE, %g7 -1: stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P - add %o0, 32, %o0 - stxa %g0, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g1] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P - stxa %g0, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P - subcc %g7, 64, %g7 +1: stxa %g0, [%o0 + 0x00] %asi + stxa %g0, [%o0 + 0x08] %asi + stxa %g0, [%o0 + 0x10] %asi + stxa %g0, [%o0 + 0x18] %asi + stxa %g0, [%o0 + 0x20] %asi + stxa %g0, [%o0 + 0x28] %asi + stxa %g0, [%o0 + 0x30] %asi + stxa %g0, [%o0 + 0x38] %asi + stxa %g0, [%o0 + 0x40] %asi + stxa %g0, [%o0 + 0x48] %asi + stxa %g0, [%o0 + 0x50] %asi + stxa %g0, [%o0 + 0x58] %asi + stxa %g0, [%o0 + 0x60] %asi + stxa %g0, [%o0 + 0x68] %asi + stxa %g0, [%o0 + 0x70] %asi + stxa %g0, [%o0 + 0x78] %asi + stxa %g0, [%o0 + 0x80] %asi + stxa %g0, [%o0 + 0x88] %asi + stxa %g0, [%o0 + 0x90] %asi + stxa %g0, [%o0 + 0x98] %asi + stxa %g0, [%o0 + 0xa0] %asi + stxa %g0, [%o0 + 0xa8] %asi + stxa %g0, [%o0 + 0xb0] %asi + stxa %g0, [%o0 + 0xb8] %asi + stxa %g0, [%o0 + 0xc0] %asi + stxa %g0, [%o0 + 0xc8] %asi + stxa %g0, [%o0 + 0xd0] %asi + stxa %g0, [%o0 + 0xd8] %asi + stxa %g0, [%o0 + 0xe0] %asi + stxa %g0, [%o0 + 0xe8] %asi + stxa %g0, [%o0 + 0xf0] %asi + stxa %g0, [%o0 + 0xf8] %asi + subcc %g7, 256, %g7 bne,pt %xcc, 1b - add %o0, 32, %o0 + add %o0, 256, %o0 + wr %g3, 0x0, %asi membar #Sync retl nop diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index bafd2fc07ac..b67142b7768 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -109,7 +109,7 @@ #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ subcc %left, 8, %left; \ bl,pn %xcc, 95f; \ - fsrc1 %f0, %f1; + fsrc2 %f0, %f1; #define UNEVEN_VISCHUNK(dest, f0, f1, left) \ UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ @@ -201,7 +201,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o1, (0x40 - 1), %o1 and %g2, 7, %g2 andncc %g3, 0x7, %g3 - fmovd %f0, %f2 + fsrc2 %f0, %f2 sub %g3, 0x8, %g3 sub %o2, %GLOBAL_SPARE, %o2 diff --git a/arch/sparc/lib/ashldi3.S b/arch/sparc/lib/ashldi3.S index 17912e60871..86f60de07b0 100644 --- a/arch/sparc/lib/ashldi3.S +++ b/arch/sparc/lib/ashldi3.S @@ -5,10 +5,10 @@ * Copyright (C) 1999 David S. Miller (davem@redhat.com) */ +#include <linux/linkage.h> + .text - .align 4 - .globl __ashldi3 -__ashldi3: +ENTRY(__ashldi3) cmp %o2, 0 be 9f mov 0x20, %g2 @@ -32,3 +32,4 @@ __ashldi3: 9: retl nop +ENDPROC(__ashldi3) diff --git a/arch/sparc/lib/ashrdi3.S b/arch/sparc/lib/ashrdi3.S index 85398fd6dcc..6eb8ba2dd50 100644 --- a/arch/sparc/lib/ashrdi3.S +++ b/arch/sparc/lib/ashrdi3.S @@ -5,10 +5,10 @@ * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/linkage.h> + .text - .align 4 - .globl __ashrdi3 -__ashrdi3: +ENTRY(__ashrdi3) tst %o2 be 3f or %g0, 32, %g2 @@ -34,3 +34,4 @@ __ashrdi3: 3: jmpl %o7 + 8, %g0 nop +ENDPROC(__ashrdi3) diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index cbddeb38ffd..1d32b54089a 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -7,7 +7,7 @@ * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf */ -#include <asm/atomic.h> +#include <linux/atomic.h> #include <linux/spinlock.h> #include <linux/module.h> @@ -16,7 +16,7 @@ #define ATOMIC_HASH(a) (&__atomic_hash[(((unsigned long)a)>>8) & (ATOMIC_HASH_SIZE-1)]) spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] = { - [0 ... (ATOMIC_HASH_SIZE-1)] = SPIN_LOCK_UNLOCKED + [0 ... (ATOMIC_HASH_SIZE-1)] = __SPIN_LOCK_UNLOCKED(__atomic_hash) }; #else /* SMP */ @@ -55,7 +55,7 @@ int atomic_cmpxchg(atomic_t *v, int old, int new) } EXPORT_SYMBOL(atomic_cmpxchg); -int atomic_add_unless(atomic_t *v, int a, int u) +int __atomic_add_unless(atomic_t *v, int a, int u) { int ret; unsigned long flags; @@ -65,9 +65,9 @@ int atomic_add_unless(atomic_t *v, int a, int u) if (ret != u) v->counter += a; spin_unlock_irqrestore(ATOMIC_HASH(v), flags); - return ret != u; + return ret; } -EXPORT_SYMBOL(atomic_add_unless); +EXPORT_SYMBOL(__atomic_add_unless); /* Atomic operations are already serializing */ void atomic_set(atomic_t *v, int i) diff --git a/arch/sparc/lib/atomic_32.S b/arch/sparc/lib/atomic_32.S deleted file mode 100644 index 178cbb8ae1b..00000000000 --- a/arch/sparc/lib/atomic_32.S +++ /dev/null @@ -1,99 +0,0 @@ -/* atomic.S: Move this stuff here for better ICACHE hit rates. - * - * Copyright (C) 1996 David S. Miller (davem@caipfs.rutgers.edu) - */ - -#include <asm/ptrace.h> -#include <asm/psr.h> - - .text - .align 4 - - .globl __atomic_begin -__atomic_begin: - -#ifndef CONFIG_SMP - .globl ___xchg32_sun4c -___xchg32_sun4c: - rd %psr, %g3 - andcc %g3, PSR_PIL, %g0 - bne 1f - nop - wr %g3, PSR_PIL, %psr - nop; nop; nop -1: - andcc %g3, PSR_PIL, %g0 - ld [%g1], %g7 - bne 1f - st %g2, [%g1] - wr %g3, 0x0, %psr - nop; nop; nop -1: - mov %g7, %g2 - jmpl %o7 + 8, %g0 - mov %g4, %o7 - - .globl ___xchg32_sun4md -___xchg32_sun4md: - swap [%g1], %g2 - jmpl %o7 + 8, %g0 - mov %g4, %o7 -#endif - - /* Read asm-sparc/atomic.h carefully to understand how this works for SMP. - * Really, some things here for SMP are overly clever, go read the header. - */ - .globl ___atomic24_add -___atomic24_add: - rd %psr, %g3 ! Keep the code small, old way was stupid - nop; nop; nop; ! Let the bits set - or %g3, PSR_PIL, %g7 ! Disable interrupts - wr %g7, 0x0, %psr ! Set %psr - nop; nop; nop; ! Let the bits set -#ifdef CONFIG_SMP -1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. - orcc %g7, 0x0, %g0 ! Did we get it? - bne 1b ! Nope... - ld [%g1], %g7 ! Load locked atomic24_t - sra %g7, 8, %g7 ! Get signed 24-bit integer - add %g7, %g2, %g2 ! Add in argument - sll %g2, 8, %g7 ! Transpose back to atomic24_t - st %g7, [%g1] ! Clever: This releases the lock as well. -#else - ld [%g1], %g7 ! Load locked atomic24_t - add %g7, %g2, %g2 ! Add in argument - st %g2, [%g1] ! Store it back -#endif - wr %g3, 0x0, %psr ! Restore original PSR_PIL - nop; nop; nop; ! Let the bits set - jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h - mov %g4, %o7 ! Restore %o7 - - .globl ___atomic24_sub -___atomic24_sub: - rd %psr, %g3 ! Keep the code small, old way was stupid - nop; nop; nop; ! Let the bits set - or %g3, PSR_PIL, %g7 ! Disable interrupts - wr %g7, 0x0, %psr ! Set %psr - nop; nop; nop; ! Let the bits set -#ifdef CONFIG_SMP -1: ldstub [%g1 + 3], %g7 ! Spin on the byte lock for SMP. - orcc %g7, 0x0, %g0 ! Did we get it? - bne 1b ! Nope... - ld [%g1], %g7 ! Load locked atomic24_t - sra %g7, 8, %g7 ! Get signed 24-bit integer - sub %g7, %g2, %g2 ! Subtract argument - sll %g2, 8, %g7 ! Transpose back to atomic24_t - st %g7, [%g1] ! Clever: This releases the lock as well -#else - ld [%g1], %g7 ! Load locked atomic24_t - sub %g7, %g2, %g2 ! Subtract argument - st %g2, [%g1] ! Store it back -#endif - wr %g3, 0x0, %psr ! Restore original PSR_PIL - nop; nop; nop; ! Let the bits set - jmpl %o7, %g0 ! NOTE: not + 8, see callers in atomic.h - mov %g4, %o7 ! Restore %o7 - - .globl __atomic_end -__atomic_end: diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index 59186e0fcf3..85c233d0a34 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -1,8 +1,9 @@ /* atomic.S: These things are too big to do inline. * - * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net) */ +#include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> @@ -13,9 +14,7 @@ * memory barriers, and a second which returns * a value and does the barriers. */ - .globl atomic_add - .type atomic_add,#function -atomic_add: /* %o0 = increment, %o1 = atomic_ptr */ +ENTRY(atomic_add) /* %o0 = increment, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 add %g1, %o0, %g7 @@ -26,11 +25,9 @@ atomic_add: /* %o0 = increment, %o1 = atomic_ptr */ retl nop 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic_add, .-atomic_add +ENDPROC(atomic_add) - .globl atomic_sub - .type atomic_sub,#function -atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ +ENTRY(atomic_sub) /* %o0 = decrement, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 sub %g1, %o0, %g7 @@ -41,11 +38,9 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */ retl nop 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic_sub, .-atomic_sub +ENDPROC(atomic_sub) - .globl atomic_add_ret - .type atomic_add_ret,#function -atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ +ENTRY(atomic_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 add %g1, %o0, %g7 @@ -56,11 +51,9 @@ atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ retl sra %g1, 0, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic_add_ret, .-atomic_add_ret +ENDPROC(atomic_add_ret) - .globl atomic_sub_ret - .type atomic_sub_ret,#function -atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ +ENTRY(atomic_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: lduw [%o1], %g1 sub %g1, %o0, %g7 @@ -71,11 +64,9 @@ atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ retl sra %g1, 0, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic_sub_ret, .-atomic_sub_ret +ENDPROC(atomic_sub_ret) - .globl atomic64_add - .type atomic64_add,#function -atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */ +ENTRY(atomic64_add) /* %o0 = increment, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 add %g1, %o0, %g7 @@ -86,11 +77,9 @@ atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */ retl nop 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic64_add, .-atomic64_add +ENDPROC(atomic64_add) - .globl atomic64_sub - .type atomic64_sub,#function -atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */ +ENTRY(atomic64_sub) /* %o0 = decrement, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 sub %g1, %o0, %g7 @@ -101,11 +90,9 @@ atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */ retl nop 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic64_sub, .-atomic64_sub +ENDPROC(atomic64_sub) - .globl atomic64_add_ret - .type atomic64_add_ret,#function -atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ +ENTRY(atomic64_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 add %g1, %o0, %g7 @@ -116,11 +103,9 @@ atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */ retl add %g1, %o0, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic64_add_ret, .-atomic64_add_ret +ENDPROC(atomic64_add_ret) - .globl atomic64_sub_ret - .type atomic64_sub_ret,#function -atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ +ENTRY(atomic64_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ BACKOFF_SETUP(%o2) 1: ldx [%o1], %g1 sub %g1, %o0, %g7 @@ -131,4 +116,18 @@ atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */ retl sub %g1, %o0, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) - .size atomic64_sub_ret, .-atomic64_sub_ret +ENDPROC(atomic64_sub_ret) + +ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o0], %g1 + brlez,pn %g1, 3f + sub %g1, 1, %g7 + casx [%o0], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + nop +3: retl + sub %g1, 1, %o0 +2: BACKOFF_SPIN(%o2, %o3, 1b) +ENDPROC(atomic64_dec_if_positive) diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c index 48d00e72ce1..8ec4e9c0251 100644 --- a/arch/sparc/lib/bitext.c +++ b/arch/sparc/lib/bitext.c @@ -119,11 +119,7 @@ void bit_map_clear(struct bit_map *t, int offset, int len) void bit_map_init(struct bit_map *t, unsigned long *map, int size) { - - if ((size & 07) != 0) - BUG(); - memset(map, 0, size>>3); - + bitmap_zero(map, size); memset(t, 0, sizeof *t); spin_lock_init(&t->lock); t->map = map; diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S index 3dc61d5537c..36f72cc0e67 100644 --- a/arch/sparc/lib/bitops.S +++ b/arch/sparc/lib/bitops.S @@ -3,14 +3,13 @@ * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net) */ +#include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> .text - .globl test_and_set_bit - .type test_and_set_bit,#function -test_and_set_bit: /* %o0=nr, %o1=addr */ +ENTRY(test_and_set_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -29,11 +28,9 @@ test_and_set_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size test_and_set_bit, .-test_and_set_bit +ENDPROC(test_and_set_bit) - .globl test_and_clear_bit - .type test_and_clear_bit,#function -test_and_clear_bit: /* %o0=nr, %o1=addr */ +ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -52,11 +49,9 @@ test_and_clear_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size test_and_clear_bit, .-test_and_clear_bit +ENDPROC(test_and_clear_bit) - .globl test_and_change_bit - .type test_and_change_bit,#function -test_and_change_bit: /* %o0=nr, %o1=addr */ +ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -75,11 +70,9 @@ test_and_change_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size test_and_change_bit, .-test_and_change_bit +ENDPROC(test_and_change_bit) - .globl set_bit - .type set_bit,#function -set_bit: /* %o0=nr, %o1=addr */ +ENTRY(set_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -96,11 +89,9 @@ set_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size set_bit, .-set_bit +ENDPROC(set_bit) - .globl clear_bit - .type clear_bit,#function -clear_bit: /* %o0=nr, %o1=addr */ +ENTRY(clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -117,11 +108,9 @@ clear_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size clear_bit, .-clear_bit +ENDPROC(clear_bit) - .globl change_bit - .type change_bit,#function -change_bit: /* %o0=nr, %o1=addr */ +ENTRY(change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) srlx %o0, 6, %g1 mov 1, %o2 @@ -138,4 +127,4 @@ change_bit: /* %o0=nr, %o1=addr */ retl nop 2: BACKOFF_SPIN(%o3, %o4, 1b) - .size change_bit, .-change_bit +ENDPROC(change_bit) diff --git a/arch/sparc/lib/blockops.S b/arch/sparc/lib/blockops.S index 804be87f9a4..3c771011ff4 100644 --- a/arch/sparc/lib/blockops.S +++ b/arch/sparc/lib/blockops.S @@ -4,6 +4,7 @@ * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ +#include <linux/linkage.h> #include <asm/page.h> /* Zero out 64 bytes of memory at (buf + offset). @@ -44,10 +45,7 @@ */ .text - .align 4 - .globl bzero_1page, __copy_1page - -bzero_1page: +ENTRY(bzero_1page) /* NOTE: If you change the number of insns of this routine, please check * arch/sparc/mm/hypersparc.S */ /* %o0 = buf */ @@ -65,8 +63,9 @@ bzero_1page: retl nop +ENDPROC(bzero_1page) -__copy_1page: +ENTRY(__copy_1page) /* NOTE: If you change the number of insns of this routine, please check * arch/sparc/mm/hypersparc.S */ /* %o0 = dst, %o1 = src */ @@ -87,3 +86,4 @@ __copy_1page: retl nop +ENDPROC(__copy_1page) diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S index 615f401edf6..8c058114b64 100644 --- a/arch/sparc/lib/bzero.S +++ b/arch/sparc/lib/bzero.S @@ -4,11 +4,11 @@ * Copyright (C) 2005 David S. Miller <davem@davemloft.net> */ +#include <linux/linkage.h> + .text - .globl memset - .type memset, #function -memset: /* %o0=buf, %o1=pat, %o2=len */ +ENTRY(memset) /* %o0=buf, %o1=pat, %o2=len */ and %o1, 0xff, %o3 mov %o2, %o1 sllx %o3, 8, %g1 @@ -19,9 +19,7 @@ memset: /* %o0=buf, %o1=pat, %o2=len */ ba,pt %xcc, 1f or %g1, %o2, %o2 - .globl __bzero - .type __bzero, #function -__bzero: /* %o0=buf, %o1=len */ +ENTRY(__bzero) /* %o0=buf, %o1=len */ clr %o2 1: mov %o0, %o3 brz,pn %o1, __bzero_done @@ -78,8 +76,8 @@ __bzero_tiny: __bzero_done: retl mov %o3, %o0 - .size __bzero, .-__bzero - .size memset, .-memset +ENDPROC(__bzero) +ENDPROC(memset) #define EX_ST(x,y) \ 98: x,y; \ @@ -89,9 +87,7 @@ __bzero_done: .text; \ .align 4; - .globl __clear_user - .type __clear_user, #function -__clear_user: /* %o0=buf, %o1=len */ +ENTRY(__clear_user) /* %o0=buf, %o1=len */ brz,pn %o1, __clear_user_done cmp %o1, 16 bl,pn %icc, __clear_user_tiny @@ -146,4 +142,4 @@ __clear_user_tiny: __clear_user_done: retl clr %o0 - .size __clear_user, .-__clear_user +ENDPROC(__clear_user) diff --git a/arch/sparc/lib/checksum_32.S b/arch/sparc/lib/checksum_32.S index 3632cb34e91..0084c3361e1 100644 --- a/arch/sparc/lib/checksum_32.S +++ b/arch/sparc/lib/checksum_32.S @@ -289,10 +289,16 @@ cc_end_cruft: /* Also, handle the alignment code out of band. */ cc_dword_align: - cmp %g1, 6 - bl,a ccte + cmp %g1, 16 + bge 1f + srl %g1, 1, %o3 +2: cmp %o3, 0 + be,a ccte andcc %g1, 0xf, %o3 - andcc %o0, 0x1, %g0 + andcc %o3, %o0, %g0 ! Check %o0 only (%o1 has the same last 2 bits) + be,a 2b + srl %o3, 1, %o3 +1: andcc %o0, 0x1, %g0 bne ccslow andcc %o0, 0x2, %g0 be 1f diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S index 77e531f6c2a..46272dfc26e 100644 --- a/arch/sparc/lib/clear_page.S +++ b/arch/sparc/lib/clear_page.S @@ -37,10 +37,10 @@ _clear_page: /* %o0=dest */ .globl clear_user_page clear_user_page: /* %o0=dest, %o1=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o2 - sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_OFFSET), %g2 sethi %hi(PAGE_SIZE), %o4 - sllx %g2, 32, %g2 + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 sethi %hi(PAGE_KERNEL_LOCKED), %g3 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S index b243d3b606b..dd16c61f326 100644 --- a/arch/sparc/lib/copy_page.S +++ b/arch/sparc/lib/copy_page.S @@ -34,10 +34,10 @@ #endif #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ - fmovd %reg0, %f48; fmovd %reg1, %f50; \ - fmovd %reg2, %f52; fmovd %reg3, %f54; \ - fmovd %reg4, %f56; fmovd %reg5, %f58; \ - fmovd %reg6, %f60; fmovd %reg7, %f62; + fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \ + fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \ + fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \ + fsrc2 %reg6, %f60; fsrc2 %reg7, %f62; .text @@ -46,10 +46,10 @@ .type copy_user_page,#function copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o4 - sethi %uhi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_OFFSET), %g2 sethi %hi(PAGE_SIZE), %o3 - sllx %g2, 32, %g2 + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 sethi %hi(PAGE_KERNEL_LOCKED), %g3 ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 @@ -104,60 +104,60 @@ cheetah_copy_page_insn: prefetch [%o1 + 0x140], #one_read ldd [%o1 + 0x010], %f4 prefetch [%o1 + 0x180], #one_read - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x018], %f6 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x020], %f8 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x028], %f10 - fmovd %f6, %f22 + fsrc2 %f6, %f22 ldd [%o1 + 0x030], %f12 - fmovd %f8, %f24 + fsrc2 %f8, %f24 ldd [%o1 + 0x038], %f14 - fmovd %f10, %f26 + fsrc2 %f10, %f26 ldd [%o1 + 0x040], %f0 1: ldd [%o1 + 0x048], %f2 - fmovd %f12, %f28 + fsrc2 %f12, %f28 ldd [%o1 + 0x050], %f4 - fmovd %f14, %f30 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P ldd [%o1 + 0x058], %f6 - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x060], %f8 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x068], %f10 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x070], %f12 - fmovd %f6, %f22 + fsrc2 %f6, %f22 ldd [%o1 + 0x078], %f14 - fmovd %f8, %f24 + fsrc2 %f8, %f24 ldd [%o1 + 0x080], %f0 prefetch [%o1 + 0x180], #one_read - fmovd %f10, %f26 + fsrc2 %f10, %f26 subcc %o2, 1, %o2 add %o0, 0x40, %o0 bne,pt %xcc, 1b add %o1, 0x40, %o1 ldd [%o1 + 0x048], %f2 - fmovd %f12, %f28 + fsrc2 %f12, %f28 ldd [%o1 + 0x050], %f4 - fmovd %f14, %f30 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P ldd [%o1 + 0x058], %f6 - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x060], %f8 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x068], %f10 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x070], %f12 - fmovd %f6, %f22 + fsrc2 %f6, %f22 add %o0, 0x40, %o0 ldd [%o1 + 0x078], %f14 - fmovd %f8, %f24 - fmovd %f10, %f26 - fmovd %f12, %f28 - fmovd %f14, %f30 + fsrc2 %f8, %f24 + fsrc2 %f10, %f26 + fsrc2 %f12, %f28 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P membar #Sync VISExitHalf diff --git a/arch/sparc/lib/divdi3.S b/arch/sparc/lib/divdi3.S index 681b3683da9..9614b48b6ef 100644 --- a/arch/sparc/lib/divdi3.S +++ b/arch/sparc/lib/divdi3.S @@ -17,21 +17,6 @@ along with GNU CC; see the file COPYING. If not, write to the Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ - .data - .align 8 - .globl __clz_tab -__clz_tab: - .byte 0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 - .byte 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6 - .byte 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 - .byte 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 - .byte 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 - .byte 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 - .byte 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 - .byte 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 - .size __clz_tab,256 - .global .udiv - .text .align 4 .globl __divdi3 @@ -97,8 +82,9 @@ __divdi3: bne .LL85 mov %i0,%o2 mov 1,%o0 - call .udiv,0 mov 0,%o1 + wr %g0, 0, %y + udiv %o0, %o1, %o0 mov %o0,%o4 mov %i0,%o2 .LL85: diff --git a/arch/sparc/lib/ffs.S b/arch/sparc/lib/ffs.S new file mode 100644 index 00000000000..b39389f6989 --- /dev/null +++ b/arch/sparc/lib/ffs.S @@ -0,0 +1,84 @@ +#include <linux/linkage.h> + + .register %g2,#scratch + + .text + .align 32 + +ENTRY(ffs) + brnz,pt %o0, 1f + mov 1, %o1 + retl + clr %o0 + nop + nop +ENTRY(__ffs) + sllx %o0, 32, %g1 /* 1 */ + srlx %o0, 32, %g2 + + clr %o1 /* 2 */ + movrz %g1, %g2, %o0 + + movrz %g1, 32, %o1 /* 3 */ +1: clr %o2 + + sllx %o0, (64 - 16), %g1 /* 4 */ + srlx %o0, 16, %g2 + + movrz %g1, %g2, %o0 /* 5 */ + clr %o3 + + movrz %g1, 16, %o2 /* 6 */ + clr %o4 + + and %o0, 0xff, %g1 /* 7 */ + srlx %o0, 8, %g2 + + movrz %g1, %g2, %o0 /* 8 */ + clr %o5 + + movrz %g1, 8, %o3 /* 9 */ + add %o2, %o1, %o2 + + and %o0, 0xf, %g1 /* 10 */ + srlx %o0, 4, %g2 + + movrz %g1, %g2, %o0 /* 11 */ + add %o2, %o3, %o2 + + movrz %g1, 4, %o4 /* 12 */ + + and %o0, 0x3, %g1 /* 13 */ + srlx %o0, 2, %g2 + + movrz %g1, %g2, %o0 /* 14 */ + add %o2, %o4, %o2 + + movrz %g1, 2, %o5 /* 15 */ + + and %o0, 0x1, %g1 /* 16 */ + + add %o2, %o5, %o2 /* 17 */ + xor %g1, 0x1, %g1 + + retl /* 18 */ + add %o2, %g1, %o0 +ENDPROC(ffs) +ENDPROC(__ffs) + + .section .popc_6insn_patch, "ax" + .word ffs + brz,pn %o0, 98f + neg %o0, %g1 + xnor %o0, %g1, %o1 + popc %o1, %o0 +98: retl + nop + .word __ffs + neg %o0, %g1 + xnor %o0, %g1, %o1 + popc %o1, %o0 + retl + sub %o0, 1, %o0 + nop + .previous diff --git a/arch/sparc/lib/hweight.S b/arch/sparc/lib/hweight.S new file mode 100644 index 00000000000..95414e0a680 --- /dev/null +++ b/arch/sparc/lib/hweight.S @@ -0,0 +1,51 @@ +#include <linux/linkage.h> + + .text + .align 32 +ENTRY(__arch_hweight8) + ba,pt %xcc, __sw_hweight8 + nop + nop +ENDPROC(__arch_hweight8) + .section .popc_3insn_patch, "ax" + .word __arch_hweight8 + sllx %o0, 64-8, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight16) + ba,pt %xcc, __sw_hweight16 + nop + nop +ENDPROC(__arch_hweight16) + .section .popc_3insn_patch, "ax" + .word __arch_hweight16 + sllx %o0, 64-16, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight32) + ba,pt %xcc, __sw_hweight32 + nop + nop +ENDPROC(__arch_hweight32) + .section .popc_3insn_patch, "ax" + .word __arch_hweight32 + sllx %o0, 64-32, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight64) + ba,pt %xcc, __sw_hweight64 + nop + nop +ENDPROC(__arch_hweight64) + .section .popc_3insn_patch, "ax" + .word __arch_hweight64 + retl + popc %o0, %o0 + nop + .previous diff --git a/arch/sparc/lib/iomap.c b/arch/sparc/lib/iomap.c index 9ef37e13a92..c4d42a50ebc 100644 --- a/arch/sparc/lib/iomap.c +++ b/arch/sparc/lib/iomap.c @@ -18,31 +18,8 @@ void ioport_unmap(void __iomem *addr) EXPORT_SYMBOL(ioport_map); EXPORT_SYMBOL(ioport_unmap); -/* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ -void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) -{ - resource_size_t start = pci_resource_start(dev, bar); - resource_size_t len = pci_resource_len(dev, bar); - unsigned long flags = pci_resource_flags(dev, bar); - - if (!len || !start) - return NULL; - if (maxlen && len > maxlen) - len = maxlen; - if (flags & IORESOURCE_IO) - return ioport_map(start, len); - if (flags & IORESOURCE_MEM) { - if (flags & IORESOURCE_CACHEABLE) - return ioremap(start, len); - return ioremap_nocache(start, len); - } - /* What? */ - return NULL; -} - void pci_iounmap(struct pci_dev *dev, void __iomem * addr) { /* nothing to do */ } -EXPORT_SYMBOL(pci_iomap); EXPORT_SYMBOL(pci_iounmap); diff --git a/arch/sparc/lib/ipcsum.S b/arch/sparc/lib/ipcsum.S index 58ca5b9a877..4742d59029e 100644 --- a/arch/sparc/lib/ipcsum.S +++ b/arch/sparc/lib/ipcsum.S @@ -1,8 +1,7 @@ +#include <linux/linkage.h> + .text - .align 32 - .globl ip_fast_csum - .type ip_fast_csum,#function -ip_fast_csum: /* %o0 = iph, %o1 = ihl */ +ENTRY(ip_fast_csum) /* %o0 = iph, %o1 = ihl */ sub %o1, 4, %g7 lduw [%o0 + 0x00], %o2 lduw [%o0 + 0x04], %g2 @@ -31,4 +30,4 @@ ip_fast_csum: /* %o0 = iph, %o1 = ihl */ set 0xffff, %o1 retl and %o2, %o1, %o0 - .size ip_fast_csum, .-ip_fast_csum +ENDPROC(ip_fast_csum) diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c index 1b30bb3bfdb..323335b9cd2 100644 --- a/arch/sparc/lib/ksyms.c +++ b/arch/sparc/lib/ksyms.c @@ -15,8 +15,6 @@ /* string functions */ EXPORT_SYMBOL(strlen); -EXPORT_SYMBOL(__strlen_user); -EXPORT_SYMBOL(__strnlen_user); EXPORT_SYMBOL(strncmp); /* mem* functions */ @@ -33,9 +31,6 @@ EXPORT_SYMBOL(memset); EXPORT_SYMBOL(memmove); EXPORT_SYMBOL(__bzero); -/* Moving data to/from/in userspace. */ -EXPORT_SYMBOL(__strncpy_from_user); - /* Networking helper routines. */ EXPORT_SYMBOL(csum_partial); @@ -56,24 +51,10 @@ extern int __divdi3(int, int); extern void (*__copy_1page)(void *, const void *); extern void (*bzero_1page)(void *); -extern int __strncmp(const char *, const char *, __kernel_size_t); - extern void ___rw_read_enter(void); extern void ___rw_read_try(void); extern void ___rw_read_exit(void); extern void ___rw_write_enter(void); -extern void ___atomic24_add(void); -extern void ___atomic24_sub(void); - -/* Alias functions whose names begin with "." and export the aliases. - * The module references will be fixed up by module_frob_arch_sections. - */ -extern int _Div(int, int); -extern int _Mul(int, int); -extern int _Rem(int, int); -extern unsigned _Udiv(unsigned, unsigned); -extern unsigned _Umul(unsigned, unsigned); -extern unsigned _Urem(unsigned, unsigned); /* Networking helper routines. */ EXPORT_SYMBOL(__csum_partial_copy_sparc_generic); @@ -83,9 +64,6 @@ EXPORT_SYMBOL(__copy_1page); EXPORT_SYMBOL(__memmove); EXPORT_SYMBOL(bzero_1page); -/* string functions */ -EXPORT_SYMBOL(__strncmp); - /* Moving data to/from/in userspace. */ EXPORT_SYMBOL(__copy_user); @@ -97,22 +75,11 @@ EXPORT_SYMBOL(___rw_read_exit); EXPORT_SYMBOL(___rw_write_enter); #endif -/* Atomic operations. */ -EXPORT_SYMBOL(___atomic24_add); -EXPORT_SYMBOL(___atomic24_sub); - EXPORT_SYMBOL(__ashrdi3); EXPORT_SYMBOL(__ashldi3); EXPORT_SYMBOL(__lshrdi3); EXPORT_SYMBOL(__muldi3); EXPORT_SYMBOL(__divdi3); - -EXPORT_SYMBOL(_Rem); -EXPORT_SYMBOL(_Urem); -EXPORT_SYMBOL(_Mul); -EXPORT_SYMBOL(_Umul); -EXPORT_SYMBOL(_Div); -EXPORT_SYMBOL(_Udiv); #endif /* @@ -131,15 +98,6 @@ EXPORT_SYMBOL(___copy_from_user); EXPORT_SYMBOL(___copy_in_user); EXPORT_SYMBOL(__clear_user); -/* RW semaphores */ -EXPORT_SYMBOL(__down_read); -EXPORT_SYMBOL(__down_read_trylock); -EXPORT_SYMBOL(__down_write); -EXPORT_SYMBOL(__down_write_trylock); -EXPORT_SYMBOL(__up_read); -EXPORT_SYMBOL(__up_write); -EXPORT_SYMBOL(__downgrade_write); - /* Atomic counter implementation. */ EXPORT_SYMBOL(atomic_add); EXPORT_SYMBOL(atomic_add_ret); @@ -149,6 +107,7 @@ EXPORT_SYMBOL(atomic64_add); EXPORT_SYMBOL(atomic64_add_ret); EXPORT_SYMBOL(atomic64_sub); EXPORT_SYMBOL(atomic64_sub_ret); +EXPORT_SYMBOL(atomic64_dec_if_positive); /* Atomic bit operations. */ EXPORT_SYMBOL(test_and_set_bit); @@ -167,6 +126,10 @@ EXPORT_SYMBOL(copy_user_page); void VISenter(void); EXPORT_SYMBOL(VISenter); +/* CRYPTO code needs this */ +void VISenterhalf(void); +EXPORT_SYMBOL(VISenterhalf); + extern void xor_vis_2(unsigned long, unsigned long *, unsigned long *); extern void xor_vis_3(unsigned long, unsigned long *, unsigned long *, unsigned long *); diff --git a/arch/sparc/lib/lshrdi3.S b/arch/sparc/lib/lshrdi3.S index 47a1354c160..60ebc7cdbee 100644 --- a/arch/sparc/lib/lshrdi3.S +++ b/arch/sparc/lib/lshrdi3.S @@ -1,6 +1,6 @@ +#include <linux/linkage.h> - .globl __lshrdi3 -__lshrdi3: +ENTRY(__lshrdi3) cmp %o2, 0 be 3f mov 0x20, %g2 @@ -24,3 +24,4 @@ __lshrdi3: 3: retl nop +ENDPROC(__lshrdi3) diff --git a/arch/sparc/lib/memcpy.S b/arch/sparc/lib/memcpy.S index 34fe6575173..4d8c497517b 100644 --- a/arch/sparc/lib/memcpy.S +++ b/arch/sparc/lib/memcpy.S @@ -7,40 +7,12 @@ * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#ifdef __KERNEL__ - -#define FUNC(x) \ +#define FUNC(x) \ .globl x; \ .type x,@function; \ - .align 4; \ + .align 4; \ x: -#undef FASTER_REVERSE -#undef FASTER_NONALIGNED -#define FASTER_ALIGNED - -/* In kernel these functions don't return a value. - * One should use macros in asm/string.h for that purpose. - * We return 0, so that bugs are more apparent. - */ -#define SETUP_RETL -#define RETL_INSN clr %o0 - -#else - -/* libc */ - -#include "DEFS.h" - -#define FASTER_REVERSE -#define FASTER_NONALIGNED -#define FASTER_ALIGNED - -#define SETUP_RETL mov %o0, %g6 -#define RETL_INSN mov %g6, %o0 - -#endif - /* Both these macros have to start with exactly the same insn */ #define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ ldd [%src + (offset) + 0x00], %t0; \ @@ -164,30 +136,6 @@ x: .text .align 4 -#ifdef FASTER_REVERSE - -70: /* rdword_align */ - - andcc %o1, 1, %g0 - be 4f - andcc %o1, 2, %g0 - - ldub [%o1 - 1], %g2 - sub %o1, 1, %o1 - stb %g2, [%o0 - 1] - sub %o2, 1, %o2 - be 3f - sub %o0, 1, %o0 -4: - lduh [%o1 - 2], %g2 - sub %o1, 2, %o1 - sth %g2, [%o0 - 2] - sub %o2, 2, %o2 - b 3f - sub %o0, 2, %o0 - -#endif /* FASTER_REVERSE */ - 0: retl nop ! Only bcopy returns here and it retuns void... @@ -198,7 +146,7 @@ FUNC(__memmove) #endif FUNC(memmove) cmp %o0, %o1 - SETUP_RETL + mov %o0, %g7 bleu 9f sub %o0, %o1, %o4 @@ -207,8 +155,6 @@ FUNC(memmove) bleu 0f andcc %o4, 3, %o5 -#ifndef FASTER_REVERSE - add %o1, %o2, %o1 add %o0, %o2, %o0 sub %o1, 1, %o1 @@ -224,295 +170,7 @@ FUNC(memmove) sub %o0, 1, %o0 retl - RETL_INSN - -#else /* FASTER_REVERSE */ - - add %o1, %o2, %o1 - add %o0, %o2, %o0 - bne 77f - cmp %o2, 15 - bleu 91f - andcc %o1, 3, %g0 - bne 70b -3: - andcc %o1, 4, %g0 - - be 2f - mov %o2, %g1 - - ld [%o1 - 4], %o4 - sub %g1, 4, %g1 - st %o4, [%o0 - 4] - sub %o1, 4, %o1 - sub %o0, 4, %o0 -2: - andcc %g1, 0xffffff80, %g7 - be 3f - andcc %o0, 4, %g0 - - be 74f + 4 -5: - RMOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) - subcc %g7, 128, %g7 - sub %o1, 128, %o1 - bne 5b - sub %o0, 128, %o0 -3: - andcc %g1, 0x70, %g7 - be 72f - andcc %g1, 8, %g0 - - sethi %hi(72f), %o5 - srl %g7, 1, %o4 - add %g7, %o4, %o4 - sub %o1, %g7, %o1 - sub %o5, %o4, %o5 - jmpl %o5 + %lo(72f), %g0 - sub %o0, %g7, %o0 - -71: /* rmemcpy_table */ - RMOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5) - RMOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5) - -72: /* rmemcpy_table_end */ - - be 73f - andcc %g1, 4, %g0 - - ldd [%o1 - 0x08], %g2 - sub %o0, 8, %o0 - sub %o1, 8, %o1 - st %g2, [%o0] - st %g3, [%o0 + 0x04] - -73: /* rmemcpy_last7 */ - - be 1f - andcc %g1, 2, %g0 - - ld [%o1 - 4], %g2 - sub %o1, 4, %o1 - st %g2, [%o0 - 4] - sub %o0, 4, %o0 -1: - be 1f - andcc %g1, 1, %g0 - - lduh [%o1 - 2], %g2 - sub %o1, 2, %o1 - sth %g2, [%o0 - 2] - sub %o0, 2, %o0 -1: - be 1f - nop - - ldub [%o1 - 1], %g2 - stb %g2, [%o0 - 1] -1: - retl - RETL_INSN - -74: /* rldd_std */ - RMOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) - RMOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) - subcc %g7, 128, %g7 - sub %o1, 128, %o1 - bne 74b - sub %o0, 128, %o0 - - andcc %g1, 0x70, %g7 - be 72b - andcc %g1, 8, %g0 - - sethi %hi(72b), %o5 - srl %g7, 1, %o4 - add %g7, %o4, %o4 - sub %o1, %g7, %o1 - sub %o5, %o4, %o5 - jmpl %o5 + %lo(72b), %g0 - sub %o0, %g7, %o0 - -75: /* rshort_end */ - - and %o2, 0xe, %o3 -2: - sethi %hi(76f), %o5 - sll %o3, 3, %o4 - sub %o0, %o3, %o0 - sub %o5, %o4, %o5 - sub %o1, %o3, %o1 - jmpl %o5 + %lo(76f), %g0 - andcc %o2, 1, %g0 - - RMOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3) - RMOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3) - -76: /* rshort_table_end */ - - be 1f - nop - ldub [%o1 - 1], %g2 - stb %g2, [%o0 - 1] -1: - retl - RETL_INSN - -91: /* rshort_aligned_end */ - - bne 75b - andcc %o2, 8, %g0 - - be 1f - andcc %o2, 4, %g0 - - ld [%o1 - 0x08], %g2 - ld [%o1 - 0x04], %g3 - sub %o1, 8, %o1 - st %g2, [%o0 - 0x08] - st %g3, [%o0 - 0x04] - sub %o0, 8, %o0 -1: - b 73b - mov %o2, %g1 - -77: /* rnon_aligned */ - cmp %o2, 15 - bleu 75b - andcc %o0, 3, %g0 - be 64f - andcc %o0, 1, %g0 - be 63f - andcc %o0, 2, %g0 - ldub [%o1 - 1], %g5 - sub %o1, 1, %o1 - stb %g5, [%o0 - 1] - sub %o0, 1, %o0 - be 64f - sub %o2, 1, %o2 -63: - ldub [%o1 - 1], %g5 - sub %o1, 2, %o1 - stb %g5, [%o0 - 1] - sub %o0, 2, %o0 - ldub [%o1], %g5 - sub %o2, 2, %o2 - stb %g5, [%o0] -64: - and %o1, 3, %g2 - and %o1, -4, %o1 - and %o2, 0xc, %g3 - add %o1, 4, %o1 - cmp %g3, 4 - sll %g2, 3, %g4 - mov 32, %g2 - be 4f - sub %g2, %g4, %g7 - - blu 3f - cmp %g3, 8 - - be 2f - srl %o2, 2, %g3 - - ld [%o1 - 4], %o3 - add %o0, -8, %o0 - ld [%o1 - 8], %o4 - add %o1, -16, %o1 - b 7f - add %g3, 1, %g3 -2: - ld [%o1 - 4], %o4 - add %o0, -4, %o0 - ld [%o1 - 8], %g1 - add %o1, -12, %o1 - b 8f - add %g3, 2, %g3 -3: - ld [%o1 - 4], %o5 - add %o0, -12, %o0 - ld [%o1 - 8], %o3 - add %o1, -20, %o1 - b 6f - srl %o2, 2, %g3 -4: - ld [%o1 - 4], %g1 - srl %o2, 2, %g3 - ld [%o1 - 8], %o5 - add %o1, -24, %o1 - add %o0, -16, %o0 - add %g3, -1, %g3 - - ld [%o1 + 12], %o3 -5: - sll %o5, %g4, %g2 - srl %g1, %g7, %g5 - or %g2, %g5, %g2 - st %g2, [%o0 + 12] -6: - ld [%o1 + 8], %o4 - sll %o3, %g4, %g2 - srl %o5, %g7, %g5 - or %g2, %g5, %g2 - st %g2, [%o0 + 8] -7: - ld [%o1 + 4], %g1 - sll %o4, %g4, %g2 - srl %o3, %g7, %g5 - or %g2, %g5, %g2 - st %g2, [%o0 + 4] -8: - ld [%o1], %o5 - sll %g1, %g4, %g2 - srl %o4, %g7, %g5 - addcc %g3, -4, %g3 - or %g2, %g5, %g2 - add %o1, -16, %o1 - st %g2, [%o0] - add %o0, -16, %o0 - bne,a 5b - ld [%o1 + 12], %o3 - sll %o5, %g4, %g2 - srl %g1, %g7, %g5 - srl %g4, 3, %g3 - or %g2, %g5, %g2 - add %o1, %g3, %o1 - andcc %o2, 2, %g0 - st %g2, [%o0 + 12] - be 1f - andcc %o2, 1, %g0 - - ldub [%o1 + 15], %g5 - add %o1, -2, %o1 - stb %g5, [%o0 + 11] - add %o0, -2, %o0 - ldub [%o1 + 16], %g5 - stb %g5, [%o0 + 12] -1: - be 1f - nop - ldub [%o1 + 15], %g5 - stb %g5, [%o0 + 11] -1: - retl - RETL_INSN - -#endif /* FASTER_REVERSE */ + mov %g7, %o0 /* NOTE: This code is executed just for the cases, where %src (=%o1) & 3 is != 0. @@ -546,7 +204,7 @@ FUNC(memmove) FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ sub %o0, %o1, %o4 - SETUP_RETL + mov %o0, %g7 9: andcc %o4, 3, %o5 0: @@ -569,7 +227,7 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ add %o1, 4, %o1 add %o0, 4, %o0 2: - andcc %g1, 0xffffff80, %g7 + andcc %g1, 0xffffff80, %g0 be 3f andcc %o0, 4, %g0 @@ -579,22 +237,23 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) - subcc %g7, 128, %g7 + sub %g1, 128, %g1 add %o1, 128, %o1 - bne 5b + cmp %g1, 128 + bge 5b add %o0, 128, %o0 3: - andcc %g1, 0x70, %g7 + andcc %g1, 0x70, %g4 be 80f andcc %g1, 8, %g0 sethi %hi(80f), %o5 - srl %g7, 1, %o4 - add %g7, %o4, %o4 - add %o1, %g7, %o1 + srl %g4, 1, %o4 + add %g4, %o4, %o4 + add %o1, %g4, %o1 sub %o5, %o4, %o5 jmpl %o5 + %lo(80f), %g0 - add %o0, %g7, %o0 + add %o0, %g4, %o0 79: /* memcpy_table */ @@ -641,43 +300,28 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ stb %g2, [%o0] 1: retl - RETL_INSN + mov %g7, %o0 82: /* ldd_std */ MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) - subcc %g7, 128, %g7 + subcc %g1, 128, %g1 add %o1, 128, %o1 - bne 82b + cmp %g1, 128 + bge 82b add %o0, 128, %o0 -#ifndef FASTER_ALIGNED - - andcc %g1, 0x70, %g7 - be 80b - andcc %g1, 8, %g0 - - sethi %hi(80b), %o5 - srl %g7, 1, %o4 - add %g7, %o4, %o4 - add %o1, %g7, %o1 - sub %o5, %o4, %o5 - jmpl %o5 + %lo(80b), %g0 - add %o0, %g7, %o0 - -#else /* FASTER_ALIGNED */ - - andcc %g1, 0x70, %g7 + andcc %g1, 0x70, %g4 be 84f andcc %g1, 8, %g0 sethi %hi(84f), %o5 - add %o1, %g7, %o1 - sub %o5, %g7, %o5 + add %o1, %g4, %o1 + sub %o5, %g4, %o5 jmpl %o5 + %lo(84f), %g0 - add %o0, %g7, %o0 + add %o0, %g4, %o0 83: /* amemcpy_table */ @@ -721,382 +365,132 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ stb %g2, [%o0] 1: retl - RETL_INSN - -#endif /* FASTER_ALIGNED */ + mov %g7, %o0 86: /* non_aligned */ cmp %o2, 6 bleu 88f + nop -#ifdef FASTER_NONALIGNED - - cmp %o2, 256 - bcc 87f - -#endif /* FASTER_NONALIGNED */ - - andcc %o0, 3, %g0 + save %sp, -96, %sp + andcc %i0, 3, %g0 be 61f - andcc %o0, 1, %g0 + andcc %i0, 1, %g0 be 60f - andcc %o0, 2, %g0 + andcc %i0, 2, %g0 - ldub [%o1], %g5 - add %o1, 1, %o1 - stb %g5, [%o0] - sub %o2, 1, %o2 + ldub [%i1], %g5 + add %i1, 1, %i1 + stb %g5, [%i0] + sub %i2, 1, %i2 bne 61f - add %o0, 1, %o0 + add %i0, 1, %i0 60: - ldub [%o1], %g3 - add %o1, 2, %o1 - stb %g3, [%o0] - sub %o2, 2, %o2 - ldub [%o1 - 1], %g3 - add %o0, 2, %o0 - stb %g3, [%o0 - 1] + ldub [%i1], %g3 + add %i1, 2, %i1 + stb %g3, [%i0] + sub %i2, 2, %i2 + ldub [%i1 - 1], %g3 + add %i0, 2, %i0 + stb %g3, [%i0 - 1] 61: - and %o1, 3, %g2 - and %o2, 0xc, %g3 - and %o1, -4, %o1 + and %i1, 3, %g2 + and %i2, 0xc, %g3 + and %i1, -4, %i1 cmp %g3, 4 sll %g2, 3, %g4 mov 32, %g2 be 4f - sub %g2, %g4, %g7 + sub %g2, %g4, %l0 blu 3f cmp %g3, 0x8 be 2f - srl %o2, 2, %g3 + srl %i2, 2, %g3 - ld [%o1], %o3 - add %o0, -8, %o0 - ld [%o1 + 4], %o4 + ld [%i1], %i3 + add %i0, -8, %i0 + ld [%i1 + 4], %i4 b 8f add %g3, 1, %g3 2: - ld [%o1], %o4 - add %o0, -12, %o0 - ld [%o1 + 4], %o5 + ld [%i1], %i4 + add %i0, -12, %i0 + ld [%i1 + 4], %i5 add %g3, 2, %g3 b 9f - add %o1, -4, %o1 + add %i1, -4, %i1 3: - ld [%o1], %g1 - add %o0, -4, %o0 - ld [%o1 + 4], %o3 - srl %o2, 2, %g3 + ld [%i1], %g1 + add %i0, -4, %i0 + ld [%i1 + 4], %i3 + srl %i2, 2, %g3 b 7f - add %o1, 4, %o1 + add %i1, 4, %i1 4: - ld [%o1], %o5 - cmp %o2, 7 - ld [%o1 + 4], %g1 - srl %o2, 2, %g3 + ld [%i1], %i5 + cmp %i2, 7 + ld [%i1 + 4], %g1 + srl %i2, 2, %g3 bleu 10f - add %o1, 8, %o1 + add %i1, 8, %i1 - ld [%o1], %o3 + ld [%i1], %i3 add %g3, -1, %g3 5: - sll %o5, %g4, %g2 - srl %g1, %g7, %g5 + sll %i5, %g4, %g2 + srl %g1, %l0, %g5 or %g2, %g5, %g2 - st %g2, [%o0] + st %g2, [%i0] 7: - ld [%o1 + 4], %o4 + ld [%i1 + 4], %i4 sll %g1, %g4, %g2 - srl %o3, %g7, %g5 + srl %i3, %l0, %g5 or %g2, %g5, %g2 - st %g2, [%o0 + 4] + st %g2, [%i0 + 4] 8: - ld [%o1 + 8], %o5 - sll %o3, %g4, %g2 - srl %o4, %g7, %g5 + ld [%i1 + 8], %i5 + sll %i3, %g4, %g2 + srl %i4, %l0, %g5 or %g2, %g5, %g2 - st %g2, [%o0 + 8] + st %g2, [%i0 + 8] 9: - ld [%o1 + 12], %g1 - sll %o4, %g4, %g2 - srl %o5, %g7, %g5 + ld [%i1 + 12], %g1 + sll %i4, %g4, %g2 + srl %i5, %l0, %g5 addcc %g3, -4, %g3 or %g2, %g5, %g2 - add %o1, 16, %o1 - st %g2, [%o0 + 12] - add %o0, 16, %o0 + add %i1, 16, %i1 + st %g2, [%i0 + 12] + add %i0, 16, %i0 bne,a 5b - ld [%o1], %o3 + ld [%i1], %i3 10: - sll %o5, %g4, %g2 - srl %g1, %g7, %g5 - srl %g7, 3, %g3 + sll %i5, %g4, %g2 + srl %g1, %l0, %g5 + srl %l0, 3, %g3 or %g2, %g5, %g2 - sub %o1, %g3, %o1 - andcc %o2, 2, %g0 - st %g2, [%o0] + sub %i1, %g3, %i1 + andcc %i2, 2, %g0 + st %g2, [%i0] be 1f - andcc %o2, 1, %g0 - - ldub [%o1], %g2 - add %o1, 2, %o1 - stb %g2, [%o0 + 4] - add %o0, 2, %o0 - ldub [%o1 - 1], %g2 - stb %g2, [%o0 + 3] + andcc %i2, 1, %g0 + + ldub [%i1], %g2 + add %i1, 2, %i1 + stb %g2, [%i0 + 4] + add %i0, 2, %i0 + ldub [%i1 - 1], %g2 + stb %g2, [%i0 + 3] 1: be 1f nop - ldub [%o1], %g2 - stb %g2, [%o0 + 4] -1: - retl - RETL_INSN - -#ifdef FASTER_NONALIGNED - -87: /* faster_nonaligned */ - - andcc %o1, 3, %g0 - be 3f - andcc %o1, 1, %g0 - - be 4f - andcc %o1, 2, %g0 - - ldub [%o1], %g2 - add %o1, 1, %o1 - stb %g2, [%o0] - sub %o2, 1, %o2 - bne 3f - add %o0, 1, %o0 -4: - lduh [%o1], %g2 - add %o1, 2, %o1 - srl %g2, 8, %g3 - sub %o2, 2, %o2 - stb %g3, [%o0] - add %o0, 2, %o0 - stb %g2, [%o0 - 1] -3: - andcc %o1, 4, %g0 - - bne 2f - cmp %o5, 1 - - ld [%o1], %o4 - srl %o4, 24, %g2 - stb %g2, [%o0] - srl %o4, 16, %g3 - stb %g3, [%o0 + 1] - srl %o4, 8, %g2 - stb %g2, [%o0 + 2] - sub %o2, 4, %o2 - stb %o4, [%o0 + 3] - add %o1, 4, %o1 - add %o0, 4, %o0 -2: - be 33f - cmp %o5, 2 - be 32f - sub %o2, 4, %o2 -31: - ld [%o1], %g2 - add %o1, 4, %o1 - srl %g2, 24, %g3 - and %o0, 7, %g5 - stb %g3, [%o0] - cmp %g5, 7 - sll %g2, 8, %g1 - add %o0, 4, %o0 - be 41f - and %o2, 0xffffffc0, %o3 - ld [%o0 - 7], %o4 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 4b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 16, %g2 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 16, %g2 -1: - st %o4, [%o0 - 7] - sth %g2, [%o0 - 3] - srl %g1, 8, %g4 - b 88f - stb %g4, [%o0 - 1] -32: - ld [%o1], %g2 - add %o1, 4, %o1 - srl %g2, 16, %g3 - and %o0, 7, %g5 - sth %g3, [%o0] - cmp %g5, 6 - sll %g2, 16, %g1 - add %o0, 4, %o0 - be 42f - and %o2, 0xffffffc0, %o3 - ld [%o0 - 6], %o4 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 4b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 16, %g2 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 16, %g2 -1: - st %o4, [%o0 - 6] - b 88f - sth %g2, [%o0 - 2] -33: - ld [%o1], %g2 - sub %o2, 4, %o2 - srl %g2, 24, %g3 - and %o0, 7, %g5 - stb %g3, [%o0] - cmp %g5, 5 - srl %g2, 8, %g4 - sll %g2, 24, %g1 - sth %g4, [%o0 + 1] - add %o1, 4, %o1 - be 43f - and %o2, 0xffffffc0, %o3 - - ld [%o0 - 1], %o4 - add %o0, 4, %o0 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1) - SMOVE_CHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1) - SMOVE_CHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1) - SMOVE_CHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 4b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 24, %g2 -4: - SMOVE_CHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, -1) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 24, %g2 -1: - st %o4, [%o0 - 5] - b 88f - stb %g2, [%o0 - 1] -41: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 41b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 16, %g2 -4: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 8, 24, -3) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 16, %g2 + ldub [%i1], %g2 + stb %g2, [%i0 + 4] 1: - sth %g2, [%o0 - 3] - srl %g1, 8, %g4 - b 88f - stb %g4, [%o0 - 1] -43: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3) - SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3) - SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3) - SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 43b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 24, %g2 -4: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 24, 8, 3) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 24, %g2 -1: - stb %g2, [%o0 + 3] - b 88f - add %o0, 4, %o0 -42: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_ALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_ALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - SMOVE_ALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - subcc %o3, 64, %o3 - add %o1, 64, %o1 - bne 42b - add %o0, 64, %o0 - - andcc %o2, 0x30, %o3 - be,a 1f - srl %g1, 16, %g2 -4: - SMOVE_ALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5, o4, o5, g7, g1, 16, 16, -2) - subcc %o3, 16, %o3 - add %o1, 16, %o1 - bne 4b - add %o0, 16, %o0 - - srl %g1, 16, %g2 -1: - sth %g2, [%o0 - 2] - - /* Fall through */ - -#endif /* FASTER_NONALIGNED */ + ret + restore %g7, %g0, %o0 88: /* short_end */ @@ -1127,7 +521,7 @@ FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ stb %g2, [%o0] 1: retl - RETL_INSN + mov %g7, %o0 90: /* short_aligned_end */ bne 88b diff --git a/arch/sparc/lib/memmove.S b/arch/sparc/lib/memmove.S index 97395802c23..b7f6334e159 100644 --- a/arch/sparc/lib/memmove.S +++ b/arch/sparc/lib/memmove.S @@ -4,11 +4,10 @@ * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) */ +#include <linux/linkage.h> + .text - .align 32 - .globl memmove - .type memmove,#function -memmove: /* o0=dst o1=src o2=len */ +ENTRY(memmove) /* o0=dst o1=src o2=len */ mov %o0, %g1 cmp %o0, %o1 bleu,pt %xcc, memcpy @@ -28,4 +27,4 @@ memmove: /* o0=dst o1=src o2=len */ retl mov %g1, %o0 - .size memmove, .-memmove +ENDPROC(memmove) diff --git a/arch/sparc/lib/mul.S b/arch/sparc/lib/mul.S deleted file mode 100644 index c45470d0b0c..00000000000 --- a/arch/sparc/lib/mul.S +++ /dev/null @@ -1,137 +0,0 @@ -/* - * mul.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - -/* - * Signed multiply, from Appendix E of the Sparc Version 8 - * Architecture Manual. - */ - -/* - * Returns %o0 * %o1 in %o1%o0 (i.e., %o1 holds the upper 32 bits of - * the 64-bit product). - * - * This code optimizes short (less than 13-bit) multiplies. - */ - - .globl .mul - .globl _Mul -.mul: -_Mul: /* needed for export */ - mov %o0, %y ! multiplier -> Y - andncc %o0, 0xfff, %g0 ! test bits 12..31 - be Lmul_shortway ! if zero, can do it the short way - andcc %g0, %g0, %o4 ! zero the partial product and clear N and V - - /* - * Long multiply. 32 steps, followed by a final shift step. - */ - mulscc %o4, %o1, %o4 ! 1 - mulscc %o4, %o1, %o4 ! 2 - mulscc %o4, %o1, %o4 ! 3 - mulscc %o4, %o1, %o4 ! 4 - mulscc %o4, %o1, %o4 ! 5 - mulscc %o4, %o1, %o4 ! 6 - mulscc %o4, %o1, %o4 ! 7 - mulscc %o4, %o1, %o4 ! 8 - mulscc %o4, %o1, %o4 ! 9 - mulscc %o4, %o1, %o4 ! 10 - mulscc %o4, %o1, %o4 ! 11 - mulscc %o4, %o1, %o4 ! 12 - mulscc %o4, %o1, %o4 ! 13 - mulscc %o4, %o1, %o4 ! 14 - mulscc %o4, %o1, %o4 ! 15 - mulscc %o4, %o1, %o4 ! 16 - mulscc %o4, %o1, %o4 ! 17 - mulscc %o4, %o1, %o4 ! 18 - mulscc %o4, %o1, %o4 ! 19 - mulscc %o4, %o1, %o4 ! 20 - mulscc %o4, %o1, %o4 ! 21 - mulscc %o4, %o1, %o4 ! 22 - mulscc %o4, %o1, %o4 ! 23 - mulscc %o4, %o1, %o4 ! 24 - mulscc %o4, %o1, %o4 ! 25 - mulscc %o4, %o1, %o4 ! 26 - mulscc %o4, %o1, %o4 ! 27 - mulscc %o4, %o1, %o4 ! 28 - mulscc %o4, %o1, %o4 ! 29 - mulscc %o4, %o1, %o4 ! 30 - mulscc %o4, %o1, %o4 ! 31 - mulscc %o4, %o1, %o4 ! 32 - mulscc %o4, %g0, %o4 ! final shift - - ! If %o0 was negative, the result is - ! (%o0 * %o1) + (%o1 << 32)) - ! We fix that here. - -#if 0 - tst %o0 - bge 1f - rd %y, %o0 - - ! %o0 was indeed negative; fix upper 32 bits of result by subtracting - ! %o1 (i.e., return %o4 - %o1 in %o1). - retl - sub %o4, %o1, %o1 - -1: - retl - mov %o4, %o1 -#else - /* Faster code adapted from tege@sics.se's code for umul.S. */ - sra %o0, 31, %o2 ! make mask from sign bit - and %o1, %o2, %o2 ! %o2 = 0 or %o1, depending on sign of %o0 - rd %y, %o0 ! get lower half of product - retl - sub %o4, %o2, %o1 ! subtract compensation - ! and put upper half in place -#endif - -Lmul_shortway: - /* - * Short multiply. 12 steps, followed by a final shift step. - * The resulting bits are off by 12 and (32-12) = 20 bit positions, - * but there is no problem with %o0 being negative (unlike above). - */ - mulscc %o4, %o1, %o4 ! 1 - mulscc %o4, %o1, %o4 ! 2 - mulscc %o4, %o1, %o4 ! 3 - mulscc %o4, %o1, %o4 ! 4 - mulscc %o4, %o1, %o4 ! 5 - mulscc %o4, %o1, %o4 ! 6 - mulscc %o4, %o1, %o4 ! 7 - mulscc %o4, %o1, %o4 ! 8 - mulscc %o4, %o1, %o4 ! 9 - mulscc %o4, %o1, %o4 ! 10 - mulscc %o4, %o1, %o4 ! 11 - mulscc %o4, %o1, %o4 ! 12 - mulscc %o4, %g0, %o4 ! final shift - - /* - * %o4 has 20 of the bits that should be in the low part of the - * result; %y has the bottom 12 (as %y's top 12). That is: - * - * %o4 %y - * +----------------+----------------+ - * | -12- | -20- | -12- | -20- | - * +------(---------+------)---------+ - * --hi-- ----low-part---- - * - * The upper 12 bits of %o4 should be sign-extended to form the - * high part of the product (i.e., highpart = %o4 >> 20). - */ - - rd %y, %o5 - sll %o4, 12, %o0 ! shift middle bits left 12 - srl %o5, 20, %o5 ! shift low bits right 20, zero fill at left - or %o5, %o0, %o0 ! construct low part of result - retl - sra %o4, 20, %o1 ! ... and extract high part of result - - .globl .mul_patch -.mul_patch: - smul %o0, %o1, %o0 - retl - rd %y, %o1 - nop diff --git a/arch/sparc/lib/muldi3.S b/arch/sparc/lib/muldi3.S index 7f17872d060..9794939d1c1 100644 --- a/arch/sparc/lib/muldi3.S +++ b/arch/sparc/lib/muldi3.S @@ -63,12 +63,12 @@ __muldi3: rd %y, %o1 mov %o1, %l3 mov %i1, %o0 - call .umul mov %i2, %o1 + umul %o0, %o1, %o0 mov %o0, %l0 mov %i0, %o0 - call .umul mov %i3, %o1 + umul %o0, %o1, %o0 add %l0, %o0, %l0 mov %l2, %i0 add %l2, %l0, %i0 diff --git a/arch/sparc/lib/rem.S b/arch/sparc/lib/rem.S deleted file mode 100644 index 42fb8625281..00000000000 --- a/arch/sparc/lib/rem.S +++ /dev/null @@ -1,384 +0,0 @@ -/* - * rem.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - - -/* This file is generated from divrem.m4; DO NOT EDIT! */ -/* - * Division and remainder, from Appendix E of the Sparc Version 8 - * Architecture Manual, with fixes from Gordon Irlam. - */ - -/* - * Input: dividend and divisor in %o0 and %o1 respectively. - * - * m4 parameters: - * .rem name of function to generate - * rem rem=div => %o0 / %o1; rem=rem => %o0 % %o1 - * true true=true => signed; true=false => unsigned - * - * Algorithm parameters: - * N how many bits per iteration we try to get (4) - * WORDSIZE total number of bits (32) - * - * Derived constants: - * TOPBITS number of bits in the top decade of a number - * - * Important variables: - * Q the partial quotient under development (initially 0) - * R the remainder so far, initially the dividend - * ITER number of main division loop iterations required; - * equal to ceil(log2(quotient) / N). Note that this - * is the log base (2^N) of the quotient. - * V the current comparand, initially divisor*2^(ITER*N-1) - * - * Cost: - * Current estimate for non-large dividend is - * ceil(log2(quotient) / N) * (10 + 7N/2) + C - * A large dividend is one greater than 2^(31-TOPBITS) and takes a - * different path, as the upper bits of the quotient must be developed - * one bit at a time. - */ - - - .globl .rem - .globl _Rem -.rem: -_Rem: /* needed for export */ - ! compute sign of result; if neither is negative, no problem - orcc %o1, %o0, %g0 ! either negative? - bge 2f ! no, go do the divide - mov %o0, %g2 ! compute sign in any case - - tst %o1 - bge 1f - tst %o0 - ! %o1 is definitely negative; %o0 might also be negative - bge 2f ! if %o0 not negative... - sub %g0, %o1, %o1 ! in any case, make %o1 nonneg -1: ! %o0 is negative, %o1 is nonnegative - sub %g0, %o0, %o0 ! make %o0 nonnegative -2: - - ! Ready to divide. Compute size of quotient; scale comparand. - orcc %o1, %g0, %o5 - bne 1f - mov %o0, %o3 - - ! Divide by zero trap. If it returns, return 0 (about as - ! wrong as possible, but that is what SunOS does...). - ta ST_DIV0 - retl - clr %o0 - -1: - cmp %o3, %o5 ! if %o1 exceeds %o0, done - blu Lgot_result ! (and algorithm fails otherwise) - clr %o2 - - sethi %hi(1 << (32 - 4 - 1)), %g1 - - cmp %o3, %g1 - blu Lnot_really_big - clr %o4 - - ! Here the dividend is >= 2**(31-N) or so. We must be careful here, - ! as our usual N-at-a-shot divide step will cause overflow and havoc. - ! The number of bits in the result here is N*ITER+SC, where SC <= N. - ! Compute ITER in an unorthodox manner: know we need to shift V into - ! the top decade: so do not even bother to compare to R. - 1: - cmp %o5, %g1 - bgeu 3f - mov 1, %g7 - - sll %o5, 4, %o5 - - b 1b - add %o4, 1, %o4 - - ! Now compute %g7. - 2: - addcc %o5, %o5, %o5 - - bcc Lnot_too_big - add %g7, 1, %g7 - - ! We get here if the %o1 overflowed while shifting. - ! This means that %o3 has the high-order bit set. - ! Restore %o5 and subtract from %o3. - sll %g1, 4, %g1 ! high order bit - srl %o5, 1, %o5 ! rest of %o5 - add %o5, %g1, %o5 - - b Ldo_single_div - sub %g7, 1, %g7 - - Lnot_too_big: - 3: - cmp %o5, %o3 - blu 2b - nop - - be Ldo_single_div - nop - /* NB: these are commented out in the V8-Sparc manual as well */ - /* (I do not understand this) */ - ! %o5 > %o3: went too far: back up 1 step - ! srl %o5, 1, %o5 - ! dec %g7 - ! do single-bit divide steps - ! - ! We have to be careful here. We know that %o3 >= %o5, so we can do the - ! first divide step without thinking. BUT, the others are conditional, - ! and are only done if %o3 >= 0. Because both %o3 and %o5 may have the high- - ! order bit set in the first step, just falling into the regular - ! division loop will mess up the first time around. - ! So we unroll slightly... - Ldo_single_div: - subcc %g7, 1, %g7 - bl Lend_regular_divide - nop - - sub %o3, %o5, %o3 - mov 1, %o2 - - b Lend_single_divloop - nop - Lsingle_divloop: - sll %o2, 1, %o2 - - bl 1f - srl %o5, 1, %o5 - ! %o3 >= 0 - sub %o3, %o5, %o3 - - b 2f - add %o2, 1, %o2 - 1: ! %o3 < 0 - add %o3, %o5, %o3 - sub %o2, 1, %o2 - 2: - Lend_single_divloop: - subcc %g7, 1, %g7 - bge Lsingle_divloop - tst %o3 - - b,a Lend_regular_divide - -Lnot_really_big: -1: - sll %o5, 4, %o5 - cmp %o5, %o3 - bleu 1b - addcc %o4, 1, %o4 - be Lgot_result - sub %o4, 1, %o4 - - tst %o3 ! set up for initial iteration -Ldivloop: - sll %o2, 4, %o2 - ! depth 1, accumulated bits 0 - bl L.1.16 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 2, accumulated bits 1 - bl L.2.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits 3 - bl L.3.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 7 - bl L.4.23 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - - b 9f - add %o2, (7*2+1), %o2 - -L.4.23: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (7*2-1), %o2 - -L.3.19: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 5 - bl L.4.21 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (5*2+1), %o2 - -L.4.21: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (5*2-1), %o2 - -L.2.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits 1 - bl L.3.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 3 - bl L.4.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (3*2+1), %o2 - -L.4.19: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (3*2-1), %o2 - -L.3.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 1 - bl L.4.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (1*2+1), %o2 - -L.4.17: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (1*2-1), %o2 - -L.1.16: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 2, accumulated bits -1 - bl L.2.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits -1 - bl L.3.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -1 - bl L.4.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2+1), %o2 - -L.4.15: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2-1), %o2 - -L.3.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -3 - bl L.4.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2+1), %o2 - -L.4.13: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2-1), %o2 - -L.2.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits -3 - bl L.3.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -5 - bl L.4.11 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2+1), %o2 - -L.4.11: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2-1), %o2 - - -L.3.13: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -7 - bl L.4.9 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2+1), %o2 - -L.4.9: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2-1), %o2 - - 9: -Lend_regular_divide: - subcc %o4, 1, %o4 - bge Ldivloop - tst %o3 - - bl,a Lgot_result - ! non-restoring fixup here (one instruction only!) - add %o3, %o1, %o3 - -Lgot_result: - ! check to see if answer should be < 0 - tst %g2 - bl,a 1f - sub %g0, %o3, %o3 -1: - retl - mov %o3, %o0 - - .globl .rem_patch -.rem_patch: - sra %o0, 0x1f, %o4 - wr %o4, 0x0, %y - nop - nop - nop - sdivcc %o0, %o1, %o2 - bvs,a 1f - xnor %o2, %g0, %o2 -1: smul %o2, %o1, %o2 - retl - sub %o0, %o2, %o0 - nop diff --git a/arch/sparc/lib/rwsem_32.S b/arch/sparc/lib/rwsem_32.S deleted file mode 100644 index 9675268e7fd..00000000000 --- a/arch/sparc/lib/rwsem_32.S +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Assembly part of rw semaphores. - * - * Copyright (C) 1999 Jakub Jelinek (jakub@redhat.com) - */ - -#include <asm/ptrace.h> -#include <asm/psr.h> - - .section .sched.text, "ax" - .align 4 - - .globl ___down_read -___down_read: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - nop - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - sub %g7, 1, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - sub %g7, 1, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - add %g7, 1, %g7 - nop - nop - subcc %g7, 1, %g7 - bneg 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - bcs 4f - mov %g5, %l5 - call down_read_failed - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba ___down_read - restore %l5, %g0, %g5 -4: call down_read_failed_biased - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .globl ___down_write -___down_write: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - sethi %hi(0x01000000), %g2 - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - sub %g7, %g2, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - sub %g7, %g2, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - add %g7, %g2, %g7 - nop - nop - subcc %g7, %g2, %g7 - bne 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - bcs 4f - mov %g5, %l5 - call down_write_failed - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba ___down_write - restore %l5, %g0, %g5 -4: call down_write_failed_biased - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .text - .globl ___up_read -___up_read: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - nop - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - add %g7, 1, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - add %g7, 1, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - nop - nop - nop - cmp %g7, 0 - be 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - mov %g5, %l5 - clr %o1 - call __rwsem_wake - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 - - .globl ___up_write -___up_write: - rd %psr, %g3 - nop - nop - nop - or %g3, PSR_PIL, %g7 - wr %g7, 0, %psr - sethi %hi(0x01000000), %g2 - nop - nop -#ifdef CONFIG_SMP -1: ldstub [%g1 + 4], %g7 - tst %g7 - bne 1b - ld [%g1], %g7 - add %g7, %g2, %g7 - st %g7, [%g1] - stb %g0, [%g1 + 4] -#else - ld [%g1], %g7 - add %g7, %g2, %g7 - st %g7, [%g1] -#endif - wr %g3, 0, %psr - sub %g7, %g2, %g7 - nop - nop - addcc %g7, %g2, %g7 - bcs 3f - nop -2: jmpl %o7, %g0 - mov %g4, %o7 -3: save %sp, -64, %sp - mov %g1, %l1 - mov %g4, %l4 - mov %g5, %l5 - mov %g7, %o1 - call __rwsem_wake - mov %l1, %o0 - mov %l1, %g1 - mov %l4, %g4 - ba 2b - restore %l5, %g0, %g5 diff --git a/arch/sparc/lib/sdiv.S b/arch/sparc/lib/sdiv.S deleted file mode 100644 index f0a0d4e4db7..00000000000 --- a/arch/sparc/lib/sdiv.S +++ /dev/null @@ -1,381 +0,0 @@ -/* - * sdiv.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - - -/* This file is generated from divrem.m4; DO NOT EDIT! */ -/* - * Division and remainder, from Appendix E of the Sparc Version 8 - * Architecture Manual, with fixes from Gordon Irlam. - */ - -/* - * Input: dividend and divisor in %o0 and %o1 respectively. - * - * m4 parameters: - * .div name of function to generate - * div div=div => %o0 / %o1; div=rem => %o0 % %o1 - * true true=true => signed; true=false => unsigned - * - * Algorithm parameters: - * N how many bits per iteration we try to get (4) - * WORDSIZE total number of bits (32) - * - * Derived constants: - * TOPBITS number of bits in the top decade of a number - * - * Important variables: - * Q the partial quotient under development (initially 0) - * R the remainder so far, initially the dividend - * ITER number of main division loop iterations required; - * equal to ceil(log2(quotient) / N). Note that this - * is the log base (2^N) of the quotient. - * V the current comparand, initially divisor*2^(ITER*N-1) - * - * Cost: - * Current estimate for non-large dividend is - * ceil(log2(quotient) / N) * (10 + 7N/2) + C - * A large dividend is one greater than 2^(31-TOPBITS) and takes a - * different path, as the upper bits of the quotient must be developed - * one bit at a time. - */ - - - .globl .div - .globl _Div -.div: -_Div: /* needed for export */ - ! compute sign of result; if neither is negative, no problem - orcc %o1, %o0, %g0 ! either negative? - bge 2f ! no, go do the divide - xor %o1, %o0, %g2 ! compute sign in any case - - tst %o1 - bge 1f - tst %o0 - ! %o1 is definitely negative; %o0 might also be negative - bge 2f ! if %o0 not negative... - sub %g0, %o1, %o1 ! in any case, make %o1 nonneg -1: ! %o0 is negative, %o1 is nonnegative - sub %g0, %o0, %o0 ! make %o0 nonnegative -2: - - ! Ready to divide. Compute size of quotient; scale comparand. - orcc %o1, %g0, %o5 - bne 1f - mov %o0, %o3 - - ! Divide by zero trap. If it returns, return 0 (about as - ! wrong as possible, but that is what SunOS does...). - ta ST_DIV0 - retl - clr %o0 - -1: - cmp %o3, %o5 ! if %o1 exceeds %o0, done - blu Lgot_result ! (and algorithm fails otherwise) - clr %o2 - - sethi %hi(1 << (32 - 4 - 1)), %g1 - - cmp %o3, %g1 - blu Lnot_really_big - clr %o4 - - ! Here the dividend is >= 2**(31-N) or so. We must be careful here, - ! as our usual N-at-a-shot divide step will cause overflow and havoc. - ! The number of bits in the result here is N*ITER+SC, where SC <= N. - ! Compute ITER in an unorthodox manner: know we need to shift V into - ! the top decade: so do not even bother to compare to R. - 1: - cmp %o5, %g1 - bgeu 3f - mov 1, %g7 - - sll %o5, 4, %o5 - - b 1b - add %o4, 1, %o4 - - ! Now compute %g7. - 2: - addcc %o5, %o5, %o5 - bcc Lnot_too_big - add %g7, 1, %g7 - - ! We get here if the %o1 overflowed while shifting. - ! This means that %o3 has the high-order bit set. - ! Restore %o5 and subtract from %o3. - sll %g1, 4, %g1 ! high order bit - srl %o5, 1, %o5 ! rest of %o5 - add %o5, %g1, %o5 - - b Ldo_single_div - sub %g7, 1, %g7 - - Lnot_too_big: - 3: - cmp %o5, %o3 - blu 2b - nop - - be Ldo_single_div - nop - /* NB: these are commented out in the V8-Sparc manual as well */ - /* (I do not understand this) */ - ! %o5 > %o3: went too far: back up 1 step - ! srl %o5, 1, %o5 - ! dec %g7 - ! do single-bit divide steps - ! - ! We have to be careful here. We know that %o3 >= %o5, so we can do the - ! first divide step without thinking. BUT, the others are conditional, - ! and are only done if %o3 >= 0. Because both %o3 and %o5 may have the high- - ! order bit set in the first step, just falling into the regular - ! division loop will mess up the first time around. - ! So we unroll slightly... - Ldo_single_div: - subcc %g7, 1, %g7 - bl Lend_regular_divide - nop - - sub %o3, %o5, %o3 - mov 1, %o2 - - b Lend_single_divloop - nop - Lsingle_divloop: - sll %o2, 1, %o2 - - bl 1f - srl %o5, 1, %o5 - ! %o3 >= 0 - sub %o3, %o5, %o3 - - b 2f - add %o2, 1, %o2 - 1: ! %o3 < 0 - add %o3, %o5, %o3 - sub %o2, 1, %o2 - 2: - Lend_single_divloop: - subcc %g7, 1, %g7 - bge Lsingle_divloop - tst %o3 - - b,a Lend_regular_divide - -Lnot_really_big: -1: - sll %o5, 4, %o5 - cmp %o5, %o3 - bleu 1b - addcc %o4, 1, %o4 - - be Lgot_result - sub %o4, 1, %o4 - - tst %o3 ! set up for initial iteration -Ldivloop: - sll %o2, 4, %o2 - ! depth 1, accumulated bits 0 - bl L.1.16 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 2, accumulated bits 1 - bl L.2.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits 3 - bl L.3.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 7 - bl L.4.23 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (7*2+1), %o2 - -L.4.23: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (7*2-1), %o2 - -L.3.19: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 5 - bl L.4.21 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (5*2+1), %o2 - -L.4.21: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (5*2-1), %o2 - -L.2.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits 1 - bl L.3.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 3 - bl L.4.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (3*2+1), %o2 - -L.4.19: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (3*2-1), %o2 - - -L.3.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 1 - bl L.4.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (1*2+1), %o2 - -L.4.17: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (1*2-1), %o2 - -L.1.16: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 2, accumulated bits -1 - bl L.2.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits -1 - bl L.3.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -1 - bl L.4.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2+1), %o2 - -L.4.15: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2-1), %o2 - -L.3.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -3 - bl L.4.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2+1), %o2 - -L.4.13: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2-1), %o2 - -L.2.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits -3 - bl L.3.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -5 - bl L.4.11 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2+1), %o2 - -L.4.11: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2-1), %o2 - -L.3.13: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -7 - bl L.4.9 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2+1), %o2 - -L.4.9: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2-1), %o2 - - 9: -Lend_regular_divide: - subcc %o4, 1, %o4 - bge Ldivloop - tst %o3 - - bl,a Lgot_result - ! non-restoring fixup here (one instruction only!) - sub %o2, 1, %o2 - -Lgot_result: - ! check to see if answer should be < 0 - tst %g2 - bl,a 1f - sub %g0, %o2, %o2 -1: - retl - mov %o2, %o0 - - .globl .div_patch -.div_patch: - sra %o0, 0x1f, %o2 - wr %o2, 0x0, %y - nop - nop - nop - sdivcc %o0, %o1, %o0 - bvs,a 1f - xnor %o0, %g0, %o0 -1: retl - nop diff --git a/arch/sparc/lib/strlen_user_32.S b/arch/sparc/lib/strlen_user_32.S deleted file mode 100644 index 8c8a371df3c..00000000000 --- a/arch/sparc/lib/strlen_user_32.S +++ /dev/null @@ -1,109 +0,0 @@ -/* strlen_user.S: Sparc optimized strlen_user code - * - * Return length of string in userspace including terminating 0 - * or 0 for error - * - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - -10: - ldub [%o0], %o5 - cmp %o5, 0 - be 1f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 4f - or %o4, %lo(HI_MAGIC), %o3 -11: - ldub [%o0], %o5 - cmp %o5, 0 - be 2f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be 5f - sethi %hi(LO_MAGIC), %o4 -12: - ldub [%o0], %o5 - cmp %o5, 0 - be 3f - add %o0, 1, %o0 - b 13f - or %o4, %lo(LO_MAGIC), %o2 -1: - retl - mov 1, %o0 -2: - retl - mov 2, %o0 -3: - retl - mov 3, %o0 - - .align 4 - .global __strlen_user, __strnlen_user -__strlen_user: - sethi %hi(32768), %o1 -__strnlen_user: - mov %o1, %g1 - mov %o0, %o1 - andcc %o0, 3, %g0 - bne 10b - sethi %hi(HI_MAGIC), %o4 - or %o4, %lo(HI_MAGIC), %o3 -4: - sethi %hi(LO_MAGIC), %o4 -5: - or %o4, %lo(LO_MAGIC), %o2 -13: - ld [%o0], %o5 -2: - sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - bne 82f - add %o0, 4, %o0 - sub %o0, %o1, %g2 -81: cmp %g2, %g1 - blu 13b - mov %o0, %o4 - ba,a 1f - - /* Check every byte. */ -82: srl %o5, 24, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o0, -3, %o4 - srl %o5, 16, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - srl %o5, 8, %g5 - andcc %g5, 0xff, %g0 - be 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne 81b - sub %o0, %o1, %g2 - - add %o4, 1, %o4 -1: - retl - sub %o4, %o1, %o0 - - .section .fixup,#alloc,#execinstr - .align 4 -9: - retl - clr %o0 - - .section __ex_table,#alloc - .align 4 - - .word 10b, 9b - .word 11b, 9b - .word 12b, 9b - .word 13b, 9b diff --git a/arch/sparc/lib/strlen_user_64.S b/arch/sparc/lib/strlen_user_64.S deleted file mode 100644 index 114ed111e25..00000000000 --- a/arch/sparc/lib/strlen_user_64.S +++ /dev/null @@ -1,95 +0,0 @@ -/* strlen_user.S: Sparc64 optimized strlen_user code - * - * Return length of string in userspace including terminating 0 - * or 0 for error - * - * Copyright (C) 1991,1996 Free Software Foundation - * Copyright (C) 1996,1999 David S. Miller (davem@redhat.com) - * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <asm/asi.h> - -#define LO_MAGIC 0x01010101 -#define HI_MAGIC 0x80808080 - - .align 4 - .global __strlen_user, __strnlen_user -__strlen_user: - sethi %hi(32768), %o1 -__strnlen_user: - mov %o1, %g1 - mov %o0, %o1 - andcc %o0, 3, %g0 - be,pt %icc, 9f - sethi %hi(HI_MAGIC), %o4 -10: lduba [%o0] %asi, %o5 - brz,pn %o5, 21f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be,pn %icc, 4f - or %o4, %lo(HI_MAGIC), %o3 -11: lduba [%o0] %asi, %o5 - brz,pn %o5, 22f - add %o0, 1, %o0 - andcc %o0, 3, %g0 - be,pt %icc, 13f - srl %o3, 7, %o2 -12: lduba [%o0] %asi, %o5 - brz,pn %o5, 23f - add %o0, 1, %o0 - ba,pt %icc, 2f -15: lda [%o0] %asi, %o5 -9: or %o4, %lo(HI_MAGIC), %o3 -4: srl %o3, 7, %o2 -13: lda [%o0] %asi, %o5 -2: sub %o5, %o2, %o4 - andcc %o4, %o3, %g0 - bne,pn %icc, 82f - add %o0, 4, %o0 - sub %o0, %o1, %g2 -81: cmp %g2, %g1 - blu,pt %icc, 13b - mov %o0, %o4 - ba,a,pt %xcc, 1f - - /* Check every byte. */ -82: srl %o5, 24, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o0, -3, %o4 - srl %o5, 16, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o4, 1, %o4 - srl %o5, 8, %g7 - andcc %g7, 0xff, %g0 - be,pn %icc, 1f - add %o4, 1, %o4 - andcc %o5, 0xff, %g0 - bne,pt %icc, 81b - sub %o0, %o1, %g2 - add %o4, 1, %o4 -1: retl - sub %o4, %o1, %o0 -21: retl - mov 1, %o0 -22: retl - mov 2, %o0 -23: retl - mov 3, %o0 - - .section .fixup,#alloc,#execinstr - .align 4 -30: - retl - clr %o0 - - .section __ex_table,"a" - .align 4 - - .word 10b, 30b - .word 11b, 30b - .word 12b, 30b - .word 15b, 30b - .word 13b, 30b diff --git a/arch/sparc/lib/strncmp_32.S b/arch/sparc/lib/strncmp_32.S index 494ec664537..c0d1b568c1c 100644 --- a/arch/sparc/lib/strncmp_32.S +++ b/arch/sparc/lib/strncmp_32.S @@ -3,11 +3,10 @@ * generic strncmp routine. */ +#include <linux/linkage.h> + .text - .align 4 - .global __strncmp, strncmp -__strncmp: -strncmp: +ENTRY(strncmp) mov %o0, %g3 mov 0, %o3 @@ -116,3 +115,4 @@ strncmp: and %g2, 0xff, %o0 retl sub %o3, %o0, %o0 +ENDPROC(strncmp) diff --git a/arch/sparc/lib/strncmp_64.S b/arch/sparc/lib/strncmp_64.S index 980e8375155..0656627166f 100644 --- a/arch/sparc/lib/strncmp_64.S +++ b/arch/sparc/lib/strncmp_64.S @@ -4,13 +4,11 @@ * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <linux/linkage.h> #include <asm/asi.h> .text - .align 32 - .globl strncmp - .type strncmp,#function -strncmp: +ENTRY(strncmp) brlez,pn %o2, 3f lduba [%o0] (ASI_PNF), %o3 1: @@ -29,4 +27,4 @@ strncmp: 3: retl clr %o0 - .size strncmp, .-strncmp +ENDPROC(strncmp) diff --git a/arch/sparc/lib/strncpy_from_user_32.S b/arch/sparc/lib/strncpy_from_user_32.S deleted file mode 100644 index d77198976a6..00000000000 --- a/arch/sparc/lib/strncpy_from_user_32.S +++ /dev/null @@ -1,47 +0,0 @@ -/* strncpy_from_user.S: Sparc strncpy from userspace. - * - * Copyright(C) 1996 David S. Miller - */ - -#include <asm/ptrace.h> -#include <asm/errno.h> - - .text - .align 4 - - /* Must return: - * - * -EFAULT for an exception - * count if we hit the buffer limit - * bytes copied if we hit a null byte - */ - - .globl __strncpy_from_user -__strncpy_from_user: - /* %o0=dest, %o1=src, %o2=count */ - mov %o2, %o3 -1: - subcc %o2, 1, %o2 - bneg 2f - nop -10: - ldub [%o1], %o4 - add %o0, 1, %o0 - cmp %o4, 0 - add %o1, 1, %o1 - bne 1b - stb %o4, [%o0 - 1] -2: - add %o2, 1, %o0 - retl - sub %o3, %o0, %o0 - - .section .fixup,#alloc,#execinstr - .align 4 -4: - retl - mov -EFAULT, %o0 - - .section __ex_table,#alloc - .align 4 - .word 10b, 4b diff --git a/arch/sparc/lib/strncpy_from_user_64.S b/arch/sparc/lib/strncpy_from_user_64.S deleted file mode 100644 index 511c8f136f9..00000000000 --- a/arch/sparc/lib/strncpy_from_user_64.S +++ /dev/null @@ -1,135 +0,0 @@ -/* - * strncpy_from_user.S: Sparc64 strncpy from userspace. - * - * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) - */ - -#include <asm/asi.h> -#include <asm/errno.h> - - .data - .align 8 -0: .xword 0x0101010101010101 - - .text - .align 32 - - /* Must return: - * - * -EFAULT for an exception - * count if we hit the buffer limit - * bytes copied if we hit a null byte - * (without the null byte) - * - * This implementation assumes: - * %o1 is 8 aligned => !(%o2 & 7) - * %o0 is 8 aligned (if not, it will be slooooow, but will work) - * - * This is optimized for the common case: - * in my stats, 90% of src are 8 aligned (even on sparc32) - * and average length is 18 or so. - */ - - .globl __strncpy_from_user - .type __strncpy_from_user,#function -__strncpy_from_user: - /* %o0=dest, %o1=src, %o2=count */ - andcc %o1, 7, %g0 ! IEU1 Group - bne,pn %icc, 30f ! CTI - add %o0, %o2, %g3 ! IEU0 -60: ldxa [%o1] %asi, %g1 ! Load Group - brlez,pn %o2, 10f ! CTI - mov %o0, %o3 ! IEU0 -50: sethi %hi(0b), %o4 ! IEU0 Group - ldx [%o4 + %lo(0b)], %o4 ! Load - sllx %o4, 7, %o5 ! IEU1 Group -1: sub %g1, %o4, %g2 ! IEU0 Group - stx %g1, [%o0] ! Store - add %o0, 8, %o0 ! IEU1 - andcc %g2, %o5, %g0 ! IEU1 Group - bne,pn %xcc, 5f ! CTI - add %o1, 8, %o1 ! IEU0 - cmp %o0, %g3 ! IEU1 Group - bl,a,pt %xcc, 1b ! CTI -61: ldxa [%o1] %asi, %g1 ! Load -10: retl ! CTI Group - mov %o2, %o0 ! IEU0 -5: srlx %g2, 32, %g7 ! IEU0 Group - sethi %hi(0xff00), %o4 ! IEU1 - andcc %g7, %o5, %g0 ! IEU1 Group - be,pn %icc, 2f ! CTI - or %o4, %lo(0xff00), %o4 ! IEU0 - srlx %g1, 48, %g7 ! IEU0 Group - andcc %g7, %o4, %g0 ! IEU1 Group - be,pn %icc, 50f ! CTI - andcc %g7, 0xff, %g0 ! IEU1 Group - be,pn %icc, 51f ! CTI - srlx %g1, 32, %g7 ! IEU0 - andcc %g7, %o4, %g0 ! IEU1 Group - be,pn %icc, 52f ! CTI - andcc %g7, 0xff, %g0 ! IEU1 Group - be,pn %icc, 53f ! CTI -2: andcc %g2, %o5, %g0 ! IEU1 Group - be,pn %icc, 2f ! CTI - srl %g1, 16, %g7 ! IEU0 - andcc %g7, %o4, %g0 ! IEU1 Group - be,pn %icc, 54f ! CTI - andcc %g7, 0xff, %g0 ! IEU1 Group - be,pn %icc, 55f ! CTI - andcc %g1, %o4, %g0 ! IEU1 Group - be,pn %icc, 56f ! CTI - andcc %g1, 0xff, %g0 ! IEU1 Group - be,a,pn %icc, 57f ! CTI - sub %o0, %o3, %o0 ! IEU0 -2: cmp %o0, %g3 ! IEU1 Group - bl,a,pt %xcc, 50b ! CTI -62: ldxa [%o1] %asi, %g1 ! Load - retl ! CTI Group - mov %o2, %o0 ! IEU0 -50: sub %o0, %o3, %o0 - retl - sub %o0, 8, %o0 -51: sub %o0, %o3, %o0 - retl - sub %o0, 7, %o0 -52: sub %o0, %o3, %o0 - retl - sub %o0, 6, %o0 -53: sub %o0, %o3, %o0 - retl - sub %o0, 5, %o0 -54: sub %o0, %o3, %o0 - retl - sub %o0, 4, %o0 -55: sub %o0, %o3, %o0 - retl - sub %o0, 3, %o0 -56: sub %o0, %o3, %o0 - retl - sub %o0, 2, %o0 -57: retl - sub %o0, 1, %o0 -30: brlez,pn %o2, 3f - sub %g0, %o2, %o3 - add %o0, %o2, %o0 -63: lduba [%o1] %asi, %o4 -1: add %o1, 1, %o1 - brz,pn %o4, 2f - stb %o4, [%o0 + %o3] - addcc %o3, 1, %o3 - bne,pt %xcc, 1b -64: lduba [%o1] %asi, %o4 -3: retl - mov %o2, %o0 -2: retl - add %o2, %o3, %o0 - .size __strncpy_from_user, .-__strncpy_from_user - - .section __ex_table,"a" - .align 4 - .word 60b, __retl_efault - .word 61b, __retl_efault - .word 62b, __retl_efault - .word 63b, __retl_efault - .word 64b, __retl_efault - .previous diff --git a/arch/sparc/lib/ucmpdi2.c b/arch/sparc/lib/ucmpdi2.c new file mode 100644 index 00000000000..1e06ed50068 --- /dev/null +++ b/arch/sparc/lib/ucmpdi2.c @@ -0,0 +1,19 @@ +#include <linux/module.h> +#include "libgcc.h" + +word_type __ucmpdi2(unsigned long long a, unsigned long long b) +{ + const DWunion au = {.ll = a}; + const DWunion bu = {.ll = b}; + + if ((unsigned int) au.s.high < (unsigned int) bu.s.high) + return 0; + else if ((unsigned int) au.s.high > (unsigned int) bu.s.high) + return 2; + if ((unsigned int) au.s.low < (unsigned int) bu.s.low) + return 0; + else if ((unsigned int) au.s.low > (unsigned int) bu.s.low) + return 2; + return 1; +} +EXPORT_SYMBOL(__ucmpdi2); diff --git a/arch/sparc/lib/udiv.S b/arch/sparc/lib/udiv.S deleted file mode 100644 index 2101405bdfc..00000000000 --- a/arch/sparc/lib/udiv.S +++ /dev/null @@ -1,357 +0,0 @@ -/* - * udiv.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - - -/* This file is generated from divrem.m4; DO NOT EDIT! */ -/* - * Division and remainder, from Appendix E of the Sparc Version 8 - * Architecture Manual, with fixes from Gordon Irlam. - */ - -/* - * Input: dividend and divisor in %o0 and %o1 respectively. - * - * m4 parameters: - * .udiv name of function to generate - * div div=div => %o0 / %o1; div=rem => %o0 % %o1 - * false false=true => signed; false=false => unsigned - * - * Algorithm parameters: - * N how many bits per iteration we try to get (4) - * WORDSIZE total number of bits (32) - * - * Derived constants: - * TOPBITS number of bits in the top decade of a number - * - * Important variables: - * Q the partial quotient under development (initially 0) - * R the remainder so far, initially the dividend - * ITER number of main division loop iterations required; - * equal to ceil(log2(quotient) / N). Note that this - * is the log base (2^N) of the quotient. - * V the current comparand, initially divisor*2^(ITER*N-1) - * - * Cost: - * Current estimate for non-large dividend is - * ceil(log2(quotient) / N) * (10 + 7N/2) + C - * A large dividend is one greater than 2^(31-TOPBITS) and takes a - * different path, as the upper bits of the quotient must be developed - * one bit at a time. - */ - - - .globl .udiv - .globl _Udiv -.udiv: -_Udiv: /* needed for export */ - - ! Ready to divide. Compute size of quotient; scale comparand. - orcc %o1, %g0, %o5 - bne 1f - mov %o0, %o3 - - ! Divide by zero trap. If it returns, return 0 (about as - ! wrong as possible, but that is what SunOS does...). - ta ST_DIV0 - retl - clr %o0 - -1: - cmp %o3, %o5 ! if %o1 exceeds %o0, done - blu Lgot_result ! (and algorithm fails otherwise) - clr %o2 - - sethi %hi(1 << (32 - 4 - 1)), %g1 - - cmp %o3, %g1 - blu Lnot_really_big - clr %o4 - - ! Here the dividend is >= 2**(31-N) or so. We must be careful here, - ! as our usual N-at-a-shot divide step will cause overflow and havoc. - ! The number of bits in the result here is N*ITER+SC, where SC <= N. - ! Compute ITER in an unorthodox manner: know we need to shift V into - ! the top decade: so do not even bother to compare to R. - 1: - cmp %o5, %g1 - bgeu 3f - mov 1, %g7 - - sll %o5, 4, %o5 - - b 1b - add %o4, 1, %o4 - - ! Now compute %g7. - 2: - addcc %o5, %o5, %o5 - bcc Lnot_too_big - add %g7, 1, %g7 - - ! We get here if the %o1 overflowed while shifting. - ! This means that %o3 has the high-order bit set. - ! Restore %o5 and subtract from %o3. - sll %g1, 4, %g1 ! high order bit - srl %o5, 1, %o5 ! rest of %o5 - add %o5, %g1, %o5 - - b Ldo_single_div - sub %g7, 1, %g7 - - Lnot_too_big: - 3: - cmp %o5, %o3 - blu 2b - nop - - be Ldo_single_div - nop - /* NB: these are commented out in the V8-Sparc manual as well */ - /* (I do not understand this) */ - ! %o5 > %o3: went too far: back up 1 step - ! srl %o5, 1, %o5 - ! dec %g7 - ! do single-bit divide steps - ! - ! We have to be careful here. We know that %o3 >= %o5, so we can do the - ! first divide step without thinking. BUT, the others are conditional, - ! and are only done if %o3 >= 0. Because both %o3 and %o5 may have the high- - ! order bit set in the first step, just falling into the regular - ! division loop will mess up the first time around. - ! So we unroll slightly... - Ldo_single_div: - subcc %g7, 1, %g7 - bl Lend_regular_divide - nop - - sub %o3, %o5, %o3 - mov 1, %o2 - - b Lend_single_divloop - nop - Lsingle_divloop: - sll %o2, 1, %o2 - bl 1f - srl %o5, 1, %o5 - ! %o3 >= 0 - sub %o3, %o5, %o3 - b 2f - add %o2, 1, %o2 - 1: ! %o3 < 0 - add %o3, %o5, %o3 - sub %o2, 1, %o2 - 2: - Lend_single_divloop: - subcc %g7, 1, %g7 - bge Lsingle_divloop - tst %o3 - - b,a Lend_regular_divide - -Lnot_really_big: -1: - sll %o5, 4, %o5 - - cmp %o5, %o3 - bleu 1b - addcc %o4, 1, %o4 - - be Lgot_result - sub %o4, 1, %o4 - - tst %o3 ! set up for initial iteration -Ldivloop: - sll %o2, 4, %o2 - ! depth 1, accumulated bits 0 - bl L.1.16 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 2, accumulated bits 1 - bl L.2.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits 3 - bl L.3.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 7 - bl L.4.23 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (7*2+1), %o2 - -L.4.23: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (7*2-1), %o2 - -L.3.19: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 5 - bl L.4.21 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (5*2+1), %o2 - -L.4.21: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (5*2-1), %o2 - -L.2.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits 1 - bl L.3.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 3 - bl L.4.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (3*2+1), %o2 - -L.4.19: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (3*2-1), %o2 - -L.3.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 1 - bl L.4.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (1*2+1), %o2 - -L.4.17: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (1*2-1), %o2 - -L.1.16: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 2, accumulated bits -1 - bl L.2.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits -1 - bl L.3.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -1 - bl L.4.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2+1), %o2 - -L.4.15: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2-1), %o2 - -L.3.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -3 - bl L.4.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2+1), %o2 - -L.4.13: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2-1), %o2 - -L.2.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits -3 - bl L.3.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -5 - bl L.4.11 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2+1), %o2 - -L.4.11: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2-1), %o2 - -L.3.13: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -7 - bl L.4.9 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2+1), %o2 - -L.4.9: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2-1), %o2 - - 9: -Lend_regular_divide: - subcc %o4, 1, %o4 - bge Ldivloop - tst %o3 - - bl,a Lgot_result - ! non-restoring fixup here (one instruction only!) - sub %o2, 1, %o2 - -Lgot_result: - - retl - mov %o2, %o0 - - .globl .udiv_patch -.udiv_patch: - wr %g0, 0x0, %y - nop - nop - retl - udiv %o0, %o1, %o0 - nop diff --git a/arch/sparc/lib/udivdi3.S b/arch/sparc/lib/udivdi3.S index b430f1f0ef6..24e0a355e2e 100644 --- a/arch/sparc/lib/udivdi3.S +++ b/arch/sparc/lib/udivdi3.S @@ -60,8 +60,9 @@ __udivdi3: bne .LL77 mov %i0,%o2 mov 1,%o0 - call .udiv,0 mov 0,%o1 + wr %g0, 0, %y + udiv %o0, %o1, %o0 mov %o0,%o3 mov %i0,%o2 .LL77: diff --git a/arch/sparc/lib/umul.S b/arch/sparc/lib/umul.S deleted file mode 100644 index 1f36ae68252..00000000000 --- a/arch/sparc/lib/umul.S +++ /dev/null @@ -1,171 +0,0 @@ -/* - * umul.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - - -/* - * Unsigned multiply. Returns %o0 * %o1 in %o1%o0 (i.e., %o1 holds the - * upper 32 bits of the 64-bit product). - * - * This code optimizes short (less than 13-bit) multiplies. Short - * multiplies require 25 instruction cycles, and long ones require - * 45 instruction cycles. - * - * On return, overflow has occurred (%o1 is not zero) if and only if - * the Z condition code is clear, allowing, e.g., the following: - * - * call .umul - * nop - * bnz overflow (or tnz) - */ - - .globl .umul - .globl _Umul -.umul: -_Umul: /* needed for export */ - or %o0, %o1, %o4 - mov %o0, %y ! multiplier -> Y - - andncc %o4, 0xfff, %g0 ! test bits 12..31 of *both* args - be Lmul_shortway ! if zero, can do it the short way - andcc %g0, %g0, %o4 ! zero the partial product and clear N and V - - /* - * Long multiply. 32 steps, followed by a final shift step. - */ - mulscc %o4, %o1, %o4 ! 1 - mulscc %o4, %o1, %o4 ! 2 - mulscc %o4, %o1, %o4 ! 3 - mulscc %o4, %o1, %o4 ! 4 - mulscc %o4, %o1, %o4 ! 5 - mulscc %o4, %o1, %o4 ! 6 - mulscc %o4, %o1, %o4 ! 7 - mulscc %o4, %o1, %o4 ! 8 - mulscc %o4, %o1, %o4 ! 9 - mulscc %o4, %o1, %o4 ! 10 - mulscc %o4, %o1, %o4 ! 11 - mulscc %o4, %o1, %o4 ! 12 - mulscc %o4, %o1, %o4 ! 13 - mulscc %o4, %o1, %o4 ! 14 - mulscc %o4, %o1, %o4 ! 15 - mulscc %o4, %o1, %o4 ! 16 - mulscc %o4, %o1, %o4 ! 17 - mulscc %o4, %o1, %o4 ! 18 - mulscc %o4, %o1, %o4 ! 19 - mulscc %o4, %o1, %o4 ! 20 - mulscc %o4, %o1, %o4 ! 21 - mulscc %o4, %o1, %o4 ! 22 - mulscc %o4, %o1, %o4 ! 23 - mulscc %o4, %o1, %o4 ! 24 - mulscc %o4, %o1, %o4 ! 25 - mulscc %o4, %o1, %o4 ! 26 - mulscc %o4, %o1, %o4 ! 27 - mulscc %o4, %o1, %o4 ! 28 - mulscc %o4, %o1, %o4 ! 29 - mulscc %o4, %o1, %o4 ! 30 - mulscc %o4, %o1, %o4 ! 31 - mulscc %o4, %o1, %o4 ! 32 - mulscc %o4, %g0, %o4 ! final shift - - - /* - * Normally, with the shift-and-add approach, if both numbers are - * positive you get the correct result. With 32-bit two's-complement - * numbers, -x is represented as - * - * x 32 - * ( 2 - ------ ) mod 2 * 2 - * 32 - * 2 - * - * (the `mod 2' subtracts 1 from 1.bbbb). To avoid lots of 2^32s, - * we can treat this as if the radix point were just to the left - * of the sign bit (multiply by 2^32), and get - * - * -x = (2 - x) mod 2 - * - * Then, ignoring the `mod 2's for convenience: - * - * x * y = xy - * -x * y = 2y - xy - * x * -y = 2x - xy - * -x * -y = 4 - 2x - 2y + xy - * - * For signed multiplies, we subtract (x << 32) from the partial - * product to fix this problem for negative multipliers (see mul.s). - * Because of the way the shift into the partial product is calculated - * (N xor V), this term is automatically removed for the multiplicand, - * so we don't have to adjust. - * - * But for unsigned multiplies, the high order bit wasn't a sign bit, - * and the correction is wrong. So for unsigned multiplies where the - * high order bit is one, we end up with xy - (y << 32). To fix it - * we add y << 32. - */ -#if 0 - tst %o1 - bl,a 1f ! if %o1 < 0 (high order bit = 1), - add %o4, %o0, %o4 ! %o4 += %o0 (add y to upper half) - -1: - rd %y, %o0 ! get lower half of product - retl - addcc %o4, %g0, %o1 ! put upper half in place and set Z for %o1==0 -#else - /* Faster code from tege@sics.se. */ - sra %o1, 31, %o2 ! make mask from sign bit - and %o0, %o2, %o2 ! %o2 = 0 or %o0, depending on sign of %o1 - rd %y, %o0 ! get lower half of product - retl - addcc %o4, %o2, %o1 ! add compensation and put upper half in place -#endif - -Lmul_shortway: - /* - * Short multiply. 12 steps, followed by a final shift step. - * The resulting bits are off by 12 and (32-12) = 20 bit positions, - * but there is no problem with %o0 being negative (unlike above), - * and overflow is impossible (the answer is at most 24 bits long). - */ - mulscc %o4, %o1, %o4 ! 1 - mulscc %o4, %o1, %o4 ! 2 - mulscc %o4, %o1, %o4 ! 3 - mulscc %o4, %o1, %o4 ! 4 - mulscc %o4, %o1, %o4 ! 5 - mulscc %o4, %o1, %o4 ! 6 - mulscc %o4, %o1, %o4 ! 7 - mulscc %o4, %o1, %o4 ! 8 - mulscc %o4, %o1, %o4 ! 9 - mulscc %o4, %o1, %o4 ! 10 - mulscc %o4, %o1, %o4 ! 11 - mulscc %o4, %o1, %o4 ! 12 - mulscc %o4, %g0, %o4 ! final shift - - /* - * %o4 has 20 of the bits that should be in the result; %y has - * the bottom 12 (as %y's top 12). That is: - * - * %o4 %y - * +----------------+----------------+ - * | -12- | -20- | -12- | -20- | - * +------(---------+------)---------+ - * -----result----- - * - * The 12 bits of %o4 left of the `result' area are all zero; - * in fact, all top 20 bits of %o4 are zero. - */ - - rd %y, %o5 - sll %o4, 12, %o0 ! shift middle bits left 12 - srl %o5, 20, %o5 ! shift low bits right 20 - or %o5, %o0, %o0 - retl - addcc %g0, %g0, %o1 ! %o1 = zero, and set Z - - .globl .umul_patch -.umul_patch: - umul %o0, %o1, %o0 - retl - rd %y, %o1 - nop diff --git a/arch/sparc/lib/urem.S b/arch/sparc/lib/urem.S deleted file mode 100644 index 77123eb83c4..00000000000 --- a/arch/sparc/lib/urem.S +++ /dev/null @@ -1,357 +0,0 @@ -/* - * urem.S: This routine was taken from glibc-1.09 and is covered - * by the GNU Library General Public License Version 2. - */ - -/* This file is generated from divrem.m4; DO NOT EDIT! */ -/* - * Division and remainder, from Appendix E of the Sparc Version 8 - * Architecture Manual, with fixes from Gordon Irlam. - */ - -/* - * Input: dividend and divisor in %o0 and %o1 respectively. - * - * m4 parameters: - * .urem name of function to generate - * rem rem=div => %o0 / %o1; rem=rem => %o0 % %o1 - * false false=true => signed; false=false => unsigned - * - * Algorithm parameters: - * N how many bits per iteration we try to get (4) - * WORDSIZE total number of bits (32) - * - * Derived constants: - * TOPBITS number of bits in the top decade of a number - * - * Important variables: - * Q the partial quotient under development (initially 0) - * R the remainder so far, initially the dividend - * ITER number of main division loop iterations required; - * equal to ceil(log2(quotient) / N). Note that this - * is the log base (2^N) of the quotient. - * V the current comparand, initially divisor*2^(ITER*N-1) - * - * Cost: - * Current estimate for non-large dividend is - * ceil(log2(quotient) / N) * (10 + 7N/2) + C - * A large dividend is one greater than 2^(31-TOPBITS) and takes a - * different path, as the upper bits of the quotient must be developed - * one bit at a time. - */ - - .globl .urem - .globl _Urem -.urem: -_Urem: /* needed for export */ - - ! Ready to divide. Compute size of quotient; scale comparand. - orcc %o1, %g0, %o5 - bne 1f - mov %o0, %o3 - - ! Divide by zero trap. If it returns, return 0 (about as - ! wrong as possible, but that is what SunOS does...). - ta ST_DIV0 - retl - clr %o0 - -1: - cmp %o3, %o5 ! if %o1 exceeds %o0, done - blu Lgot_result ! (and algorithm fails otherwise) - clr %o2 - - sethi %hi(1 << (32 - 4 - 1)), %g1 - - cmp %o3, %g1 - blu Lnot_really_big - clr %o4 - - ! Here the dividend is >= 2**(31-N) or so. We must be careful here, - ! as our usual N-at-a-shot divide step will cause overflow and havoc. - ! The number of bits in the result here is N*ITER+SC, where SC <= N. - ! Compute ITER in an unorthodox manner: know we need to shift V into - ! the top decade: so do not even bother to compare to R. - 1: - cmp %o5, %g1 - bgeu 3f - mov 1, %g7 - - sll %o5, 4, %o5 - - b 1b - add %o4, 1, %o4 - - ! Now compute %g7. - 2: - addcc %o5, %o5, %o5 - bcc Lnot_too_big - add %g7, 1, %g7 - - ! We get here if the %o1 overflowed while shifting. - ! This means that %o3 has the high-order bit set. - ! Restore %o5 and subtract from %o3. - sll %g1, 4, %g1 ! high order bit - srl %o5, 1, %o5 ! rest of %o5 - add %o5, %g1, %o5 - - b Ldo_single_div - sub %g7, 1, %g7 - - Lnot_too_big: - 3: - cmp %o5, %o3 - blu 2b - nop - - be Ldo_single_div - nop - /* NB: these are commented out in the V8-Sparc manual as well */ - /* (I do not understand this) */ - ! %o5 > %o3: went too far: back up 1 step - ! srl %o5, 1, %o5 - ! dec %g7 - ! do single-bit divide steps - ! - ! We have to be careful here. We know that %o3 >= %o5, so we can do the - ! first divide step without thinking. BUT, the others are conditional, - ! and are only done if %o3 >= 0. Because both %o3 and %o5 may have the high- - ! order bit set in the first step, just falling into the regular - ! division loop will mess up the first time around. - ! So we unroll slightly... - Ldo_single_div: - subcc %g7, 1, %g7 - bl Lend_regular_divide - nop - - sub %o3, %o5, %o3 - mov 1, %o2 - - b Lend_single_divloop - nop - Lsingle_divloop: - sll %o2, 1, %o2 - bl 1f - srl %o5, 1, %o5 - ! %o3 >= 0 - sub %o3, %o5, %o3 - b 2f - add %o2, 1, %o2 - 1: ! %o3 < 0 - add %o3, %o5, %o3 - sub %o2, 1, %o2 - 2: - Lend_single_divloop: - subcc %g7, 1, %g7 - bge Lsingle_divloop - tst %o3 - - b,a Lend_regular_divide - -Lnot_really_big: -1: - sll %o5, 4, %o5 - - cmp %o5, %o3 - bleu 1b - addcc %o4, 1, %o4 - - be Lgot_result - sub %o4, 1, %o4 - - tst %o3 ! set up for initial iteration -Ldivloop: - sll %o2, 4, %o2 - ! depth 1, accumulated bits 0 - bl L.1.16 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 2, accumulated bits 1 - bl L.2.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits 3 - bl L.3.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 7 - bl L.4.23 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (7*2+1), %o2 - -L.4.23: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (7*2-1), %o2 - -L.3.19: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 5 - bl L.4.21 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (5*2+1), %o2 - -L.4.21: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (5*2-1), %o2 - -L.2.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits 1 - bl L.3.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits 3 - bl L.4.19 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (3*2+1), %o2 - -L.4.19: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (3*2-1), %o2 - -L.3.17: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits 1 - bl L.4.17 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (1*2+1), %o2 - -L.4.17: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (1*2-1), %o2 - -L.1.16: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 2, accumulated bits -1 - bl L.2.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 3, accumulated bits -1 - bl L.3.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -1 - bl L.4.15 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2+1), %o2 - -L.4.15: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-1*2-1), %o2 - -L.3.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -3 - bl L.4.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2+1), %o2 - -L.4.13: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-3*2-1), %o2 - -L.2.15: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 3, accumulated bits -3 - bl L.3.13 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - ! depth 4, accumulated bits -5 - bl L.4.11 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2+1), %o2 - -L.4.11: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-5*2-1), %o2 - -L.3.13: - ! remainder is negative - addcc %o3,%o5,%o3 - ! depth 4, accumulated bits -7 - bl L.4.9 - srl %o5,1,%o5 - ! remainder is positive - subcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2+1), %o2 - -L.4.9: - ! remainder is negative - addcc %o3,%o5,%o3 - b 9f - add %o2, (-7*2-1), %o2 - - 9: -Lend_regular_divide: - subcc %o4, 1, %o4 - bge Ldivloop - tst %o3 - - bl,a Lgot_result - ! non-restoring fixup here (one instruction only!) - add %o3, %o1, %o3 - -Lgot_result: - - retl - mov %o3, %o0 - - .globl .urem_patch -.urem_patch: - wr %g0, 0x0, %y - nop - nop - nop - udiv %o0, %o1, %o2 - umul %o2, %o1, %o2 - retl - sub %o0, %o2, %o0 diff --git a/arch/sparc/lib/usercopy.c b/arch/sparc/lib/usercopy.c deleted file mode 100644 index 14b363fec8a..00000000000 --- a/arch/sparc/lib/usercopy.c +++ /dev/null @@ -1,8 +0,0 @@ -#include <linux/module.h> -#include <linux/bug.h> - -void copy_from_user_overflow(void) -{ - WARN(1, "Buffer overflow detected!\n"); -} -EXPORT_SYMBOL(copy_from_user_overflow); diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S index f44f58f4023..2c05641c326 100644 --- a/arch/sparc/lib/xor.S +++ b/arch/sparc/lib/xor.S @@ -8,6 +8,7 @@ * Copyright (C) 2006 David S. Miller <davem@davemloft.net> */ +#include <linux/linkage.h> #include <asm/visasm.h> #include <asm/asi.h> #include <asm/dcu.h> @@ -19,12 +20,9 @@ * !(len & 127) && len >= 256 */ .text - .align 32 /* VIS versions. */ - .globl xor_vis_2 - .type xor_vis_2,#function -xor_vis_2: +ENTRY(xor_vis_2) rd %fprs, %o5 andcc %o5, FPRS_FEF|FPRS_DU, %g0 be,pt %icc, 0f @@ -91,11 +89,9 @@ xor_vis_2: wr %g1, %g0, %asi retl wr %g0, 0, %fprs - .size xor_vis_2, .-xor_vis_2 +ENDPROC(xor_vis_2) - .globl xor_vis_3 - .type xor_vis_3,#function -xor_vis_3: +ENTRY(xor_vis_3) rd %fprs, %o5 andcc %o5, FPRS_FEF|FPRS_DU, %g0 be,pt %icc, 0f @@ -159,11 +155,9 @@ xor_vis_3: wr %g1, %g0, %asi retl wr %g0, 0, %fprs - .size xor_vis_3, .-xor_vis_3 +ENDPROC(xor_vis_3) - .globl xor_vis_4 - .type xor_vis_4,#function -xor_vis_4: +ENTRY(xor_vis_4) rd %fprs, %o5 andcc %o5, FPRS_FEF|FPRS_DU, %g0 be,pt %icc, 0f @@ -246,11 +240,9 @@ xor_vis_4: wr %g1, %g0, %asi retl wr %g0, 0, %fprs - .size xor_vis_4, .-xor_vis_4 +ENDPROC(xor_vis_4) - .globl xor_vis_5 - .type xor_vis_5,#function -xor_vis_5: +ENTRY(xor_vis_5) save %sp, -192, %sp rd %fprs, %o5 andcc %o5, FPRS_FEF|FPRS_DU, %g0 @@ -354,12 +346,10 @@ xor_vis_5: wr %g0, 0, %fprs ret restore - .size xor_vis_5, .-xor_vis_5 +ENDPROC(xor_vis_5) /* Niagara versions. */ - .globl xor_niagara_2 - .type xor_niagara_2,#function -xor_niagara_2: /* %o0=bytes, %o1=dest, %o2=src */ +ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ save %sp, -192, %sp prefetch [%i1], #n_writes prefetch [%i2], #one_read @@ -402,11 +392,9 @@ xor_niagara_2: /* %o0=bytes, %o1=dest, %o2=src */ wr %g7, 0x0, %asi ret restore - .size xor_niagara_2, .-xor_niagara_2 +ENDPROC(xor_niagara_2) - .globl xor_niagara_3 - .type xor_niagara_3,#function -xor_niagara_3: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ +ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ save %sp, -192, %sp prefetch [%i1], #n_writes prefetch [%i2], #one_read @@ -465,11 +453,9 @@ xor_niagara_3: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ wr %g7, 0x0, %asi ret restore - .size xor_niagara_3, .-xor_niagara_3 +ENDPROC(xor_niagara_3) - .globl xor_niagara_4 - .type xor_niagara_4,#function -xor_niagara_4: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ +ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ save %sp, -192, %sp prefetch [%i1], #n_writes prefetch [%i2], #one_read @@ -549,11 +535,9 @@ xor_niagara_4: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ wr %g7, 0x0, %asi ret restore - .size xor_niagara_4, .-xor_niagara_4 +ENDPROC(xor_niagara_4) - .globl xor_niagara_5 - .type xor_niagara_5,#function -xor_niagara_5: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ +ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ save %sp, -192, %sp prefetch [%i1], #n_writes prefetch [%i2], #one_read @@ -649,4 +633,4 @@ xor_niagara_5: /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 * wr %g7, 0x0, %asi ret restore - .size xor_niagara_5, .-xor_niagara_5 +ENDPROC(xor_niagara_5) diff --git a/arch/sparc/math-emu/Makefile b/arch/sparc/math-emu/Makefile index b9085ecbb27..825dbee94d8 100644 --- a/arch/sparc/math-emu/Makefile +++ b/arch/sparc/math-emu/Makefile @@ -2,7 +2,7 @@ # Makefile for the FPU instruction emulation. # -# supress all warnings - as math.c produces a lot! +# suppress all warnings - as math.c produces a lot! ccflags-y := -w obj-y := math_$(BITS).o diff --git a/arch/sparc/math-emu/math_32.c b/arch/sparc/math-emu/math_32.c index a3fccde894e..aa4d55b0bdf 100644 --- a/arch/sparc/math-emu/math_32.c +++ b/arch/sparc/math-emu/math_32.c @@ -164,7 +164,7 @@ int do_mathemu(struct pt_regs *regs, struct task_struct *fpt) int retcode = 0; /* assume all succeed */ unsigned long insn; - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); #ifdef DEBUG_MATHEMU printk("In do_mathemu()... pc is %08lx\n", regs->pc); diff --git a/arch/sparc/math-emu/math_64.c b/arch/sparc/math-emu/math_64.c index 56d2c44747b..034aadbff03 100644 --- a/arch/sparc/math-emu/math_64.c +++ b/arch/sparc/math-emu/math_64.c @@ -16,6 +16,7 @@ #include <asm/fpumacro.h> #include <asm/ptrace.h> #include <asm/uaccess.h> +#include <asm/cacheflush.h> #include "sfp-util_64.h" #include <math-emu/soft-fp.h> @@ -162,7 +163,7 @@ typedef union { u64 q[2]; } *argp; -int do_mathemu(struct pt_regs *regs, struct fpustate *f) +int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap) { unsigned long pc = regs->tpc; unsigned long tstate = regs->tstate; @@ -184,7 +185,7 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) if (tstate & TSTATE_PRIV) die_if_kernel("unfinished/unimplemented FPop from kernel", regs); - perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, 0); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); if (test_thread_flag(TIF_32BIT)) pc = (u32)pc; if (get_user(insn, (u32 __user *) pc) != -EFAULT) { @@ -217,7 +218,7 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) case FSQRTS: { unsigned long x = current_thread_info()->xfsr[0]; - x = (x >> 14) & 0xf; + x = (x >> 14) & 0x7; TYPE(x,1,1,1,1,0,0); break; } @@ -225,7 +226,7 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) case FSQRTD: { unsigned long x = current_thread_info()->xfsr[0]; - x = (x >> 14) & 0xf; + x = (x >> 14) & 0x7; TYPE(x,2,1,2,1,0,0); break; } @@ -319,7 +320,7 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) XR = 0; else if (freg < 16) XR = regs->u_regs[freg]; - else if (test_thread_flag(TIF_32BIT)) { + else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) { struct reg_window32 __user *win32; flushw_user (); win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); @@ -356,9 +357,17 @@ int do_mathemu(struct pt_regs *regs, struct fpustate *f) if (type) { argp rs1 = NULL, rs2 = NULL, rd = NULL; - freg = (current_thread_info()->xfsr[0] >> 14) & 0xf; - if (freg != (type >> 9)) - goto err; + /* Starting with UltraSPARC-T2, the cpu does not set the FP Trap + * Type field in the %fsr to unimplemented_FPop. Nor does it + * use the fp_exception_other trap. Instead it signals an + * illegal instruction and leaves the FP trap type field of + * the %fsr unchanged. + */ + if (!illegal_insn_trap) { + int ftt = (current_thread_info()->xfsr[0] >> 14) & 0x7; + if (ftt != (type >> 9)) + goto err; + } current_thread_info()->xfsr[0] &= ~0x1c000; freg = ((insn >> 14) & 0x1f); switch (type & 0x3) { diff --git a/arch/sparc/math-emu/sfp-util_32.h b/arch/sparc/math-emu/sfp-util_32.h index d1b2aff3c25..bb587d5f3d9 100644 --- a/arch/sparc/math-emu/sfp-util_32.h +++ b/arch/sparc/math-emu/sfp-util_32.h @@ -4,20 +4,20 @@ #include <asm/byteorder.h> #define add_ssaaaa(sh, sl, ah, al, bh, bl) \ - __asm__ ("addcc %r4,%5,%1\n\t" \ + __asm__ ("addcc %r4,%5,%1\n\t" \ "addx %r2,%3,%0\n" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "%rJ" ((USItype)(ah)), \ "rI" ((USItype)(bh)), \ "%rJ" ((USItype)(al)), \ "rI" ((USItype)(bl)) \ : "cc") #define sub_ddmmss(sh, sl, ah, al, bh, bl) \ - __asm__ ("subcc %r4,%5,%1\n\t" \ + __asm__ ("subcc %r4,%5,%1\n\t" \ "subx %r2,%3,%0\n" \ - : "=r" ((USItype)(sh)), \ - "=&r" ((USItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "rJ" ((USItype)(ah)), \ "rI" ((USItype)(bh)), \ "rJ" ((USItype)(al)), \ @@ -65,8 +65,8 @@ "mulscc %%g1,0,%%g1\n\t" \ "add %%g1,%%g2,%0\n\t" \ "rd %%y,%1\n" \ - : "=r" ((USItype)(w1)), \ - "=r" ((USItype)(w0)) \ + : "=r" (w1), \ + "=r" (w0) \ : "%rI" ((USItype)(u)), \ "r" ((USItype)(v)) \ : "%g1", "%g2", "cc") @@ -98,8 +98,8 @@ "sub %1,%2,%1\n\t" \ "3: xnor %0,0,%0\n\t" \ "! End of inline udiv_qrnnd\n" \ - : "=&r" ((USItype)(q)), \ - "=&r" ((USItype)(r)) \ + : "=&r" (q), \ + "=&r" (r) \ : "r" ((USItype)(d)), \ "1" ((USItype)(n1)), \ "0" ((USItype)(n0)) : "%g1", "cc") diff --git a/arch/sparc/math-emu/sfp-util_64.h b/arch/sparc/math-emu/sfp-util_64.h index 425d3cf01af..51320a861cc 100644 --- a/arch/sparc/math-emu/sfp-util_64.h +++ b/arch/sparc/math-emu/sfp-util_64.h @@ -17,8 +17,8 @@ "bcs,a,pn %%xcc, 1f\n\t" \ "add %0, 1, %0\n" \ "1:" \ - : "=r" ((UDItype)(sh)), \ - "=&r" ((UDItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((UDItype)(ah)), \ "r" ((UDItype)(bh)), \ "r" ((UDItype)(al)), \ @@ -31,8 +31,8 @@ "bcs,a,pn %%xcc, 1f\n\t" \ "sub %0, 1, %0\n" \ "1:" \ - : "=r" ((UDItype)(sh)), \ - "=&r" ((UDItype)(sl)) \ + : "=r" (sh), \ + "=&r" (sl) \ : "r" ((UDItype)(ah)), \ "r" ((UDItype)(bh)), \ "r" ((UDItype)(al)), \ @@ -64,8 +64,8 @@ "sllx %3,32,%3\n\t" \ "add %1,%3,%1\n\t" \ "add %5,%2,%0" \ - : "=r" ((UDItype)(wh)), \ - "=&r" ((UDItype)(wl)), \ + : "=r" (wh), \ + "=&r" (wl), \ "=&r" (tmp1), "=&r" (tmp2), "=&r" (tmp3), "=&r" (tmp4) \ : "r" ((UDItype)(u)), \ "r" ((UDItype)(v)) \ diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile index 79836a7dd00..30c3eccfdf5 100644 --- a/arch/sparc/mm/Makefile +++ b/arch/sparc/mm/Makefile @@ -4,23 +4,16 @@ asflags-y := -ansi ccflags-y := -Werror -obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o +obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o obj-y += fault_$(BITS).o obj-y += init_$(BITS).o -obj-$(CONFIG_SPARC32) += loadmmu.o -obj-y += generic_$(BITS).o -obj-$(CONFIG_SPARC32) += extable.o btfixup.o srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += extable.o srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += srmmu_access.o obj-$(CONFIG_SPARC32) += hypersparc.o viking.o tsunami.o swift.o -obj-$(CONFIG_SPARC_LEON)+= leon_mm.o +obj-$(CONFIG_SPARC32) += leon_mm.o # Only used by sparc64 obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o # Only used by sparc32 obj-$(CONFIG_HIGHMEM) += highmem.o - -ifdef CONFIG_SMP -obj-$(CONFIG_SPARC32) += nosun4c.o -else -obj-$(CONFIG_SPARC32) += sun4c.o -endif diff --git a/arch/sparc/mm/btfixup.c b/arch/sparc/mm/btfixup.c deleted file mode 100644 index 5175ac2f482..00000000000 --- a/arch/sparc/mm/btfixup.c +++ /dev/null @@ -1,330 +0,0 @@ -/* btfixup.c: Boot time code fixup and relocator, so that - * we can get rid of most indirect calls to achieve single - * image sun4c and srmmu kernel. - * - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/init.h> -#include <asm/btfixup.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/oplib.h> -#include <asm/system.h> -#include <asm/cacheflush.h> - -#define BTFIXUP_OPTIMIZE_NOP -#define BTFIXUP_OPTIMIZE_OTHER - -extern char *srmmu_name; -static char version[] __initdata = "Boot time fixup v1.6. 4/Mar/98 Jakub Jelinek (jj@ultra.linux.cz). Patching kernel for "; -static char str_sun4c[] __initdata = "sun4c\n"; -static char str_srmmu[] __initdata = "srmmu[%s]/"; -static char str_iommu[] __initdata = "iommu\n"; -static char str_iounit[] __initdata = "io-unit\n"; - -static int visited __initdata = 0; -extern unsigned int ___btfixup_start[], ___btfixup_end[], __init_begin[], __init_end[], __init_text_end[]; -extern unsigned int _stext[], _end[], __start___ksymtab[], __stop___ksymtab[]; -static char wrong_f[] __initdata = "Trying to set f fixup %p to invalid function %08x\n"; -static char wrong_b[] __initdata = "Trying to set b fixup %p to invalid function %08x\n"; -static char wrong_s[] __initdata = "Trying to set s fixup %p to invalid value %08x\n"; -static char wrong_h[] __initdata = "Trying to set h fixup %p to invalid value %08x\n"; -static char wrong_a[] __initdata = "Trying to set a fixup %p to invalid value %08x\n"; -static char wrong[] __initdata = "Wrong address for %c fixup %p\n"; -static char insn_f[] __initdata = "Fixup f %p refers to weird instructions at %p[%08x,%08x]\n"; -static char insn_b[] __initdata = "Fixup b %p doesn't refer to a SETHI at %p[%08x]\n"; -static char insn_s[] __initdata = "Fixup s %p doesn't refer to an OR at %p[%08x]\n"; -static char insn_h[] __initdata = "Fixup h %p doesn't refer to a SETHI at %p[%08x]\n"; -static char insn_a[] __initdata = "Fixup a %p doesn't refer to a SETHI nor OR at %p[%08x]\n"; -static char insn_i[] __initdata = "Fixup i %p doesn't refer to a valid instruction at %p[%08x]\n"; -static char fca_und[] __initdata = "flush_cache_all undefined in btfixup()\n"; -static char wrong_setaddr[] __initdata = "Garbled CALL/INT patch at %p[%08x,%08x,%08x]=%08x\n"; - -#ifdef BTFIXUP_OPTIMIZE_OTHER -static void __init set_addr(unsigned int *addr, unsigned int q1, int fmangled, unsigned int value) -{ - if (!fmangled) - *addr = value; - else { - unsigned int *q = (unsigned int *)q1; - if (*addr == 0x01000000) { - /* Noped */ - *q = value; - } else if (addr[-1] == *q) { - /* Moved */ - addr[-1] = value; - *q = value; - } else { - prom_printf(wrong_setaddr, addr-1, addr[-1], *addr, *q, value); - prom_halt(); - } - } -} -#else -static inline void set_addr(unsigned int *addr, unsigned int q1, int fmangled, unsigned int value) -{ - *addr = value; -} -#endif - -void __init btfixup(void) -{ - unsigned int *p, *q; - int type, count; - unsigned insn; - unsigned *addr; - int fmangled = 0; - void (*flush_cacheall)(void); - - if (!visited) { - visited++; - printk(version); - if (ARCH_SUN4C) - printk(str_sun4c); - else { - printk(str_srmmu, srmmu_name); - if (sparc_cpu_model == sun4d) - printk(str_iounit); - else - printk(str_iommu); - } - } - for (p = ___btfixup_start; p < ___btfixup_end; ) { - count = p[2]; - q = p + 3; - switch (type = *(unsigned char *)p) { - case 'f': - count = p[3]; - q = p + 4; - if (((p[0] & 1) || p[1]) - && ((p[1] & 3) || (unsigned *)(p[1]) < _stext || (unsigned *)(p[1]) >= _end)) { - prom_printf(wrong_f, p, p[1]); - prom_halt(); - } - break; - case 'b': - if (p[1] < (unsigned long)__init_begin || p[1] >= (unsigned long)__init_text_end || (p[1] & 3)) { - prom_printf(wrong_b, p, p[1]); - prom_halt(); - } - break; - case 's': - if (p[1] + 0x1000 >= 0x2000) { - prom_printf(wrong_s, p, p[1]); - prom_halt(); - } - break; - case 'h': - if (p[1] & 0x3ff) { - prom_printf(wrong_h, p, p[1]); - prom_halt(); - } - break; - case 'a': - if (p[1] + 0x1000 >= 0x2000 && (p[1] & 0x3ff)) { - prom_printf(wrong_a, p, p[1]); - prom_halt(); - } - break; - } - if (p[0] & 1) { - p[0] &= ~1; - while (count) { - fmangled = 0; - addr = (unsigned *)*q; - if (addr < _stext || addr >= _end) { - prom_printf(wrong, type, p); - prom_halt(); - } - insn = *addr; -#ifdef BTFIXUP_OPTIMIZE_OTHER - if (type != 'f' && q[1]) { - insn = *(unsigned int *)q[1]; - if (!insn || insn == 1) - insn = *addr; - else - fmangled = 1; - } -#endif - switch (type) { - case 'f': /* CALL */ - if (addr >= __start___ksymtab && addr < __stop___ksymtab) { - *addr = p[1]; - break; - } else if (!q[1]) { - if ((insn & 0xc1c00000) == 0x01000000) { /* SETHI */ - *addr = (insn & 0xffc00000) | (p[1] >> 10); break; - } else if ((insn & 0xc1f82000) == 0x80102000) { /* OR X, %LO(i), Y */ - *addr = (insn & 0xffffe000) | (p[1] & 0x3ff); break; - } else if ((insn & 0xc0000000) != 0x40000000) { /* !CALL */ - bad_f: - prom_printf(insn_f, p, addr, insn, addr[1]); - prom_halt(); - } - } else if (q[1] != 1) - addr[1] = q[1]; - if (p[2] == BTFIXUPCALL_NORM) { - norm_f: - *addr = 0x40000000 | ((p[1] - (unsigned)addr) >> 2); - q[1] = 0; - break; - } -#ifndef BTFIXUP_OPTIMIZE_NOP - goto norm_f; -#else - if (!(addr[1] & 0x80000000)) { - if ((addr[1] & 0xc1c00000) != 0x01000000) /* !SETHI */ - goto bad_f; /* CALL, Bicc, FBfcc, CBccc are weird in delay slot, aren't they? */ - } else { - if ((addr[1] & 0x01800000) == 0x01800000) { - if ((addr[1] & 0x01f80000) == 0x01e80000) { - /* RESTORE */ - goto norm_f; /* It is dangerous to patch that */ - } - goto bad_f; - } - if ((addr[1] & 0xffffe003) == 0x9e03e000) { - /* ADD %O7, XX, %o7 */ - int displac = (addr[1] << 19); - - displac = (displac >> 21) + 2; - *addr = (0x10800000) + (displac & 0x3fffff); - q[1] = addr[1]; - addr[1] = p[2]; - break; - } - if ((addr[1] & 0x201f) == 0x200f || (addr[1] & 0x7c000) == 0x3c000) - goto norm_f; /* Someone is playing bad tricks with us: rs1 or rs2 is o7 */ - if ((addr[1] & 0x3e000000) == 0x1e000000) - goto norm_f; /* rd is %o7. We'd better take care. */ - } - if (p[2] == BTFIXUPCALL_NOP) { - *addr = 0x01000000; - q[1] = 1; - break; - } -#ifndef BTFIXUP_OPTIMIZE_OTHER - goto norm_f; -#else - if (addr[1] == 0x01000000) { /* NOP in the delay slot */ - q[1] = addr[1]; - *addr = p[2]; - break; - } - if ((addr[1] & 0xc0000000) != 0xc0000000) { - /* Not a memory operation */ - if ((addr[1] & 0x30000000) == 0x10000000) { - /* Ok, non-memory op with rd %oX */ - if ((addr[1] & 0x3e000000) == 0x1c000000) - goto bad_f; /* Aiee. Someone is playing strange %sp tricks */ - if ((addr[1] & 0x3e000000) > 0x12000000 || - ((addr[1] & 0x3e000000) == 0x12000000 && - p[2] != BTFIXUPCALL_STO1O0 && p[2] != BTFIXUPCALL_SWAPO0O1) || - ((p[2] & 0xffffe000) == BTFIXUPCALL_RETINT(0))) { - /* Nobody uses the result. We can nop it out. */ - *addr = p[2]; - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - if ((addr[1] & 0xf1ffffe0) == 0x90100000) { - /* MOV %reg, %Ox */ - if ((addr[1] & 0x3e000000) == 0x10000000 && - (p[2] & 0x7c000) == 0x20000) { - /* Ok, it is call xx; mov reg, %o0 and call optimizes - to doing something on %o0. Patch the patch. */ - *addr = (p[2] & ~0x7c000) | ((addr[1] & 0x1f) << 14); - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - if ((addr[1] & 0x3e000000) == 0x12000000 && - p[2] == BTFIXUPCALL_STO1O0) { - *addr = (p[2] & ~0x3e000000) | ((addr[1] & 0x1f) << 25); - q[1] = addr[1]; - addr[1] = 0x01000000; - break; - } - } - } - } - *addr = addr[1]; - q[1] = addr[1]; - addr[1] = p[2]; - break; -#endif /* BTFIXUP_OPTIMIZE_OTHER */ -#endif /* BTFIXUP_OPTIMIZE_NOP */ - case 'b': /* BLACKBOX */ - /* Has to be sethi i, xx */ - if ((insn & 0xc1c00000) != 0x01000000) { - prom_printf(insn_b, p, addr, insn); - prom_halt(); - } else { - void (*do_fixup)(unsigned *); - - do_fixup = (void (*)(unsigned *))p[1]; - do_fixup(addr); - } - break; - case 's': /* SIMM13 */ - /* Has to be or %g0, i, xx */ - if ((insn & 0xc1ffe000) != 0x80102000) { - prom_printf(insn_s, p, addr, insn); - prom_halt(); - } - set_addr(addr, q[1], fmangled, (insn & 0xffffe000) | (p[1] & 0x1fff)); - break; - case 'h': /* SETHI */ - /* Has to be sethi i, xx */ - if ((insn & 0xc1c00000) != 0x01000000) { - prom_printf(insn_h, p, addr, insn); - prom_halt(); - } - set_addr(addr, q[1], fmangled, (insn & 0xffc00000) | (p[1] >> 10)); - break; - case 'a': /* HALF */ - /* Has to be sethi i, xx or or %g0, i, xx */ - if ((insn & 0xc1c00000) != 0x01000000 && - (insn & 0xc1ffe000) != 0x80102000) { - prom_printf(insn_a, p, addr, insn); - prom_halt(); - } - if (p[1] & 0x3ff) - set_addr(addr, q[1], fmangled, - (insn & 0x3e000000) | 0x80102000 | (p[1] & 0x1fff)); - else - set_addr(addr, q[1], fmangled, - (insn & 0x3e000000) | 0x01000000 | (p[1] >> 10)); - break; - case 'i': /* INT */ - if ((insn & 0xc1c00000) == 0x01000000) /* %HI */ - set_addr(addr, q[1], fmangled, (insn & 0xffc00000) | (p[1] >> 10)); - else if ((insn & 0x80002000) == 0x80002000 && - (insn & 0x01800000) != 0x01800000) /* %LO */ - set_addr(addr, q[1], fmangled, (insn & 0xffffe000) | (p[1] & 0x3ff)); - else { - prom_printf(insn_i, p, addr, insn); - prom_halt(); - } - break; - } - count -= 2; - q += 2; - } - } else - p = q + count; - } -#ifdef CONFIG_SMP - flush_cacheall = (void (*)(void))BTFIXUPVAL_CALL(local_flush_cache_all); -#else - flush_cacheall = (void (*)(void))BTFIXUPVAL_CALL(flush_cache_all); -#endif - if (!flush_cacheall) { - prom_printf(fca_und); - prom_halt(); - } - (*flush_cacheall)(); -} diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 5b836f5aea9..908e8c17c90 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -20,97 +20,48 @@ #include <linux/smp.h> #include <linux/perf_event.h> #include <linux/interrupt.h> -#include <linux/module.h> #include <linux/kdebug.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgtable.h> -#include <asm/memreg.h> #include <asm/openprom.h> #include <asm/oplib.h> +#include <asm/setup.h> #include <asm/smp.h> #include <asm/traps.h> #include <asm/uaccess.h> -extern int prom_node_root; +#include "mm_32.h" int show_unhandled_signals = 1; -/* At boot time we determine these two values necessary for setting - * up the segment maps and page table entries (pte's). - */ - -int num_segmaps, num_contexts; -int invalid_segment; - -/* various Virtual Address Cache parameters we find at boot time... */ - -int vac_size, vac_linesize, vac_do_hw_vac_flushes; -int vac_entries_per_context, vac_entries_per_segment; -int vac_entries_per_page; - -/* Return how much physical memory we have. */ -unsigned long probe_memory(void) -{ - unsigned long total = 0; - int i; - - for (i = 0; sp_banks[i].num_bytes; i++) - total += sp_banks[i].num_bytes; - - return total; -} - -extern void sun4c_complete_all_stores(void); - -/* Whee, a level 15 NMI interrupt memory error. Let's have fun... */ -asmlinkage void sparc_lvl15_nmi(struct pt_regs *regs, unsigned long serr, - unsigned long svaddr, unsigned long aerr, - unsigned long avaddr) -{ - sun4c_complete_all_stores(); - printk("FAULT: NMI received\n"); - printk("SREGS: Synchronous Error %08lx\n", serr); - printk(" Synchronous Vaddr %08lx\n", svaddr); - printk(" Asynchronous Error %08lx\n", aerr); - printk(" Asynchronous Vaddr %08lx\n", avaddr); - if (sun4c_memerr_reg) - printk(" Memory Parity Error %08lx\n", *sun4c_memerr_reg); - printk("REGISTER DUMP:\n"); - show_regs(regs); - prom_halt(); -} - -static void unhandled_fault(unsigned long, struct task_struct *, - struct pt_regs *) __attribute__ ((noreturn)); - -static void unhandled_fault(unsigned long address, struct task_struct *tsk, - struct pt_regs *regs) +static void __noreturn unhandled_fault(unsigned long address, + struct task_struct *tsk, + struct pt_regs *regs) { - if((unsigned long) address < PAGE_SIZE) { + if ((unsigned long) address < PAGE_SIZE) { printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference\n"); } else { - printk(KERN_ALERT "Unable to handle kernel paging request " - "at virtual address %08lx\n", address); + printk(KERN_ALERT "Unable to handle kernel paging request at virtual address %08lx\n", + address); } printk(KERN_ALERT "tsk->{mm,active_mm}->context = %08lx\n", (tsk->mm ? tsk->mm->context : tsk->active_mm->context)); printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %08lx\n", (tsk->mm ? (unsigned long) tsk->mm->pgd : - (unsigned long) tsk->active_mm->pgd)); + (unsigned long) tsk->active_mm->pgd)); die_if_kernel("Oops", regs); } -asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, +asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, unsigned long address) { struct pt_regs regs; unsigned long g2; unsigned int insn; int i; - + i = search_extables_range(ret_pc, &g2); switch (i) { case 3: @@ -130,14 +81,14 @@ asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, /* for _from_ macros */ insn = *((unsigned int *) pc); if (!((insn >> 21) & 1) || ((insn>>19)&0x3f) == 15) - return 2; - break; + return 2; + break; default: break; - }; + } - memset(®s, 0, sizeof (regs)); + memset(®s, 0, sizeof(regs)); regs.pc = pc; regs.npc = pc + 4; __asm__ __volatile__( @@ -190,9 +141,6 @@ static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs, force_sig_info (sig, &info, current); } -extern unsigned long safe_compute_effective_address(struct pt_regs *, - unsigned int); - static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) { unsigned int insn; @@ -200,11 +148,10 @@ static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) if (text_fault) return regs->pc; - if (regs->psr & PSR_PS) { + if (regs->psr & PSR_PS) insn = *(unsigned int *) regs->pc; - } else { + else __get_user(insn, (unsigned int *) regs->pc); - } return safe_compute_effective_address(regs, insn); } @@ -227,8 +174,9 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, unsigned long g2; int from_user = !(regs->psr & PSR_PS); int fault, code; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; - if(text_fault) + if (text_fault) address = regs->pc; /* @@ -240,37 +188,33 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, * only copy the information from the master page table, * nothing more. */ - if (!ARCH_SUN4C && address >= TASK_SIZE) - goto vmalloc_fault; - code = SEGV_MAPERR; + if (address >= TASK_SIZE) + goto vmalloc_fault; /* * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm) - goto no_context; + if (in_atomic() || !mm) + goto no_context; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); +retry: down_read(&mm->mmap_sem); - /* - * The kernel referencing a bad kernel pointer can lock up - * a sun4c machine completely, so we must attempt recovery. - */ - if(!from_user && address >= PAGE_OFFSET) + if (!from_user && address >= PAGE_OFFSET) goto bad_area; vma = find_vma(mm, address); - if(!vma) + if (!vma) goto bad_area; - if(vma->vm_start <= address) + if (vma->vm_start <= address) goto good_area; - if(!(vma->vm_flags & VM_GROWSDOWN)) + if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; - if(expand_stack(vma, address)) + if (expand_stack(vma, address)) goto bad_area; /* * Ok, we have a good vm_area for this memory access, so @@ -278,21 +222,30 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, */ good_area: code = SEGV_ACCERR; - if(write) { - if(!(vma->vm_flags & VM_WRITE)) + if (write) { + if (!(vma->vm_flags & VM_WRITE)) goto bad_area; } else { /* Allow reads even for write-only mappings */ - if(!(vma->vm_flags & (VM_READ | VM_EXEC))) + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } + if (from_user) + flags |= FAULT_FLAG_USER; + if (write) + flags |= FAULT_FLAG_WRITE; + /* * If for any reason at all we couldn't handle the fault, * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0); + fault = handle_mm_fault(mm, vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + return; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -300,15 +253,30 @@ good_area: goto do_sigbus; BUG(); } - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) { + current->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, + 1, regs, address); + } else { + current->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, + 1, regs, address); + } + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } } + up_read(&mm->mmap_sem); return; @@ -331,14 +299,16 @@ no_context: g2 = regs->u_regs[UREG_G2]; if (!from_user) { fixup = search_extables_range(regs->pc, &g2); - if (fixup > 10) { /* Values below are reserved for other things */ + /* Values below 10 are reserved for other things */ + if (fixup > 10) { extern const unsigned __memset_start[]; extern const unsigned __memset_end[]; extern const unsigned __csum_partial_copy_start[]; extern const unsigned __csum_partial_copy_end[]; #ifdef DEBUG_EXCEPTIONS - printk("Exception: PC<%08lx> faddr<%08lx>\n", regs->pc, address); + printk("Exception: PC<%08lx> faddr<%08lx>\n", + regs->pc, address); printk("EX_TABLE: insn<%08lx> fixup<%08x> g2<%08lx>\n", regs->pc, fixup, g2); #endif @@ -346,7 +316,7 @@ no_context: regs->pc < (unsigned long)__memset_end) || (regs->pc >= (unsigned long)__csum_partial_copy_start && regs->pc < (unsigned long)__csum_partial_copy_end)) { - regs->u_regs[UREG_I4] = address; + regs->u_regs[UREG_I4] = address; regs->u_regs[UREG_I5] = regs->pc; } regs->u_regs[UREG_G2] = g2; @@ -355,8 +325,8 @@ no_context: return; } } - - unhandled_fault (address, tsk, regs); + + unhandled_fault(address, tsk, regs); do_exit(SIGKILL); /* @@ -402,127 +372,44 @@ vmalloc_fault: if (pmd_present(*pmd) || !pmd_present(*pmd_k)) goto bad_area_nosemaphore; + *pmd = *pmd_k; return; } } -asmlinkage void do_sun4c_fault(struct pt_regs *regs, int text_fault, int write, - unsigned long address) -{ - extern void sun4c_update_mmu_cache(struct vm_area_struct *, - unsigned long,pte_t *); - extern pte_t *sun4c_pte_offset_kernel(pmd_t *,unsigned long); - struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; - pgd_t *pgdp; - pte_t *ptep; - - if (text_fault) { - address = regs->pc; - } else if (!write && - !(regs->psr & PSR_PS)) { - unsigned int insn, __user *ip; - - ip = (unsigned int __user *)regs->pc; - if (!get_user(insn, ip)) { - if ((insn & 0xc1680000) == 0xc0680000) - write = 1; - } - } - - if (!mm) { - /* We are oopsing. */ - do_sparc_fault(regs, text_fault, write, address); - BUG(); /* P3 Oops already, you bitch */ - } - - pgdp = pgd_offset(mm, address); - ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, address); - - if (pgd_val(*pgdp)) { - if (write) { - if ((pte_val(*ptep) & (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_PRESENT)) - == (_SUN4C_PAGE_WRITE|_SUN4C_PAGE_PRESENT)) { - unsigned long flags; - - *ptep = __pte(pte_val(*ptep) | _SUN4C_PAGE_ACCESSED | - _SUN4C_PAGE_MODIFIED | - _SUN4C_PAGE_VALID | - _SUN4C_PAGE_DIRTY); - - local_irq_save(flags); - if (sun4c_get_segmap(address) != invalid_segment) { - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); - return; - } - local_irq_restore(flags); - } - } else { - if ((pte_val(*ptep) & (_SUN4C_PAGE_READ|_SUN4C_PAGE_PRESENT)) - == (_SUN4C_PAGE_READ|_SUN4C_PAGE_PRESENT)) { - unsigned long flags; - - *ptep = __pte(pte_val(*ptep) | _SUN4C_PAGE_ACCESSED | - _SUN4C_PAGE_VALID); - - local_irq_save(flags); - if (sun4c_get_segmap(address) != invalid_segment) { - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); - return; - } - local_irq_restore(flags); - } - } - } - - /* This conditional is 'interesting'. */ - if (pgd_val(*pgdp) && !(write && !(pte_val(*ptep) & _SUN4C_PAGE_WRITE)) - && (pte_val(*ptep) & _SUN4C_PAGE_VALID)) - /* Note: It is safe to not grab the MMAP semaphore here because - * we know that update_mmu_cache() will not sleep for - * any reason (at least not in the current implementation) - * and therefore there is no danger of another thread getting - * on the CPU and doing a shrink_mmap() on this vma. - */ - sun4c_update_mmu_cache (find_vma(current->mm, address), address, - ptep); - else - do_sparc_fault(regs, text_fault, write, address); -} - /* This always deals with user addresses. */ static void force_user_fault(unsigned long address, int write) { struct vm_area_struct *vma; struct task_struct *tsk = current; struct mm_struct *mm = tsk->mm; + unsigned int flags = FAULT_FLAG_USER; int code; code = SEGV_MAPERR; down_read(&mm->mmap_sem); vma = find_vma(mm, address); - if(!vma) + if (!vma) goto bad_area; - if(vma->vm_start <= address) + if (vma->vm_start <= address) goto good_area; - if(!(vma->vm_flags & VM_GROWSDOWN)) + if (!(vma->vm_flags & VM_GROWSDOWN)) goto bad_area; - if(expand_stack(vma, address)) + if (expand_stack(vma, address)) goto bad_area; good_area: code = SEGV_ACCERR; - if(write) { - if(!(vma->vm_flags & VM_WRITE)) + if (write) { + if (!(vma->vm_flags & VM_WRITE)) goto bad_area; + flags |= FAULT_FLAG_WRITE; } else { - if(!(vma->vm_flags & (VM_READ | VM_EXEC))) + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0)) { + switch (handle_mm_fault(mm, vma, address, flags)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; @@ -550,7 +437,7 @@ void window_overflow_fault(void) unsigned long sp; sp = current_thread_info()->rwbuf_stkptrs[0]; - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 1); force_user_fault(sp, 1); @@ -559,7 +446,7 @@ void window_overflow_fault(void) void window_underflow_fault(unsigned long sp) { - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 0); force_user_fault(sp, 0); @@ -571,7 +458,7 @@ void window_ret_fault(struct pt_regs *regs) unsigned long sp; sp = regs->u_regs[UREG_FP]; - if(((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) force_user_fault(sp + 0x38, 0); force_user_fault(sp, 0); diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index f92ce56a8b2..587cd056512 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -21,6 +21,7 @@ #include <linux/kprobes.h> #include <linux/kdebug.h> #include <linux/percpu.h> +#include <linux/context_tracking.h> #include <asm/page.h> #include <asm/pgtable.h> @@ -31,6 +32,7 @@ #include <asm/lsu.h> #include <asm/sections.h> #include <asm/mmu_context.h> +#include <asm/setup.h> int show_unhandled_signals = 1; @@ -95,38 +97,51 @@ static unsigned int get_user_insn(unsigned long tpc) pte_t *ptep, pte; unsigned long pa; u32 insn = 0; - unsigned long pstate; - if (pgd_none(*pgdp)) - goto outret; + if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) + goto out; pudp = pud_offset(pgdp, tpc); - if (pud_none(*pudp)) - goto outret; - pmdp = pmd_offset(pudp, tpc); - if (pmd_none(*pmdp)) - goto outret; - - /* This disables preemption for us as well. */ - __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); - __asm__ __volatile__("wrpr %0, %1, %%pstate" - : : "r" (pstate), "i" (PSTATE_IE)); - ptep = pte_offset_map(pmdp, tpc); - pte = *ptep; - if (!pte_present(pte)) + if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) goto out; - pa = (pte_pfn(pte) << PAGE_SHIFT); - pa += (tpc & ~PAGE_MASK); - - /* Use phys bypass so we don't pollute dtlb/dcache. */ - __asm__ __volatile__("lduwa [%1] %2, %0" - : "=r" (insn) - : "r" (pa), "i" (ASI_PHYS_USE_EC)); + /* This disables preemption for us as well. */ + local_irq_disable(); + pmdp = pmd_offset(pudp, tpc); + if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) + goto out_irq_enable; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + if (pmd_trans_huge(*pmdp)) { + if (pmd_trans_splitting(*pmdp)) + goto out_irq_enable; + + pa = pmd_pfn(*pmdp) << PAGE_SHIFT; + pa += tpc & ~HPAGE_MASK; + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } else +#endif + { + ptep = pte_offset_map(pmdp, tpc); + pte = *ptep; + if (pte_present(pte)) { + pa = (pte_pfn(pte) << PAGE_SHIFT); + pa += (tpc & ~PAGE_MASK); + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } + pte_unmap(ptep); + } +out_irq_enable: + local_irq_enable(); out: - pte_unmap(ptep); - __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); -outret: return insn; } @@ -151,10 +166,9 @@ show_signal_msg(struct pt_regs *regs, int sig, int code, printk(KERN_CONT "\n"); } -extern unsigned long compute_effective_address(struct pt_regs *, unsigned int, unsigned int); - static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, - unsigned int insn, int fault_code) + unsigned long fault_addr, unsigned int insn, + int fault_code) { unsigned long addr; siginfo_t info; @@ -162,10 +176,18 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, info.si_code = code; info.si_signo = sig; info.si_errno = 0; - if (fault_code & FAULT_CODE_ITLB) + if (fault_code & FAULT_CODE_ITLB) { addr = regs->tpc; - else - addr = compute_effective_address(regs, insn, 0); + } else { + /* If we were able to probe the faulting instruction, use it + * to compute a precise fault address. Otherwise use the fault + * time provided address which may only have page granularity. + */ + if (insn) + addr = compute_effective_address(regs, insn, 0); + else + addr = fault_addr; + } info.si_addr = (void __user *) addr; info.si_trapno = 0; @@ -175,9 +197,6 @@ static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, force_sig_info(sig, &info, current); } -extern int handle_ldf_stq(u32, struct pt_regs *); -extern int handle_ld_nf(u32, struct pt_regs *); - static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) { if (!insn) { @@ -240,7 +259,7 @@ static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code, /* The si_code was set to make clear whether * this was a SEGV_MAPERR or SEGV_ACCERR fault. */ - do_fault_siginfo(si_code, SIGSEGV, regs, insn, fault_code); + do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code); return; } @@ -260,30 +279,20 @@ static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) show_regs(regs); } -static void noinline __kprobes bogus_32bit_fault_address(struct pt_regs *regs, - unsigned long addr) -{ - static int times; - - if (times++ < 10) - printk(KERN_ERR "FAULT[%s:%d]: 32-bit process " - "reports 64-bit fault address [%lx]\n", - current->comm, current->pid, addr); - show_regs(regs); -} - asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) { + enum ctx_state prev_state = exception_enter(); struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned int insn = 0; int si_code, fault_code, fault; unsigned long address, mm_rss; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; fault_code = get_thread_fault_code(); if (notify_page_fault(regs)) - return; + goto exit_exception; si_code = SEGV_MAPERR; address = current_thread_info()->fault_address; @@ -299,10 +308,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) goto intr_or_no_mm; } } - if (unlikely((address >> 32) != 0)) { - bogus_32bit_fault_address(regs, address); + if (unlikely((address >> 32) != 0)) goto intr_or_no_mm; - } } if (regs->tstate & TSTATE_PRIV) { @@ -314,9 +321,10 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) /* Valid, no problems... */ } else { bad_kernel_pc(regs, address); - return; + goto exit_exception; } - } + } else + flags |= FAULT_FLAG_USER; /* * If we're in an interrupt or have no user @@ -325,7 +333,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) if (in_atomic() || !mm) goto intr_or_no_mm; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); if (!down_read_trylock(&mm->mmap_sem)) { if ((regs->tstate & TSTATE_PRIV) && @@ -333,6 +341,8 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) insn = get_fault_insn(regs, insn); goto handle_kernel_fault; } + +retry: down_read(&mm->mmap_sem); } @@ -417,13 +427,19 @@ good_area: vma->vm_file != NULL) set_thread_fault_code(fault_code | FAULT_CODE_BLKCOMMIT); + + flags |= FAULT_FLAG_WRITE; } else { /* Allow reads even for write-only mappings */ if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - fault = handle_mm_fault(mm, vma, address, (fault_code & FAULT_CODE_WRITE) ? FAULT_FLAG_WRITE : 0); + fault = handle_mm_fault(mm, vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + goto exit_exception; + if (unlikely(fault & VM_FAULT_ERROR)) { if (fault & VM_FAULT_OOM) goto out_of_memory; @@ -431,30 +447,51 @@ good_area: goto do_sigbus; BUG(); } - if (fault & VM_FAULT_MAJOR) { - current->maj_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, - regs, address); - } else { - current->min_flt++; - perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, - regs, address); + + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) { + current->maj_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, + 1, regs, address); + } else { + current->min_flt++; + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, + 1, regs, address); + } + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } } up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); #endif if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) mm_rss = mm->context.huge_pte_count; if (unlikely(mm_rss > - mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) - tsb_grow(mm, MM_TSB_HUGE, mm_rss); + mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) + tsb_grow(mm, MM_TSB_HUGE, mm_rss); + else + hugetlb_setup(regs); + + } #endif +exit_exception: + exception_exit(prev_state); return; /* @@ -467,7 +504,7 @@ bad_area: handle_kernel_fault: do_kernel_fault(regs, si_code, fault_code, insn, address); - return; + goto exit_exception; /* * We ran out of memory, or some other thing happened to us that made @@ -478,7 +515,7 @@ out_of_memory: up_read(&mm->mmap_sem); if (!(regs->tstate & TSTATE_PRIV)) { pagefault_out_of_memory(); - return; + goto exit_exception; } goto handle_kernel_fault; @@ -494,7 +531,7 @@ do_sigbus: * Send a sigbus, regardless of whether we were in kernel * or user mode. */ - do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, insn, fault_code); + do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code); /* Kernel mode? Handle exceptions or die */ if (regs->tstate & TSTATE_PRIV) diff --git a/arch/sparc/mm/generic_32.c b/arch/sparc/mm/generic_32.c deleted file mode 100644 index e6067b75f11..00000000000 --- a/arch/sparc/mm/generic_32.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * generic.c: Generic Sparc mm routines that are not dependent upon - * MMU type but are Sparc specific. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/swap.h> -#include <linux/pagemap.h> - -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/page.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> - -/* Remap IO memory, the same way as remap_pfn_range(), but use - * the obio memory space. - * - * They use a pgprot that sets PAGE_IO and does not check the - * mem_map table as this is independent of normal memory. - */ -static inline void io_remap_pte_range(struct mm_struct *mm, pte_t * pte, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; - do { - set_pte_at(mm, address, pte, mk_pte_io(offset, prot, space)); - address += PAGE_SIZE; - offset += PAGE_SIZE; - pte++; - } while (address < end); -} - -static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - offset -= address; - do { - pte_t *pte = pte_alloc_map(mm, NULL, pmd, address); - if (!pte) - return -ENOMEM; - io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address < end); - return 0; -} - -int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, - unsigned long pfn, unsigned long size, pgprot_t prot) -{ - int error = 0; - pgd_t * dir; - unsigned long beg = from; - unsigned long end = from + size; - struct mm_struct *mm = vma->vm_mm; - int space = GET_IOSPACE(pfn); - unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; - - /* See comment in mm/memory.c remap_pfn_range */ - vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; - vma->vm_pgoff = (offset >> PAGE_SHIFT) | - ((unsigned long)space << 28UL); - - offset -= from; - dir = pgd_offset(mm, from); - flush_cache_range(vma, beg, end); - - while (from < end) { - pmd_t *pmd = pmd_alloc(mm, dir, from); - error = -ENOMEM; - if (!pmd) - break; - error = io_remap_pmd_range(mm, pmd, from, end - from, offset + from, prot, space); - if (error) - break; - from = (from + PGDIR_SIZE) & PGDIR_MASK; - dir++; - } - - flush_tlb_range(vma, beg, end); - return error; -} -EXPORT_SYMBOL(io_remap_pfn_range); diff --git a/arch/sparc/mm/generic_64.c b/arch/sparc/mm/generic_64.c deleted file mode 100644 index 3cb00dfd4bd..00000000000 --- a/arch/sparc/mm/generic_64.c +++ /dev/null @@ -1,164 +0,0 @@ -/* - * generic.c: Generic Sparc mm routines that are not dependent upon - * MMU type but are Sparc specific. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/swap.h> -#include <linux/pagemap.h> - -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/page.h> -#include <asm/tlbflush.h> - -/* Remap IO memory, the same way as remap_pfn_range(), but use - * the obio memory space. - * - * They use a pgprot that sets PAGE_IO and does not check the - * mem_map table as this is independent of normal memory. - */ -static inline void io_remap_pte_range(struct mm_struct *mm, pte_t * pte, - unsigned long address, - unsigned long size, - unsigned long offset, pgprot_t prot, - int space) -{ - unsigned long end; - - /* clear hack bit that was used as a write_combine side-effect flag */ - offset &= ~0x1UL; - address &= ~PMD_MASK; - end = address + size; - if (end > PMD_SIZE) - end = PMD_SIZE; - do { - pte_t entry; - unsigned long curend = address + PAGE_SIZE; - - entry = mk_pte_io(offset, prot, space, PAGE_SIZE); - if (!(address & 0xffff)) { - if (PAGE_SIZE < (4 * 1024 * 1024) && - !(address & 0x3fffff) && - !(offset & 0x3ffffe) && - end >= address + 0x400000) { - entry = mk_pte_io(offset, prot, space, - 4 * 1024 * 1024); - curend = address + 0x400000; - offset += 0x400000; - } else if (PAGE_SIZE < (512 * 1024) && - !(address & 0x7ffff) && - !(offset & 0x7fffe) && - end >= address + 0x80000) { - entry = mk_pte_io(offset, prot, space, - 512 * 1024 * 1024); - curend = address + 0x80000; - offset += 0x80000; - } else if (PAGE_SIZE < (64 * 1024) && - !(offset & 0xfffe) && - end >= address + 0x10000) { - entry = mk_pte_io(offset, prot, space, - 64 * 1024); - curend = address + 0x10000; - offset += 0x10000; - } else - offset += PAGE_SIZE; - } else - offset += PAGE_SIZE; - - if (pte_write(entry)) - entry = pte_mkdirty(entry); - do { - BUG_ON(!pte_none(*pte)); - set_pte_at(mm, address, pte, entry); - address += PAGE_SIZE; - pte_val(entry) += PAGE_SIZE; - pte++; - } while (address < curend); - } while (address < end); -} - -static inline int io_remap_pmd_range(struct mm_struct *mm, pmd_t * pmd, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PGDIR_MASK; - end = address + size; - if (end > PGDIR_SIZE) - end = PGDIR_SIZE; - offset -= address; - do { - pte_t *pte = pte_alloc_map(mm, NULL, pmd, address); - if (!pte) - return -ENOMEM; - io_remap_pte_range(mm, pte, address, end - address, address + offset, prot, space); - pte_unmap(pte); - address = (address + PMD_SIZE) & PMD_MASK; - pmd++; - } while (address < end); - return 0; -} - -static inline int io_remap_pud_range(struct mm_struct *mm, pud_t * pud, unsigned long address, unsigned long size, - unsigned long offset, pgprot_t prot, int space) -{ - unsigned long end; - - address &= ~PUD_MASK; - end = address + size; - if (end > PUD_SIZE) - end = PUD_SIZE; - offset -= address; - do { - pmd_t *pmd = pmd_alloc(mm, pud, address); - if (!pud) - return -ENOMEM; - io_remap_pmd_range(mm, pmd, address, end - address, address + offset, prot, space); - address = (address + PUD_SIZE) & PUD_MASK; - pud++; - } while (address < end); - return 0; -} - -int io_remap_pfn_range(struct vm_area_struct *vma, unsigned long from, - unsigned long pfn, unsigned long size, pgprot_t prot) -{ - int error = 0; - pgd_t * dir; - unsigned long beg = from; - unsigned long end = from + size; - struct mm_struct *mm = vma->vm_mm; - int space = GET_IOSPACE(pfn); - unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; - unsigned long phys_base; - - phys_base = offset | (((unsigned long) space) << 32UL); - - /* See comment in mm/memory.c remap_pfn_range */ - vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP; - vma->vm_pgoff = phys_base >> PAGE_SHIFT; - - offset -= from; - dir = pgd_offset(mm, from); - flush_cache_range(vma, beg, end); - - while (from < end) { - pud_t *pud = pud_alloc(mm, dir, from); - error = -ENOMEM; - if (!pud) - break; - error = io_remap_pud_range(mm, pud, from, end - from, offset + from, prot, space); - if (error) - break; - from = (from + PGDIR_SIZE) & PGDIR_MASK; - dir++; - } - - flush_tlb_range(vma, beg, end); - return error; -} -EXPORT_SYMBOL(io_remap_pfn_range); diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c new file mode 100644 index 00000000000..1aed0432c64 --- /dev/null +++ b/arch/sparc/mm/gup.c @@ -0,0 +1,237 @@ +/* + * Lockless get_user_pages_fast for sparc, cribbed from powerpc + * + * Copyright (C) 2008 Nick Piggin + * Copyright (C) 2008 Novell Inc. + */ + +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/vmstat.h> +#include <linux/pagemap.h> +#include <linux/rwsem.h> +#include <asm/pgtable.h> + +/* + * The performance critical leaf functions are made noinline otherwise gcc + * inlines everything into a single function which results in too much + * register pressure. + */ +static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, + unsigned long end, int write, struct page **pages, int *nr) +{ + unsigned long mask, result; + pte_t *ptep; + + if (tlb_type == hypervisor) { + result = _PAGE_PRESENT_4V|_PAGE_P_4V; + if (write) + result |= _PAGE_WRITE_4V; + } else { + result = _PAGE_PRESENT_4U|_PAGE_P_4U; + if (write) + result |= _PAGE_WRITE_4U; + } + mask = result | _PAGE_SPECIAL; + + ptep = pte_offset_kernel(&pmd, addr); + do { + struct page *page, *head; + pte_t pte = *ptep; + + if ((pte_val(pte) & mask) != result) + return 0; + VM_BUG_ON(!pfn_valid(pte_pfn(pte))); + + /* The hugepage case is simplified on sparc64 because + * we encode the sub-page pfn offsets into the + * hugepage PTEs. We could optimize this in the future + * use page_cache_add_speculative() for the hugepage case. + */ + page = pte_page(pte); + head = compound_head(page); + if (!page_cache_get_speculative(head)) + return 0; + if (unlikely(pte_val(pte) != pte_val(*ptep))) { + put_page(head); + return 0; + } + if (head != page) + get_huge_page_tail(page); + + pages[*nr] = page; + (*nr)++; + } while (ptep++, addr += PAGE_SIZE, addr != end); + + return 1; +} + +static int gup_huge_pmd(pmd_t *pmdp, pmd_t pmd, unsigned long addr, + unsigned long end, int write, struct page **pages, + int *nr) +{ + struct page *head, *page, *tail; + int refs; + + if (!(pmd_val(pmd) & _PAGE_VALID)) + return 0; + + if (write && !pmd_write(pmd)) + return 0; + + refs = 0; + head = pmd_page(pmd); + page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + tail = page; + do { + VM_BUG_ON(compound_head(page) != head); + pages[*nr] = page; + (*nr)++; + page++; + refs++; + } while (addr += PAGE_SIZE, addr != end); + + if (!page_cache_add_speculative(head, refs)) { + *nr -= refs; + return 0; + } + + if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { + *nr -= refs; + while (refs--) + put_page(head); + return 0; + } + + /* Any tail page need their mapcount reference taken before we + * return. + */ + while (refs--) { + if (PageTail(tail)) + get_huge_page_tail(tail); + tail++; + } + + return 1; +} + +static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + unsigned long next; + pmd_t *pmdp; + + pmdp = pmd_offset(&pud, addr); + do { + pmd_t pmd = *pmdp; + + next = pmd_addr_end(addr, end); + if (pmd_none(pmd) || pmd_trans_splitting(pmd)) + return 0; + if (unlikely(pmd_large(pmd))) { + if (!gup_huge_pmd(pmdp, pmd, addr, next, + write, pages, nr)) + return 0; + } else if (!gup_pte_range(pmd, addr, next, write, + pages, nr)) + return 0; + } while (pmdp++, addr = next, addr != end); + + return 1; +} + +static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end, + int write, struct page **pages, int *nr) +{ + unsigned long next; + pud_t *pudp; + + pudp = pud_offset(&pgd, addr); + do { + pud_t pud = *pudp; + + next = pud_addr_end(addr, end); + if (pud_none(pud)) + return 0; + if (!gup_pmd_range(pud, addr, next, write, pages, nr)) + return 0; + } while (pudp++, addr = next, addr != end); + + return 1; +} + +int get_user_pages_fast(unsigned long start, int nr_pages, int write, + struct page **pages) +{ + struct mm_struct *mm = current->mm; + unsigned long addr, len, end; + unsigned long next; + pgd_t *pgdp; + int nr = 0; + + start &= PAGE_MASK; + addr = start; + len = (unsigned long) nr_pages << PAGE_SHIFT; + end = start + len; + + /* + * XXX: batch / limit 'nr', to avoid large irq off latency + * needs some instrumenting to determine the common sizes used by + * important workloads (eg. DB2), and whether limiting the batch size + * will decrease performance. + * + * It seems like we're in the clear for the moment. Direct-IO is + * the main guy that batches up lots of get_user_pages, and even + * they are limited to 64-at-a-time which is not so many. + */ + /* + * This doesn't prevent pagetable teardown, but does prevent + * the pagetables from being freed on sparc. + * + * So long as we atomically load page table pointers versus teardown, + * we can follow the address down to the the page and take a ref on it. + */ + local_irq_disable(); + + pgdp = pgd_offset(mm, addr); + do { + pgd_t pgd = *pgdp; + + next = pgd_addr_end(addr, end); + if (pgd_none(pgd)) + goto slow; + if (!gup_pud_range(pgd, addr, next, write, pages, &nr)) + goto slow; + } while (pgdp++, addr = next, addr != end); + + local_irq_enable(); + + VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT); + return nr; + + { + int ret; + +slow: + local_irq_enable(); + + /* Try to get the remaining pages with get_user_pages */ + start += nr << PAGE_SHIFT; + pages += nr; + + down_read(&mm->mmap_sem); + ret = get_user_pages(current, mm, start, + (end - start) >> PAGE_SHIFT, write, 0, pages, NULL); + up_read(&mm->mmap_sem); + + /* Have to be a bit careful with return values */ + if (nr > 0) { + if (ret < 0) + ret = nr; + else + ret += nr; + } + + return ret; + } +} diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 4730eac0747..449f864f0ce 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -22,14 +22,33 @@ * shared by CPUs, and so precious, and establishing them requires IPI. * Atomic kmaps are lightweight and we may have NCPUS more of them. */ -#include <linux/mm.h> #include <linux/highmem.h> -#include <asm/pgalloc.h> +#include <linux/export.h> +#include <linux/mm.h> + #include <asm/cacheflush.h> #include <asm/tlbflush.h> -#include <asm/fixmap.h> +#include <asm/pgalloc.h> +#include <asm/vaddrs.h> + +pgprot_t kmap_prot; -void *__kmap_atomic(struct page *page) +static pte_t *kmap_pte; + +void __init kmap_init(void) +{ + unsigned long address; + pmd_t *dir; + + address = __fix_to_virt(FIX_KMAP_BEGIN); + dir = pmd_offset(pgd_offset_k(address), address); + + /* cache the first kmap pte */ + kmap_pte = pte_offset_kernel(dir, address); + kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); +} + +void *kmap_atomic(struct page *page) { unsigned long vaddr; long idx, type; @@ -63,7 +82,7 @@ void *__kmap_atomic(struct page *page) return (void*) vaddr; } -EXPORT_SYMBOL(__kmap_atomic); +EXPORT_SYMBOL(kmap_atomic); void __kunmap_atomic(void *kvaddr) { @@ -109,21 +128,3 @@ void __kunmap_atomic(void *kvaddr) pagefault_enable(); } EXPORT_SYMBOL(__kunmap_atomic); - -/* We may be fed a pagetable here by ptep_to_xxx and others. */ -struct page *kmap_atomic_to_page(void *ptr) -{ - unsigned long idx, vaddr = (unsigned long)ptr; - pte_t *pte; - - if (vaddr < SRMMU_NOCACHE_VADDR) - return virt_to_page(ptr); - if (vaddr < PKMAP_BASE) - return pfn_to_page(__nocache_pa(vaddr) >> PAGE_SHIFT); - BUG_ON(vaddr < FIXADDR_START); - BUG_ON(vaddr > FIXADDR_TOP); - - idx = virt_to_fix(vaddr); - pte = kmap_pte - (idx - FIX_KMAP_BEGIN); - return pte_page(*pte); -} diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index f4e97646ce2..d329537739c 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -4,8 +4,6 @@ * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) */ -#include <linux/init.h> -#include <linux/module.h> #include <linux/fs.h> #include <linux/mm.h> #include <linux/hugetlb.h> @@ -22,8 +20,6 @@ /* Slightly simplified from the non-hugepage variant because by * definition we don't have to worry about any page coloring stuff */ -#define VA_EXCLUDE_START (0x0000080000000000UL - (1UL << 32UL)) -#define VA_EXCLUDE_END (0xfffff80000000000UL + (1UL << 32UL)) static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long addr, @@ -31,55 +27,28 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, unsigned long pgoff, unsigned long flags) { - struct mm_struct *mm = current->mm; - struct vm_area_struct * vma; unsigned long task_size = TASK_SIZE; - unsigned long start_addr; + struct vm_unmapped_area_info info; if (test_thread_flag(TIF_32BIT)) task_size = STACK_TOP32; - if (unlikely(len >= VA_EXCLUDE_START)) - return -ENOMEM; - if (len > mm->cached_hole_size) { - start_addr = addr = mm->free_area_cache; - } else { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; + info.flags = 0; + info.length = len; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = min(task_size, VA_EXCLUDE_START); + info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_offset = 0; + addr = vm_unmapped_area(&info); + + if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { + VM_BUG_ON(addr != -ENOMEM); + info.low_limit = VA_EXCLUDE_END; + info.high_limit = task_size; + addr = vm_unmapped_area(&info); } - task_size -= len; - -full_search: - addr = ALIGN(addr, HPAGE_SIZE); - - for (vma = find_vma(mm, addr); ; vma = vma->vm_next) { - /* At this point: (!vma || addr < vma->vm_end). */ - if (addr < VA_EXCLUDE_START && - (addr + len) >= VA_EXCLUDE_START) { - addr = VA_EXCLUDE_END; - vma = find_vma(mm, VA_EXCLUDE_END); - } - if (unlikely(task_size < addr)) { - if (start_addr != TASK_UNMAPPED_BASE) { - start_addr = addr = TASK_UNMAPPED_BASE; - mm->cached_hole_size = 0; - goto full_search; - } - return -ENOMEM; - } - if (likely(!vma || addr + len <= vma->vm_start)) { - /* - * Remember the place where we stopped the search: - */ - mm->free_area_cache = addr + len; - return addr; - } - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - addr = ALIGN(vma->vm_end, HPAGE_SIZE); - } + return addr; } static unsigned long @@ -88,71 +57,34 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, const unsigned long pgoff, const unsigned long flags) { - struct vm_area_struct *vma; struct mm_struct *mm = current->mm; unsigned long addr = addr0; + struct vm_unmapped_area_info info; /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); - /* check if free_area_cache is useful for us */ - if (len <= mm->cached_hole_size) { - mm->cached_hole_size = 0; - mm->free_area_cache = mm->mmap_base; - } - - /* either no address requested or can't fit in requested address hole */ - addr = mm->free_area_cache & HPAGE_MASK; - - /* make sure it can fit in the remaining address space */ - if (likely(addr > len)) { - vma = find_vma(mm, addr-len); - if (!vma || addr <= vma->vm_start) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr-len); - } - } - - if (unlikely(mm->mmap_base < len)) - goto bottomup; - - addr = (mm->mmap_base-len) & HPAGE_MASK; - - do { - /* - * Lookup failure means no vma is above this address, - * else if new region fits below vma->vm_start, - * return with success: - */ - vma = find_vma(mm, addr); - if (likely(!vma || addr+len <= vma->vm_start)) { - /* remember the address as a hint for next time */ - return (mm->free_area_cache = addr); - } + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = mm->mmap_base; + info.align_mask = PAGE_MASK & ~HPAGE_MASK; + info.align_offset = 0; + addr = vm_unmapped_area(&info); - /* remember the largest hole we saw so far */ - if (addr + mm->cached_hole_size < vma->vm_start) - mm->cached_hole_size = vma->vm_start - addr; - - /* try just below the current vma->vm_start */ - addr = (vma->vm_start-len) & HPAGE_MASK; - } while (likely(len < vma->vm_start)); - -bottomup: /* * A failed mmap() very likely causes application failure, * so fall back to the bottom-up function here. This scenario * can happen with large stack limits and large mmap() * allocations. */ - mm->cached_hole_size = ~0UL; - mm->free_area_cache = TASK_UNMAPPED_BASE; - addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags); - /* - * Restore the topdown base: - */ - mm->free_area_cache = mm->mmap_base; - mm->cached_hole_size = ~0UL; + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = STACK_TOP32; + addr = vm_unmapped_area(&info); + } return addr; } @@ -304,53 +236,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, { return NULL; } - -static void context_reload(void *__data) -{ - struct mm_struct *mm = __data; - - if (mm == current->mm) - load_secondary_context(mm); -} - -void hugetlb_prefault_arch_hook(struct mm_struct *mm) -{ - struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE]; - - if (likely(tp->tsb != NULL)) - return; - - tsb_grow(mm, MM_TSB_HUGE, 0); - tsb_context_switch(mm); - smp_tsb_sync(mm); - - /* On UltraSPARC-III+ and later, configure the second half of - * the Data-TLB for huge pages. - */ - if (tlb_type == cheetah_plus) { - unsigned long ctx; - - spin_lock(&ctx_alloc_lock); - ctx = mm->context.sparc64_ctx_val; - ctx &= ~CTX_PGSZ_MASK; - ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; - ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; - - if (ctx != mm->context.sparc64_ctx_val) { - /* When changing the page size fields, we - * must perform a context flush so that no - * stale entries match. This flush must - * occur with the original context register - * settings. - */ - do_flush_tlb_mm(mm); - - /* Reload the context register of all processors - * also executing in this address space. - */ - mm->context.sparc64_ctx_val = ctx; - on_each_cpu(context_reload, mm, 0); - } - spin_unlock(&ctx_alloc_lock); - } -} diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S index 44aad32eeb4..969f96450f6 100644 --- a/arch/sparc/mm/hypersparc.S +++ b/arch/sparc/mm/hypersparc.S @@ -74,7 +74,7 @@ hypersparc_flush_cache_mm_out: /* The things we do for performance... */ hypersparc_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 #ifndef CONFIG_SMP ld [%o0 + AOFF_mm_context], %g1 cmp %g1, -1 @@ -163,7 +163,7 @@ hypersparc_flush_cache_range_out: */ /* Verified, my ass... */ hypersparc_flush_cache_page: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %g2 #ifndef CONFIG_SMP cmp %g2, -1 @@ -284,7 +284,7 @@ hypersparc_flush_tlb_mm_out: sta %g5, [%g1] ASI_M_MMUREGS hypersparc_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -307,7 +307,7 @@ hypersparc_flush_tlb_range_out: sta %g5, [%g1] ASI_M_MMUREGS hypersparc_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index 6d0e02c4fe0..eb828715527 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -27,17 +27,16 @@ #include <linux/gfp.h> #include <asm/sections.h> -#include <asm/system.h> -#include <asm/vac-ops.h> #include <asm/page.h> #include <asm/pgtable.h> #include <asm/vaddrs.h> #include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */ +#include <asm/setup.h> #include <asm/tlb.h> #include <asm/prom.h> #include <asm/leon.h> -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); +#include "mm_32.h" unsigned long *sparc_valid_addr_bitmap; EXPORT_SYMBOL(sparc_valid_addr_bitmap); @@ -48,13 +47,7 @@ EXPORT_SYMBOL(phys_base); unsigned long pfn_base; EXPORT_SYMBOL(pfn_base); -unsigned long page_kernel; -EXPORT_SYMBOL(page_kernel); - struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; -unsigned long sparc_unmapped_base; - -struct pgtable_cache_struct pgt_quicklists; /* Initial ramdisk setup */ extern unsigned int sparc_ramdisk_image; @@ -62,56 +55,17 @@ extern unsigned int sparc_ramdisk_size; unsigned long highstart_pfn, highend_pfn; -pte_t *kmap_pte; -pgprot_t kmap_prot; - -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)) - -void __init kmap_init(void) -{ - /* cache the first kmap pte */ - kmap_pte = kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN)); - kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); -} - -void show_mem(void) +void show_mem(unsigned int filter) { printk("Mem-info:\n"); - show_free_areas(); + show_free_areas(filter); printk("Free swap: %6ldkB\n", - nr_swap_pages << (PAGE_SHIFT-10)); + get_nr_swap_pages() << (PAGE_SHIFT-10)); printk("%ld pages of RAM\n", totalram_pages); printk("%ld free pages\n", nr_free_pages()); -#if 0 /* undefined pgtable_cache_size, pgd_cache_size */ - printk("%ld pages in page table cache\n",pgtable_cache_size); -#ifndef CONFIG_SMP - if (sparc_cpu_model == sun4m || sparc_cpu_model == sun4d) - printk("%ld entries in page dir cache\n",pgd_cache_size); -#endif -#endif } -void __init sparc_context_init(int numctx) -{ - int ctx; - - ctx_list_pool = __alloc_bootmem(numctx * sizeof(struct ctx_list), SMP_CACHE_BYTES, 0UL); - - for(ctx = 0; ctx < numctx; ctx++) { - struct ctx_list *clist; - clist = (ctx_list_pool + ctx); - clist->ctx_number = ctx; - clist->ctx_mm = NULL; - } - ctx_free.next = ctx_free.prev = &ctx_free; - ctx_used.next = ctx_used.prev = &ctx_used; - for(ctx = 0; ctx < numctx; ctx++) - add_to_free_ctxlist(ctx_list_pool + ctx); -} - -extern unsigned long cmdline_memory_size; unsigned long last_valid_pfn; unsigned long calc_highpages(void) @@ -290,78 +244,13 @@ unsigned long __init bootmem_init(unsigned long *pages_avail) } /* - * check_pgt_cache - * - * This is called at the end of unmapping of VMA (zap_page_range), - * to rescan the page cache for architecture specific things, - * presumably something like sun4/sun4c PMEGs. Most architectures - * define check_pgt_cache empty. - * - * We simply copy the 2.4 implementation for now. - */ -static int pgt_cache_water[2] = { 25, 50 }; - -void check_pgt_cache(void) -{ - do_check_pgt_cache(pgt_cache_water[0], pgt_cache_water[1]); -} - -/* * paging_init() sets up the page tables: We call the MMU specific * init routine based upon the Sun model type on the Sparc. * */ -extern void sun4c_paging_init(void); -extern void srmmu_paging_init(void); -extern void device_scan(void); - -pgprot_t PAGE_SHARED __read_mostly; -EXPORT_SYMBOL(PAGE_SHARED); - void __init paging_init(void) { - switch(sparc_cpu_model) { - case sun4c: - case sun4e: - case sun4: - sun4c_paging_init(); - sparc_unmapped_base = 0xe0000000; - BTFIXUPSET_SETHI(sparc_unmapped_base, 0xe0000000); - break; - case sparc_leon: - leon_init(); - /* fall through */ - case sun4m: - case sun4d: - srmmu_paging_init(); - sparc_unmapped_base = 0x50000000; - BTFIXUPSET_SETHI(sparc_unmapped_base, 0x50000000); - break; - default: - prom_printf("paging_init: Cannot init paging on this Sparc\n"); - prom_printf("paging_init: sparc_cpu_model = %d\n", sparc_cpu_model); - prom_printf("paging_init: Halting...\n"); - prom_halt(); - }; - - /* Initialize the protection map with non-constant, MMU dependent values. */ - protection_map[0] = PAGE_NONE; - protection_map[1] = PAGE_READONLY; - protection_map[2] = PAGE_COPY; - protection_map[3] = PAGE_COPY; - protection_map[4] = PAGE_READONLY; - protection_map[5] = PAGE_READONLY; - protection_map[6] = PAGE_COPY; - protection_map[7] = PAGE_COPY; - protection_map[8] = PAGE_NONE; - protection_map[9] = PAGE_READONLY; - protection_map[10] = PAGE_SHARED; - protection_map[11] = PAGE_SHARED; - protection_map[12] = PAGE_READONLY; - protection_map[13] = PAGE_READONLY; - protection_map[14] = PAGE_SHARED; - protection_map[15] = PAGE_SHARED; - btfixup(); + srmmu_paging_init(); prom_build_devicetree(); of_fill_in_cpu_data(); device_scan(); @@ -392,22 +281,12 @@ static void map_high_region(unsigned long start_pfn, unsigned long end_pfn) printk("mapping high region %08lx - %08lx\n", start_pfn, end_pfn); #endif - for (tmp = start_pfn; tmp < end_pfn; tmp++) { - struct page *page = pfn_to_page(tmp); - - ClearPageReserved(page); - init_page_count(page); - __free_page(page); - totalhigh_pages++; - } + for (tmp = start_pfn; tmp < end_pfn; tmp++) + free_highmem_page(pfn_to_page(tmp)); } void __init mem_init(void) { - int codepages = 0; - int datapages = 0; - int initpages = 0; - int reservedpages = 0; int i; if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) { @@ -439,15 +318,12 @@ void __init mem_init(void) max_mapnr = last_valid_pfn - pfn_base; high_memory = __va(max_low_pfn << PAGE_SHIFT); - - totalram_pages = free_all_bootmem(); + free_all_bootmem(); for (i = 0; sp_banks[i].num_bytes != 0; i++) { unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; - num_physpages += sp_banks[i].num_bytes >> PAGE_SHIFT; - if (end_pfn <= highstart_pfn) continue; @@ -457,72 +333,19 @@ void __init mem_init(void) map_high_region(start_pfn, end_pfn); } - totalram_pages += totalhigh_pages; - - codepages = (((unsigned long) &_etext) - ((unsigned long)&_start)); - codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT; - datapages = (((unsigned long) &_edata) - ((unsigned long)&_etext)); - datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT; - initpages = (((unsigned long) &__init_end) - ((unsigned long) &__init_begin)); - initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT; - - /* Ignore memory holes for the purpose of counting reserved pages */ - for (i=0; i < max_low_pfn; i++) - if (test_bit(i >> (20 - PAGE_SHIFT), sparc_valid_addr_bitmap) - && PageReserved(pfn_to_page(i))) - reservedpages++; - - printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n", - nr_free_pages() << (PAGE_SHIFT-10), - num_physpages << (PAGE_SHIFT - 10), - codepages << (PAGE_SHIFT-10), - reservedpages << (PAGE_SHIFT - 10), - datapages << (PAGE_SHIFT-10), - initpages << (PAGE_SHIFT-10), - totalhigh_pages << (PAGE_SHIFT-10)); + mem_init_print_info(NULL); } void free_initmem (void) { - unsigned long addr; - unsigned long freed; - - addr = (unsigned long)(&__init_begin); - freed = (unsigned long)(&__init_end) - addr; - for (; addr < (unsigned long)(&__init_end); addr += PAGE_SIZE) { - struct page *p; - - memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); - p = virt_to_page(addr); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - totalram_pages++; - num_physpages++; - } - printk(KERN_INFO "Freeing unused kernel memory: %ldk freed\n", - freed >> 10); + free_initmem_default(POISON_FREE_INITMEM); } #ifdef CONFIG_BLK_DEV_INITRD void free_initrd_mem(unsigned long start, unsigned long end) { - if (start < end) - printk(KERN_INFO "Freeing initrd memory: %ldk freed\n", - (end - start) >> 10); - for (; start < end; start += PAGE_SIZE) { - struct page *p; - - memset((void *)start, POISON_FREE_INITMEM, PAGE_SIZE); - p = virt_to_page(start); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - totalram_pages++; - num_physpages++; - } + free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, + "initrd"); } #endif diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 2f6ae1d1fb6..16b58ff11e6 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -28,7 +28,6 @@ #include <linux/gfp.h> #include <asm/head.h> -#include <asm/system.h> #include <asm/page.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> @@ -48,30 +47,49 @@ #include <asm/prom.h> #include <asm/mdesc.h> #include <asm/cpudata.h> +#include <asm/setup.h> #include <asm/irq.h> #include "init_64.h" -unsigned long kern_linear_pte_xor[2] __read_mostly; +unsigned long kern_linear_pte_xor[4] __read_mostly; -/* A bitmap, one bit for every 256MB of physical memory. If the bit - * is clear, we should use a 4MB page (via kern_linear_pte_xor[0]) else - * if set we should use a 256MB page (via kern_linear_pte_xor[1]). +/* A bitmap, two bits for every 256MB of physical memory. These two + * bits determine what page size we use for kernel linear + * translations. They form an index into kern_linear_pte_xor[]. The + * value in the indexed slot is XOR'd with the TLB miss virtual + * address to form the resulting TTE. The mapping is: + * + * 0 ==> 4MB + * 1 ==> 256MB + * 2 ==> 2GB + * 3 ==> 16GB + * + * All sun4v chips support 256MB pages. Only SPARC-T4 and later + * support 2GB pages, and hopefully future cpus will support the 16GB + * pages as well. For slots 2 and 3, we encode a 256MB TTE xor there + * if these larger page sizes are not supported by the cpu. + * + * It would be nice to determine this from the machine description + * 'cpu' properties, but we need to have this table setup before the + * MDESC is initialized. */ unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; #ifndef CONFIG_DEBUG_PAGEALLOC -/* A special kernel TSB for 4MB and 256MB linear mappings. - * Space is allocated for this right after the trap table - * in arch/sparc64/kernel/head.S +/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. + * Space is allocated for this right after the trap table in + * arch/sparc64/kernel/head.S */ extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; #endif +static unsigned long cpu_pgsz_mask; + #define MAX_BANKS 32 -static struct linux_prom64_registers pavail[MAX_BANKS] __devinitdata; -static int pavail_ents __devinitdata; +static struct linux_prom64_registers pavail[MAX_BANKS]; +static int pavail_ents; static int cmp_p64(const void *a, const void *b) { @@ -102,7 +120,8 @@ static void __init read_obp_memory(const char *property, ret = prom_getproperty(node, property, (char *) regs, prop_size); if (ret == -1) { - prom_printf("Couldn't get %s property from /memory.\n"); + prom_printf("Couldn't get %s property from /memory.\n", + property); prom_halt(); } @@ -258,7 +277,6 @@ static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long } unsigned long _PAGE_ALL_SZ_BITS __read_mostly; -unsigned long _PAGE_SZBITS __read_mostly; static void flush_dcache(unsigned long pfn) { @@ -289,12 +307,39 @@ static void flush_dcache(unsigned long pfn) } } +/* mm->context.lock must be held */ +static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index, + unsigned long tsb_hash_shift, unsigned long address, + unsigned long tte) +{ + struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb; + unsigned long tag; + + if (unlikely(!tsb)) + return; + + tsb += ((address >> tsb_hash_shift) & + (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); + tag = (address >> 22UL); + tsb_insert(tsb, tag, tte); +} + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static inline bool is_hugetlb_pte(pte_t pte) +{ + if ((tlb_type == hypervisor && + (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || + (tlb_type != hypervisor && + (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) + return true; + return false; +} +#endif + void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) { struct mm_struct *mm; - struct tsb *tsb; - unsigned long tag, flags; - unsigned long tsb_index, tsb_hash_shift; + unsigned long flags; pte_t pte = *ptep; if (tlb_type != hypervisor) { @@ -306,28 +351,16 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * mm = vma->vm_mm; - tsb_index = MM_TSB_BASE; - tsb_hash_shift = PAGE_SHIFT; - spin_lock_irqsave(&mm->context.lock, flags); -#ifdef CONFIG_HUGETLB_PAGE - if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) { - if ((tlb_type == hypervisor && - (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) || - (tlb_type != hypervisor && - (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U)) { - tsb_index = MM_TSB_HUGE; - tsb_hash_shift = HPAGE_SHIFT; - } - } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, + address, pte_val(pte)); + else #endif - - tsb = mm->context.tsb_block[tsb_index].tsb; - tsb += ((address >> tsb_hash_shift) & - (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); - tag = (address >> 22UL); - tsb_insert(tsb, tag, pte_val(pte)); + __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, + address, pte_val(pte)); spin_unlock_irqrestore(&mm->context.lock, flags); } @@ -404,6 +437,12 @@ EXPORT_SYMBOL(flush_icache_range); void mmu_info(struct seq_file *m) { + static const char *pgsz_strings[] = { + "8K", "64K", "512K", "4MB", "32MB", + "256MB", "2GB", "16GB", + }; + int i, printed; + if (tlb_type == cheetah) seq_printf(m, "MMU Type\t: Cheetah\n"); else if (tlb_type == cheetah_plus) @@ -415,6 +454,17 @@ void mmu_info(struct seq_file *m) else seq_printf(m, "MMU Type\t: ???\n"); + seq_printf(m, "MMU PGSZs\t: "); + printed = 0; + for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) { + if (cpu_pgsz_mask & (1UL << i)) { + seq_printf(m, "%s%s", + printed ? "," : "", pgsz_strings[i]); + printed++; + } + } + seq_putc(m, '\n'); + #ifdef CONFIG_DEBUG_DCFLUSH seq_printf(m, "DCPageFlushes\t: %d\n", atomic_read(&dcpage_flushes)); @@ -463,7 +513,7 @@ static void __init read_obp_translations(void) prom_halt(); } if (unlikely(n > sizeof(prom_trans))) { - prom_printf("prom_mappings: Size %Zd is too big.\n", n); + prom_printf("prom_mappings: Size %d is too big.\n", n); prom_halt(); } @@ -511,6 +561,11 @@ static void __init read_obp_translations(void) for (i = 0; i < prom_trans_ents; i++) prom_trans[i].data &= ~0x0003fe0000000000UL; } + + /* Force execute bit on. */ + for (i = 0; i < prom_trans_ents; i++) + prom_trans[i].data |= (tlb_type == hypervisor ? + _PAGE_EXEC_4V : _PAGE_EXEC_4U); } static void __init hypervisor_tlb_lock(unsigned long vaddr, @@ -520,7 +575,7 @@ static void __init hypervisor_tlb_lock(unsigned long vaddr, unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu); if (ret != 0) { - prom_printf("hypervisor_tlb_lock[%lx:%lx:%lx:%lx]: " + prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: " "errors with %lx\n", vaddr, 0, pte, mmu, ret); prom_halt(); } @@ -534,7 +589,7 @@ static void __init remap_kernel(void) int i, tlb_ent = sparc64_highest_locked_tlbent(); tte_vaddr = (unsigned long) KERNBASE; - phys_page = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; tte_data = kern_large_tte(phys_page); kern_locked_tte_data = tte_data; @@ -576,7 +631,7 @@ static void __init inherit_prom_mappings(void) void prom_world(int enter) { if (!enter) - set_fs((mm_segment_t) { get_thread_current_ds() }); + set_fs(get_fs()); __asm__ __volatile__("flushw"); } @@ -627,10 +682,9 @@ void get_new_mmu_context(struct mm_struct *mm) { unsigned long ctx, new_ctx; unsigned long orig_pgsz_bits; - unsigned long flags; int new_version; - spin_lock_irqsave(&ctx_alloc_lock, flags); + spin_lock(&ctx_alloc_lock); orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK); ctx = (tlb_context_cache + 1) & CTX_NR_MASK; new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx); @@ -666,7 +720,7 @@ void get_new_mmu_context(struct mm_struct *mm) out: tlb_context_cache = new_ctx; mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits; - spin_unlock_irqrestore(&ctx_alloc_lock, flags); + spin_unlock(&ctx_alloc_lock); if (unlikely(new_version)) smp_new_mmu_context_version(); @@ -737,16 +791,15 @@ static void __init find_ramdisk(unsigned long phys_base) struct node_mem_mask { unsigned long mask; unsigned long val; - unsigned long bootmem_paddr; }; static struct node_mem_mask node_masks[MAX_NUMNODES]; static int num_node_masks; +#ifdef CONFIG_NEED_MULTIPLE_NODES + int numa_cpu_lookup_table[NR_CPUS]; cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; -#ifdef CONFIG_NEED_MULTIPLE_NODES - struct mdesc_mblock { u64 base; u64 size; @@ -785,7 +838,7 @@ static int find_node(unsigned long addr) return -1; } -u64 memblock_nid_range(u64 start, u64 end, int *nid) +static u64 memblock_nid_range(u64 start, u64 end, int *nid) { *nid = find_node(start); start += PAGE_SIZE; @@ -802,24 +855,19 @@ u64 memblock_nid_range(u64 start, u64 end, int *nid) return start; } -#else -u64 memblock_nid_range(u64 start, u64 end, int *nid) -{ - *nid = 0; - return end; -} #endif /* This must be invoked after performing all of the necessary - * add_active_range() calls for 'nid'. We need to be able to get + * memblock_set_node() calls for 'nid'. We need to be able to get * correct data from get_pfn_range_for_nid(). */ static void __init allocate_node_data(int nid) { - unsigned long paddr, num_pages, start_pfn, end_pfn; struct pglist_data *p; - + unsigned long start_pfn, end_pfn; #ifdef CONFIG_NEED_MULTIPLE_NODES + unsigned long paddr; + paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid); if (!paddr) { prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid); @@ -828,7 +876,7 @@ static void __init allocate_node_data(int nid) NODE_DATA(nid) = __va(paddr); memset(NODE_DATA(nid), 0, sizeof(struct pglist_data)); - NODE_DATA(nid)->bdata = &bootmem_node_data[nid]; + NODE_DATA(nid)->node_id = nid; #endif p = NODE_DATA(nid); @@ -836,33 +884,25 @@ static void __init allocate_node_data(int nid) get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); p->node_start_pfn = start_pfn; p->node_spanned_pages = end_pfn - start_pfn; - - if (p->node_spanned_pages) { - num_pages = bootmem_bootmap_pages(p->node_spanned_pages); - - paddr = memblock_alloc_try_nid(num_pages << PAGE_SHIFT, PAGE_SIZE, nid); - if (!paddr) { - prom_printf("Cannot allocate bootmap for nid[%d]\n", - nid); - prom_halt(); - } - node_masks[nid].bootmem_paddr = paddr; - } } static void init_node_masks_nonnuma(void) { +#ifdef CONFIG_NEED_MULTIPLE_NODES int i; +#endif numadbg("Initializing tables for non-numa.\n"); node_masks[0].mask = node_masks[0].val = 0; num_node_masks = 1; +#ifdef CONFIG_NEED_MULTIPLE_NODES for (i = 0; i < NR_CPUS; i++) numa_cpu_lookup_table[i] = 0; - numa_cpumask_lookup_table[0] = CPU_MASK_ALL; + cpumask_setall(&numa_cpumask_lookup_table[0]); +#endif } #ifdef CONFIG_NEED_MULTIPLE_NODES @@ -982,14 +1022,12 @@ static void __init add_node_ranges(void) this_end = memblock_nid_range(start, end, &nid); - numadbg("Adding active range nid[%d] " + numadbg("Setting memblock NUMA node nid[%d] " "start[%lx] end[%lx]\n", nid, start, this_end); - add_active_range(nid, - start >> PAGE_SHIFT, - this_end >> PAGE_SHIFT); - + memblock_set_node(start, this_end - start, + &memblock.memory, nid); start = this_end; } } @@ -1066,7 +1104,14 @@ static int __init grab_mblocks(struct mdesc_handle *md) m->size = *val; val = mdesc_get_property(md, node, "address-congruence-offset", NULL); - m->offset = *val; + + /* The address-congruence-offset property is optional. + * Explicity zero it be identifty this. + */ + if (val) + m->offset = *val; + else + m->offset = 0UL; numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n", count - 1, m->base, m->size, m->offset); @@ -1080,7 +1125,7 @@ static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, { u64 arc; - cpus_clear(*mask); + cpumask_clear(mask); mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) { u64 target = mdesc_arc_target(md, arc); @@ -1091,7 +1136,7 @@ static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, continue; id = mdesc_get_property(md, target, "id", NULL); if (*id < nr_cpu_ids) - cpu_set(*id, *mask); + cpumask_set_cpu(*id, mask); } } @@ -1153,13 +1198,13 @@ static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp, numa_parse_mdesc_group_cpus(md, grp, &mask); - for_each_cpu_mask(cpu, mask) + for_each_cpu(cpu, &mask) numa_cpu_lookup_table[cpu] = index; - numa_cpumask_lookup_table[index] = mask; + cpumask_copy(&numa_cpumask_lookup_table[index], &mask); if (numa_debug) { printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index); - for_each_cpu_mask(cpu, mask) + for_each_cpu(cpu, &mask) printk("%d ", cpu); printk("]\n"); } @@ -1218,7 +1263,7 @@ static int __init numa_parse_jbus(void) index = 0; for_each_present_cpu(cpu) { numa_cpu_lookup_table[cpu] = index; - numa_cpumask_lookup_table[index] = cpumask_of_cpu(cpu); + cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu)); node_masks[index].mask = ~((1UL << 36UL) - 1UL); node_masks[index].val = cpu << 36UL; @@ -1277,7 +1322,6 @@ static void __init bootmem_init_nonnuma(void) { unsigned long top_of_ram = memblock_end_of_DRAM(); unsigned long total_ram = memblock_phys_mem_size(); - struct memblock_region *reg; numadbg("bootmem_init_nonnuma()\n"); @@ -1287,92 +1331,14 @@ static void __init bootmem_init_nonnuma(void) (top_of_ram - total_ram) >> 20); init_node_masks_nonnuma(); - - for_each_memblock(memory, reg) { - unsigned long start_pfn, end_pfn; - - if (!reg->size) - continue; - - start_pfn = memblock_region_memory_base_pfn(reg); - end_pfn = memblock_region_memory_end_pfn(reg); - add_active_range(0, start_pfn, end_pfn); - } - + memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0); allocate_node_data(0); - node_set_online(0); } -static void __init reserve_range_in_node(int nid, unsigned long start, - unsigned long end) -{ - numadbg(" reserve_range_in_node(nid[%d],start[%lx],end[%lx]\n", - nid, start, end); - while (start < end) { - unsigned long this_end; - int n; - - this_end = memblock_nid_range(start, end, &n); - if (n == nid) { - numadbg(" MATCH reserving range [%lx:%lx]\n", - start, this_end); - reserve_bootmem_node(NODE_DATA(nid), start, - (this_end - start), BOOTMEM_DEFAULT); - } else - numadbg(" NO MATCH, advancing start to %lx\n", - this_end); - - start = this_end; - } -} - -static void __init trim_reserved_in_node(int nid) -{ - struct memblock_region *reg; - - numadbg(" trim_reserved_in_node(%d)\n", nid); - - for_each_memblock(reserved, reg) - reserve_range_in_node(nid, reg->base, reg->base + reg->size); -} - -static void __init bootmem_init_one_node(int nid) -{ - struct pglist_data *p; - - numadbg("bootmem_init_one_node(%d)\n", nid); - - p = NODE_DATA(nid); - - if (p->node_spanned_pages) { - unsigned long paddr = node_masks[nid].bootmem_paddr; - unsigned long end_pfn; - - end_pfn = p->node_start_pfn + p->node_spanned_pages; - - numadbg(" init_bootmem_node(%d, %lx, %lx, %lx)\n", - nid, paddr >> PAGE_SHIFT, p->node_start_pfn, end_pfn); - - init_bootmem_node(p, paddr >> PAGE_SHIFT, - p->node_start_pfn, end_pfn); - - numadbg(" free_bootmem_with_active_regions(%d, %lx)\n", - nid, end_pfn); - free_bootmem_with_active_regions(nid, end_pfn); - - trim_reserved_in_node(nid); - - numadbg(" sparse_memory_present_with_active_regions(%d)\n", - nid); - sparse_memory_present_with_active_regions(nid); - } -} - static unsigned long __init bootmem_init(unsigned long phys_base) { unsigned long end_pfn; - int nid; end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; max_pfn = max_low_pfn = end_pfn; @@ -1381,11 +1347,12 @@ static unsigned long __init bootmem_init(unsigned long phys_base) if (bootmem_init_numa() < 0) bootmem_init_nonnuma(); - /* XXX cpu notifier XXX */ + /* Dump memblock with node info. */ + memblock_dump_all(); - for_each_online_node(nid) - bootmem_init_one_node(nid); + /* XXX cpu notifier XXX */ + sparse_memory_present_with_active_regions(MAX_NUMNODES); sparse_init(); return end_pfn; @@ -1453,32 +1420,75 @@ static unsigned long __ref kernel_map_range(unsigned long pstart, extern unsigned int kvmap_linear_patch[1]; #endif /* CONFIG_DEBUG_PAGEALLOC */ -static void __init mark_kpte_bitmap(unsigned long start, unsigned long end) +static void __init kpte_set_val(unsigned long index, unsigned long val) { - const unsigned long shift_256MB = 28; - const unsigned long mask_256MB = ((1UL << shift_256MB) - 1UL); - const unsigned long size_256MB = (1UL << shift_256MB); + unsigned long *ptr = kpte_linear_bitmap; - while (start < end) { - long remains; + val <<= ((index % (BITS_PER_LONG / 2)) * 2); + ptr += (index / (BITS_PER_LONG / 2)); - remains = end - start; - if (remains < size_256MB) - break; + *ptr |= val; +} - if (start & mask_256MB) { - start = (start + size_256MB) & ~mask_256MB; - continue; - } +static const unsigned long kpte_shift_min = 28; /* 256MB */ +static const unsigned long kpte_shift_max = 34; /* 16GB */ +static const unsigned long kpte_shift_incr = 3; + +static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end, + unsigned long shift) +{ + unsigned long size = (1UL << shift); + unsigned long mask = (size - 1UL); + unsigned long remains = end - start; + unsigned long val; - while (remains >= size_256MB) { - unsigned long index = start >> shift_256MB; + if (remains < size || (start & mask)) + return start; - __set_bit(index, kpte_linear_bitmap); + /* VAL maps: + * + * shift 28 --> kern_linear_pte_xor index 1 + * shift 31 --> kern_linear_pte_xor index 2 + * shift 34 --> kern_linear_pte_xor index 3 + */ + val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1; + + remains &= ~mask; + if (shift != kpte_shift_max) + remains = size; - start += size_256MB; - remains -= size_256MB; + while (remains) { + unsigned long index = start >> kpte_shift_min; + + kpte_set_val(index, val); + + start += 1UL << kpte_shift_min; + remains -= 1UL << kpte_shift_min; + } + + return start; +} + +static void __init mark_kpte_bitmap(unsigned long start, unsigned long end) +{ + unsigned long smallest_size, smallest_mask; + unsigned long s; + + smallest_size = (1UL << kpte_shift_min); + smallest_mask = (smallest_size - 1UL); + + while (start < end) { + unsigned long orig_start = start; + + for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) { + start = kpte_mark_using_shift(start, end, s); + + if (start != orig_start) + break; } + + if (start == orig_start) + start = (start + smallest_size) & ~smallest_mask; } } @@ -1553,6 +1563,96 @@ unsigned long __init find_ecache_flush_span(unsigned long size) return ~0UL; } +unsigned long PAGE_OFFSET; +EXPORT_SYMBOL(PAGE_OFFSET); + +static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits) +{ + unsigned long final_shift; + unsigned int val = *insn; + unsigned int cnt; + + /* We are patching in ilog2(max_supported_phys_address), and + * we are doing so in a manner similar to a relocation addend. + * That is, we are adding the shift value to whatever value + * is in the shift instruction count field already. + */ + cnt = (val & 0x3f); + val &= ~0x3f; + + /* If we are trying to shift >= 64 bits, clear the destination + * register. This can happen when phys_bits ends up being equal + * to MAX_PHYS_ADDRESS_BITS. + */ + final_shift = (cnt + (64 - phys_bits)); + if (final_shift >= 64) { + unsigned int rd = (val >> 25) & 0x1f; + + val = 0x80100000 | (rd << 25); + } else { + val |= final_shift; + } + *insn = val; + + __asm__ __volatile__("flush %0" + : /* no outputs */ + : "r" (insn)); +} + +static void __init page_offset_shift_patch(unsigned long phys_bits) +{ + extern unsigned int __page_offset_shift_patch; + extern unsigned int __page_offset_shift_patch_end; + unsigned int *p; + + p = &__page_offset_shift_patch; + while (p < &__page_offset_shift_patch_end) { + unsigned int *insn = (unsigned int *)(unsigned long)*p; + + page_offset_shift_patch_one(insn, phys_bits); + + p++; + } +} + +static void __init setup_page_offset(void) +{ + unsigned long max_phys_bits = 40; + + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + max_phys_bits = 42; + } else if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + max_phys_bits = 39; + break; + case SUN4V_CHIP_NIAGARA3: + max_phys_bits = 43; + break; + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC64X: + default: + max_phys_bits = 47; + break; + } + } + + if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) { + prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n", + max_phys_bits); + prom_halt(); + } + + PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits); + + pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", + PAGE_OFFSET, max_phys_bits); + + page_offset_shift_patch(max_phys_bits); +} + static void __init tsb_phys_patch(void) { struct tsb_ldquad_phys_patch_entry *pquad; @@ -1597,6 +1697,44 @@ static void __init tsb_phys_patch(void) static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; +static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) +{ + pa >>= KTSB_PHYS_SHIFT; + + while (start < end) { + unsigned int *ia = (unsigned int *)(unsigned long)*start; + + ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10); + __asm__ __volatile__("flush %0" : : "r" (ia)); + + ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 1)); + + start++; + } +} + +static void ktsb_phys_patch(void) +{ + extern unsigned int __swapper_tsb_phys_patch; + extern unsigned int __swapper_tsb_phys_patch_end; + unsigned long ktsb_pa; + + ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE); + patch_one_ktsb_phys(&__swapper_tsb_phys_patch, + &__swapper_tsb_phys_patch_end, ktsb_pa); +#ifndef CONFIG_DEBUG_PAGEALLOC + { + extern unsigned int __swapper_4m_tsb_phys_patch; + extern unsigned int __swapper_4m_tsb_phys_patch_end; + ktsb_pa = (kern_base + + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); + patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch, + &__swapper_4m_tsb_phys_patch_end, ktsb_pa); + } +#endif +} + static void __init sun4v_ktsb_init(void) { unsigned long ktsb_pa; @@ -1625,7 +1763,7 @@ static void __init sun4v_ktsb_init(void) ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB; ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB; break; - }; + } ktsb_descr[0].assoc = 1; ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES; @@ -1634,13 +1772,16 @@ static void __init sun4v_ktsb_init(void) ktsb_descr[0].resv = 0; #ifndef CONFIG_DEBUG_PAGEALLOC - /* Second KTSB for 4MB/256MB mappings. */ + /* Second KTSB for 4MB/256MB/2GB/16GB mappings. */ ktsb_pa = (kern_base + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB; - ktsb_descr[1].pgsz_mask = (HV_PGSZ_MASK_4MB | - HV_PGSZ_MASK_256MB); + ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB | + HV_PGSZ_MASK_256MB | + HV_PGSZ_MASK_2GB | + HV_PGSZ_MASK_16GB) & + cpu_pgsz_mask); ktsb_descr[1].assoc = 1; ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES; ktsb_descr[1].ctx_idx = 0; @@ -1649,7 +1790,7 @@ static void __init sun4v_ktsb_init(void) #endif } -void __cpuinit sun4v_ktsb_register(void) +void sun4v_ktsb_register(void) { unsigned long pa, ret; @@ -1663,10 +1804,51 @@ void __cpuinit sun4v_ktsb_register(void) } } +static void __init sun4u_linear_pte_xor_finalize(void) +{ +#ifndef CONFIG_DEBUG_PAGEALLOC + /* This is where we would add Panther support for + * 32MB and 256MB pages. + */ +#endif +} + +static void __init sun4v_linear_pte_xor_finalize(void) +{ +#ifndef CONFIG_DEBUG_PAGEALLOC + if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { + kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { + kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[2] = kern_linear_pte_xor[1]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { + kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[3] = kern_linear_pte_xor[2]; + } +#endif +} + /* paging_init() sets up the page tables */ static unsigned long last_valid_pfn; -pgd_t swapper_pg_dir[2048]; +pgd_t swapper_pg_dir[PTRS_PER_PGD]; static void sun4u_pgprot_init(void); static void sun4v_pgprot_init(void); @@ -1675,6 +1857,9 @@ void __init paging_init(void) { unsigned long end_pfn, shift, phys_base; unsigned long real_end, i; + int node; + + setup_page_offset(); /* These build time checkes make sure that the dcache_dirty_cpu() * page->flags usage will work. @@ -1701,7 +1886,7 @@ void __init paging_init(void) BUILD_BUG_ON(NR_CPUS > 4096); - kern_base = (prom_boot_mapping_phys_low >> 22UL) << 22UL; + kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; kern_size = (unsigned long)&_end - (unsigned long)KERNBASE; /* Invalidate both kernel TSBs. */ @@ -1716,15 +1901,13 @@ void __init paging_init(void) sun4u_pgprot_init(); if (tlb_type == cheetah_plus || - tlb_type == hypervisor) + tlb_type == hypervisor) { tsb_phys_patch(); - - if (tlb_type == hypervisor) { - sun4v_patch_tlb_handlers(); - sun4v_ktsb_init(); + ktsb_phys_patch(); } - memblock_init(); + if (tlb_type == hypervisor) + sun4v_patch_tlb_handlers(); /* Find available physical memory... * @@ -1751,7 +1934,7 @@ void __init paging_init(void) memblock_enforce_memory_limit(cmdline_memory_size); - memblock_analyze(); + memblock_allow_resize(); memblock_dump_all(); set_bit(0, mmu_context_bmap); @@ -1759,7 +1942,7 @@ void __init paging_init(void) shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE); real_end = (unsigned long)_end; - num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << 22); + num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB); printk("Kernel: Using %d locked TLB entries for main kernel image.\n", num_kernel_image_mappings); @@ -1783,9 +1966,6 @@ void __init paging_init(void) __flush_tlb_all(); - if (tlb_type == hypervisor) - sun4v_ktsb_register(); - prom_build_devicetree(); of_populate_present_mask(); #ifndef CONFIG_SMP @@ -1798,24 +1978,54 @@ void __init paging_init(void) #ifndef CONFIG_SMP mdesc_fill_in_cpu_data(cpu_all_mask); #endif + mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask); + + sun4v_linear_pte_xor_finalize(); + + sun4v_ktsb_init(); + sun4v_ktsb_register(); + } else { + unsigned long impl, ver; + + cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K | + HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB); + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + impl = ((ver >> 32) & 0xffff); + if (impl == PANTHER_IMPL) + cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB | + HV_PGSZ_MASK_256MB); + + sun4u_linear_pte_xor_finalize(); } + /* Flush the TLBs and the 4M TSB so that the updated linear + * pte XOR settings are realized for all mappings. + */ + __flush_tlb_all(); +#ifndef CONFIG_DEBUG_PAGEALLOC + memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb)); +#endif + __flush_tlb_all(); + + /* Setup bootmem... */ + last_valid_pfn = end_pfn = bootmem_init(phys_base); + /* Once the OF device tree and MDESC have been setup, we know * the list of possible cpus. Therefore we can allocate the * IRQ stacks. */ for_each_possible_cpu(i) { - /* XXX Use node local allocations... XXX */ - softirq_stack[i] = __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE)); - hardirq_stack[i] = __va(memblock_alloc(THREAD_SIZE, THREAD_SIZE)); - } + node = cpu_to_node(i); - /* Setup bootmem... */ - last_valid_pfn = end_pfn = bootmem_init(phys_base); + softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + } -#ifndef CONFIG_NEED_MULTIPLE_NODES - max_mapnr = last_valid_pfn; -#endif kernel_physical_mapping_init(); { @@ -1831,7 +2041,7 @@ void __init paging_init(void) printk("Booting Linux...\n"); } -int __devinit page_in_phys_avail(unsigned long paddr) +int page_in_phys_avail(unsigned long paddr) { int i; @@ -1889,7 +2099,7 @@ static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap) if (new_start <= old_start && new_end >= (old_start + PAGE_SIZE)) { - set_bit(old_start >> 22, bitmap); + set_bit(old_start >> ILOG2_4MB, bitmap); goto do_next_page; } } @@ -1921,15 +2131,24 @@ static void __init patch_tlb_miss_handler_bitmap(void) flushi(&valid_addr_bitmap_insn[0]); } +static void __init register_page_bootmem_info(void) +{ +#ifdef CONFIG_NEED_MULTIPLE_NODES + int i; + + for_each_online_node(i) + if (NODE_DATA(i)->node_spanned_pages) + register_page_bootmem_info_node(NODE_DATA(i)); +#endif +} void __init mem_init(void) { - unsigned long codepages, datapages, initpages; unsigned long addr, last; addr = PAGE_OFFSET + kern_base; last = PAGE_ALIGN(kern_size) + addr; while (addr < last) { - set_bit(__pa(addr) >> 22, sparc64_valid_addr_bitmap); + set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap); addr += PAGE_SIZE; } @@ -1938,25 +2157,8 @@ void __init mem_init(void) high_memory = __va(last_valid_pfn << PAGE_SHIFT); -#ifdef CONFIG_NEED_MULTIPLE_NODES - { - int i; - for_each_online_node(i) { - if (NODE_DATA(i)->node_spanned_pages != 0) { - totalram_pages += - free_all_bootmem_node(NODE_DATA(i)); - } - } - } -#else - totalram_pages = free_all_bootmem(); -#endif - - /* We subtract one to account for the mem_map_zero page - * allocated below. - */ - totalram_pages -= 1; - num_physpages = totalram_pages; + register_page_bootmem_info(); + free_all_bootmem(); /* * Set up the zero page, mark it reserved, so that page count @@ -1967,21 +2169,9 @@ void __init mem_init(void) prom_printf("paging_init: Cannot alloc zero page.\n"); prom_halt(); } - SetPageReserved(mem_map_zero); + mark_page_reserved(mem_map_zero); - codepages = (((unsigned long) _etext) - ((unsigned long) _start)); - codepages = PAGE_ALIGN(codepages) >> PAGE_SHIFT; - datapages = (((unsigned long) _edata) - ((unsigned long) _etext)); - datapages = PAGE_ALIGN(datapages) >> PAGE_SHIFT; - initpages = (((unsigned long) __init_end) - ((unsigned long) __init_begin)); - initpages = PAGE_ALIGN(initpages) >> PAGE_SHIFT; - - printk("Memory: %luk available (%ldk kernel code, %ldk data, %ldk init) [%016lx,%016lx]\n", - nr_free_pages() << (PAGE_SHIFT-10), - codepages << (PAGE_SHIFT-10), - datapages << (PAGE_SHIFT-10), - initpages << (PAGE_SHIFT-10), - PAGE_OFFSET, (last_valid_pfn << PAGE_SHIFT)); + mem_init_print_info(NULL); if (tlb_type == cheetah || tlb_type == cheetah_plus) cheetah_ecache_flush_init(); @@ -2007,39 +2197,22 @@ void free_initmem(void) initend = (unsigned long)(__init_end) & PAGE_MASK; for (; addr < initend; addr += PAGE_SIZE) { unsigned long page; - struct page *p; page = (addr + ((unsigned long) __va(kern_base)) - ((unsigned long) KERNBASE)); memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); - if (do_free) { - p = virt_to_page(page); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - num_physpages++; - totalram_pages++; - } + if (do_free) + free_reserved_page(virt_to_page(page)); } } #ifdef CONFIG_BLK_DEV_INITRD void free_initrd_mem(unsigned long start, unsigned long end) { - if (start < end) - printk ("Freeing initrd memory: %ldk freed\n", (end - start) >> 10); - for (; start < end; start += PAGE_SIZE) { - struct page *p = virt_to_page(start); - - ClearPageReserved(p); - init_page_count(p); - __free_page(p); - num_physpages++; - totalram_pages++; - } + free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM, + "initrd"); } #endif @@ -2073,10 +2246,12 @@ EXPORT_SYMBOL(_PAGE_CACHE); #ifdef CONFIG_SPARSEMEM_VMEMMAP unsigned long vmemmap_table[VMEMMAP_SIZE]; -int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) +static long __meminitdata addr_start, addr_end; +static int __meminitdata node_start; + +int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, + int node) { - unsigned long vstart = (unsigned long) start; - unsigned long vend = (unsigned long) (start + nr); unsigned long phys_start = (vstart - VMEMMAP_BASE); unsigned long phys_end = (vend - VMEMMAP_BASE); unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK; @@ -2097,21 +2272,41 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node) void *block; if (!(*vmem_pp & _PAGE_VALID)) { - block = vmemmap_alloc_block(1UL << 22, node); + block = vmemmap_alloc_block(1UL << ILOG2_4MB, node); if (!block) return -ENOMEM; *vmem_pp = pte_base | __pa(block); - printk(KERN_INFO "[%p-%p] page_structs=%lu " - "node=%d entry=%lu/%lu\n", start, block, nr, - node, - addr >> VMEMMAP_CHUNK_SHIFT, - VMEMMAP_SIZE); + /* check to see if we have contiguous blocks */ + if (addr_end != addr || node_start != node) { + if (addr_start) + printk(KERN_DEBUG " [%lx-%lx] on node %d\n", + addr_start, addr_end-1, node_start); + addr_start = addr; + node_start = node; + } + addr_end = addr + VMEMMAP_CHUNK; } } return 0; } + +void __meminit vmemmap_populate_print_last(void) +{ + if (addr_start) { + printk(KERN_DEBUG " [%lx-%lx] on node %d\n", + addr_start, addr_end-1, node_start); + addr_start = 0; + addr_end = 0; + node_start = 0; + } +} + +void vmemmap_free(unsigned long start, unsigned long end) +{ +} + #endif /* CONFIG_SPARSEMEM_VMEMMAP */ static void prot_init_common(unsigned long page_none, @@ -2145,6 +2340,7 @@ static void __init sun4u_pgprot_init(void) { unsigned long page_none, page_shared, page_copy, page_readonly; unsigned long page_exec_bit; + int i; PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID | _PAGE_CACHE_4U | _PAGE_P_4U | @@ -2163,19 +2359,17 @@ static void __init sun4u_pgprot_init(void) __ACCESS_BITS_4U | _PAGE_E_4U); #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4U) ^ - 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U | _PAGE_W_4U); - /* XXX Should use 256MB on Panther. XXX */ - kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; - _PAGE_SZBITS = _PAGE_SZBITS_4U; _PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U | _PAGE_SZ64K_4U | _PAGE_SZ8K_4U | _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U); @@ -2199,6 +2393,7 @@ static void __init sun4v_pgprot_init(void) { unsigned long page_none, page_shared, page_copy, page_readonly; unsigned long page_exec_bit; + int i; PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID | _PAGE_CACHE_4V | _PAGE_P_4V | @@ -2211,29 +2406,20 @@ static void __init sun4v_pgprot_init(void) _PAGE_CACHE = _PAGE_CACHE_4V; #ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^ - 0xfffff80000000000UL; + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; #else kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ - 0xfffff80000000000UL; + PAGE_OFFSET; #endif kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V | _PAGE_W_4V); -#ifdef CONFIG_DEBUG_PAGEALLOC - kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZBITS_4V) ^ - 0xfffff80000000000UL; -#else - kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ - 0xfffff80000000000UL; -#endif - kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V | - _PAGE_P_4V | _PAGE_W_4V); + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V | __ACCESS_BITS_4V | _PAGE_E_4V); - _PAGE_SZBITS = _PAGE_SZBITS_4V; _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V | _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V | _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V | @@ -2266,7 +2452,7 @@ unsigned long pte_sz_bits(unsigned long sz) return _PAGE_SZ512K_4V; case 4 * 1024 * 1024: return _PAGE_SZ4MB_4V; - }; + } } else { switch (sz) { case 8 * 1024: @@ -2278,7 +2464,7 @@ unsigned long pte_sz_bits(unsigned long sz) return _PAGE_SZ512K_4U; case 4 * 1024 * 1024: return _PAGE_SZ4MB_4U; - }; + } } } @@ -2366,3 +2552,150 @@ void __flush_tlb_all(void) __asm__ __volatile__("wrpr %0, 0, %%pstate" : : "r" (pstate)); } + +pte_t *pte_alloc_one_kernel(struct mm_struct *mm, + unsigned long address) +{ + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | + __GFP_REPEAT | __GFP_ZERO); + pte_t *pte = NULL; + + if (page) + pte = (pte_t *) page_address(page); + + return pte; +} + +pgtable_t pte_alloc_one(struct mm_struct *mm, + unsigned long address) +{ + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | + __GFP_REPEAT | __GFP_ZERO); + if (!page) + return NULL; + if (!pgtable_page_ctor(page)) { + free_hot_cold_page(page, 0); + return NULL; + } + return (pte_t *) page_address(page); +} + +void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + free_page((unsigned long)pte); +} + +static void __pte_free(pgtable_t pte) +{ + struct page *page = virt_to_page(pte); + + pgtable_page_dtor(page); + __free_page(page); +} + +void pte_free(struct mm_struct *mm, pgtable_t pte) +{ + __pte_free(pte); +} + +void pgtable_free(void *table, bool is_page) +{ + if (is_page) + __pte_free(table); + else + kmem_cache_free(pgtable_cache, table); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd) +{ + unsigned long pte, flags; + struct mm_struct *mm; + pmd_t entry = *pmd; + + if (!pmd_large(entry) || !pmd_young(entry)) + return; + + pte = pmd_val(entry); + + /* We are fabricating 8MB pages using 4MB real hw pages. */ + pte |= (addr & (1UL << REAL_HPAGE_SHIFT)); + + mm = vma->vm_mm; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, + addr, pte); + + spin_unlock_irqrestore(&mm->context.lock, flags); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void context_reload(void *__data) +{ + struct mm_struct *mm = __data; + + if (mm == current->mm) + load_secondary_context(mm); +} + +void hugetlb_setup(struct pt_regs *regs) +{ + struct mm_struct *mm = current->mm; + struct tsb_config *tp; + + if (in_atomic() || !mm) { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + pr_alert("Unexpected HugeTLB setup in atomic context.\n"); + die_if_kernel("HugeTSB in atomic", regs); + } + + tp = &mm->context.tsb_block[MM_TSB_HUGE]; + if (likely(tp->tsb == NULL)) + tsb_grow(mm, MM_TSB_HUGE, 0); + + tsb_context_switch(mm); + smp_tsb_sync(mm); + + /* On UltraSPARC-III+ and later, configure the second half of + * the Data-TLB for huge pages. + */ + if (tlb_type == cheetah_plus) { + unsigned long ctx; + + spin_lock(&ctx_alloc_lock); + ctx = mm->context.sparc64_ctx_val; + ctx &= ~CTX_PGSZ_MASK; + ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; + ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; + + if (ctx != mm->context.sparc64_ctx_val) { + /* When changing the page size fields, we + * must perform a context flush so that no + * stale entries match. This flush must + * occur with the original context register + * settings. + */ + do_flush_tlb_mm(mm); + + /* Reload the context register of all processors + * also executing in this address space. + */ + mm->context.sparc64_ctx_val = ctx; + on_each_cpu(context_reload, mm, 0); + } + spin_unlock(&ctx_alloc_lock); + } +} +#endif diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h index 77d1b313e34..0668b364f44 100644 --- a/arch/sparc/mm/init_64.h +++ b/arch/sparc/mm/init_64.h @@ -1,25 +1,27 @@ #ifndef _SPARC64_MM_INIT_H #define _SPARC64_MM_INIT_H +#include <asm/page.h> + /* Most of the symbols in this file are defined in init.c and * marked non-static so that assembler code can get at them. */ -#define MAX_PHYS_ADDRESS (1UL << 41UL) +#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) #define KPTE_BITMAP_CHUNK_SZ (256UL * 1024UL * 1024UL) #define KPTE_BITMAP_BYTES \ - ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8) + ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 4) #define VALID_ADDR_BITMAP_CHUNK_SZ (4UL * 1024UL * 1024UL) #define VALID_ADDR_BITMAP_BYTES \ ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8) -extern unsigned long kern_linear_pte_xor[2]; +extern unsigned long kern_linear_pte_xor[4]; extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)]; extern unsigned int sparc64_highest_unlocked_tlb_ent; extern unsigned long sparc64_kern_pri_context; extern unsigned long sparc64_kern_pri_nuc_bits; extern unsigned long sparc64_kern_sec_context; -extern void mmu_info(struct seq_file *m); +void mmu_info(struct seq_file *m); struct linux_prom_translation { unsigned long virt; @@ -34,9 +36,7 @@ extern unsigned int prom_trans_ents; /* Exported for SMP bootup purposes. */ extern unsigned long kern_locked_tte_data; -extern void prom_world(int enter); - -extern void free_initmem(void); +void prom_world(int enter); #ifdef CONFIG_SPARSEMEM_VMEMMAP #define VMEMMAP_CHUNK_SHIFT 22 diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index fc58c3e917d..f311bf21901 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -25,6 +25,8 @@ #include <asm/dma.h> #include <asm/oplib.h> +#include "mm_32.h" + /* #define IOUNIT_DEBUG */ #ifdef IOUNIT_DEBUG #define IOD(x) printk(x) @@ -38,7 +40,8 @@ static void __init iounit_iommu_init(struct platform_device *op) { struct iounit_struct *iounit; - iopte_t *xpt, *xptend; + iopte_t __iomem *xpt; + iopte_t __iomem *xptend; iounit = kzalloc(sizeof(struct iounit_struct), GFP_ATOMIC); if (!iounit) { @@ -62,10 +65,10 @@ static void __init iounit_iommu_init(struct platform_device *op) op->dev.archdata.iommu = iounit; iounit->page_table = xpt; spin_lock_init(&iounit->lock); - - for (xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); - xpt < xptend;) - iopte_val(*xpt++) = 0; + + xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); + for (; xpt < xptend; xpt++) + sbus_writel(0, xpt); } static int __init iounit_init(void) @@ -130,7 +133,7 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); vaddr = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + (vaddr & ~PAGE_MASK); for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) { set_bit(scan, iounit->bmap); - iounit->page_table[scan] = iopte; + sbus_writel(iopte, &iounit->page_table[scan]); } IOD(("%08lx\n", vaddr)); return vaddr; @@ -197,12 +200,12 @@ static void iounit_release_scsi_sgl(struct device *dev, struct scatterlist *sg, } #ifdef CONFIG_SBUS -static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, __u32 addr, int len) +static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, unsigned long addr, int len) { struct iounit_struct *iounit = dev->archdata.iommu; unsigned long page, end; pgprot_t dvma_prot; - iopte_t *iopte; + iopte_t __iomem *iopte; *pba = addr; @@ -224,8 +227,8 @@ static int iounit_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned lon i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); - iopte = (iopte_t *)(iounit->page_table + i); - *iopte = MKIOPTE(__pa(page)); + iopte = iounit->page_table + i; + sbus_writel(MKIOPTE(__pa(page)), iopte); } addr += PAGE_SIZE; va += PAGE_SIZE; @@ -242,29 +245,18 @@ static void iounit_unmap_dma_area(struct device *dev, unsigned long addr, int le } #endif -static char *iounit_lockarea(char *vaddr, unsigned long len) -{ -/* FIXME: Write this */ - return vaddr; -} - -static void iounit_unlockarea(char *vaddr, unsigned long len) -{ -/* FIXME: Write this */ -} +static const struct sparc32_dma_ops iounit_dma_ops = { + .get_scsi_one = iounit_get_scsi_one, + .get_scsi_sgl = iounit_get_scsi_sgl, + .release_scsi_one = iounit_release_scsi_one, + .release_scsi_sgl = iounit_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iounit_map_dma_area, + .unmap_dma_area = iounit_unmap_dma_area, +#endif +}; void __init ld_mmu_iounit(void) { - BTFIXUPSET_CALL(mmu_lockarea, iounit_lockarea, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_unlockarea, iounit_unlockarea, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(mmu_get_scsi_one, iounit_get_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iounit_get_scsi_sgl, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_one, iounit_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, iounit_release_scsi_sgl, BTFIXUPCALL_NORM); - -#ifdef CONFIG_SBUS - BTFIXUPSET_CALL(mmu_map_dma_area, iounit_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, iounit_unmap_dma_area, BTFIXUPCALL_NORM); -#endif + sparc32_dma_ops = &iounit_dma_ops; } diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index 07fc6a65d9b..491511d37e3 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -27,6 +27,8 @@ #include <asm/iommu.h> #include <asm/dma.h> +#include "mm_32.h" + /* * This can be sized dynamically, but we will do this * only when we have a guidance about actual I/O pressures. @@ -34,14 +36,9 @@ #define IOMMU_RNGE IOMMU_RNGE_256MB #define IOMMU_START 0xF0000000 #define IOMMU_WINSIZE (256*1024*1024U) -#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */ +#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ #define IOMMU_ORDER 6 /* 4096 * (1<<6) */ -/* srmmu.c */ -extern int viking_mxcc_present; -BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long) -#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page) -extern int flush_page_for_dma_global; static int viking_flush; /* viking.S */ extern void viking_flush_page(unsigned long page); @@ -61,6 +58,8 @@ static void __init sbus_iommu_init(struct platform_device *op) struct iommu_struct *iommu; unsigned int impl, vers; unsigned long *bitmap; + unsigned long control; + unsigned long base; unsigned long tmp; iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL); @@ -75,12 +74,14 @@ static void __init sbus_iommu_init(struct platform_device *op) prom_printf("Cannot map IOMMU registers\n"); prom_halt(); } - impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28; - vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24; - tmp = iommu->regs->control; - tmp &= ~(IOMMU_CTRL_RNGE); - tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); - iommu->regs->control = tmp; + + control = sbus_readl(&iommu->regs->control); + impl = (control & IOMMU_CTRL_IMPL) >> 28; + vers = (control & IOMMU_CTRL_VERS) >> 24; + control &= ~(IOMMU_CTRL_RNGE); + control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); + sbus_writel(control, &iommu->regs->control); + iommu_invalidate(iommu->regs); iommu->start = IOMMU_START; iommu->end = 0xffffffff; @@ -92,8 +93,8 @@ static void __init sbus_iommu_init(struct platform_device *op) it to us. */ tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); if (!tmp) { - prom_printf("Unable to allocate iommu table [0x%08x]\n", - IOMMU_NPTES*sizeof(iopte_t)); + prom_printf("Unable to allocate iommu table [0x%lx]\n", + IOMMU_NPTES * sizeof(iopte_t)); prom_halt(); } iommu->page_table = (iopte_t *)tmp; @@ -102,7 +103,9 @@ static void __init sbus_iommu_init(struct platform_device *op) memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); flush_cache_all(); flush_tlb_all(); - iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4; + + base = __pa((unsigned long)iommu->page_table) >> 4; + sbus_writel(base, &iommu->regs->base); iommu_invalidate(iommu->regs); bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); @@ -143,7 +146,6 @@ static int __init iommu_init(void) subsys_initcall(iommu_init); -/* This begs to be btfixup-ed by srmmu. */ /* Flush the iotlb entries to ram. */ /* This could be better if we didn't have to flush whole pages. */ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) @@ -216,11 +218,6 @@ static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len) return busa + off; } -static __u32 iommu_get_scsi_one_noflush(struct device *dev, char *vaddr, unsigned long len) -{ - return iommu_get_scsi_one(dev, vaddr, len); -} - static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len) { flush_page_for_dma(0); @@ -238,19 +235,6 @@ static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned return iommu_get_scsi_one(dev, vaddr, len); } -static void iommu_get_scsi_sgl_noflush(struct device *dev, struct scatterlist *sg, int sz) -{ - int n; - - while (sz != 0) { - --sz; - n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT; - sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset; - sg->dma_length = sg->length; - sg = sg_next(sg); - } -} - static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz) { int n; @@ -426,40 +410,36 @@ static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len } #endif -static char *iommu_lockarea(char *vaddr, unsigned long len) -{ - return vaddr; -} +static const struct sparc32_dma_ops iommu_dma_gflush_ops = { + .get_scsi_one = iommu_get_scsi_one_gflush, + .get_scsi_sgl = iommu_get_scsi_sgl_gflush, + .release_scsi_one = iommu_release_scsi_one, + .release_scsi_sgl = iommu_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iommu_map_dma_area, + .unmap_dma_area = iommu_unmap_dma_area, +#endif +}; -static void iommu_unlockarea(char *vaddr, unsigned long len) -{ -} +static const struct sparc32_dma_ops iommu_dma_pflush_ops = { + .get_scsi_one = iommu_get_scsi_one_pflush, + .get_scsi_sgl = iommu_get_scsi_sgl_pflush, + .release_scsi_one = iommu_release_scsi_one, + .release_scsi_sgl = iommu_release_scsi_sgl, +#ifdef CONFIG_SBUS + .map_dma_area = iommu_map_dma_area, + .unmap_dma_area = iommu_unmap_dma_area, +#endif +}; void __init ld_mmu_iommu(void) { - viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page); - BTFIXUPSET_CALL(mmu_lockarea, iommu_lockarea, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_unlockarea, iommu_unlockarea, BTFIXUPCALL_NOP); - - if (!BTFIXUPVAL_CALL(flush_page_for_dma)) { - /* IO coherent chip */ - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_noflush, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_noflush, BTFIXUPCALL_NORM); - } else if (flush_page_for_dma_global) { + if (flush_page_for_dma_global) { /* flush_page_for_dma flushes everything, no matter of what page is it */ - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_gflush, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_gflush, BTFIXUPCALL_NORM); + sparc32_dma_ops = &iommu_dma_gflush_ops; } else { - BTFIXUPSET_CALL(mmu_get_scsi_one, iommu_get_scsi_one_pflush, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, iommu_get_scsi_sgl_pflush, BTFIXUPCALL_NORM); + sparc32_dma_ops = &iommu_dma_pflush_ops; } - BTFIXUPSET_CALL(mmu_release_scsi_one, iommu_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, iommu_release_scsi_sgl, BTFIXUPCALL_NORM); - -#ifdef CONFIG_SBUS - BTFIXUPSET_CALL(mmu_map_dma_area, iommu_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, iommu_unmap_dma_area, BTFIXUPCALL_NORM); -#endif if (viking_mxcc_present || srmmu_modtype == HyperSparc) { dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c index c0e01297e64..3b17b6f7895 100644 --- a/arch/sparc/mm/leon_mm.c +++ b/arch/sparc/mm/leon_mm.c @@ -15,10 +15,24 @@ #include <asm/leon.h> #include <asm/tlbflush.h> +#include "mm_32.h" + int leon_flush_during_switch = 1; -int srmmu_swprobe_trace; +static int srmmu_swprobe_trace; + +static inline unsigned long leon_get_ctable_ptr(void) +{ + unsigned int retval; + + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (SRMMU_CTXTBL_PTR), + "i" (ASI_LEON_MMUREGS)); + return (retval & SRMMU_CTX_PMASK) << 4; +} + -unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr) +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) { unsigned int ctxtbl; @@ -33,10 +47,10 @@ unsigned long srmmu_swprobe(unsigned long vaddr, unsigned long *paddr) if (srmmu_swprobe_trace) printk(KERN_INFO "swprobe: trace on\n"); - ctxtbl = srmmu_get_ctable_ptr(); + ctxtbl = leon_get_ctable_ptr(); if (!(ctxtbl)) { if (srmmu_swprobe_trace) - printk(KERN_INFO "swprobe: srmmu_get_ctable_ptr returned 0=>0\n"); + printk(KERN_INFO "swprobe: leon_get_ctable_ptr returned 0=>0\n"); return 0; } if (!_pfn_valid(PFN(ctxtbl))) { @@ -162,7 +176,7 @@ ready: printk(KERN_INFO "swprobe: padde %x\n", paddr_calc); if (paddr) *paddr = paddr_calc; - return paddrbase; + return pte; } void leon_flush_icache_all(void) @@ -226,7 +240,7 @@ void leon3_getCacheRegs(struct leon3_cacheregs *regs) * Leon2 and Leon3 differ in their way of telling cache information * */ -int leon_flush_needed(void) +int __init leon_flush_needed(void) { int flush_needed = -1; unsigned int ssize, sets; @@ -258,3 +272,80 @@ void leon_switch_mm(void) if (leon_flush_during_switch) leon_flush_cache_all(); } + +static void leon_flush_cache_mm(struct mm_struct *mm) +{ + leon_flush_cache_all(); +} + +static void leon_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + leon_flush_pcache_all(vma, page); +} + +static void leon_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_cache_all(); +} + +static void leon_flush_tlb_mm(struct mm_struct *mm) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_page(struct vm_area_struct *vma, + unsigned long page) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_page_to_ram(unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_sig_insns(struct mm_struct *mm, unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_page_for_dma(unsigned long page) +{ + leon_flush_dcache_all(); +} + +void __init poke_leonsparc(void) +{ +} + +static const struct sparc32_cachetlb_ops leon_ops = { + .cache_all = leon_flush_cache_all, + .cache_mm = leon_flush_cache_mm, + .cache_page = leon_flush_cache_page, + .cache_range = leon_flush_cache_range, + .tlb_all = leon_flush_tlb_all, + .tlb_mm = leon_flush_tlb_mm, + .tlb_page = leon_flush_tlb_page, + .tlb_range = leon_flush_tlb_range, + .page_to_ram = leon_flush_page_to_ram, + .sig_insns = leon_flush_sig_insns, + .page_for_dma = leon_flush_page_for_dma, +}; + +void __init init_leon(void) +{ + srmmu_name = "LEON"; + sparc32_cachetlb_ops = &leon_ops; + poke_srmmu = poke_leonsparc; + + leon_flush_during_switch = leon_flush_needed(); +} diff --git a/arch/sparc/mm/loadmmu.c b/arch/sparc/mm/loadmmu.c deleted file mode 100644 index 82ec8f66603..00000000000 --- a/arch/sparc/mm/loadmmu.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * loadmmu.c: This code loads up all the mm function pointers once the - * machine type has been determined. It also sets the static - * mmu values such as PAGE_NONE, etc. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> - -#include <asm/system.h> -#include <asm/page.h> -#include <asm/pgtable.h> -#include <asm/mmu_context.h> -#include <asm/oplib.h> - -struct ctx_list *ctx_list_pool; -struct ctx_list ctx_free; -struct ctx_list ctx_used; - -extern void ld_mmu_sun4c(void); -extern void ld_mmu_srmmu(void); - -void __init load_mmu(void) -{ - switch(sparc_cpu_model) { - case sun4c: - case sun4: - ld_mmu_sun4c(); - break; - case sun4m: - case sun4d: - case sparc_leon: - ld_mmu_srmmu(); - break; - default: - prom_printf("load_mmu: %d unsupported\n", (int)sparc_cpu_model); - prom_halt(); - } - btfixup(); -} diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h new file mode 100644 index 00000000000..a6c27ca9a72 --- /dev/null +++ b/arch/sparc/mm/mm_32.h @@ -0,0 +1,24 @@ +/* fault_32.c - visible as they are called from assembler */ +asmlinkage int lookup_fault(unsigned long pc, unsigned long ret_pc, + unsigned long address); +asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, + unsigned long address); + +void window_overflow_fault(void); +void window_underflow_fault(unsigned long sp); +void window_ret_fault(struct pt_regs *regs); + +/* srmmu.c */ +extern char *srmmu_name; +extern int viking_mxcc_present; +extern int flush_page_for_dma_global; + +extern void (*poke_srmmu)(void); + +void __init srmmu_paging_init(void); + +/* iommu.c */ +void ld_mmu_iommu(void); + +/* io-unit.c */ +void ld_mmu_iounit(void); diff --git a/arch/sparc/mm/nosun4c.c b/arch/sparc/mm/nosun4c.c deleted file mode 100644 index 4e62c27147c..00000000000 --- a/arch/sparc/mm/nosun4c.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * nosun4c.c: This file is a bunch of dummies for SMP compiles, - * so that it does not need sun4c and avoid ifdefs. - * - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <asm/pgtable.h> - -static char shouldnothappen[] __initdata = "32bit SMP kernel only supports sun4m and sun4d\n"; - -/* Dummies */ -struct sun4c_mmu_ring { - unsigned long xxx1[3]; - unsigned char xxx2[2]; - int xxx3; -}; -struct sun4c_mmu_ring sun4c_kernel_ring; -struct sun4c_mmu_ring sun4c_kfree_ring; -unsigned long sun4c_kernel_faults; -unsigned long *sun4c_memerr_reg; - -static void __init should_not_happen(void) -{ - prom_printf(shouldnothappen); - prom_halt(); -} - -unsigned long __init sun4c_paging_init(unsigned long start_mem, unsigned long end_mem) -{ - should_not_happen(); - return 0; -} - -void __init ld_mmu_sun4c(void) -{ - should_not_happen(); -} - -void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr, int bus_type, int rdonly) -{ -} - -void sun4c_unmapioaddr(unsigned long virt_addr) -{ -} - -void sun4c_complete_all_stores(void) -{ -} - -pte_t *sun4c_pte_offset(pmd_t * dir, unsigned long address) -{ - return NULL; -} - -pte_t *sun4c_pte_offset_kernel(pmd_t *dir, unsigned long address) -{ - return NULL; -} - -void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) -{ -} - -void __init sun4c_probe_vac(void) -{ - should_not_happen(); -} - -void __init sun4c_probe_memerr_reg(void) -{ - should_not_happen(); -} diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 92319aa8b66..be65f035d18 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -8,47 +8,48 @@ * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) */ -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <linux/pagemap.h> -#include <linux/init.h> +#include <linux/seq_file.h> #include <linux/spinlock.h> #include <linux/bootmem.h> -#include <linux/fs.h> -#include <linux/seq_file.h> +#include <linux/pagemap.h> +#include <linux/vmalloc.h> #include <linux/kdebug.h> +#include <linux/export.h> +#include <linux/kernel.h> +#include <linux/init.h> #include <linux/log2.h> #include <linux/gfp.h> +#include <linux/fs.h> +#include <linux/mm.h> -#include <asm/bitext.h> -#include <asm/page.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/io-unit.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> -#include <asm/io.h> +#include <asm/bitext.h> #include <asm/vaddrs.h> -#include <asm/traps.h> -#include <asm/smp.h> -#include <asm/mbus.h> #include <asm/cache.h> +#include <asm/traps.h> #include <asm/oplib.h> +#include <asm/mbus.h> +#include <asm/page.h> #include <asm/asi.h> #include <asm/msi.h> -#include <asm/mmu_context.h> -#include <asm/io-unit.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> +#include <asm/smp.h> +#include <asm/io.h> /* Now the cpu specific definitions. */ -#include <asm/viking.h> -#include <asm/mxcc.h> -#include <asm/ross.h> +#include <asm/turbosparc.h> #include <asm/tsunami.h> +#include <asm/viking.h> #include <asm/swift.h> -#include <asm/turbosparc.h> #include <asm/leon.h> +#include <asm/mxcc.h> +#include <asm/ross.h> -#include <asm/btfixup.h> +#include "mm_32.h" enum mbus_module srmmu_modtype; static unsigned int hwbug_bitmask; @@ -59,28 +60,23 @@ extern struct resource sparc_iomap; extern unsigned long last_valid_pfn; -extern unsigned long page_kernel; - static pgd_t *srmmu_swapper_pg_dir; +const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +EXPORT_SYMBOL(sparc32_cachetlb_ops); + #ifdef CONFIG_SMP +const struct sparc32_cachetlb_ops *local_ops; + #define FLUSH_BEGIN(mm) #define FLUSH_END #else -#define FLUSH_BEGIN(mm) if((mm)->context != NO_CONTEXT) { +#define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) { #define FLUSH_END } #endif -BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long) -#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page) - int flush_page_for_dma_global = 1; -#ifdef CONFIG_SMP -BTFIXUPDEF_CALL(void, local_flush_page_for_dma, unsigned long) -#define local_flush_page_for_dma(page) BTFIXUP_CALL(local_flush_page_for_dma)(page) -#endif - char *srmmu_name; ctxd_t *srmmu_ctx_table_phys; @@ -91,28 +87,6 @@ static DEFINE_SPINLOCK(srmmu_context_spinlock); static int is_hypersparc; -/* - * In general all page table modifications should use the V8 atomic - * swap instruction. This insures the mmu and the cpu are in sync - * with respect to ref/mod bits in the page tables. - */ -static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value) -{ - __asm__ __volatile__("swap [%2], %0" : "=&r" (value) : "0" (value), "r" (addr)); - return value; -} - -static inline void srmmu_set_pte(pte_t *ptep, pte_t pteval) -{ - srmmu_swap((unsigned long *)ptep, pte_val(pteval)); -} - -/* The very generic SRMMU page table operations. */ -static inline int srmmu_device_memory(unsigned long x) -{ - return ((x & 0xF0000000) != 0); -} - static int srmmu_cache_pagetables; /* these will be initialized in srmmu_nocache_calcsize() */ @@ -126,148 +100,41 @@ static unsigned long srmmu_nocache_end; #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS) void *srmmu_nocache_pool; -void *srmmu_nocache_bitmap; static struct bit_map srmmu_nocache_map; -static unsigned long srmmu_pte_pfn(pte_t pte) -{ - if (srmmu_device_memory(pte_val(pte))) { - /* Just return something that will cause - * pfn_valid() to return false. This makes - * copy_one_pte() to just directly copy to - * PTE over. - */ - return ~0UL; - } - return (pte_val(pte) & SRMMU_PTE_PMASK) >> (PAGE_SHIFT-4); -} - -static struct page *srmmu_pmd_page(pmd_t pmd) -{ - - if (srmmu_device_memory(pmd_val(pmd))) - BUG(); - return pfn_to_page((pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4)); -} - -static inline unsigned long srmmu_pgd_page(pgd_t pgd) -{ return srmmu_device_memory(pgd_val(pgd))?~0:(unsigned long)__nocache_va((pgd_val(pgd) & SRMMU_PTD_PMASK) << 4); } - - -static inline int srmmu_pte_none(pte_t pte) -{ return !(pte_val(pte) & 0xFFFFFFF); } - -static inline int srmmu_pte_present(pte_t pte) -{ return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); } - -static inline void srmmu_pte_clear(pte_t *ptep) -{ srmmu_set_pte(ptep, __pte(0)); } - static inline int srmmu_pmd_none(pmd_t pmd) { return !(pmd_val(pmd) & 0xFFFFFFF); } -static inline int srmmu_pmd_bad(pmd_t pmd) -{ return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } - -static inline int srmmu_pmd_present(pmd_t pmd) -{ return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } - -static inline void srmmu_pmd_clear(pmd_t *pmdp) { - int i; - for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], __pte(0)); -} - -static inline int srmmu_pgd_none(pgd_t pgd) -{ return !(pgd_val(pgd) & 0xFFFFFFF); } - -static inline int srmmu_pgd_bad(pgd_t pgd) -{ return (pgd_val(pgd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; } - -static inline int srmmu_pgd_present(pgd_t pgd) -{ return ((pgd_val(pgd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); } - -static inline void srmmu_pgd_clear(pgd_t * pgdp) -{ srmmu_set_pte((pte_t *)pgdp, __pte(0)); } - -static inline pte_t srmmu_pte_wrprotect(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_WRITE);} - -static inline pte_t srmmu_pte_mkclean(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_DIRTY);} - -static inline pte_t srmmu_pte_mkold(pte_t pte) -{ return __pte(pte_val(pte) & ~SRMMU_REF);} - -static inline pte_t srmmu_pte_mkwrite(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_WRITE);} - -static inline pte_t srmmu_pte_mkdirty(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_DIRTY);} - -static inline pte_t srmmu_pte_mkyoung(pte_t pte) -{ return __pte(pte_val(pte) | SRMMU_REF);} - -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -static pte_t srmmu_mk_pte(struct page *page, pgprot_t pgprot) -{ return __pte((page_to_pfn(page) << (PAGE_SHIFT-4)) | pgprot_val(pgprot)); } - -static pte_t srmmu_mk_pte_phys(unsigned long page, pgprot_t pgprot) -{ return __pte(((page) >> 4) | pgprot_val(pgprot)); } - -static pte_t srmmu_mk_pte_io(unsigned long page, pgprot_t pgprot, int space) -{ return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); } - /* XXX should we hyper_flush_whole_icache here - Anton */ static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) -{ srmmu_set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } - -static inline void srmmu_pgd_set(pgd_t * pgdp, pmd_t * pmdp) -{ srmmu_set_pte((pte_t *)pgdp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pmdp) >> 4))); } +{ set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } -static void srmmu_pmd_set(pmd_t *pmdp, pte_t *ptep) +void pmd_set(pmd_t *pmdp, pte_t *ptep) { unsigned long ptp; /* Physical address, shifted right by 4 */ int i; ptp = __nocache_pa((unsigned long) ptep) >> 4; for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); + set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); } } -static void srmmu_pmd_populate(pmd_t *pmdp, struct page *ptep) +void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) { unsigned long ptp; /* Physical address, shifted right by 4 */ int i; ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */ for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { - srmmu_set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); + set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); } } -static inline pte_t srmmu_pte_modify(pte_t pte, pgprot_t newprot) -{ return __pte((pte_val(pte) & SRMMU_CHG_MASK) | pgprot_val(newprot)); } - -/* to find an entry in a top-level page table... */ -static inline pgd_t *srmmu_pgd_offset(struct mm_struct * mm, unsigned long address) -{ return mm->pgd + (address >> SRMMU_PGDIR_SHIFT); } - -/* Find an entry in the second-level page table.. */ -static inline pmd_t *srmmu_pmd_offset(pgd_t * dir, unsigned long address) -{ - return (pmd_t *) srmmu_pgd_page(*dir) + - ((address >> PMD_SHIFT) & (PTRS_PER_PMD - 1)); -} - -/* Find an entry in the third-level page table.. */ -static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address) +/* Find an entry in the third-level page table.. */ +pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address) { void *pte; @@ -276,77 +143,66 @@ static inline pte_t *srmmu_pte_offset(pmd_t * dir, unsigned long address) ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); } -static unsigned long srmmu_swp_type(swp_entry_t entry) -{ - return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK; -} - -static unsigned long srmmu_swp_offset(swp_entry_t entry) -{ - return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK; -} - -static swp_entry_t srmmu_swp_entry(unsigned long type, unsigned long offset) -{ - return (swp_entry_t) { - (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT - | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT }; -} - /* * size: bytes to allocate in the nocache area. * align: bytes, number to align at. * Returns the virtual address of the allocated area. */ -static unsigned long __srmmu_get_nocache(int size, int align) +static void *__srmmu_get_nocache(int size, int align) { int offset; + unsigned long addr; if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { - printk("Size 0x%x too small for nocache request\n", size); + printk(KERN_ERR "Size 0x%x too small for nocache request\n", + size); size = SRMMU_NOCACHE_BITMAP_SHIFT; } - if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) { - printk("Size 0x%x unaligned int nocache request\n", size); - size += SRMMU_NOCACHE_BITMAP_SHIFT-1; + if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) { + printk(KERN_ERR "Size 0x%x unaligned int nocache request\n", + size); + size += SRMMU_NOCACHE_BITMAP_SHIFT - 1; } BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); offset = bit_map_string_get(&srmmu_nocache_map, - size >> SRMMU_NOCACHE_BITMAP_SHIFT, - align >> SRMMU_NOCACHE_BITMAP_SHIFT); + size >> SRMMU_NOCACHE_BITMAP_SHIFT, + align >> SRMMU_NOCACHE_BITMAP_SHIFT); if (offset == -1) { - printk("srmmu: out of nocache %d: %d/%d\n", - size, (int) srmmu_nocache_size, - srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); - return 0; + printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n", + size, (int) srmmu_nocache_size, + srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); + return NULL; } - return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT)); + addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT); + return (void *)addr; } -static unsigned long srmmu_get_nocache(int size, int align) +void *srmmu_get_nocache(int size, int align) { - unsigned long tmp; + void *tmp; tmp = __srmmu_get_nocache(size, align); if (tmp) - memset((void *)tmp, 0, size); + memset(tmp, 0, size); return tmp; } -static void srmmu_free_nocache(unsigned long vaddr, int size) +void srmmu_free_nocache(void *addr, int size) { + unsigned long vaddr; int offset; + vaddr = (unsigned long)addr; if (vaddr < SRMMU_NOCACHE_VADDR) { printk("Vaddr %lx is smaller than nocache base 0x%lx\n", vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); BUG(); } - if (vaddr+size > srmmu_nocache_end) { + if (vaddr + size > srmmu_nocache_end) { printk("Vaddr %lx is bigger than nocache end 0x%lx\n", vaddr, srmmu_nocache_end); BUG(); @@ -359,7 +215,7 @@ static void srmmu_free_nocache(unsigned long vaddr, int size) printk("Size 0x%x is too small\n", size); BUG(); } - if (vaddr & (size-1)) { + if (vaddr & (size - 1)) { printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); BUG(); } @@ -373,13 +229,23 @@ static void srmmu_free_nocache(unsigned long vaddr, int size) static void srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end); -extern unsigned long probe_memory(void); /* in fault.c */ +/* Return how much physical memory we have. */ +static unsigned long __init probe_memory(void) +{ + unsigned long total = 0; + int i; + + for (i = 0; sp_banks[i].num_bytes; i++) + total += sp_banks[i].num_bytes; + + return total; +} /* * Reserve nocache dynamically proportionally to the amount of * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002 */ -static void srmmu_nocache_calcsize(void) +static void __init srmmu_nocache_calcsize(void) { unsigned long sysmemavail = probe_memory() / 1024; int srmmu_nocache_npages; @@ -402,6 +268,7 @@ static void srmmu_nocache_calcsize(void) static void __init srmmu_nocache_init(void) { + void *srmmu_nocache_bitmap; unsigned int bitmap_bits; pgd_t *pgd; pmd_t *pmd; @@ -415,10 +282,12 @@ static void __init srmmu_nocache_init(void) SRMMU_NOCACHE_ALIGN_MAX, 0UL); memset(srmmu_nocache_pool, 0, srmmu_nocache_size); - srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); + srmmu_nocache_bitmap = + __alloc_bootmem(BITS_TO_LONGS(bitmap_bits) * sizeof(long), + SMP_CACHE_BYTES, 0UL); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); - srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); init_mm.pgd = srmmu_swapper_pg_dir; @@ -429,15 +298,15 @@ static void __init srmmu_nocache_init(void) while (vaddr < srmmu_nocache_end) { pgd = pgd_offset_k(vaddr); - pmd = srmmu_pmd_offset(__nocache_fix(pgd), vaddr); - pte = srmmu_pte_offset(__nocache_fix(pmd), vaddr); + pmd = pmd_offset(__nocache_fix(pgd), vaddr); + pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV); if (srmmu_cache_pagetables) pteval |= SRMMU_CACHE; - srmmu_set_pte(__nocache_fix(pte), __pte(pteval)); + set_pte(__nocache_fix(pte), __pte(pteval)); vaddr += PAGE_SIZE; paddr += PAGE_SIZE; @@ -447,11 +316,11 @@ static void __init srmmu_nocache_init(void) flush_tlb_all(); } -static inline pgd_t *srmmu_get_pgd_fast(void) +pgd_t *get_pgd_fast(void) { pgd_t *pgd = NULL; - pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); if (pgd) { pgd_t *init = pgd_offset_k(0); memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); @@ -462,21 +331,6 @@ static inline pgd_t *srmmu_get_pgd_fast(void) return pgd; } -static void srmmu_free_pgd_fast(pgd_t *pgd) -{ - srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE); -} - -static pmd_t *srmmu_pmd_alloc_one(struct mm_struct *mm, unsigned long address) -{ - return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); -} - -static void srmmu_pmd_free(pmd_t * pmd) -{ - srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE); -} - /* * Hardware needs alignment to 256 only, but we align to whole page size * to reduce fragmentation problems due to the buddy principle. @@ -485,31 +339,22 @@ static void srmmu_pmd_free(pmd_t * pmd) * Alignments up to the page size are the same for physical and virtual * addresses of the nocache area. */ -static pte_t * -srmmu_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) -{ - return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE); -} - -static pgtable_t -srmmu_pte_alloc_one(struct mm_struct *mm, unsigned long address) +pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { unsigned long pte; struct page *page; - if ((pte = (unsigned long)srmmu_pte_alloc_one_kernel(mm, address)) == 0) + if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) return NULL; - page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); - pgtable_page_ctor(page); + page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT); + if (!pgtable_page_ctor(page)) { + __free_page(page); + return NULL; + } return page; } -static void srmmu_free_pte_fast(pte_t *pte) -{ - srmmu_free_nocache((unsigned long)pte, PTE_SIZE); -} - -static void srmmu_pte_free(pgtable_t pte) +void pte_free(struct mm_struct *mm, pgtable_t pte) { unsigned long p; @@ -518,18 +363,50 @@ static void srmmu_pte_free(pgtable_t pte) if (p == 0) BUG(); p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ - p = (unsigned long) __nocache_va(p); /* Nocached virtual */ - srmmu_free_nocache(p, PTE_SIZE); + + /* free non cached virtual address*/ + srmmu_free_nocache(__nocache_va(p), PTE_SIZE); } -/* - */ +/* context handling - a dynamically sized pool is used */ +#define NO_CONTEXT -1 + +struct ctx_list { + struct ctx_list *next; + struct ctx_list *prev; + unsigned int ctx_number; + struct mm_struct *ctx_mm; +}; + +static struct ctx_list *ctx_list_pool; +static struct ctx_list ctx_free; +static struct ctx_list ctx_used; + +/* At boot time we determine the number of contexts */ +static int num_contexts; + +static inline void remove_from_ctx_list(struct ctx_list *entry) +{ + entry->next->prev = entry->prev; + entry->prev->next = entry->next; +} + +static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) +{ + entry->next = head; + (entry->prev = head->prev)->next = entry; + head->prev = entry; +} +#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) +#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) + + static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) { struct ctx_list *ctxp; ctxp = ctx_free.next; - if(ctxp != &ctx_free) { + if (ctxp != &ctx_free) { remove_from_ctx_list(ctxp); add_to_used_ctxlist(ctxp); mm->context = ctxp->ctx_number; @@ -537,9 +414,9 @@ static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) return; } ctxp = ctx_used.next; - if(ctxp->ctx_mm == old_mm) + if (ctxp->ctx_mm == old_mm) ctxp = ctxp->next; - if(ctxp == &ctx_used) + if (ctxp == &ctx_used) panic("out of mmu contexts"); flush_cache_mm(ctxp->ctx_mm); flush_tlb_mm(ctxp->ctx_mm); @@ -559,11 +436,31 @@ static inline void free_context(int context) add_to_free_ctxlist(ctx_old); } +static void __init sparc_context_init(int numctx) +{ + int ctx; + unsigned long size; -static void srmmu_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, - struct task_struct *tsk, int cpu) + size = numctx * sizeof(struct ctx_list); + ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); + + for (ctx = 0; ctx < numctx; ctx++) { + struct ctx_list *clist; + + clist = (ctx_list_pool + ctx); + clist->ctx_number = ctx; + clist->ctx_mm = NULL; + } + ctx_free.next = ctx_free.prev = &ctx_free; + ctx_used.next = ctx_used.prev = &ctx_used; + for (ctx = 0; ctx < numctx; ctx++) + add_to_free_ctxlist(ctx_list_pool + ctx); +} + +void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, + struct task_struct *tsk) { - if(mm->context == NO_CONTEXT) { + if (mm->context == NO_CONTEXT) { spin_lock(&srmmu_context_spinlock); alloc_context(old_mm, mm); spin_unlock(&srmmu_context_spinlock); @@ -581,7 +478,7 @@ static void srmmu_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, /* Low level IO area allocation on the SRMMU. */ static inline void srmmu_mapioaddr(unsigned long physaddr, - unsigned long virt_addr, int bus_type) + unsigned long virt_addr, int bus_type) { pgd_t *pgdp; pmd_t *pmdp; @@ -590,23 +487,22 @@ static inline void srmmu_mapioaddr(unsigned long physaddr, physaddr &= PAGE_MASK; pgdp = pgd_offset_k(virt_addr); - pmdp = srmmu_pmd_offset(pgdp, virt_addr); - ptep = srmmu_pte_offset(pmdp, virt_addr); + pmdp = pmd_offset(pgdp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); tmp = (physaddr >> 4) | SRMMU_ET_PTE; - /* - * I need to test whether this is consistent over all + /* I need to test whether this is consistent over all * sun4m's. The bus_type represents the upper 4 bits of * 36-bit physical address on the I/O space lines... */ tmp |= (bus_type << 28); tmp |= SRMMU_PRIV; __flush_page_to_ram(virt_addr); - srmmu_set_pte(ptep, __pte(tmp)); + set_pte(ptep, __pte(tmp)); } -static void srmmu_mapiorange(unsigned int bus, unsigned long xpa, - unsigned long xva, unsigned int len) +void srmmu_mapiorange(unsigned int bus, unsigned long xpa, + unsigned long xva, unsigned int len) { while (len != 0) { len -= PAGE_SIZE; @@ -624,14 +520,14 @@ static inline void srmmu_unmapioaddr(unsigned long virt_addr) pte_t *ptep; pgdp = pgd_offset_k(virt_addr); - pmdp = srmmu_pmd_offset(pgdp, virt_addr); - ptep = srmmu_pte_offset(pmdp, virt_addr); + pmdp = pmd_offset(pgdp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); /* No need to flush uncacheable page. */ - srmmu_pte_clear(ptep); + __pte_clear(ptep); } -static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) +void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) { while (len != 0) { len -= PAGE_SIZE; @@ -641,34 +537,6 @@ static void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) flush_tlb_all(); } -/* - * On the SRMMU we do not have the problems with limited tlb entries - * for mapping kernel pages, so we just take things from the free page - * pool. As a side effect we are putting a little too much pressure - * on the gfp() subsystem. This setup also makes the logic of the - * iommu mapping code a lot easier as we can transparently handle - * mappings on the kernel stack without any special code as we did - * need on the sun4c. - */ -static struct thread_info *srmmu_alloc_thread_info(void) -{ - struct thread_info *ret; - - ret = (struct thread_info *)__get_free_pages(GFP_KERNEL, - THREAD_INFO_ORDER); -#ifdef CONFIG_DEBUG_STACK_USAGE - if (ret) - memset(ret, 0, PAGE_SIZE << THREAD_INFO_ORDER); -#endif /* DEBUG_STACK_USAGE */ - - return ret; -} - -static void srmmu_free_thread_info(struct thread_info *ti) -{ - free_pages((unsigned long)ti, THREAD_INFO_ORDER); -} - /* tsunami.S */ extern void tsunami_flush_cache_all(void); extern void tsunami_flush_cache_mm(struct mm_struct *mm); @@ -683,38 +551,6 @@ extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long st extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); extern void tsunami_setup_blockops(void); -/* - * Workaround, until we find what's going on with Swift. When low on memory, - * it sometimes loops in fault/handle_mm_fault incl. flush_tlb_page to find - * out it is already in page tables/ fault again on the same instruction. - * I really don't understand it, have checked it and contexts - * are right, flush_tlb_all is done as well, and it faults again... - * Strange. -jj - * - * The following code is a deadwood that may be necessary when - * we start to make precise page flushes again. --zaitcev - */ -static void swift_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t *ptep) -{ -#if 0 - static unsigned long last; - unsigned int val; - /* unsigned int n; */ - - if (address == last) { - val = srmmu_hwprobe(address); - if (val != 0 && pte_val(*ptep) != val) { - printk("swift_update_mmu_cache: " - "addr %lx put %08x probed %08x from %p\n", - address, pte_val(*ptep), val, - __builtin_return_address(0)); - srmmu_flush_whole_tlb(); - } - } - last = address; -#endif -} - /* swift.S */ extern void swift_flush_cache_all(void); extern void swift_flush_cache_mm(struct mm_struct *mm); @@ -767,244 +603,6 @@ void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) * with respect to cache coherency. */ -/* Cypress flushes. */ -static void cypress_flush_cache_all(void) -{ - volatile unsigned long cypress_sucks; - unsigned long faddr, tagval; - - flush_user_windows(); - for(faddr = 0; faddr < 0x10000; faddr += 0x20) { - __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" : - "=r" (tagval) : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - - /* If modified and valid, kick it. */ - if((tagval & 0x60) == 0x60) - cypress_sucks = *(unsigned long *)(0xf0020000 + faddr); - } -} - -static void cypress_flush_cache_mm(struct mm_struct *mm) -{ - register unsigned long a, b, c, d, e, f, g; - unsigned long flags, faddr; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - faddr = (0x10000 - 0x100); - goto inside; - do { - faddr -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (faddr), "i" (ASI_M_FLUSH_CTX), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(faddr); - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -static void cypress_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - register unsigned long a, b, c, d, e, f, g; - unsigned long flags, faddr; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - start &= SRMMU_REAL_PMD_MASK; - while(start < end) { - faddr = (start + (0x10000 - 0x100)); - goto inside; - do { - faddr -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (faddr), - "i" (ASI_M_FLUSH_SEG), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while (faddr != start); - start += SRMMU_REAL_PMD_SIZE; - } - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -static void cypress_flush_cache_page(struct vm_area_struct *vma, unsigned long page) -{ - register unsigned long a, b, c, d, e, f, g; - struct mm_struct *mm = vma->vm_mm; - unsigned long flags, line; - int octx; - - FLUSH_BEGIN(mm) - flush_user_windows(); - local_irq_save(flags); - octx = srmmu_get_context(); - srmmu_set_context(mm->context); - a = 0x20; b = 0x40; c = 0x60; - d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - - page &= PAGE_MASK; - line = (page + PAGE_SIZE) - 0x100; - goto inside; - do { - line -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (line), - "i" (ASI_M_FLUSH_PAGE), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(line != page); - srmmu_set_context(octx); - local_irq_restore(flags); - FLUSH_END -} - -/* Cypress is copy-back, at least that is how we configure it. */ -static void cypress_flush_page_to_ram(unsigned long page) -{ - register unsigned long a, b, c, d, e, f, g; - unsigned long line; - - a = 0x20; b = 0x40; c = 0x60; d = 0x80; e = 0xa0; f = 0xc0; g = 0xe0; - page &= PAGE_MASK; - line = (page + PAGE_SIZE) - 0x100; - goto inside; - do { - line -= 0x100; - inside: - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" - "sta %%g0, [%0 + %2] %1\n\t" - "sta %%g0, [%0 + %3] %1\n\t" - "sta %%g0, [%0 + %4] %1\n\t" - "sta %%g0, [%0 + %5] %1\n\t" - "sta %%g0, [%0 + %6] %1\n\t" - "sta %%g0, [%0 + %7] %1\n\t" - "sta %%g0, [%0 + %8] %1\n\t" : : - "r" (line), - "i" (ASI_M_FLUSH_PAGE), - "r" (a), "r" (b), "r" (c), "r" (d), - "r" (e), "r" (f), "r" (g)); - } while(line != page); -} - -/* Cypress is also IO cache coherent. */ -static void cypress_flush_page_for_dma(unsigned long page) -{ -} - -/* Cypress has unified L2 VIPT, from which both instructions and data - * are stored. It does not have an onboard icache of any sort, therefore - * no flush is necessary. - */ -static void cypress_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) -{ -} - -static void cypress_flush_tlb_all(void) -{ - srmmu_flush_whole_tlb(); -} - -static void cypress_flush_tlb_mm(struct mm_struct *mm) -{ - FLUSH_BEGIN(mm) - __asm__ __volatile__( - "lda [%0] %3, %%g5\n\t" - "sta %2, [%0] %3\n\t" - "sta %%g0, [%1] %4\n\t" - "sta %%g5, [%0] %3\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (0x300), "r" (mm->context), - "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE) - : "g5"); - FLUSH_END -} - -static void cypress_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - unsigned long size; - - FLUSH_BEGIN(mm) - start &= SRMMU_PGDIR_MASK; - size = SRMMU_PGDIR_ALIGN(end) - start; - __asm__ __volatile__( - "lda [%0] %5, %%g5\n\t" - "sta %1, [%0] %5\n" - "1:\n\t" - "subcc %3, %4, %3\n\t" - "bne 1b\n\t" - " sta %%g0, [%2 + %3] %6\n\t" - "sta %%g5, [%0] %5\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (start | 0x200), - "r" (size), "r" (SRMMU_PGDIR_SIZE), "i" (ASI_M_MMUREGS), - "i" (ASI_M_FLUSH_PROBE) - : "g5", "cc"); - FLUSH_END -} - -static void cypress_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - - FLUSH_BEGIN(mm) - __asm__ __volatile__( - "lda [%0] %3, %%g5\n\t" - "sta %1, [%0] %3\n\t" - "sta %%g0, [%2] %4\n\t" - "sta %%g5, [%0] %3\n" - : /* no outputs */ - : "r" (SRMMU_CTX_REG), "r" (mm->context), "r" (page & PAGE_MASK), - "i" (ASI_M_MMUREGS), "i" (ASI_M_FLUSH_PROBE) - : "g5"); - FLUSH_END -} - /* viking.S */ extern void viking_flush_cache_all(void); extern void viking_flush_cache_mm(struct mm_struct *mm); @@ -1063,23 +661,23 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); - if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *) __srmmu_get_nocache( + if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(__nocache_fix(pgdp), pmdp); + pgd_set(__nocache_fix(pgdp), pmdp); } - pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + pmdp = pmd_offset(__nocache_fix(pgdp), start); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); - srmmu_pmd_set(__nocache_fix(pmdp), ptep); + pmd_set(__nocache_fix(pmdp), ptep); } if (start > (0xffffffffUL - PMD_SIZE)) break; @@ -1094,23 +692,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); - if(srmmu_pgd_none(*pgdp)) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pgd_none(*pgdp)) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(pgdp, pmdp); + pgd_set(pgdp, pmdp); } - pmdp = srmmu_pmd_offset(pgdp, start); - if(srmmu_pmd_none(*pmdp)) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, + pmdp = pmd_offset(pgdp, start); + if (srmmu_pmd_none(*pmdp)) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(ptep, 0, PTE_SIZE); - srmmu_pmd_set(pmdp, ptep); + pmd_set(pmdp, ptep); } if (start > (0xffffffffUL - PMD_SIZE)) break; @@ -1118,6 +716,23 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, } } +/* These flush types are not available on all chips... */ +static inline unsigned long srmmu_probe(unsigned long vaddr) +{ + unsigned long retval; + + if (sparc_cpu_model != sparc_leon) { + + vaddr &= PAGE_MASK; + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); + } else { + retval = leon_swprobe(vaddr, NULL); + } + return retval; +} + /* * This is much cleaner than poking around physical address space * looking at the prom's page table directly which is what most @@ -1126,72 +741,76 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, static void __init srmmu_inherit_prom_mappings(unsigned long start, unsigned long end) { + unsigned long probed; + unsigned long addr; pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - unsigned long prompte; + int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - while(start <= end) { + while (start <= end) { if (start == 0) break; /* probably wrap around */ - if(start == 0xfef00000) + if (start == 0xfef00000) start = KADB_DEBUGGER_BEGVM; - if(!(prompte = srmmu_hwprobe(start))) { + probed = srmmu_probe(start); + if (!probed) { + /* continue probing until we find an entry */ start += PAGE_SIZE; continue; } - + /* A red snapper, see what it really is. */ what = 0; - - if(!(start & ~(SRMMU_REAL_PMD_MASK))) { - if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte) + addr = start - PAGE_SIZE; + + if (!(start & ~(SRMMU_REAL_PMD_MASK))) { + if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed) what = 1; } - - if(!(start & ~(SRMMU_PGDIR_MASK))) { - if(srmmu_hwprobe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) == - prompte) + + if (!(start & ~(SRMMU_PGDIR_MASK))) { + if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed) what = 2; } - + pgdp = pgd_offset_k(start); - if(what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte); + if (what == 2) { + *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); start += SRMMU_PGDIR_SIZE; continue; } - if(srmmu_pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); - srmmu_pgd_set(__nocache_fix(pgdp), pmdp); + pgd_set(__nocache_fix(pgdp), pmdp); } - pmdp = srmmu_pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, - PTE_SIZE); + pmdp = pmd_offset(__nocache_fix(pgdp), start); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); - srmmu_pmd_set(__nocache_fix(pmdp), ptep); + pmd_set(__nocache_fix(pmdp), ptep); } - if(what == 1) { - /* - * We bend the rule where all 16 PTPs in a pmd_t point + if (what == 1) { + /* We bend the rule where all 16 PTPs in a pmd_t point * inside the same PTE page, and we leak a perfectly * good hardware PTE piece. Alternatives seem worse. */ unsigned int x; /* Index of HW PMD in soft cluster */ + unsigned long *val; x = (start >> PMD_SHIFT) & 15; - *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte; + val = &pmdp->pmdv[x]; + *(unsigned long *)__nocache_fix(val) = probed; start += SRMMU_REAL_PMD_SIZE; continue; } - ptep = srmmu_pte_offset(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(prompte); + ptep = pte_offset_kernel(__nocache_fix(pmdp), start); + *(pte_t *)__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -1220,25 +839,18 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) if (vstart < min_vaddr || vstart >= max_vaddr) return vstart; - + if (vend > max_vaddr || vend < min_vaddr) vend = max_vaddr; - while(vstart < vend) { + while (vstart < vend) { do_large_mapping(vstart, pstart); vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; } return vstart; } -static inline void memprobe_error(char *msg) -{ - prom_printf(msg); - prom_printf("Halting now...\n"); - prom_halt(); -} - -static inline void map_kernel(void) +static void __init map_kernel(void) { int i; @@ -1249,16 +861,9 @@ static inline void map_kernel(void) for (i = 0; sp_banks[i].num_bytes != 0; i++) { map_spbank((unsigned long)__va(sp_banks[i].base_addr), i); } - - BTFIXUPSET_SIMM13(user_ptrs_per_pgd, PAGE_OFFSET / SRMMU_PGDIR_SIZE); } -/* Paging initialization on the Sparc Reference MMU. */ -extern void sparc_context_init(int); - -void (*poke_srmmu)(void) __cpuinitdata = NULL; - -extern unsigned long bootmem_init(unsigned long *pages_avail); +void (*poke_srmmu)(void) = NULL; void __init srmmu_paging_init(void) { @@ -1270,6 +875,7 @@ void __init srmmu_paging_init(void) pte_t *pte; unsigned long pages_avail; + init_mm.context = (unsigned long) NO_CONTEXT; sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ if (sparc_cpu_model == sun4d) @@ -1278,9 +884,9 @@ void __init srmmu_paging_init(void) /* Find the number of contexts on the srmmu. */ cpunode = prom_getchild(prom_root_node); num_contexts = 0; - while(cpunode != 0) { + while (cpunode != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); break; } @@ -1288,7 +894,7 @@ void __init srmmu_paging_init(void) } } - if(!num_contexts) { + if (!num_contexts) { prom_printf("Something wrong, can't find cpu node in paging_init.\n"); prom_halt(); } @@ -1298,21 +904,21 @@ void __init srmmu_paging_init(void) srmmu_nocache_calcsize(); srmmu_nocache_init(); - srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE)); + srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE)); map_kernel(); /* ctx table has to be physically aligned to its size */ - srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t)); + srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t)); srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); - for(i = 0; i < num_contexts; i++) + for (i = 0; i < num_contexts; i++) srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); #ifdef CONFIG_SMP /* Stop from hanging here... */ - local_flush_tlb_all(); + local_ops->tlb_all(); #else flush_tlb_all(); #endif @@ -1326,8 +932,8 @@ void __init srmmu_paging_init(void) srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); pgd = pgd_offset_k(PKMAP_BASE); - pmd = srmmu_pmd_offset(pgd, PKMAP_BASE); - pte = srmmu_pte_offset(pmd, PKMAP_BASE); + pmd = pmd_offset(pgd, PKMAP_BASE); + pte = pte_offset_kernel(pmd, PKMAP_BASE); pkmap_page_table = pte; flush_cache_all(); @@ -1359,9 +965,9 @@ void __init srmmu_paging_init(void) } } -static void srmmu_mmu_info(struct seq_file *m) +void mmu_info(struct seq_file *m) { - seq_printf(m, + seq_printf(m, "MMU type\t: %s\n" "contexts\t: %d\n" "nocache total\t: %ld\n" @@ -1372,14 +978,16 @@ static void srmmu_mmu_info(struct seq_file *m) srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); } -static void srmmu_update_mmu_cache(struct vm_area_struct * vma, unsigned long address, pte_t pte) +int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { + mm->context = NO_CONTEXT; + return 0; } -static void srmmu_destroy_context(struct mm_struct *mm) +void destroy_context(struct mm_struct *mm) { - if(mm->context != NO_CONTEXT) { + if (mm->context != NO_CONTEXT) { flush_cache_mm(mm); srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); flush_tlb_mm(mm); @@ -1409,13 +1017,12 @@ static void __init init_vac_layout(void) #endif nd = prom_getchild(prom_root_node); - while((nd = prom_getsibling(nd)) != 0) { + while ((nd = prom_getsibling(nd)) != 0) { prom_getstring(nd, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { vac_line_size = prom_getint(nd, "cache-line-size"); if (vac_line_size == -1) { - prom_printf("can't determine cache-line-size, " - "halting.\n"); + prom_printf("can't determine cache-line-size, halting.\n"); prom_halt(); } cache_lines = prom_getint(nd, "cache-nlines"); @@ -1426,9 +1033,9 @@ static void __init init_vac_layout(void) vac_cache_size = cache_lines * vac_line_size; #ifdef CONFIG_SMP - if(vac_cache_size > max_size) + if (vac_cache_size > max_size) max_size = vac_cache_size; - if(vac_line_size < min_line_size) + if (vac_line_size < min_line_size) min_line_size = vac_line_size; //FIXME: cpus not contiguous!! cpu++; @@ -1439,7 +1046,7 @@ static void __init init_vac_layout(void) #endif } } - if(nd == 0) { + if (nd == 0) { prom_printf("No CPU nodes found, halting.\n"); prom_halt(); } @@ -1451,7 +1058,7 @@ static void __init init_vac_layout(void) (int)vac_cache_size, (int)vac_line_size); } -static void __cpuinit poke_hypersparc(void) +static void poke_hypersparc(void) { volatile unsigned long clear; unsigned long mreg = srmmu_get_mmureg(); @@ -1474,6 +1081,20 @@ static void __cpuinit poke_hypersparc(void) clear = srmmu_get_fstatus(); } +static const struct sparc32_cachetlb_ops hypersparc_ops = { + .cache_all = hypersparc_flush_cache_all, + .cache_mm = hypersparc_flush_cache_mm, + .cache_page = hypersparc_flush_cache_page, + .cache_range = hypersparc_flush_cache_range, + .tlb_all = hypersparc_flush_tlb_all, + .tlb_mm = hypersparc_flush_tlb_mm, + .tlb_page = hypersparc_flush_tlb_page, + .tlb_range = hypersparc_flush_tlb_range, + .page_to_ram = hypersparc_flush_page_to_ram, + .sig_insns = hypersparc_flush_sig_insns, + .page_for_dma = hypersparc_flush_page_for_dma, +}; + static void __init init_hypersparc(void) { srmmu_name = "ROSS HyperSparc"; @@ -1482,119 +1103,14 @@ static void __init init_hypersparc(void) init_vac_layout(); is_hypersparc = 1; - - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_all, hypersparc_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, hypersparc_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, hypersparc_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, hypersparc_flush_cache_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, hypersparc_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, hypersparc_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, hypersparc_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, hypersparc_flush_tlb_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, hypersparc_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, hypersparc_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, hypersparc_flush_page_for_dma, BTFIXUPCALL_NOP); - + sparc32_cachetlb_ops = &hypersparc_ops; poke_srmmu = poke_hypersparc; hypersparc_setup_blockops(); } -static void __cpuinit poke_cypress(void) -{ - unsigned long mreg = srmmu_get_mmureg(); - unsigned long faddr, tagval; - volatile unsigned long cypress_sucks; - volatile unsigned long clear; - - clear = srmmu_get_faddr(); - clear = srmmu_get_fstatus(); - - if (!(mreg & CYPRESS_CENABLE)) { - for(faddr = 0x0; faddr < 0x10000; faddr += 20) { - __asm__ __volatile__("sta %%g0, [%0 + %1] %2\n\t" - "sta %%g0, [%0] %2\n\t" : : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - } - } else { - for(faddr = 0; faddr < 0x10000; faddr += 0x20) { - __asm__ __volatile__("lda [%1 + %2] %3, %0\n\t" : - "=r" (tagval) : - "r" (faddr), "r" (0x40000), - "i" (ASI_M_DATAC_TAG)); - - /* If modified and valid, kick it. */ - if((tagval & 0x60) == 0x60) - cypress_sucks = *(unsigned long *) - (0xf0020000 + faddr); - } - } - - /* And one more, for our good neighbor, Mr. Broken Cypress. */ - clear = srmmu_get_faddr(); - clear = srmmu_get_fstatus(); - - mreg |= (CYPRESS_CENABLE | CYPRESS_CMODE); - srmmu_set_mmureg(mreg); -} - -static void __init init_cypress_common(void) -{ - init_vac_layout(); - - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_all, cypress_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, cypress_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, cypress_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, cypress_flush_cache_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, cypress_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, cypress_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, cypress_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, cypress_flush_tlb_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(__flush_page_to_ram, cypress_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, cypress_flush_sig_insns, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_page_for_dma, cypress_flush_page_for_dma, BTFIXUPCALL_NOP); - - poke_srmmu = poke_cypress; -} - -static void __init init_cypress_604(void) -{ - srmmu_name = "ROSS Cypress-604(UP)"; - srmmu_modtype = Cypress; - init_cypress_common(); -} - -static void __init init_cypress_605(unsigned long mrev) -{ - srmmu_name = "ROSS Cypress-605(MP)"; - if(mrev == 0xe) { - srmmu_modtype = Cypress_vE; - hwbug_bitmask |= HWBUG_COPYBACK_BROKEN; - } else { - if(mrev == 0xd) { - srmmu_modtype = Cypress_vD; - hwbug_bitmask |= HWBUG_ASIFLUSH_BROKEN; - } else { - srmmu_modtype = Cypress; - } - } - init_cypress_common(); -} - -static void __cpuinit poke_swift(void) +static void poke_swift(void) { unsigned long mreg; @@ -1617,6 +1133,20 @@ static void __cpuinit poke_swift(void) srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops swift_ops = { + .cache_all = swift_flush_cache_all, + .cache_mm = swift_flush_cache_mm, + .cache_page = swift_flush_cache_page, + .cache_range = swift_flush_cache_range, + .tlb_all = swift_flush_tlb_all, + .tlb_mm = swift_flush_tlb_mm, + .tlb_page = swift_flush_tlb_page, + .tlb_range = swift_flush_tlb_range, + .page_to_ram = swift_flush_page_to_ram, + .sig_insns = swift_flush_sig_insns, + .page_for_dma = swift_flush_page_for_dma, +}; + #define SWIFT_MASKID_ADDR 0x10003018 static void __init init_swift(void) { @@ -1627,7 +1157,7 @@ static void __init init_swift(void) "=r" (swift_rev) : "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); srmmu_name = "Fujitsu Swift"; - switch(swift_rev) { + switch (swift_rev) { case 0x11: case 0x20: case 0x23: @@ -1665,25 +1195,9 @@ static void __init init_swift(void) default: srmmu_modtype = Swift_ok; break; - }; - - BTFIXUPSET_CALL(flush_cache_all, swift_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, swift_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, swift_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, swift_flush_cache_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(flush_tlb_all, swift_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, swift_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, swift_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, swift_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, swift_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, swift_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, swift_flush_page_for_dma, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(update_mmu_cache, swift_update_mmu_cache, BTFIXUPCALL_NORM); + } + sparc32_cachetlb_ops = &swift_ops; flush_page_for_dma_global = 0; /* @@ -1734,7 +1248,7 @@ static void turbosparc_flush_page_to_ram(unsigned long page) #ifdef TURBOSPARC_WRITEBACK volatile unsigned long clear; - if (srmmu_hwprobe(page)) + if (srmmu_probe(page)) turbosparc_flush_page_cache(page); clear = srmmu_get_fstatus(); #endif @@ -1776,17 +1290,18 @@ static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long } -static void __cpuinit poke_turbosparc(void) +static void poke_turbosparc(void) { unsigned long mreg = srmmu_get_mmureg(); unsigned long ccreg; /* Clear any crap from the cache or else... */ turbosparc_flush_cache_all(); - mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */ + /* Temporarily disable I & D caches */ + mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ srmmu_set_mmureg(mreg); - + ccreg = turbosparc_get_ccreg(); #ifdef TURBOSPARC_WRITEBACK @@ -1809,37 +1324,36 @@ static void __cpuinit poke_turbosparc(void) default: ccreg |= (TURBOSPARC_SCENABLE); } - turbosparc_set_ccreg (ccreg); + turbosparc_set_ccreg(ccreg); mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops turbosparc_ops = { + .cache_all = turbosparc_flush_cache_all, + .cache_mm = turbosparc_flush_cache_mm, + .cache_page = turbosparc_flush_cache_page, + .cache_range = turbosparc_flush_cache_range, + .tlb_all = turbosparc_flush_tlb_all, + .tlb_mm = turbosparc_flush_tlb_mm, + .tlb_page = turbosparc_flush_tlb_page, + .tlb_range = turbosparc_flush_tlb_range, + .page_to_ram = turbosparc_flush_page_to_ram, + .sig_insns = turbosparc_flush_sig_insns, + .page_for_dma = turbosparc_flush_page_for_dma, +}; + static void __init init_turbosparc(void) { srmmu_name = "Fujitsu TurboSparc"; srmmu_modtype = TurboSparc; - - BTFIXUPSET_CALL(flush_cache_all, turbosparc_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, turbosparc_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, turbosparc_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, turbosparc_flush_cache_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, turbosparc_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, turbosparc_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, turbosparc_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, turbosparc_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, turbosparc_flush_page_to_ram, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_sig_insns, turbosparc_flush_sig_insns, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_page_for_dma, turbosparc_flush_page_for_dma, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = &turbosparc_ops; poke_srmmu = poke_turbosparc; } -static void __cpuinit poke_tsunami(void) +static void poke_tsunami(void) { unsigned long mreg = srmmu_get_mmureg(); @@ -1850,6 +1364,20 @@ static void __cpuinit poke_tsunami(void) srmmu_set_mmureg(mreg); } +static const struct sparc32_cachetlb_ops tsunami_ops = { + .cache_all = tsunami_flush_cache_all, + .cache_mm = tsunami_flush_cache_mm, + .cache_page = tsunami_flush_cache_page, + .cache_range = tsunami_flush_cache_range, + .tlb_all = tsunami_flush_tlb_all, + .tlb_mm = tsunami_flush_tlb_mm, + .tlb_page = tsunami_flush_tlb_page, + .tlb_range = tsunami_flush_tlb_range, + .page_to_ram = tsunami_flush_page_to_ram, + .sig_insns = tsunami_flush_sig_insns, + .page_for_dma = tsunami_flush_page_for_dma, +}; + static void __init init_tsunami(void) { /* @@ -1860,33 +1388,18 @@ static void __init init_tsunami(void) srmmu_name = "TI Tsunami"; srmmu_modtype = Tsunami; - - BTFIXUPSET_CALL(flush_cache_all, tsunami_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, tsunami_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, tsunami_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, tsunami_flush_cache_range, BTFIXUPCALL_NORM); - - - BTFIXUPSET_CALL(flush_tlb_all, tsunami_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, tsunami_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, tsunami_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, tsunami_flush_tlb_range, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, tsunami_flush_page_to_ram, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, tsunami_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, tsunami_flush_page_for_dma, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = &tsunami_ops; poke_srmmu = poke_tsunami; tsunami_setup_blockops(); } -static void __cpuinit poke_viking(void) +static void poke_viking(void) { unsigned long mreg = srmmu_get_mmureg(); static int smp_catch; - if(viking_mxcc_present) { + if (viking_mxcc_present) { unsigned long mxcc_control = mxcc_get_creg(); mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE); @@ -1905,7 +1418,7 @@ static void __cpuinit poke_viking(void) unsigned long bpreg; mreg &= ~(VIKING_TCENABLE); - if(smp_catch++) { + if (smp_catch++) { /* Must disable mixed-cmd mode here for other cpu's. */ bpreg = viking_get_bpreg(); bpreg &= ~(VIKING_ACTION_MIX); @@ -1923,20 +1436,62 @@ static void __cpuinit poke_viking(void) srmmu_set_mmureg(mreg); } +static struct sparc32_cachetlb_ops viking_ops = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = viking_flush_tlb_all, + .tlb_mm = viking_flush_tlb_mm, + .tlb_page = viking_flush_tlb_page, + .tlb_range = viking_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; + +#ifdef CONFIG_SMP +/* On sun4d the cpu broadcasts local TLB flushes, so we can just + * perform the local TLB flush and all the other cpus will see it. + * But, unfortunately, there is a bug in the sun4d XBUS backplane + * that requires that we add some synchronization to these flushes. + * + * The bug is that the fifo which keeps track of all the pending TLB + * broadcasts in the system is an entry or two too small, so if we + * have too many going at once we'll overflow that fifo and lose a TLB + * flush resulting in corruption. + * + * Our workaround is to take a global spinlock around the TLB flushes, + * which guarentees we won't ever have too many pending. It's a big + * hammer, but a semaphore like system to make sure we only have N TLB + * flushes going at once will require SMP locking anyways so there's + * no real value in trying any harder than this. + */ +static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = sun4dsmp_flush_tlb_all, + .tlb_mm = sun4dsmp_flush_tlb_mm, + .tlb_page = sun4dsmp_flush_tlb_page, + .tlb_range = sun4dsmp_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; +#endif + static void __init init_viking(void) { unsigned long mreg = srmmu_get_mmureg(); /* Ahhh, the viking. SRMMU VLSI abortion number two... */ - if(mreg & VIKING_MMODE) { + if (mreg & VIKING_MMODE) { srmmu_name = "TI Viking"; viking_mxcc_present = 0; msi_set_sync(); - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_NORM); - /* * We need this to make sure old viking takes no hits * on it's cache for dma snoops to workaround the @@ -1944,84 +1499,28 @@ static void __init init_viking(void) * This is only necessary because of the new way in * which we use the IOMMU. */ - BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page, BTFIXUPCALL_NORM); - + viking_ops.page_for_dma = viking_flush_page; +#ifdef CONFIG_SMP + viking_sun4d_smp_ops.page_for_dma = viking_flush_page; +#endif flush_page_for_dma_global = 0; } else { srmmu_name = "TI Viking/MXCC"; viking_mxcc_present = 1; - srmmu_cache_pagetables = 1; - - /* MXCC vikings lack the DMA snooping bug. */ - BTFIXUPSET_CALL(flush_page_for_dma, viking_flush_page_for_dma, BTFIXUPCALL_NOP); } - BTFIXUPSET_CALL(flush_cache_all, viking_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, viking_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, viking_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, viking_flush_cache_range, BTFIXUPCALL_NORM); - + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_ops; #ifdef CONFIG_SMP - if (sparc_cpu_model == sun4d) { - BTFIXUPSET_CALL(flush_tlb_all, sun4dsmp_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, sun4dsmp_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, sun4dsmp_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, sun4dsmp_flush_tlb_range, BTFIXUPCALL_NORM); - } else + if (sparc_cpu_model == sun4d) + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_sun4d_smp_ops; #endif - { - BTFIXUPSET_CALL(flush_tlb_all, viking_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, viking_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, viking_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, viking_flush_tlb_range, BTFIXUPCALL_NORM); - } - - BTFIXUPSET_CALL(__flush_page_to_ram, viking_flush_page_to_ram, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, viking_flush_sig_insns, BTFIXUPCALL_NOP); poke_srmmu = poke_viking; } -#ifdef CONFIG_SPARC_LEON - -void __init poke_leonsparc(void) -{ -} - -void __init init_leon(void) -{ - - srmmu_name = "LEON"; - - BTFIXUPSET_CALL(flush_cache_all, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, leon_flush_pcache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, leon_flush_cache_all, - BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, leon_flush_dcache_all, - BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_tlb_all, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, leon_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, leon_flush_tlb_all, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__flush_page_to_ram, leon_flush_cache_all, - BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(flush_sig_insns, leon_flush_cache_all, BTFIXUPCALL_NOP); - - poke_srmmu = poke_leonsparc; - - srmmu_cache_pagetables = 0; - - leon_flush_during_switch = leon_flush_needed(); -} -#endif - /* Probe for the srmmu chip version. */ static void __init get_srmmu_type(void) { @@ -2044,37 +1543,29 @@ static void __init get_srmmu_type(void) } /* Second, check for HyperSparc or Cypress. */ - if(mod_typ == 1) { - switch(mod_rev) { + if (mod_typ == 1) { + switch (mod_rev) { case 7: /* UP or MP Hypersparc */ init_hypersparc(); break; case 0: case 2: - /* Uniprocessor Cypress */ - init_cypress_604(); - break; case 10: case 11: case 12: - /* _REALLY OLD_ Cypress MP chips... */ case 13: case 14: case 15: - /* MP Cypress mmu/cache-controller */ - init_cypress_605(mod_rev); - break; default: - /* Some other Cypress revision, assume a 605. */ - init_cypress_605(mod_rev); + prom_printf("Sparc-Linux Cypress support does not longer exit.\n"); + prom_halt(); break; - }; + } return; } - - /* - * Now Fujitsu TurboSparc. It might happen that it is + + /* Now Fujitsu TurboSparc. It might happen that it is * in Swift emulation mode, so we will check later... */ if (psr_typ == 0 && psr_vers == 5) { @@ -2083,15 +1574,15 @@ static void __init get_srmmu_type(void) } /* Next check for Fujitsu Swift. */ - if(psr_typ == 0 && psr_vers == 4) { + if (psr_typ == 0 && psr_vers == 4) { phandle cpunode; char node_str[128]; /* Look if it is not a TurboSparc emulating Swift... */ cpunode = prom_getchild(prom_root_node); - while((cpunode = prom_getsibling(cpunode)) != 0) { + while ((cpunode = prom_getsibling(cpunode)) != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { if (!prom_getintdefault(cpunode, "psr-implementation", 1) && prom_getintdefault(cpunode, "psr-version", 1) == 5) { init_turbosparc(); @@ -2100,13 +1591,13 @@ static void __init get_srmmu_type(void) break; } } - + init_swift(); return; } /* Now the Viking family of srmmu. */ - if(psr_typ == 4 && + if (psr_typ == 4 && ((psr_vers == 0) || ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { init_viking(); @@ -2114,7 +1605,7 @@ static void __init get_srmmu_type(void) } /* Finally the Tsunami. */ - if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { + if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { init_tsunami(); return; } @@ -2123,203 +1614,190 @@ static void __init get_srmmu_type(void) srmmu_is_bad(); } -/* don't laugh, static pagetables */ -static void srmmu_check_pgt_cache(int low, int high) +#ifdef CONFIG_SMP +/* Local cross-calls. */ +static void smp_flush_page_for_dma(unsigned long page) { + xc1((smpfunc_t) local_ops->page_for_dma, page); + local_ops->page_for_dma(page); } -extern unsigned long spwin_mmu_patchme, fwin_mmu_patchme, - tsetup_mmu_patchme, rtrap_mmu_patchme; - -extern unsigned long spwin_srmmu_stackchk, srmmu_fwin_stackchk, - tsetup_srmmu_stackchk, srmmu_rett_stackchk; - -extern unsigned long srmmu_fault; - -#define PATCH_BRANCH(insn, dest) do { \ - iaddr = &(insn); \ - daddr = &(dest); \ - *iaddr = SPARC_BRANCH((unsigned long) daddr, (unsigned long) iaddr); \ - } while(0) - -static void __init patch_window_trap_handlers(void) +static void smp_flush_cache_all(void) { - unsigned long *iaddr, *daddr; - - PATCH_BRANCH(spwin_mmu_patchme, spwin_srmmu_stackchk); - PATCH_BRANCH(fwin_mmu_patchme, srmmu_fwin_stackchk); - PATCH_BRANCH(tsetup_mmu_patchme, tsetup_srmmu_stackchk); - PATCH_BRANCH(rtrap_mmu_patchme, srmmu_rett_stackchk); - PATCH_BRANCH(sparc_ttable[SP_TRAP_TFLT].inst_three, srmmu_fault); - PATCH_BRANCH(sparc_ttable[SP_TRAP_DFLT].inst_three, srmmu_fault); - PATCH_BRANCH(sparc_ttable[SP_TRAP_DACC].inst_three, srmmu_fault); + xc0((smpfunc_t) local_ops->cache_all); + local_ops->cache_all(); } -#ifdef CONFIG_SMP -/* Local cross-calls. */ -static void smp_flush_page_for_dma(unsigned long page) +static void smp_flush_tlb_all(void) { - xc1((smpfunc_t) BTFIXUP_CALL(local_flush_page_for_dma), page); - local_flush_page_for_dma(page); + xc0((smpfunc_t) local_ops->tlb_all); + local_ops->tlb_all(); } -#endif +static void smp_flush_cache_mm(struct mm_struct *mm) +{ + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm); + local_ops->cache_mm(mm); + } +} -static pte_t srmmu_pgoff_to_pte(unsigned long pgoff) +static void smp_flush_tlb_mm(struct mm_struct *mm) { - return __pte((pgoff << SRMMU_PTE_FILE_SHIFT) | SRMMU_FILE); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) { + xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm); + if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) + cpumask_copy(mm_cpumask(mm), + cpumask_of(smp_processor_id())); + } + local_ops->tlb_mm(mm); + } } -static unsigned long srmmu_pte_to_pgoff(pte_t pte) +static void smp_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) { - return pte_val(pte) >> SRMMU_PTE_FILE_SHIFT; + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc3((smpfunc_t) local_ops->cache_range, + (unsigned long) vma, start, end); + local_ops->cache_range(vma, start, end); + } } -static pgprot_t srmmu_pgprot_noncached(pgprot_t prot) +static void smp_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) { - prot &= ~__pgprot(SRMMU_CACHE); + struct mm_struct *mm = vma->vm_mm; - return prot; + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc3((smpfunc_t) local_ops->tlb_range, + (unsigned long) vma, start, end); + local_ops->tlb_range(vma, start, end); + } } -/* Load up routines and constants for sun4m and sun4d mmu */ -void __init ld_mmu_srmmu(void) +static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) { - extern void ld_mmu_iommu(void); - extern void ld_mmu_iounit(void); - extern void ___xchg32_sun4md(void); + struct mm_struct *mm = vma->vm_mm; - BTFIXUPSET_SIMM13(pgdir_shift, SRMMU_PGDIR_SHIFT); - BTFIXUPSET_SETHI(pgdir_size, SRMMU_PGDIR_SIZE); - BTFIXUPSET_SETHI(pgdir_mask, SRMMU_PGDIR_MASK); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->cache_page, + (unsigned long) vma, page); + local_ops->cache_page(vma, page); + } +} - BTFIXUPSET_SIMM13(ptrs_per_pmd, SRMMU_PTRS_PER_PMD); - BTFIXUPSET_SIMM13(ptrs_per_pgd, SRMMU_PTRS_PER_PGD); +static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; - BTFIXUPSET_INT(page_none, pgprot_val(SRMMU_PAGE_NONE)); - PAGE_SHARED = pgprot_val(SRMMU_PAGE_SHARED); - BTFIXUPSET_INT(page_copy, pgprot_val(SRMMU_PAGE_COPY)); - BTFIXUPSET_INT(page_readonly, pgprot_val(SRMMU_PAGE_RDONLY)); - BTFIXUPSET_INT(page_kernel, pgprot_val(SRMMU_PAGE_KERNEL)); - page_kernel = pgprot_val(SRMMU_PAGE_KERNEL); + if (mm->context != NO_CONTEXT) { + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->tlb_page, + (unsigned long) vma, page); + local_ops->tlb_page(vma, page); + } +} - /* Functions */ - BTFIXUPSET_CALL(pgprot_noncached, srmmu_pgprot_noncached, BTFIXUPCALL_NORM); -#ifndef CONFIG_SMP - BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4md, BTFIXUPCALL_SWAPG1G2); +static void smp_flush_page_to_ram(unsigned long page) +{ + /* Current theory is that those who call this are the one's + * who have just dirtied their cache with the pages contents + * in kernel space, therefore we only run this on local cpu. + * + * XXX This experiment failed, research further... -DaveM + */ +#if 1 + xc1((smpfunc_t) local_ops->page_to_ram, page); +#endif + local_ops->page_to_ram(page); +} + +static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) +{ + cpumask_t cpu_mask; + cpumask_copy(&cpu_mask, mm_cpumask(mm)); + cpumask_clear_cpu(smp_processor_id(), &cpu_mask); + if (!cpumask_empty(&cpu_mask)) + xc2((smpfunc_t) local_ops->sig_insns, + (unsigned long) mm, insn_addr); + local_ops->sig_insns(mm, insn_addr); +} + +static struct sparc32_cachetlb_ops smp_cachetlb_ops = { + .cache_all = smp_flush_cache_all, + .cache_mm = smp_flush_cache_mm, + .cache_page = smp_flush_cache_page, + .cache_range = smp_flush_cache_range, + .tlb_all = smp_flush_tlb_all, + .tlb_mm = smp_flush_tlb_mm, + .tlb_page = smp_flush_tlb_page, + .tlb_range = smp_flush_tlb_range, + .page_to_ram = smp_flush_page_to_ram, + .sig_insns = smp_flush_sig_insns, + .page_for_dma = smp_flush_page_for_dma, +}; #endif - BTFIXUPSET_CALL(do_check_pgt_cache, srmmu_check_pgt_cache, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(set_pte, srmmu_set_pte, BTFIXUPCALL_SWAPO0O1); - BTFIXUPSET_CALL(switch_mm, srmmu_switch_mm, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_pfn, srmmu_pte_pfn, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_page, srmmu_pmd_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_page_vaddr, srmmu_pgd_page, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_present, srmmu_pte_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_clear, srmmu_pte_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(pmd_bad, srmmu_pmd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_present, srmmu_pmd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, srmmu_pmd_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(pgd_none, srmmu_pgd_none, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_bad, srmmu_pgd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_present, srmmu_pgd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_clear, srmmu_pgd_clear, BTFIXUPCALL_SWAPO0G0); - - BTFIXUPSET_CALL(mk_pte, srmmu_mk_pte, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_phys, srmmu_mk_pte_phys, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_io, srmmu_mk_pte_io, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgd_set, srmmu_pgd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_set, srmmu_pmd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_populate, srmmu_pmd_populate, BTFIXUPCALL_NORM); - - BTFIXUPSET_INT(pte_modify_mask, SRMMU_CHG_MASK); - BTFIXUPSET_CALL(pmd_offset, srmmu_pmd_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_offset_kernel, srmmu_pte_offset, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(free_pte_fast, srmmu_free_pte_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_free, srmmu_pte_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one_kernel, srmmu_pte_alloc_one_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one, srmmu_pte_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pmd_fast, srmmu_pmd_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_alloc_one, srmmu_pmd_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pgd_fast, srmmu_free_pgd_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(get_pgd_fast, srmmu_get_pgd_fast, BTFIXUPCALL_NORM); - - BTFIXUPSET_HALF(pte_writei, SRMMU_WRITE); - BTFIXUPSET_HALF(pte_dirtyi, SRMMU_DIRTY); - BTFIXUPSET_HALF(pte_youngi, SRMMU_REF); - BTFIXUPSET_HALF(pte_filei, SRMMU_FILE); - BTFIXUPSET_HALF(pte_wrprotecti, SRMMU_WRITE); - BTFIXUPSET_HALF(pte_mkcleani, SRMMU_DIRTY); - BTFIXUPSET_HALF(pte_mkoldi, SRMMU_REF); - BTFIXUPSET_CALL(pte_mkwrite, srmmu_pte_mkwrite, BTFIXUPCALL_ORINT(SRMMU_WRITE)); - BTFIXUPSET_CALL(pte_mkdirty, srmmu_pte_mkdirty, BTFIXUPCALL_ORINT(SRMMU_DIRTY)); - BTFIXUPSET_CALL(pte_mkyoung, srmmu_pte_mkyoung, BTFIXUPCALL_ORINT(SRMMU_REF)); - BTFIXUPSET_CALL(update_mmu_cache, srmmu_update_mmu_cache, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(destroy_context, srmmu_destroy_context, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(sparc_mapiorange, srmmu_mapiorange, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sparc_unmapiorange, srmmu_unmapiorange, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__swp_type, srmmu_swp_type, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_offset, srmmu_swp_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_entry, srmmu_swp_entry, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_info, srmmu_mmu_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(alloc_thread_info, srmmu_alloc_thread_info, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_thread_info, srmmu_free_thread_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_to_pgoff, srmmu_pte_to_pgoff, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgoff_to_pte, srmmu_pgoff_to_pte, BTFIXUPCALL_NORM); +/* Load up routines and constants for sun4m and sun4d mmu */ +void __init load_mmu(void) +{ + /* Functions */ get_srmmu_type(); - patch_window_trap_handlers(); #ifdef CONFIG_SMP /* El switcheroo... */ + local_ops = sparc32_cachetlb_ops; - BTFIXUPCOPY_CALL(local_flush_cache_all, flush_cache_all); - BTFIXUPCOPY_CALL(local_flush_cache_mm, flush_cache_mm); - BTFIXUPCOPY_CALL(local_flush_cache_range, flush_cache_range); - BTFIXUPCOPY_CALL(local_flush_cache_page, flush_cache_page); - BTFIXUPCOPY_CALL(local_flush_tlb_all, flush_tlb_all); - BTFIXUPCOPY_CALL(local_flush_tlb_mm, flush_tlb_mm); - BTFIXUPCOPY_CALL(local_flush_tlb_range, flush_tlb_range); - BTFIXUPCOPY_CALL(local_flush_tlb_page, flush_tlb_page); - BTFIXUPCOPY_CALL(local_flush_page_to_ram, __flush_page_to_ram); - BTFIXUPCOPY_CALL(local_flush_sig_insns, flush_sig_insns); - BTFIXUPCOPY_CALL(local_flush_page_for_dma, flush_page_for_dma); - - BTFIXUPSET_CALL(flush_cache_all, smp_flush_cache_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, smp_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, smp_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, smp_flush_cache_page, BTFIXUPCALL_NORM); - if (sparc_cpu_model != sun4d && - sparc_cpu_model != sparc_leon) { - BTFIXUPSET_CALL(flush_tlb_all, smp_flush_tlb_all, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_mm, smp_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, smp_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, smp_flush_tlb_page, BTFIXUPCALL_NORM); + if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) { + smp_cachetlb_ops.tlb_all = local_ops->tlb_all; + smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm; + smp_cachetlb_ops.tlb_range = local_ops->tlb_range; + smp_cachetlb_ops.tlb_page = local_ops->tlb_page; } - BTFIXUPSET_CALL(__flush_page_to_ram, smp_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_sig_insns, smp_flush_sig_insns, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_page_for_dma, smp_flush_page_for_dma, BTFIXUPCALL_NORM); if (poke_srmmu == poke_viking) { /* Avoid unnecessary cross calls. */ - BTFIXUPCOPY_CALL(flush_cache_all, local_flush_cache_all); - BTFIXUPCOPY_CALL(flush_cache_mm, local_flush_cache_mm); - BTFIXUPCOPY_CALL(flush_cache_range, local_flush_cache_range); - BTFIXUPCOPY_CALL(flush_cache_page, local_flush_cache_page); - BTFIXUPCOPY_CALL(__flush_page_to_ram, local_flush_page_to_ram); - BTFIXUPCOPY_CALL(flush_sig_insns, local_flush_sig_insns); - BTFIXUPCOPY_CALL(flush_page_for_dma, local_flush_page_for_dma); + smp_cachetlb_ops.cache_all = local_ops->cache_all; + smp_cachetlb_ops.cache_mm = local_ops->cache_mm; + smp_cachetlb_ops.cache_range = local_ops->cache_range; + smp_cachetlb_ops.cache_page = local_ops->cache_page; + + smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram; + smp_cachetlb_ops.sig_insns = local_ops->sig_insns; + smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma; } + + /* It really is const after this point. */ + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &smp_cachetlb_ops; #endif if (sparc_cpu_model == sun4d) diff --git a/arch/sparc/mm/srmmu_access.S b/arch/sparc/mm/srmmu_access.S new file mode 100644 index 00000000000..d0a67b2c238 --- /dev/null +++ b/arch/sparc/mm/srmmu_access.S @@ -0,0 +1,82 @@ +/* Assembler variants of srmmu access functions. + * Implemented in assembler to allow run-time patching. + * LEON uses a different ASI for MMUREGS than SUN. + * + * The leon_1insn_patch infrastructure is used + * for the run-time patching. + */ + +#include <linux/linkage.h> + +#include <asm/asmmacro.h> +#include <asm/pgtsrmmu.h> +#include <asm/asi.h> + +/* unsigned int srmmu_get_mmureg(void) */ +ENTRY(srmmu_get_mmureg) +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_mmureg) + +/* void srmmu_set_mmureg(unsigned long regval) */ +ENTRY(srmmu_set_mmureg) +LEON_PI(sta %o0, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g0] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_mmureg) + +/* void srmmu_set_ctable_ptr(unsigned long paddr) */ +ENTRY(srmmu_set_ctable_ptr) + /* paddr = ((paddr >> 4) & SRMMU_CTX_PMASK); */ + srl %o0, 4, %g1 + and %g1, SRMMU_CTX_PMASK, %g1 + + mov SRMMU_CTXTBL_PTR, %g2 +LEON_PI(sta %g1, [%g2] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g2] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_ctable_ptr) + + +/* void srmmu_set_context(int context) */ +ENTRY(srmmu_set_context) + mov SRMMU_CTX_REG, %g1 +LEON_PI(sta %o0, [%g1] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g1] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_context) + + +/* int srmmu_get_context(void) */ +ENTRY(srmmu_get_context) + mov SRMMU_CTX_REG, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_context) + + +/* unsigned int srmmu_get_fstatus(void) */ +ENTRY(srmmu_get_fstatus) + mov SRMMU_FAULT_STATUS, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_fstatus) + + +/* unsigned int srmmu_get_faddr(void) */ +ENTRY(srmmu_get_faddr) + mov SRMMU_FAULT_ADDR, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_faddr) diff --git a/arch/sparc/mm/sun4c.c b/arch/sparc/mm/sun4c.c deleted file mode 100644 index b5137cc2aba..00000000000 --- a/arch/sparc/mm/sun4c.c +++ /dev/null @@ -1,2166 +0,0 @@ -/* sun4c.c: Doing in software what should be done in hardware. - * - * Copyright (C) 1996 David S. Miller (davem@davemloft.net) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au) - * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org) - * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) - */ - -#define NR_TASK_BUCKETS 512 - -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/init.h> -#include <linux/slab.h> -#include <linux/bootmem.h> -#include <linux/highmem.h> -#include <linux/fs.h> -#include <linux/seq_file.h> -#include <linux/scatterlist.h> -#include <linux/bitmap.h> - -#include <asm/sections.h> -#include <asm/page.h> -#include <asm/pgalloc.h> -#include <asm/pgtable.h> -#include <asm/vaddrs.h> -#include <asm/idprom.h> -#include <asm/machines.h> -#include <asm/memreg.h> -#include <asm/processor.h> -#include <asm/auxio.h> -#include <asm/io.h> -#include <asm/oplib.h> -#include <asm/openprom.h> -#include <asm/mmu_context.h> -#include <asm/highmem.h> -#include <asm/btfixup.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> - -/* Because of our dynamic kernel TLB miss strategy, and how - * our DVMA mapping allocation works, you _MUST_: - * - * 1) Disable interrupts _and_ not touch any dynamic kernel - * memory while messing with kernel MMU state. By - * dynamic memory I mean any object which is not in - * the kernel image itself or a thread_union (both of - * which are locked into the MMU). - * 2) Disable interrupts while messing with user MMU state. - */ - -extern int num_segmaps, num_contexts; - -extern unsigned long page_kernel; - -/* That's it, we prom_halt() on sun4c if the cache size is something other than 65536. - * So let's save some cycles and just use that everywhere except for that bootup - * sanity check. - */ -#define SUN4C_VAC_SIZE 65536 - -#define SUN4C_KERNEL_BUCKETS 32 - -/* Flushing the cache. */ -struct sun4c_vac_props sun4c_vacinfo; -unsigned long sun4c_kernel_faults; - -/* Invalidate every sun4c cache line tag. */ -static void __init sun4c_flush_all(void) -{ - unsigned long begin, end; - - if (sun4c_vacinfo.on) - panic("SUN4C: AIEEE, trying to invalidate vac while it is on."); - - /* Clear 'valid' bit in all cache line tags */ - begin = AC_CACHETAGS; - end = (AC_CACHETAGS + SUN4C_VAC_SIZE); - while (begin < end) { - __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : - "r" (begin), "i" (ASI_CONTROL)); - begin += sun4c_vacinfo.linesize; - } -} - -static void sun4c_flush_context_hw(void) -{ - unsigned long end = SUN4C_VAC_SIZE; - - __asm__ __volatile__( - "1: addcc %0, -4096, %0\n\t" - " bne 1b\n\t" - " sta %%g0, [%0] %2" - : "=&r" (end) - : "0" (end), "i" (ASI_HWFLUSHCONTEXT) - : "cc"); -} - -/* Must be called minimally with IRQs disabled. */ -static void sun4c_flush_segment_hw(unsigned long addr) -{ - if (sun4c_get_segmap(addr) != invalid_segment) { - unsigned long vac_size = SUN4C_VAC_SIZE; - - __asm__ __volatile__( - "1: addcc %0, -4096, %0\n\t" - " bne 1b\n\t" - " sta %%g0, [%2 + %0] %3" - : "=&r" (vac_size) - : "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG) - : "cc"); - } -} - -/* File local boot time fixups. */ -BTFIXUPDEF_CALL(void, sun4c_flush_page, unsigned long) -BTFIXUPDEF_CALL(void, sun4c_flush_segment, unsigned long) -BTFIXUPDEF_CALL(void, sun4c_flush_context, void) - -#define sun4c_flush_page(addr) BTFIXUP_CALL(sun4c_flush_page)(addr) -#define sun4c_flush_segment(addr) BTFIXUP_CALL(sun4c_flush_segment)(addr) -#define sun4c_flush_context() BTFIXUP_CALL(sun4c_flush_context)() - -/* Must be called minimally with interrupts disabled. */ -static void sun4c_flush_page_hw(unsigned long addr) -{ - addr &= PAGE_MASK; - if ((int)sun4c_get_pte(addr) < 0) - __asm__ __volatile__("sta %%g0, [%0] %1" - : : "r" (addr), "i" (ASI_HWFLUSHPAGE)); -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_context_sw(void) -{ - unsigned long nbytes = SUN4C_VAC_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %0, %%o5, %0\n\t" - "sta %%g0, [%0] %3\n\t" - "sta %%g0, [%0 + %2] %3\n\t" - "sta %%g0, [%0 + %%g1] %3\n\t" - "sta %%g0, [%0 + %%g2] %3\n\t" - "sta %%g0, [%0 + %%g3] %3\n\t" - "sta %%g0, [%0 + %%g4] %3\n\t" - "sta %%g0, [%0 + %%g5] %3\n\t" - "bg 1b\n\t" - " sta %%g0, [%1 + %%o4] %3\n" - : "=&r" (nbytes) - : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_segment_sw(unsigned long addr) -{ - if (sun4c_get_segmap(addr) != invalid_segment) { - unsigned long nbytes = SUN4C_VAC_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %1, %%o5, %1\n\t" - "sta %%g0, [%0] %6\n\t" - "sta %%g0, [%0 + %2] %6\n\t" - "sta %%g0, [%0 + %%g1] %6\n\t" - "sta %%g0, [%0 + %%g2] %6\n\t" - "sta %%g0, [%0 + %%g3] %6\n\t" - "sta %%g0, [%0 + %%g4] %6\n\t" - "sta %%g0, [%0 + %%g5] %6\n\t" - "sta %%g0, [%0 + %%o4] %6\n\t" - "bg 1b\n\t" - " add %0, %%o5, %0\n" - : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize) - : "0" (addr), "1" (nbytes), "2" (lsize), - "i" (ASI_FLUSHSEG) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); - } -} - -/* Don't inline the software version as it eats too many cache lines if expanded. */ -static void sun4c_flush_page_sw(unsigned long addr) -{ - addr &= PAGE_MASK; - if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) == - _SUN4C_PAGE_VALID) { - unsigned long left = PAGE_SIZE; - unsigned long lsize = sun4c_vacinfo.linesize; - - __asm__ __volatile__( - "add %2, %2, %%g1\n\t" - "add %2, %%g1, %%g2\n\t" - "add %2, %%g2, %%g3\n\t" - "add %2, %%g3, %%g4\n\t" - "add %2, %%g4, %%g5\n\t" - "add %2, %%g5, %%o4\n\t" - "add %2, %%o4, %%o5\n" - "1:\n\t" - "subcc %1, %%o5, %1\n\t" - "sta %%g0, [%0] %6\n\t" - "sta %%g0, [%0 + %2] %6\n\t" - "sta %%g0, [%0 + %%g1] %6\n\t" - "sta %%g0, [%0 + %%g2] %6\n\t" - "sta %%g0, [%0 + %%g3] %6\n\t" - "sta %%g0, [%0 + %%g4] %6\n\t" - "sta %%g0, [%0 + %%g5] %6\n\t" - "sta %%g0, [%0 + %%o4] %6\n\t" - "bg 1b\n\t" - " add %0, %%o5, %0\n" - : "=&r" (addr), "=&r" (left), "=&r" (lsize) - : "0" (addr), "1" (left), "2" (lsize), - "i" (ASI_FLUSHPG) - : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); - } -} - -/* The sun4c's do have an on chip store buffer. And the way you - * clear them out isn't so obvious. The only way I can think of - * to accomplish this is to read the current context register, - * store the same value there, then read an external hardware - * register. - */ -void sun4c_complete_all_stores(void) -{ - volatile int _unused; - - _unused = sun4c_get_context(); - sun4c_set_context(_unused); - _unused = get_auxio(); -} - -/* Bootup utility functions. */ -static inline void sun4c_init_clean_segmap(unsigned char pseg) -{ - unsigned long vaddr; - - sun4c_put_segmap(0, pseg); - for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE) - sun4c_put_pte(vaddr, 0); - sun4c_put_segmap(0, invalid_segment); -} - -static inline void sun4c_init_clean_mmu(unsigned long kernel_end) -{ - unsigned long vaddr; - unsigned char savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE) - sun4c_put_segmap(vaddr, invalid_segment); - } - sun4c_set_context(savectx); -} - -void __init sun4c_probe_vac(void) -{ - sun4c_disable_vac(); - - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { - /* PROM on SS1 lacks this info, to be super safe we - * hard code it here since this arch is cast in stone. - */ - sun4c_vacinfo.num_bytes = 65536; - sun4c_vacinfo.linesize = 16; - } else { - sun4c_vacinfo.num_bytes = - prom_getintdefault(prom_root_node, "vac-size", 65536); - sun4c_vacinfo.linesize = - prom_getintdefault(prom_root_node, "vac-linesize", 16); - } - sun4c_vacinfo.do_hwflushes = - prom_getintdefault(prom_root_node, "vac-hwflush", 0); - - if (sun4c_vacinfo.do_hwflushes == 0) - sun4c_vacinfo.do_hwflushes = - prom_getintdefault(prom_root_node, "vac_hwflush", 0); - - if (sun4c_vacinfo.num_bytes != 65536) { - prom_printf("WEIRD Sun4C VAC cache size, " - "tell sparclinux@vger.kernel.org"); - prom_halt(); - } - - switch (sun4c_vacinfo.linesize) { - case 16: - sun4c_vacinfo.log2lsize = 4; - break; - case 32: - sun4c_vacinfo.log2lsize = 5; - break; - default: - prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n", - sun4c_vacinfo.linesize); - prom_halt(); - }; - - sun4c_flush_all(); - sun4c_enable_vac(); -} - -/* Patch instructions for the low level kernel fault handler. */ -extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff; -extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff; -extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff; -extern unsigned long num_context_patch1, num_context_patch1_16; -extern unsigned long num_context_patch2_16; -extern unsigned long vac_linesize_patch, vac_linesize_patch_32; -extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on; -extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on; - -#define PATCH_INSN(src, dst) do { \ - daddr = &(dst); \ - iaddr = &(src); \ - *daddr = *iaddr; \ - } while (0) - -static void __init patch_kernel_fault_handler(void) -{ - unsigned long *iaddr, *daddr; - - switch (num_segmaps) { - case 128: - /* Default, nothing to do. */ - break; - case 256: - PATCH_INSN(invalid_segment_patch1_ff, - invalid_segment_patch1); - PATCH_INSN(invalid_segment_patch2_ff, - invalid_segment_patch2); - break; - case 512: - PATCH_INSN(invalid_segment_patch1_1ff, - invalid_segment_patch1); - PATCH_INSN(invalid_segment_patch2_1ff, - invalid_segment_patch2); - break; - default: - prom_printf("Unhandled number of segmaps: %d\n", - num_segmaps); - prom_halt(); - }; - switch (num_contexts) { - case 8: - /* Default, nothing to do. */ - break; - case 16: - PATCH_INSN(num_context_patch1_16, - num_context_patch1); - break; - default: - prom_printf("Unhandled number of contexts: %d\n", - num_contexts); - prom_halt(); - }; - - if (sun4c_vacinfo.do_hwflushes != 0) { - PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1); - PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2); - } else { - switch (sun4c_vacinfo.linesize) { - case 16: - /* Default, nothing to do. */ - break; - case 32: - PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch); - break; - default: - prom_printf("Impossible VAC linesize %d, halting...\n", - sun4c_vacinfo.linesize); - prom_halt(); - }; - } -} - -static void __init sun4c_probe_mmu(void) -{ - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { - /* Hardcode these just to be safe, PROM on SS1 does - * not have this info available in the root node. - */ - num_segmaps = 128; - num_contexts = 8; - } else { - num_segmaps = - prom_getintdefault(prom_root_node, "mmu-npmg", 128); - num_contexts = - prom_getintdefault(prom_root_node, "mmu-nctx", 0x8); - } - patch_kernel_fault_handler(); -} - -volatile unsigned long __iomem *sun4c_memerr_reg = NULL; - -void __init sun4c_probe_memerr_reg(void) -{ - phandle node; - struct linux_prom_registers regs[1]; - - node = prom_getchild(prom_root_node); - node = prom_searchsiblings(prom_root_node, "memory-error"); - if (!node) - return; - if (prom_getproperty(node, "reg", (char *)regs, sizeof(regs)) <= 0) - return; - /* hmm I think regs[0].which_io is zero here anyways */ - sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size); -} - -static inline void sun4c_init_ss2_cache_bug(void) -{ - if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) || - (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) { - /* Whee.. */ - printk("SS2 cache bug detected, uncaching trap table page\n"); - sun4c_flush_page((unsigned int) &_start); - sun4c_put_pte(((unsigned long) &_start), - (sun4c_get_pte((unsigned long) &_start) | _SUN4C_PAGE_NOCACHE)); - } -} - -/* Addr is always aligned on a page boundary for us already. */ -static int sun4c_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, - unsigned long addr, int len) -{ - unsigned long page, end; - - *pba = addr; - - end = PAGE_ALIGN((addr + len)); - while (addr < end) { - page = va; - sun4c_flush_page(page); - page -= PAGE_OFFSET; - page >>= PAGE_SHIFT; - page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY | - _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV); - sun4c_put_pte(addr, page); - addr += PAGE_SIZE; - va += PAGE_SIZE; - } - - return 0; -} - -static void sun4c_unmap_dma_area(struct device *dev, unsigned long busa, int len) -{ - /* Fortunately for us, bus_addr == uncached_virt in sun4c. */ - /* XXX Implement this */ -} - -/* TLB management. */ - -/* Don't change this struct without changing entry.S. This is used - * in the in-window kernel fault handler, and you don't want to mess - * with that. (See sun4c_fault in entry.S). - */ -struct sun4c_mmu_entry { - struct sun4c_mmu_entry *next; - struct sun4c_mmu_entry *prev; - unsigned long vaddr; - unsigned char pseg; - unsigned char locked; - - /* For user mappings only, and completely hidden from kernel - * TLB miss code. - */ - unsigned char ctx; - struct sun4c_mmu_entry *lru_next; - struct sun4c_mmu_entry *lru_prev; -}; - -static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS]; - -static void __init sun4c_init_mmu_entry_pool(void) -{ - int i; - - for (i=0; i < SUN4C_MAX_SEGMAPS; i++) { - mmu_entry_pool[i].pseg = i; - mmu_entry_pool[i].next = NULL; - mmu_entry_pool[i].prev = NULL; - mmu_entry_pool[i].vaddr = 0; - mmu_entry_pool[i].locked = 0; - mmu_entry_pool[i].ctx = 0; - mmu_entry_pool[i].lru_next = NULL; - mmu_entry_pool[i].lru_prev = NULL; - } - mmu_entry_pool[invalid_segment].locked = 1; -} - -static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on, - unsigned long bits_off) -{ - unsigned long start, end; - - end = vaddr + SUN4C_REAL_PGDIR_SIZE; - for (start = vaddr; start < end; start += PAGE_SIZE) - if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID) - sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) & - ~bits_off); -} - -static inline void sun4c_init_map_kernelprom(unsigned long kernel_end) -{ - unsigned long vaddr; - unsigned char pseg, ctx; - - for (vaddr = KADB_DEBUGGER_BEGVM; - vaddr < LINUX_OPPROM_ENDVM; - vaddr += SUN4C_REAL_PGDIR_SIZE) { - pseg = sun4c_get_segmap(vaddr); - if (pseg != invalid_segment) { - mmu_entry_pool[pseg].locked = 1; - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, vaddr, pseg); - fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0); - } - } - - for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) { - pseg = sun4c_get_segmap(vaddr); - mmu_entry_pool[pseg].locked = 1; - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, vaddr, pseg); - fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE); - } -} - -static void __init sun4c_init_lock_area(unsigned long start, unsigned long end) -{ - int i, ctx; - - while (start < end) { - for (i = 0; i < invalid_segment; i++) - if (!mmu_entry_pool[i].locked) - break; - mmu_entry_pool[i].locked = 1; - sun4c_init_clean_segmap(i); - for (ctx = 0; ctx < num_contexts; ctx++) - prom_putsegment(ctx, start, mmu_entry_pool[i].pseg); - start += SUN4C_REAL_PGDIR_SIZE; - } -} - -/* Don't change this struct without changing entry.S. This is used - * in the in-window kernel fault handler, and you don't want to mess - * with that. (See sun4c_fault in entry.S). - */ -struct sun4c_mmu_ring { - struct sun4c_mmu_entry ringhd; - int num_entries; -}; - -static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */ -static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */ -static struct sun4c_mmu_ring sun4c_ulru_ring; /* LRU user entries */ -struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */ -struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */ - -static inline void sun4c_init_rings(void) -{ - int i; - - for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) { - sun4c_context_ring[i].ringhd.next = - sun4c_context_ring[i].ringhd.prev = - &sun4c_context_ring[i].ringhd; - sun4c_context_ring[i].num_entries = 0; - } - sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev = - &sun4c_ufree_ring.ringhd; - sun4c_ufree_ring.num_entries = 0; - sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev = - &sun4c_ulru_ring.ringhd; - sun4c_ulru_ring.num_entries = 0; - sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev = - &sun4c_kernel_ring.ringhd; - sun4c_kernel_ring.num_entries = 0; - sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev = - &sun4c_kfree_ring.ringhd; - sun4c_kfree_ring.num_entries = 0; -} - -static void add_ring(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *head = &ring->ringhd; - - entry->prev = head; - (entry->next = head->next)->prev = entry; - head->next = entry; - ring->num_entries++; -} - -static inline void add_lru(struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_ring *ring = &sun4c_ulru_ring; - struct sun4c_mmu_entry *head = &ring->ringhd; - - entry->lru_next = head; - (entry->lru_prev = head->lru_prev)->lru_next = entry; - head->lru_prev = entry; -} - -static void add_ring_ordered(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *head = &ring->ringhd; - unsigned long addr = entry->vaddr; - - while ((head->next != &ring->ringhd) && (head->next->vaddr < addr)) - head = head->next; - - entry->prev = head; - (entry->next = head->next)->prev = entry; - head->next = entry; - ring->num_entries++; - - add_lru(entry); -} - -static inline void remove_ring(struct sun4c_mmu_ring *ring, - struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *next = entry->next; - - (next->prev = entry->prev)->next = next; - ring->num_entries--; -} - -static void remove_lru(struct sun4c_mmu_entry *entry) -{ - struct sun4c_mmu_entry *next = entry->lru_next; - - (next->lru_prev = entry->lru_prev)->lru_next = next; -} - -static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry) -{ - remove_ring(sun4c_context_ring+ctx, entry); - remove_lru(entry); - add_ring(&sun4c_ufree_ring, entry); -} - -static void free_kernel_entry(struct sun4c_mmu_entry *entry, - struct sun4c_mmu_ring *ring) -{ - remove_ring(ring, entry); - add_ring(&sun4c_kfree_ring, entry); -} - -static void __init sun4c_init_fill_kernel_ring(int howmany) -{ - int i; - - while (howmany) { - for (i = 0; i < invalid_segment; i++) - if (!mmu_entry_pool[i].locked) - break; - mmu_entry_pool[i].locked = 1; - sun4c_init_clean_segmap(i); - add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]); - howmany--; - } -} - -static void __init sun4c_init_fill_user_ring(void) -{ - int i; - - for (i = 0; i < invalid_segment; i++) { - if (mmu_entry_pool[i].locked) - continue; - sun4c_init_clean_segmap(i); - add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]); - } -} - -static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry) -{ - int savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(kentry->vaddr, invalid_segment); - } - sun4c_set_context(savectx); -} - -static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry) -{ - int savectx, ctx; - - savectx = sun4c_get_context(); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(kentry->vaddr, kentry->pseg); - } - sun4c_set_context(savectx); -} - -#define sun4c_user_unmap(__entry) \ - sun4c_put_segmap((__entry)->vaddr, invalid_segment) - -static void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx) -{ - struct sun4c_mmu_entry *head = &crp->ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - flush_user_windows(); - sun4c_set_context(ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); -} - -static int sun4c_user_taken_entries; /* This is how much we have. */ -static int max_user_taken_entries; /* This limits us and prevents deadlock. */ - -static struct sun4c_mmu_entry *sun4c_kernel_strategy(void) -{ - struct sun4c_mmu_entry *this_entry; - - /* If some are free, return first one. */ - if (sun4c_kfree_ring.num_entries) { - this_entry = sun4c_kfree_ring.ringhd.next; - return this_entry; - } - - /* Else free one up. */ - this_entry = sun4c_kernel_ring.ringhd.prev; - sun4c_flush_segment(this_entry->vaddr); - sun4c_kernel_unmap(this_entry); - free_kernel_entry(this_entry, &sun4c_kernel_ring); - this_entry = sun4c_kfree_ring.ringhd.next; - - return this_entry; -} - -/* Using this method to free up mmu entries eliminates a lot of - * potential races since we have a kernel that incurs tlb - * replacement faults. There may be performance penalties. - * - * NOTE: Must be called with interrupts disabled. - */ -static struct sun4c_mmu_entry *sun4c_user_strategy(void) -{ - struct sun4c_mmu_entry *entry; - unsigned char ctx; - int savectx; - - /* If some are free, return first one. */ - if (sun4c_ufree_ring.num_entries) { - entry = sun4c_ufree_ring.ringhd.next; - goto unlink_out; - } - - if (sun4c_user_taken_entries) { - entry = sun4c_kernel_strategy(); - sun4c_user_taken_entries--; - goto kunlink_out; - } - - /* Grab from the beginning of the LRU list. */ - entry = sun4c_ulru_ring.ringhd.lru_next; - ctx = entry->ctx; - - savectx = sun4c_get_context(); - flush_user_windows(); - sun4c_set_context(ctx); - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - remove_ring(sun4c_context_ring + ctx, entry); - remove_lru(entry); - sun4c_set_context(savectx); - - return entry; - -unlink_out: - remove_ring(&sun4c_ufree_ring, entry); - return entry; -kunlink_out: - remove_ring(&sun4c_kfree_ring, entry); - return entry; -} - -/* NOTE: Must be called with interrupts disabled. */ -void sun4c_grow_kernel_ring(void) -{ - struct sun4c_mmu_entry *entry; - - /* Prevent deadlock condition. */ - if (sun4c_user_taken_entries >= max_user_taken_entries) - return; - - if (sun4c_ufree_ring.num_entries) { - entry = sun4c_ufree_ring.ringhd.next; - remove_ring(&sun4c_ufree_ring, entry); - add_ring(&sun4c_kfree_ring, entry); - sun4c_user_taken_entries++; - } -} - -/* 2 page buckets for task struct and kernel stack allocation. - * - * TASK_STACK_BEGIN - * bucket[0] - * bucket[1] - * [ ... ] - * bucket[NR_TASK_BUCKETS-1] - * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS) - * - * Each slot looks like: - * - * page 1 -- task struct + beginning of kernel stack - * page 2 -- rest of kernel stack - */ - -union task_union *sun4c_bucket[NR_TASK_BUCKETS]; - -static int sun4c_lowbucket_avail; - -#define BUCKET_EMPTY ((union task_union *) 0) -#define BUCKET_SHIFT (PAGE_SHIFT + 1) /* log2(sizeof(struct task_bucket)) */ -#define BUCKET_SIZE (1 << BUCKET_SHIFT) -#define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT)) -#define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR) -#define BUCKET_PTE(page) \ - ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL)) -#define BUCKET_PTE_PAGE(pte) \ - (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT)) - -static void get_locked_segment(unsigned long addr) -{ - struct sun4c_mmu_entry *stolen; - unsigned long flags; - - local_irq_save(flags); - addr &= SUN4C_REAL_PGDIR_MASK; - stolen = sun4c_user_strategy(); - max_user_taken_entries--; - stolen->vaddr = addr; - flush_user_windows(); - sun4c_kernel_map(stolen); - local_irq_restore(flags); -} - -static void free_locked_segment(unsigned long addr) -{ - struct sun4c_mmu_entry *entry; - unsigned long flags; - unsigned char pseg; - - local_irq_save(flags); - addr &= SUN4C_REAL_PGDIR_MASK; - pseg = sun4c_get_segmap(addr); - entry = &mmu_entry_pool[pseg]; - - flush_user_windows(); - sun4c_flush_segment(addr); - sun4c_kernel_unmap(entry); - add_ring(&sun4c_ufree_ring, entry); - max_user_taken_entries++; - local_irq_restore(flags); -} - -static inline void garbage_collect(int entry) -{ - int start, end; - - /* 32 buckets per segment... */ - entry &= ~31; - start = entry; - for (end = (start + 32); start < end; start++) - if (sun4c_bucket[start] != BUCKET_EMPTY) - return; - - /* Entire segment empty, release it. */ - free_locked_segment(BUCKET_ADDR(entry)); -} - -static struct thread_info *sun4c_alloc_thread_info(void) -{ - unsigned long addr, pages; - int entry; - - pages = __get_free_pages(GFP_KERNEL, THREAD_INFO_ORDER); - if (!pages) - return NULL; - - for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++) - if (sun4c_bucket[entry] == BUCKET_EMPTY) - break; - if (entry == NR_TASK_BUCKETS) { - free_pages(pages, THREAD_INFO_ORDER); - return NULL; - } - if (entry >= sun4c_lowbucket_avail) - sun4c_lowbucket_avail = entry + 1; - - addr = BUCKET_ADDR(entry); - sun4c_bucket[entry] = (union task_union *) addr; - if(sun4c_get_segmap(addr) == invalid_segment) - get_locked_segment(addr); - - /* We are changing the virtual color of the page(s) - * so we must flush the cache to guarantee consistency. - */ - sun4c_flush_page(pages); - sun4c_flush_page(pages + PAGE_SIZE); - - sun4c_put_pte(addr, BUCKET_PTE(pages)); - sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE)); - -#ifdef CONFIG_DEBUG_STACK_USAGE - memset((void *)addr, 0, PAGE_SIZE << THREAD_INFO_ORDER); -#endif /* DEBUG_STACK_USAGE */ - - return (struct thread_info *) addr; -} - -static void sun4c_free_thread_info(struct thread_info *ti) -{ - unsigned long tiaddr = (unsigned long) ti; - unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tiaddr)); - int entry = BUCKET_NUM(tiaddr); - - /* We are deleting a mapping, so the flush here is mandatory. */ - sun4c_flush_page(tiaddr); - sun4c_flush_page(tiaddr + PAGE_SIZE); - - sun4c_put_pte(tiaddr, 0); - sun4c_put_pte(tiaddr + PAGE_SIZE, 0); - - sun4c_bucket[entry] = BUCKET_EMPTY; - if (entry < sun4c_lowbucket_avail) - sun4c_lowbucket_avail = entry; - - free_pages(pages, THREAD_INFO_ORDER); - garbage_collect(entry); -} - -static void __init sun4c_init_buckets(void) -{ - int entry; - - if (sizeof(union thread_union) != (PAGE_SIZE << THREAD_INFO_ORDER)) { - extern void thread_info_size_is_bolixed_pete(void); - thread_info_size_is_bolixed_pete(); - } - - for (entry = 0; entry < NR_TASK_BUCKETS; entry++) - sun4c_bucket[entry] = BUCKET_EMPTY; - sun4c_lowbucket_avail = 0; -} - -static unsigned long sun4c_iobuffer_start; -static unsigned long sun4c_iobuffer_end; -static unsigned long sun4c_iobuffer_high; -static unsigned long *sun4c_iobuffer_map; -static int iobuffer_map_size; - -/* - * Alias our pages so they do not cause a trap. - * Also one page may be aliased into several I/O areas and we may - * finish these I/O separately. - */ -static char *sun4c_lockarea(char *vaddr, unsigned long size) -{ - unsigned long base, scan; - unsigned long npages; - unsigned long vpage; - unsigned long pte; - unsigned long apage; - unsigned long high; - unsigned long flags; - - npages = (((unsigned long)vaddr & ~PAGE_MASK) + - size + (PAGE_SIZE-1)) >> PAGE_SHIFT; - - local_irq_save(flags); - base = bitmap_find_next_zero_area(sun4c_iobuffer_map, iobuffer_map_size, - 0, npages, 0); - if (base >= iobuffer_map_size) - goto abend; - - high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start; - high = SUN4C_REAL_PGDIR_ALIGN(high); - while (high > sun4c_iobuffer_high) { - get_locked_segment(sun4c_iobuffer_high); - sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE; - } - - vpage = ((unsigned long) vaddr) & PAGE_MASK; - for (scan = base; scan < base+npages; scan++) { - pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT); - pte |= pgprot_val(SUN4C_PAGE_KERNEL); - pte |= _SUN4C_PAGE_NOCACHE; - set_bit(scan, sun4c_iobuffer_map); - apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start; - - /* Flush original mapping so we see the right things later. */ - sun4c_flush_page(vpage); - - sun4c_put_pte(apage, pte); - vpage += PAGE_SIZE; - } - local_irq_restore(flags); - return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start + - (((unsigned long) vaddr) & ~PAGE_MASK)); - -abend: - local_irq_restore(flags); - printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size); - panic("Out of iobuffer table"); - return NULL; -} - -static void sun4c_unlockarea(char *vaddr, unsigned long size) -{ - unsigned long vpage, npages; - unsigned long flags; - int scan, high; - - vpage = (unsigned long)vaddr & PAGE_MASK; - npages = (((unsigned long)vaddr & ~PAGE_MASK) + - size + (PAGE_SIZE-1)) >> PAGE_SHIFT; - - local_irq_save(flags); - while (npages != 0) { - --npages; - - /* This mapping is marked non-cachable, no flush necessary. */ - sun4c_put_pte(vpage, 0); - clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT, - sun4c_iobuffer_map); - vpage += PAGE_SIZE; - } - - /* garbage collect */ - scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT; - while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5]) - scan -= 32; - scan += 32; - high = sun4c_iobuffer_start + (scan << PAGE_SHIFT); - high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE; - while (high < sun4c_iobuffer_high) { - sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE; - free_locked_segment(sun4c_iobuffer_high); - } - local_irq_restore(flags); -} - -/* Note the scsi code at init time passes to here buffers - * which sit on the kernel stack, those are already locked - * by implication and fool the page locking code above - * if passed to by mistake. - */ -static __u32 sun4c_get_scsi_one(struct device *dev, char *bufptr, unsigned long len) -{ - unsigned long page; - - page = ((unsigned long)bufptr) & PAGE_MASK; - if (!virt_addr_valid(page)) { - sun4c_flush_page(page); - return (__u32)bufptr; /* already locked */ - } - return (__u32)sun4c_lockarea(bufptr, len); -} - -static void sun4c_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) -{ - while (sz != 0) { - --sz; - sg->dma_address = (__u32)sun4c_lockarea(sg_virt(sg), sg->length); - sg->dma_length = sg->length; - sg = sg_next(sg); - } -} - -static void sun4c_release_scsi_one(struct device *dev, __u32 bufptr, unsigned long len) -{ - if (bufptr < sun4c_iobuffer_start) - return; /* On kernel stack or similar, see above */ - sun4c_unlockarea((char *)bufptr, len); -} - -static void sun4c_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) -{ - while (sz != 0) { - --sz; - sun4c_unlockarea((char *)sg->dma_address, sg->length); - sg = sg_next(sg); - } -} - -#define TASK_ENTRY_SIZE BUCKET_SIZE /* see above */ -#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) - -struct vm_area_struct sun4c_kstack_vma; - -static void __init sun4c_init_lock_areas(void) -{ - unsigned long sun4c_taskstack_start; - unsigned long sun4c_taskstack_end; - int bitmap_size; - - sun4c_init_buckets(); - sun4c_taskstack_start = SUN4C_LOCK_VADDR; - sun4c_taskstack_end = (sun4c_taskstack_start + - (TASK_ENTRY_SIZE * NR_TASK_BUCKETS)); - if (sun4c_taskstack_end >= SUN4C_LOCK_END) { - prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n"); - prom_halt(); - } - - sun4c_iobuffer_start = sun4c_iobuffer_high = - SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end); - sun4c_iobuffer_end = SUN4C_LOCK_END; - bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT; - bitmap_size = (bitmap_size + 7) >> 3; - bitmap_size = LONG_ALIGN(bitmap_size); - iobuffer_map_size = bitmap_size << 3; - sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL); - memset((void *) sun4c_iobuffer_map, 0, bitmap_size); - - sun4c_kstack_vma.vm_mm = &init_mm; - sun4c_kstack_vma.vm_start = sun4c_taskstack_start; - sun4c_kstack_vma.vm_end = sun4c_taskstack_end; - sun4c_kstack_vma.vm_page_prot = PAGE_SHARED; - sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC; - insert_vm_struct(&init_mm, &sun4c_kstack_vma); -} - -/* Cache flushing on the sun4c. */ -static void sun4c_flush_cache_all(void) -{ - unsigned long begin, end; - - flush_user_windows(); - begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE); - end = (begin + SUN4C_VAC_SIZE); - - if (sun4c_vacinfo.linesize == 32) { - while (begin < end) { - __asm__ __volatile__( - "ld [%0 + 0x00], %%g0\n\t" - "ld [%0 + 0x20], %%g0\n\t" - "ld [%0 + 0x40], %%g0\n\t" - "ld [%0 + 0x60], %%g0\n\t" - "ld [%0 + 0x80], %%g0\n\t" - "ld [%0 + 0xa0], %%g0\n\t" - "ld [%0 + 0xc0], %%g0\n\t" - "ld [%0 + 0xe0], %%g0\n\t" - "ld [%0 + 0x100], %%g0\n\t" - "ld [%0 + 0x120], %%g0\n\t" - "ld [%0 + 0x140], %%g0\n\t" - "ld [%0 + 0x160], %%g0\n\t" - "ld [%0 + 0x180], %%g0\n\t" - "ld [%0 + 0x1a0], %%g0\n\t" - "ld [%0 + 0x1c0], %%g0\n\t" - "ld [%0 + 0x1e0], %%g0\n" - : : "r" (begin)); - begin += 512; - } - } else { - while (begin < end) { - __asm__ __volatile__( - "ld [%0 + 0x00], %%g0\n\t" - "ld [%0 + 0x10], %%g0\n\t" - "ld [%0 + 0x20], %%g0\n\t" - "ld [%0 + 0x30], %%g0\n\t" - "ld [%0 + 0x40], %%g0\n\t" - "ld [%0 + 0x50], %%g0\n\t" - "ld [%0 + 0x60], %%g0\n\t" - "ld [%0 + 0x70], %%g0\n\t" - "ld [%0 + 0x80], %%g0\n\t" - "ld [%0 + 0x90], %%g0\n\t" - "ld [%0 + 0xa0], %%g0\n\t" - "ld [%0 + 0xb0], %%g0\n\t" - "ld [%0 + 0xc0], %%g0\n\t" - "ld [%0 + 0xd0], %%g0\n\t" - "ld [%0 + 0xe0], %%g0\n\t" - "ld [%0 + 0xf0], %%g0\n" - : : "r" (begin)); - begin += 256; - } - } -} - -static void sun4c_flush_cache_mm(struct mm_struct *mm) -{ - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - flush_user_windows(); - - if (sun4c_context_ring[new_ctx].num_entries) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); - } - } -} - -static void sun4c_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - struct sun4c_mmu_entry *entry; - unsigned long flags; - - flush_user_windows(); - - local_irq_save(flags); - /* All user segmap chains are ordered on entry->vaddr. */ - for (entry = head->next; - (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); - entry = entry->next) - ; - - /* Tracing various job mixtures showed that this conditional - * only passes ~35% of the time for most worse case situations, - * therefore we avoid all of this gross overhead ~65% of the time. - */ - if ((entry != head) && (entry->vaddr < end)) { - int octx = sun4c_get_context(); - sun4c_set_context(new_ctx); - - /* At this point, always, (start >= entry->vaddr) and - * (entry->vaddr < end), once the latter condition - * ceases to hold, or we hit the end of the list, we - * exit the loop. The ordering of all user allocated - * segmaps makes this all work out so beautifully. - */ - do { - struct sun4c_mmu_entry *next = entry->next; - unsigned long realend; - - /* "realstart" is always >= entry->vaddr */ - realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE; - if (end < realend) - realend = end; - if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) { - unsigned long page = entry->vaddr; - while (page < realend) { - sun4c_flush_page(page); - page += PAGE_SIZE; - } - } else { - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - } - entry = next; - } while ((entry != head) && (entry->vaddr < end)); - sun4c_set_context(octx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - /* Sun4c has no separate I/D caches so cannot optimize for non - * text page flushes. - */ - if (new_ctx != NO_CONTEXT) { - int octx = sun4c_get_context(); - unsigned long flags; - - flush_user_windows(); - local_irq_save(flags); - sun4c_set_context(new_ctx); - sun4c_flush_page(page); - sun4c_set_context(octx); - local_irq_restore(flags); - } -} - -static void sun4c_flush_page_to_ram(unsigned long page) -{ - unsigned long flags; - - local_irq_save(flags); - sun4c_flush_page(page); - local_irq_restore(flags); -} - -/* Sun4c cache is unified, both instructions and data live there, so - * no need to flush the on-stack instructions for new signal handlers. - */ -static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) -{ -} - -/* TLB flushing on the sun4c. These routines count on the cache - * flushing code to flush the user register windows so that we need - * not do so when we get here. - */ - -static void sun4c_flush_tlb_all(void) -{ - struct sun4c_mmu_entry *this_entry, *next_entry; - unsigned long flags; - int savectx, ctx; - - local_irq_save(flags); - this_entry = sun4c_kernel_ring.ringhd.next; - savectx = sun4c_get_context(); - flush_user_windows(); - while (sun4c_kernel_ring.num_entries) { - next_entry = this_entry->next; - sun4c_flush_segment(this_entry->vaddr); - for (ctx = 0; ctx < num_contexts; ctx++) { - sun4c_set_context(ctx); - sun4c_put_segmap(this_entry->vaddr, invalid_segment); - } - free_kernel_entry(this_entry, &sun4c_kernel_ring); - this_entry = next_entry; - } - sun4c_set_context(savectx); - local_irq_restore(flags); -} - -static void sun4c_flush_tlb_mm(struct mm_struct *mm) -{ - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - unsigned long flags; - - local_irq_save(flags); - if (head->next != head) { - struct sun4c_mmu_entry *entry = head->next; - int savectx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - sun4c_flush_context(); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while (entry != head); - sun4c_set_context(savectx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; - struct sun4c_mmu_entry *entry; - unsigned long flags; - - local_irq_save(flags); - /* See commentary in sun4c_flush_cache_range(). */ - for (entry = head->next; - (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); - entry = entry->next) - ; - - if ((entry != head) && (entry->vaddr < end)) { - int octx = sun4c_get_context(); - - sun4c_set_context(new_ctx); - do { - struct sun4c_mmu_entry *next = entry->next; - - sun4c_flush_segment(entry->vaddr); - sun4c_user_unmap(entry); - free_user_entry(new_ctx, entry); - - entry = next; - } while ((entry != head) && (entry->vaddr < end)); - sun4c_set_context(octx); - } - local_irq_restore(flags); - } -} - -static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) -{ - struct mm_struct *mm = vma->vm_mm; - int new_ctx = mm->context; - - if (new_ctx != NO_CONTEXT) { - int savectx = sun4c_get_context(); - unsigned long flags; - - local_irq_save(flags); - sun4c_set_context(new_ctx); - page &= PAGE_MASK; - sun4c_flush_page(page); - sun4c_put_pte(page, 0); - sun4c_set_context(savectx); - local_irq_restore(flags); - } -} - -static inline void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr) -{ - unsigned long page_entry, pg_iobits; - - pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE | - _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE; - - page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK); - page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT)); - sun4c_put_pte(virt_addr, page_entry); -} - -static void sun4c_mapiorange(unsigned int bus, unsigned long xpa, - unsigned long xva, unsigned int len) -{ - while (len != 0) { - len -= PAGE_SIZE; - sun4c_mapioaddr(xpa, xva); - xva += PAGE_SIZE; - xpa += PAGE_SIZE; - } -} - -static void sun4c_unmapiorange(unsigned long virt_addr, unsigned int len) -{ - while (len != 0) { - len -= PAGE_SIZE; - sun4c_put_pte(virt_addr, 0); - virt_addr += PAGE_SIZE; - } -} - -static void sun4c_alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) -{ - struct ctx_list *ctxp; - - ctxp = ctx_free.next; - if (ctxp != &ctx_free) { - remove_from_ctx_list(ctxp); - add_to_used_ctxlist(ctxp); - mm->context = ctxp->ctx_number; - ctxp->ctx_mm = mm; - return; - } - ctxp = ctx_used.next; - if (ctxp->ctx_mm == old_mm) - ctxp = ctxp->next; - remove_from_ctx_list(ctxp); - add_to_used_ctxlist(ctxp); - ctxp->ctx_mm->context = NO_CONTEXT; - ctxp->ctx_mm = mm; - mm->context = ctxp->ctx_number; - sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number], - ctxp->ctx_number); -} - -/* Switch the current MM context. */ -static void sun4c_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu) -{ - struct ctx_list *ctx; - int dirty = 0; - - if (mm->context == NO_CONTEXT) { - dirty = 1; - sun4c_alloc_context(old_mm, mm); - } else { - /* Update the LRU ring of contexts. */ - ctx = ctx_list_pool + mm->context; - remove_from_ctx_list(ctx); - add_to_used_ctxlist(ctx); - } - if (dirty || old_mm != mm) - sun4c_set_context(mm->context); -} - -static void sun4c_destroy_context(struct mm_struct *mm) -{ - struct ctx_list *ctx_old; - - if (mm->context != NO_CONTEXT) { - sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context); - ctx_old = ctx_list_pool + mm->context; - remove_from_ctx_list(ctx_old); - add_to_free_ctxlist(ctx_old); - mm->context = NO_CONTEXT; - } -} - -static void sun4c_mmu_info(struct seq_file *m) -{ - int used_user_entries, i; - - used_user_entries = 0; - for (i = 0; i < num_contexts; i++) - used_user_entries += sun4c_context_ring[i].num_entries; - - seq_printf(m, - "vacsize\t\t: %d bytes\n" - "vachwflush\t: %s\n" - "vaclinesize\t: %d bytes\n" - "mmuctxs\t\t: %d\n" - "mmupsegs\t: %d\n" - "kernelpsegs\t: %d\n" - "kfreepsegs\t: %d\n" - "usedpsegs\t: %d\n" - "ufreepsegs\t: %d\n" - "user_taken\t: %d\n" - "max_taken\t: %d\n", - sun4c_vacinfo.num_bytes, - (sun4c_vacinfo.do_hwflushes ? "yes" : "no"), - sun4c_vacinfo.linesize, - num_contexts, - (invalid_segment + 1), - sun4c_kernel_ring.num_entries, - sun4c_kfree_ring.num_entries, - used_user_entries, - sun4c_ufree_ring.num_entries, - sun4c_user_taken_entries, - max_user_taken_entries); -} - -/* Nothing below here should touch the mmu hardware nor the mmu_entry - * data structures. - */ - -/* First the functions which the mid-level code uses to directly - * manipulate the software page tables. Some defines since we are - * emulating the i386 page directory layout. - */ -#define PGD_PRESENT 0x001 -#define PGD_RW 0x002 -#define PGD_USER 0x004 -#define PGD_ACCESSED 0x020 -#define PGD_DIRTY 0x040 -#define PGD_TABLE (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY) - -static void sun4c_set_pte(pte_t *ptep, pte_t pte) -{ - *ptep = pte; -} - -static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp) -{ -} - -static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep) -{ - pmdp->pmdv[0] = PGD_TABLE | (unsigned long) ptep; -} - -static void sun4c_pmd_populate(pmd_t * pmdp, struct page * ptep) -{ - if (page_address(ptep) == NULL) BUG(); /* No highmem on sun4c */ - pmdp->pmdv[0] = PGD_TABLE | (unsigned long) page_address(ptep); -} - -static int sun4c_pte_present(pte_t pte) -{ - return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0); -} -static void sun4c_pte_clear(pte_t *ptep) { *ptep = __pte(0); } - -static int sun4c_pmd_bad(pmd_t pmd) -{ - return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) || - (!virt_addr_valid(pmd_val(pmd)))); -} - -static int sun4c_pmd_present(pmd_t pmd) -{ - return ((pmd_val(pmd) & PGD_PRESENT) != 0); -} - -#if 0 /* if PMD takes one word */ -static void sun4c_pmd_clear(pmd_t *pmdp) { *pmdp = __pmd(0); } -#else /* if pmd_t is a longish aggregate */ -static void sun4c_pmd_clear(pmd_t *pmdp) { - memset((void *)pmdp, 0, sizeof(pmd_t)); -} -#endif - -static int sun4c_pgd_none(pgd_t pgd) { return 0; } -static int sun4c_pgd_bad(pgd_t pgd) { return 0; } -static int sun4c_pgd_present(pgd_t pgd) { return 1; } -static void sun4c_pgd_clear(pgd_t * pgdp) { } - -/* - * The following only work if pte_present() is true. - * Undefined behaviour if not.. - */ -static pte_t sun4c_pte_mkwrite(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE); - if (pte_val(pte) & _SUN4C_PAGE_MODIFIED) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); - return pte; -} - -static pte_t sun4c_pte_mkdirty(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED); - if (pte_val(pte) & _SUN4C_PAGE_WRITE) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); - return pte; -} - -static pte_t sun4c_pte_mkyoung(pte_t pte) -{ - pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED); - if (pte_val(pte) & _SUN4C_PAGE_READ) - pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ); - return pte; -} - -/* - * Conversion functions: convert a page and protection to a page entry, - * and a page entry and page directory to the page they refer to. - */ -static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot) -{ - return __pte(page_to_pfn(page) | pgprot_val(pgprot)); -} - -static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot) -{ - return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot)); -} - -static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space) -{ - return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot)); -} - -static unsigned long sun4c_pte_pfn(pte_t pte) -{ - return pte_val(pte) & SUN4C_PFN_MASK; -} - -static pte_t sun4c_pgoff_to_pte(unsigned long pgoff) -{ - return __pte(pgoff | _SUN4C_PAGE_FILE); -} - -static unsigned long sun4c_pte_to_pgoff(pte_t pte) -{ - return pte_val(pte) & ((1UL << PTE_FILE_MAX_BITS) - 1); -} - - -static inline unsigned long sun4c_pmd_page_v(pmd_t pmd) -{ - return (pmd_val(pmd) & PAGE_MASK); -} - -static struct page *sun4c_pmd_page(pmd_t pmd) -{ - return virt_to_page(sun4c_pmd_page_v(pmd)); -} - -static unsigned long sun4c_pgd_page(pgd_t pgd) { return 0; } - -/* to find an entry in a page-table-directory */ -static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address) -{ - return mm->pgd + (address >> SUN4C_PGDIR_SHIFT); -} - -/* Find an entry in the second-level page table.. */ -static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address) -{ - return (pmd_t *) dir; -} - -/* Find an entry in the third-level page table.. */ -pte_t *sun4c_pte_offset_kernel(pmd_t * dir, unsigned long address) -{ - return (pte_t *) sun4c_pmd_page_v(*dir) + - ((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1)); -} - -static unsigned long sun4c_swp_type(swp_entry_t entry) -{ - return (entry.val & SUN4C_SWP_TYPE_MASK); -} - -static unsigned long sun4c_swp_offset(swp_entry_t entry) -{ - return (entry.val >> SUN4C_SWP_OFF_SHIFT) & SUN4C_SWP_OFF_MASK; -} - -static swp_entry_t sun4c_swp_entry(unsigned long type, unsigned long offset) -{ - return (swp_entry_t) { - (offset & SUN4C_SWP_OFF_MASK) << SUN4C_SWP_OFF_SHIFT - | (type & SUN4C_SWP_TYPE_MASK) }; -} - -static void sun4c_free_pte_slow(pte_t *pte) -{ - free_page((unsigned long)pte); -} - -static void sun4c_free_pgd_slow(pgd_t *pgd) -{ - free_page((unsigned long)pgd); -} - -static pgd_t *sun4c_get_pgd_fast(void) -{ - unsigned long *ret; - - if ((ret = pgd_quicklist) != NULL) { - pgd_quicklist = (unsigned long *)(*ret); - ret[0] = ret[1]; - pgtable_cache_size--; - } else { - pgd_t *init; - - ret = (unsigned long *)__get_free_page(GFP_KERNEL); - memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t)); - init = sun4c_pgd_offset(&init_mm, 0); - memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, - (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); - } - return (pgd_t *)ret; -} - -static void sun4c_free_pgd_fast(pgd_t *pgd) -{ - *(unsigned long *)pgd = (unsigned long) pgd_quicklist; - pgd_quicklist = (unsigned long *) pgd; - pgtable_cache_size++; -} - - -static inline pte_t * -sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address) -{ - unsigned long *ret; - - if ((ret = (unsigned long *)pte_quicklist) != NULL) { - pte_quicklist = (unsigned long *)(*ret); - ret[0] = ret[1]; - pgtable_cache_size--; - } - return (pte_t *)ret; -} - -static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) -{ - pte_t *pte; - - if ((pte = sun4c_pte_alloc_one_fast(mm, address)) != NULL) - return pte; - - pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); - return pte; -} - -static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address) -{ - pte_t *pte; - struct page *page; - - pte = sun4c_pte_alloc_one_kernel(mm, address); - if (pte == NULL) - return NULL; - page = virt_to_page(pte); - pgtable_page_ctor(page); - return page; -} - -static inline void sun4c_free_pte_fast(pte_t *pte) -{ - *(unsigned long *)pte = (unsigned long) pte_quicklist; - pte_quicklist = (unsigned long *) pte; - pgtable_cache_size++; -} - -static void sun4c_pte_free(pgtable_t pte) -{ - pgtable_page_dtor(pte); - sun4c_free_pte_fast(page_address(pte)); -} - -/* - * allocating and freeing a pmd is trivial: the 1-entry pmd is - * inside the pgd, so has no extra memory associated with it. - */ -static pmd_t *sun4c_pmd_alloc_one(struct mm_struct *mm, unsigned long address) -{ - BUG(); - return NULL; -} - -static void sun4c_free_pmd_fast(pmd_t * pmd) { } - -static void sun4c_check_pgt_cache(int low, int high) -{ - if (pgtable_cache_size > high) { - do { - if (pgd_quicklist) - sun4c_free_pgd_slow(sun4c_get_pgd_fast()); - if (pte_quicklist) - sun4c_free_pte_slow(sun4c_pte_alloc_one_fast(NULL, 0)); - } while (pgtable_cache_size > low); - } -} - -/* An experiment, turn off by default for now... -DaveM */ -#define SUN4C_PRELOAD_PSEG - -void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) -{ - unsigned long flags; - int pseg; - - if (vma->vm_mm->context == NO_CONTEXT) - return; - - local_irq_save(flags); - address &= PAGE_MASK; - if ((pseg = sun4c_get_segmap(address)) == invalid_segment) { - struct sun4c_mmu_entry *entry = sun4c_user_strategy(); - struct mm_struct *mm = vma->vm_mm; - unsigned long start, end; - - entry->vaddr = start = (address & SUN4C_REAL_PGDIR_MASK); - entry->ctx = mm->context; - add_ring_ordered(sun4c_context_ring + mm->context, entry); - sun4c_put_segmap(entry->vaddr, entry->pseg); - end = start + SUN4C_REAL_PGDIR_SIZE; - while (start < end) { -#ifdef SUN4C_PRELOAD_PSEG - pgd_t *pgdp = sun4c_pgd_offset(mm, start); - pte_t *ptep; - - if (!pgdp) - goto no_mapping; - ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, start); - if (!ptep || !(pte_val(*ptep) & _SUN4C_PAGE_PRESENT)) - goto no_mapping; - sun4c_put_pte(start, pte_val(*ptep)); - goto next; - - no_mapping: -#endif - sun4c_put_pte(start, 0); -#ifdef SUN4C_PRELOAD_PSEG - next: -#endif - start += PAGE_SIZE; - } -#ifndef SUN4C_PRELOAD_PSEG - sun4c_put_pte(address, pte_val(*ptep)); -#endif - local_irq_restore(flags); - return; - } else { - struct sun4c_mmu_entry *entry = &mmu_entry_pool[pseg]; - - remove_lru(entry); - add_lru(entry); - } - - sun4c_put_pte(address, pte_val(*ptep)); - local_irq_restore(flags); -} - -extern void sparc_context_init(int); -extern unsigned long bootmem_init(unsigned long *pages_avail); -extern unsigned long last_valid_pfn; - -void __init sun4c_paging_init(void) -{ - int i, cnt; - unsigned long kernel_end, vaddr; - extern struct resource sparc_iomap; - unsigned long end_pfn, pages_avail; - - kernel_end = (unsigned long) &_end; - kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end); - - pages_avail = 0; - last_valid_pfn = bootmem_init(&pages_avail); - end_pfn = last_valid_pfn; - - sun4c_probe_mmu(); - invalid_segment = (num_segmaps - 1); - sun4c_init_mmu_entry_pool(); - sun4c_init_rings(); - sun4c_init_map_kernelprom(kernel_end); - sun4c_init_clean_mmu(kernel_end); - sun4c_init_fill_kernel_ring(SUN4C_KERNEL_BUCKETS); - sun4c_init_lock_area(sparc_iomap.start, IOBASE_END); - sun4c_init_lock_area(DVMA_VADDR, DVMA_END); - sun4c_init_lock_areas(); - sun4c_init_fill_user_ring(); - - sun4c_set_context(0); - memset(swapper_pg_dir, 0, PAGE_SIZE); - memset(pg0, 0, PAGE_SIZE); - memset(pg1, 0, PAGE_SIZE); - memset(pg2, 0, PAGE_SIZE); - memset(pg3, 0, PAGE_SIZE); - - /* Save work later. */ - vaddr = VMALLOC_START; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg0); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg1); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg2); - vaddr += SUN4C_PGDIR_SIZE; - swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg3); - sun4c_init_ss2_cache_bug(); - sparc_context_init(num_contexts); - - { - unsigned long zones_size[MAX_NR_ZONES]; - unsigned long zholes_size[MAX_NR_ZONES]; - unsigned long npages; - int znum; - - for (znum = 0; znum < MAX_NR_ZONES; znum++) - zones_size[znum] = zholes_size[znum] = 0; - - npages = max_low_pfn - pfn_base; - - zones_size[ZONE_DMA] = npages; - zholes_size[ZONE_DMA] = npages - pages_avail; - - npages = highend_pfn - max_low_pfn; - zones_size[ZONE_HIGHMEM] = npages; - zholes_size[ZONE_HIGHMEM] = npages - calc_highpages(); - - free_area_init_node(0, zones_size, pfn_base, zholes_size); - } - - cnt = 0; - for (i = 0; i < num_segmaps; i++) - if (mmu_entry_pool[i].locked) - cnt++; - - max_user_taken_entries = num_segmaps - cnt - 40 - 1; - - printk("SUN4C: %d mmu entries for the kernel\n", cnt); -} - -static pgprot_t sun4c_pgprot_noncached(pgprot_t prot) -{ - prot |= __pgprot(_SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE); - - return prot; -} - -/* Load up routines and constants for sun4c mmu */ -void __init ld_mmu_sun4c(void) -{ - extern void ___xchg32_sun4c(void); - - printk("Loading sun4c MMU routines\n"); - - /* First the constants */ - BTFIXUPSET_SIMM13(pgdir_shift, SUN4C_PGDIR_SHIFT); - BTFIXUPSET_SETHI(pgdir_size, SUN4C_PGDIR_SIZE); - BTFIXUPSET_SETHI(pgdir_mask, SUN4C_PGDIR_MASK); - - BTFIXUPSET_SIMM13(ptrs_per_pmd, SUN4C_PTRS_PER_PMD); - BTFIXUPSET_SIMM13(ptrs_per_pgd, SUN4C_PTRS_PER_PGD); - BTFIXUPSET_SIMM13(user_ptrs_per_pgd, KERNBASE / SUN4C_PGDIR_SIZE); - - BTFIXUPSET_INT(page_none, pgprot_val(SUN4C_PAGE_NONE)); - PAGE_SHARED = pgprot_val(SUN4C_PAGE_SHARED); - BTFIXUPSET_INT(page_copy, pgprot_val(SUN4C_PAGE_COPY)); - BTFIXUPSET_INT(page_readonly, pgprot_val(SUN4C_PAGE_READONLY)); - BTFIXUPSET_INT(page_kernel, pgprot_val(SUN4C_PAGE_KERNEL)); - page_kernel = pgprot_val(SUN4C_PAGE_KERNEL); - - /* Functions */ - BTFIXUPSET_CALL(pgprot_noncached, sun4c_pgprot_noncached, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4c, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(do_check_pgt_cache, sun4c_check_pgt_cache, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_cache_all, sun4c_flush_cache_all, BTFIXUPCALL_NORM); - - if (sun4c_vacinfo.do_hwflushes) { - BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_hw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_hw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_hw, BTFIXUPCALL_NORM); - } else { - BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_sw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_sw, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_sw, BTFIXUPCALL_NORM); - } - - BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(flush_tlb_all, sun4c_flush_tlb_all, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(flush_sig_insns, sun4c_flush_sig_insns, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(set_pte, sun4c_set_pte, BTFIXUPCALL_STO1O0); - - BTFIXUPSET_CALL(pte_pfn, sun4c_pte_pfn, BTFIXUPCALL_NORM); -#if 0 /* PAGE_SHIFT <= 12 */ /* Eek. Investigate. XXX */ - BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_ANDNINT(PAGE_SIZE - 1)); -#else - BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_NORM); -#endif - BTFIXUPSET_CALL(pmd_set, sun4c_pmd_set, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_populate, sun4c_pmd_populate, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_present, sun4c_pte_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_clear, sun4c_pte_clear, BTFIXUPCALL_STG0O0); - - BTFIXUPSET_CALL(pmd_bad, sun4c_pmd_bad, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_present, sun4c_pmd_present, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pmd_clear, sun4c_pmd_clear, BTFIXUPCALL_STG0O0); - - BTFIXUPSET_CALL(pgd_none, sun4c_pgd_none, BTFIXUPCALL_RETINT(0)); - BTFIXUPSET_CALL(pgd_bad, sun4c_pgd_bad, BTFIXUPCALL_RETINT(0)); - BTFIXUPSET_CALL(pgd_present, sun4c_pgd_present, BTFIXUPCALL_RETINT(1)); - BTFIXUPSET_CALL(pgd_clear, sun4c_pgd_clear, BTFIXUPCALL_NOP); - - BTFIXUPSET_CALL(mk_pte, sun4c_mk_pte, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_phys, sun4c_mk_pte_phys, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mk_pte_io, sun4c_mk_pte_io, BTFIXUPCALL_NORM); - - BTFIXUPSET_INT(pte_modify_mask, _SUN4C_PAGE_CHG_MASK); - BTFIXUPSET_CALL(pmd_offset, sun4c_pmd_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_offset_kernel, sun4c_pte_offset_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pte_fast, sun4c_free_pte_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_free, sun4c_pte_free, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one_kernel, sun4c_pte_alloc_one_kernel, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_alloc_one, sun4c_pte_alloc_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_pmd_fast, sun4c_free_pmd_fast, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(pmd_alloc_one, sun4c_pmd_alloc_one, BTFIXUPCALL_RETO0); - BTFIXUPSET_CALL(free_pgd_fast, sun4c_free_pgd_fast, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(get_pgd_fast, sun4c_get_pgd_fast, BTFIXUPCALL_NORM); - - BTFIXUPSET_HALF(pte_writei, _SUN4C_PAGE_WRITE); - BTFIXUPSET_HALF(pte_dirtyi, _SUN4C_PAGE_MODIFIED); - BTFIXUPSET_HALF(pte_youngi, _SUN4C_PAGE_ACCESSED); - BTFIXUPSET_HALF(pte_filei, _SUN4C_PAGE_FILE); - BTFIXUPSET_HALF(pte_wrprotecti, _SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE); - BTFIXUPSET_HALF(pte_mkcleani, _SUN4C_PAGE_MODIFIED|_SUN4C_PAGE_SILENT_WRITE); - BTFIXUPSET_HALF(pte_mkoldi, _SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_SILENT_READ); - BTFIXUPSET_CALL(pte_mkwrite, sun4c_pte_mkwrite, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_mkdirty, sun4c_pte_mkdirty, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pte_mkyoung, sun4c_pte_mkyoung, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(update_mmu_cache, sun4c_update_mmu_cache, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(pte_to_pgoff, sun4c_pte_to_pgoff, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(pgoff_to_pte, sun4c_pgoff_to_pte, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_lockarea, sun4c_lockarea, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unlockarea, sun4c_unlockarea, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_get_scsi_one, sun4c_get_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_get_scsi_sgl, sun4c_get_scsi_sgl, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_one, sun4c_release_scsi_one, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_release_scsi_sgl, sun4c_release_scsi_sgl, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_map_dma_area, sun4c_map_dma_area, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(mmu_unmap_dma_area, sun4c_unmap_dma_area, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(sparc_mapiorange, sun4c_mapiorange, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(sparc_unmapiorange, sun4c_unmapiorange, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(__swp_type, sun4c_swp_type, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_offset, sun4c_swp_offset, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(__swp_entry, sun4c_swp_entry, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(alloc_thread_info, sun4c_alloc_thread_info, BTFIXUPCALL_NORM); - BTFIXUPSET_CALL(free_thread_info, sun4c_free_thread_info, BTFIXUPCALL_NORM); - - BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM); - - /* These should _never_ get called with two level tables. */ - BTFIXUPSET_CALL(pgd_set, sun4c_pgd_set, BTFIXUPCALL_NOP); - BTFIXUPSET_CALL(pgd_page_vaddr, sun4c_pgd_page, BTFIXUPCALL_RETO0); -} diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S index c801c3953a0..5d2b88d3942 100644 --- a/arch/sparc/mm/swift.S +++ b/arch/sparc/mm/swift.S @@ -105,7 +105,7 @@ swift_flush_cache_mm_out: .globl swift_flush_cache_range swift_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 sub %o2, %o1, %o2 sethi %hi(4096), %o3 cmp %o2, %o3 @@ -116,7 +116,7 @@ swift_flush_cache_range: .globl swift_flush_cache_page swift_flush_cache_page: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 70: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -219,7 +219,7 @@ swift_flush_sig_insns: .globl swift_flush_tlb_range .globl swift_flush_tlb_all swift_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 swift_flush_tlb_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -233,7 +233,7 @@ swift_flush_tlb_all_out: .globl swift_flush_tlb_page swift_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index d8f21e24a82..b89aba217e3 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -4,7 +4,6 @@ */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/percpu.h> #include <linux/mm.h> #include <linux/swap.h> @@ -19,39 +18,92 @@ /* Heavily inspired by the ppc64 code. */ -DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); +static DEFINE_PER_CPU(struct tlb_batch, tlb_batch); void flush_tlb_pending(void) { - struct mmu_gather *mp = &get_cpu_var(mmu_gathers); + struct tlb_batch *tb = &get_cpu_var(tlb_batch); + struct mm_struct *mm = tb->mm; - if (mp->tlb_nr) { - flush_tsb_user(mp); + if (!tb->tlb_nr) + goto out; - if (CTX_VALID(mp->mm->context)) { + flush_tsb_user(tb); + + if (CTX_VALID(mm->context)) { + if (tb->tlb_nr == 1) { + global_flush_tlb_page(mm, tb->vaddrs[0]); + } else { #ifdef CONFIG_SMP - smp_flush_tlb_pending(mp->mm, mp->tlb_nr, - &mp->vaddrs[0]); + smp_flush_tlb_pending(tb->mm, tb->tlb_nr, + &tb->vaddrs[0]); #else - __flush_tlb_pending(CTX_HWBITS(mp->mm->context), - mp->tlb_nr, &mp->vaddrs[0]); + __flush_tlb_pending(CTX_HWBITS(tb->mm->context), + tb->tlb_nr, &tb->vaddrs[0]); #endif } - mp->tlb_nr = 0; } - put_cpu_var(mmu_gathers); + tb->tlb_nr = 0; + +out: + put_cpu_var(tlb_batch); +} + +void arch_enter_lazy_mmu_mode(void) +{ + struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + + tb->active = 1; } -void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t orig) +void arch_leave_lazy_mmu_mode(void) { - struct mmu_gather *mp = &__get_cpu_var(mmu_gathers); + struct tlb_batch *tb = &__get_cpu_var(tlb_batch); + + if (tb->tlb_nr) + flush_tlb_pending(); + tb->active = 0; +} + +static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, + bool exec) +{ + struct tlb_batch *tb = &get_cpu_var(tlb_batch); unsigned long nr; vaddr &= PAGE_MASK; - if (pte_exec(orig)) + if (exec) vaddr |= 0x1UL; + nr = tb->tlb_nr; + + if (unlikely(nr != 0 && mm != tb->mm)) { + flush_tlb_pending(); + nr = 0; + } + + if (!tb->active) { + flush_tsb_user_page(mm, vaddr); + global_flush_tlb_page(mm, vaddr); + goto out; + } + + if (nr == 0) + tb->mm = mm; + + tb->vaddrs[nr] = vaddr; + tb->tlb_nr = ++nr; + if (nr >= TLB_BATCH_NR) + flush_tlb_pending(); + +out: + put_cpu_var(tlb_batch); +} + +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm) +{ if (tlb_type != hypervisor && pte_dirty(orig)) { unsigned long paddr, pfn = pte_pfn(orig); @@ -76,22 +128,116 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, pte_t *ptep, pte_t } no_cache_flush: + if (!fullmm) + tlb_batch_add_one(mm, vaddr, pte_exec(orig)); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, + pmd_t pmd) +{ + unsigned long end; + pte_t *pte; - if (mp->fullmm) + pte = pte_offset_map(&pmd, vaddr); + end = vaddr + HPAGE_SIZE; + while (vaddr < end) { + if (pte_val(*pte) & _PAGE_VALID) { + bool exec = pte_exec(*pte); + + tlb_batch_add_one(mm, vaddr, exec); + } + pte++; + vaddr += PAGE_SIZE; + } + pte_unmap(pte); +} + +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + pmd_t orig = *pmdp; + + *pmdp = pmd; + + if (mm == &init_mm) return; - nr = mp->tlb_nr; + if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { + if (pmd_val(pmd) & _PAGE_PMD_HUGE) + mm->context.huge_pte_count++; + else + mm->context.huge_pte_count--; + + /* Do not try to allocate the TSB hash table if we + * don't have one already. We have various locks held + * and thus we'll end up doing a GFP_KERNEL allocation + * in an atomic context. + * + * Instead, we let the first TLB miss on a hugepage + * take care of this. + */ + } - if (unlikely(nr != 0 && mm != mp->mm)) { - flush_tlb_pending(); - nr = 0; + if (!pmd_none(orig)) { + addr &= HPAGE_MASK; + if (pmd_trans_huge(orig)) { + pte_t orig_pte = __pte(pmd_val(orig)); + bool exec = pte_exec(orig_pte); + + tlb_batch_add_one(mm, addr, exec); + tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec); + } else { + tlb_batch_pmd_scan(mm, addr, orig); + } } +} - if (nr == 0) - mp->mm = mm; +void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp) +{ + pmd_t entry = *pmdp; - mp->vaddrs[nr] = vaddr; - mp->tlb_nr = ++nr; - if (nr >= TLB_BATCH_NR) - flush_tlb_pending(); + pmd_val(entry) &= ~_PAGE_VALID; + + set_pmd_at(vma->vm_mm, address, pmdp, entry); + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); +} + +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable) +{ + struct list_head *lh = (struct list_head *) pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + if (!pmd_huge_pte(mm, pmdp)) + INIT_LIST_HEAD(lh); + else + list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); + pmd_huge_pte(mm, pmdp) = pgtable; +} + +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) +{ + struct list_head *lh; + pgtable_t pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + pgtable = pmd_huge_pte(mm, pmdp); + lh = (struct list_head *) pgtable; + if (list_empty(lh)) + pmd_huge_pte(mm, pmdp) = NULL; + else { + pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; + list_del(lh); + } + pte_val(pgtable[0]) = 0; + pte_val(pgtable[1]) = 0; + + return pgtable; } +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index 101d7c82870..a06576683c3 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -6,13 +6,12 @@ #include <linux/kernel.h> #include <linux/preempt.h> #include <linux/slab.h> -#include <asm/system.h> #include <asm/page.h> -#include <asm/tlbflush.h> -#include <asm/tlb.h> -#include <asm/mmu_context.h> #include <asm/pgtable.h> +#include <asm/mmu_context.h> +#include <asm/setup.h> #include <asm/tsb.h> +#include <asm/tlb.h> #include <asm/oplib.h> extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; @@ -47,27 +46,56 @@ void flush_tsb_kernel_range(unsigned long start, unsigned long end) } } -static void __flush_tsb_one(struct mmu_gather *mp, unsigned long hash_shift, unsigned long tsb, unsigned long nentries) +static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries) +{ + unsigned long tag, ent, hash; + + v &= ~0x1UL; + hash = tsb_hash(v, hash_shift, nentries); + ent = tsb + (hash * sizeof(struct tsb)); + tag = (v >> 22UL); + + tsb_flush(ent, tag); +} + +static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries) { unsigned long i; - for (i = 0; i < mp->tlb_nr; i++) { - unsigned long v = mp->vaddrs[i]; - unsigned long tag, ent, hash; + for (i = 0; i < tb->tlb_nr; i++) + __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); +} + +void flush_tsb_user(struct tlb_batch *tb) +{ + struct mm_struct *mm = tb->mm; + unsigned long nentries, base, flags; - v &= ~0x1UL; + spin_lock_irqsave(&mm->context.lock, flags); - hash = tsb_hash(v, hash_shift, nentries); - ent = tsb + (hash * sizeof(struct tsb)); - tag = (v >> 22UL); + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); - tsb_flush(ent, tag); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; + nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries); } +#endif + spin_unlock_irqrestore(&mm->context.lock, flags); } -void flush_tsb_user(struct mmu_gather *mp) +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr) { - struct mm_struct *mm = mp->mm; unsigned long nentries, base, flags; spin_lock_irqsave(&mm->context.lock, flags); @@ -76,43 +104,26 @@ void flush_tsb_user(struct mmu_gather *mp) nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(mp, PAGE_SHIFT, base, nentries); + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, nentries); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; if (tlb_type == cheetah_plus || tlb_type == hypervisor) base = __pa(base); - __flush_tsb_one(mp, HPAGE_SHIFT, base, nentries); + __flush_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, nentries); } #endif spin_unlock_irqrestore(&mm->context.lock, flags); } -#if defined(CONFIG_SPARC64_PAGE_SIZE_8KB) #define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K #define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K -#elif defined(CONFIG_SPARC64_PAGE_SIZE_64KB) -#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_64K -#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_64K -#else -#error Broken base page size setting... -#endif -#ifdef CONFIG_HUGETLB_PAGE -#if defined(CONFIG_HUGETLB_PAGE_SIZE_64K) -#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_64K -#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_64K -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_512K) -#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_512K -#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_512K -#elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB) +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB #define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB -#else -#error Broken huge page size setting... -#endif #endif static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) @@ -123,7 +134,19 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign mm->context.tsb_block[tsb_idx].tsb_nentries = tsb_bytes / sizeof(struct tsb); - base = TSBMAP_BASE; + switch (tsb_idx) { + case MM_TSB_BASE: + base = TSBMAP_8K_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + base = TSBMAP_4M_BASE; + break; +#endif + default: + BUG(); + } + tte = pgprot_val(PAGE_KERNEL_LOCKED); tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); @@ -179,7 +202,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", current->comm, current->pid, tsb_bytes); do_exit(SIGSEGV); - }; + } tte |= pte_sz_bits(page_sz); if (tlb_type == cheetah_plus || tlb_type == hypervisor) { @@ -207,14 +230,14 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign case MM_TSB_BASE: hp->pgsz_idx = HV_PGSZ_IDX_BASE; break; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) case MM_TSB_HUGE: hp->pgsz_idx = HV_PGSZ_IDX_HUGE; break; #endif default: BUG(); - }; + } hp->assoc = 1; hp->num_ttes = tsb_bytes / 16; hp->ctx_idx = 0; @@ -222,19 +245,21 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign case MM_TSB_BASE: hp->pgsz_mask = HV_PGSZ_MASK_BASE; break; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) case MM_TSB_HUGE: hp->pgsz_mask = HV_PGSZ_MASK_HUGE; break; #endif default: BUG(); - }; + } hp->tsb_base = tsb_paddr; hp->resv = 0; } } +struct kmem_cache *pgtable_cache __read_mostly; + static struct kmem_cache *tsb_caches[8] __read_mostly; static const char *tsb_cache_names[8] = { @@ -252,7 +277,16 @@ void __init pgtable_cache_init(void) { unsigned long i; - for (i = 0; i < 8; i++) { + pgtable_cache = kmem_cache_create("pgtable_cache", + PAGE_SIZE, PAGE_SIZE, + 0, + _clear_page); + if (!pgtable_cache) { + prom_printf("pgtable_cache_init(): Could not create!\n"); + prom_halt(); + } + + for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) { unsigned long size = 8192 << i; const char *name = tsb_cache_names[i]; @@ -320,7 +354,7 @@ void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) retry_tsb_alloc: gfp_flags = GFP_KERNEL; if (new_size > (PAGE_SIZE * 2)) - gfp_flags = __GFP_NOWARN | __GFP_NORETRY; + gfp_flags |= __GFP_NOWARN | __GFP_NORETRY; new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index], gfp_flags, numa_node_id()); @@ -433,7 +467,7 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) unsigned long huge_pte_count; #endif unsigned int i; @@ -442,7 +476,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) /* We reset it to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. @@ -463,7 +497,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) */ tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); -#ifdef CONFIG_HUGETLB_PAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) if (unlikely(huge_pte_count)) tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); #endif diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S index 4e55e8f7664..bf10a345fa8 100644 --- a/arch/sparc/mm/tsunami.S +++ b/arch/sparc/mm/tsunami.S @@ -24,7 +24,7 @@ /* Sliiick... */ tsunami_flush_cache_page: tsunami_flush_cache_range: - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 tsunami_flush_cache_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -46,7 +46,7 @@ tsunami_flush_sig_insns: /* More slick stuff... */ tsunami_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 tsunami_flush_tlb_mm: ld [%o0 + AOFF_mm_context], %g2 cmp %g2, -1 @@ -65,7 +65,7 @@ tsunami_flush_tlb_out: /* This one can be done in a fine grained manner... */ tsunami_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 andn %o1, (PAGE_SIZE - 1), %o1 diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S index b57a5942ba6..b4f4733abc6 100644 --- a/arch/sparc/mm/ultra.S +++ b/arch/sparc/mm/ultra.S @@ -53,6 +53,33 @@ __flush_tlb_mm: /* 18 insns */ nop .align 32 + .globl __flush_tlb_page +__flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, %pstate + mov SECONDARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + andn %o1, 1, %o3 + be,pn %icc, 1f + or %o3, 0x10, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + retl + wrpr %g7, 0x0, %pstate + nop + nop + nop + nop + + .align 32 .globl __flush_tlb_pending __flush_tlb_pending: /* 26 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ @@ -126,10 +153,10 @@ __spitfire_flush_tlb_mm_slow: .globl __flush_icache_page __flush_icache_page: /* %o0 = phys_page */ srlx %o0, PAGE_SHIFT, %o0 - sethi %uhi(PAGE_OFFSET), %g1 + sethi %hi(PAGE_OFFSET), %g1 sllx %o0, PAGE_SHIFT, %o0 sethi %hi(PAGE_SIZE), %g2 - sllx %g1, 32, %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 add %o0, %g1, %o0 1: subcc %g2, 32, %g2 bne,pt %icc, 1b @@ -151,8 +178,8 @@ __flush_icache_page: /* %o0 = phys_page */ .align 64 .globl __flush_dcache_page __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 ! physical address srlx %o0, 11, %o0 ! make D-cache TAG sethi %hi(1 << 14), %o2 ! D-cache size @@ -203,6 +230,31 @@ __cheetah_flush_tlb_mm: /* 19 insns */ retl wrpr %g7, 0x0, %pstate +__cheetah_flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 + sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 + or %o0, %o3, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + be,pn %icc, 1f + andn %o1, 1, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + wrpr %g0, 0, %tl + retl + wrpr %g7, 0x0, %pstate + __cheetah_flush_tlb_pending: /* 27 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ rdpr %pstate, %g7 @@ -235,8 +287,8 @@ __cheetah_flush_tlb_pending: /* 27 insns */ #ifdef DCACHE_ALIASING_POSSIBLE __cheetah_flush_dcache_page: /* 11 insns */ - sethi %uhi(PAGE_OFFSET), %g1 - sllx %g1, 32, %g1 + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 sub %o0, %g1, %o0 sethi %hi(PAGE_SIZE), %o4 1: subcc %o4, (1 << 5), %o4 @@ -269,6 +321,20 @@ __hypervisor_flush_tlb_mm: /* 10 insns */ retl nop +__hypervisor_flush_tlb_page: /* 11 insns */ + /* %o0 = context, %o1 = vaddr */ + mov %o0, %g2 + mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ + mov %g2, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, __hypervisor_tlb_tl0_error + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + retl + nop + __hypervisor_flush_tlb_pending: /* 16 insns */ /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ sllx %o1, 3, %g1 @@ -339,6 +405,13 @@ cheetah_patch_cachetlbops: call tlb_patch_one mov 19, %o2 + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__cheetah_flush_tlb_pending), %o1 @@ -397,10 +470,9 @@ xcall_flush_tlb_mm: /* 21 insns */ nop nop - .globl xcall_flush_tlb_pending -xcall_flush_tlb_pending: /* 21 insns */ - /* %g5=context, %g1=nr, %g7=vaddrs[] */ - sllx %g1, 3, %g1 + .globl xcall_flush_tlb_page +xcall_flush_tlb_page: /* 17 insns */ + /* %g5=context, %g1=vaddr */ mov PRIMARY_CONTEXT, %g4 ldxa [%g4] ASI_DMMU, %g2 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 @@ -408,20 +480,16 @@ xcall_flush_tlb_pending: /* 21 insns */ or %g5, %g4, %g5 mov PRIMARY_CONTEXT, %g4 stxa %g5, [%g4] ASI_DMMU -1: sub %g1, (1 << 3), %g1 - ldx [%g7 + %g1], %g5 - andcc %g5, 0x1, %g0 + andcc %g1, 0x1, %g0 be,pn %icc, 2f - - andn %g5, 0x1, %g5 + andn %g1, 0x1, %g5 stxa %g0, [%g5] ASI_IMMU_DEMAP 2: stxa %g0, [%g5] ASI_DMMU_DEMAP membar #Sync - brnz,pt %g1, 1b - nop stxa %g2, [%g4] ASI_DMMU retry nop + nop .globl xcall_flush_tlb_kernel_range xcall_flush_tlb_kernel_range: /* 25 insns */ @@ -481,8 +549,8 @@ xcall_sync_tick: .globl xcall_fetch_glob_regs xcall_fetch_glob_regs: - sethi %hi(global_reg_snapshot), %g1 - or %g1, %lo(global_reg_snapshot), %g1 + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 __GET_CPUID(%g2) sllx %g2, 6, %g3 add %g1, %g3, %g1 @@ -495,11 +563,11 @@ xcall_fetch_glob_regs: stx %o7, [%g1 + GR_SNAP_O7] stx %i7, [%g1 + GR_SNAP_I7] /* Don't try this at home kids... */ - rdpr %cwp, %g2 - sub %g2, 1, %g7 + rdpr %cwp, %g3 + sub %g3, 1, %g7 wrpr %g7, %cwp mov %i7, %g7 - wrpr %g2, %cwp + wrpr %g3, %cwp stx %g7, [%g1 + GR_SNAP_RPC] sethi %hi(trap_block), %g7 or %g7, %lo(trap_block), %g7 @@ -509,6 +577,66 @@ xcall_fetch_glob_regs: stx %g3, [%g1 + GR_SNAP_THREAD] retry + .globl xcall_fetch_glob_pmu +xcall_fetch_glob_pmu: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + rd %pic, %g7 + stx %g7, [%g1 + (4 * 8)] + rd %pcr, %g7 + stx %g7, [%g1 + (0 * 8)] + retry + + .globl xcall_fetch_glob_pmu_n4 +xcall_fetch_glob_pmu_n4: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + + ldxa [%g0] ASI_PIC, %g7 + stx %g7, [%g1 + (4 * 8)] + mov 0x08, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (5 * 8)] + mov 0x10, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (6 * 8)] + mov 0x18, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (7 * 8)] + + mov %o0, %g2 + mov %o1, %g3 + mov %o5, %g7 + + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 3, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (3 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 2, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (2 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 1, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (1 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 0, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (0 * 8)] + + mov %g2, %o0 + mov %g3, %o1 + mov %g7, %o5 + + retry + #ifdef DCACHE_ALIASING_POSSIBLE .align 32 .globl xcall_flush_dcache_page_cheetah @@ -596,15 +724,13 @@ __hypervisor_xcall_flush_tlb_mm: /* 21 insns */ membar #Sync retry - .globl __hypervisor_xcall_flush_tlb_pending -__hypervisor_xcall_flush_tlb_pending: /* 21 insns */ - /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4,g6=scratch */ - sllx %g1, 3, %g1 + .globl __hypervisor_xcall_flush_tlb_page +__hypervisor_xcall_flush_tlb_page: /* 17 insns */ + /* %g5=ctx, %g1=vaddr */ mov %o0, %g2 mov %o1, %g3 mov %o2, %g4 -1: sub %g1, (1 << 3), %g1 - ldx [%g7 + %g1], %o0 /* ARG0: virtual address */ + mov %g1, %o0 /* ARG0: virtual address */ mov %g5, %o1 /* ARG1: mmu context */ mov HV_MMU_ALL, %o2 /* ARG2: flags */ srlx %o0, PAGE_SHIFT, %o0 @@ -613,8 +739,6 @@ __hypervisor_xcall_flush_tlb_pending: /* 21 insns */ mov HV_MMU_UNMAP_ADDR_TRAP, %g6 brnz,a,pn %o0, __hypervisor_tlb_xcall_error mov %o0, %g5 - brnz,pt %g1, 1b - nop mov %g2, %o0 mov %g3, %o1 mov %g4, %o2 @@ -697,6 +821,13 @@ hypervisor_patch_cachetlbops: call tlb_patch_one mov 10, %o2 + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__hypervisor_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_flush_tlb_page), %o1 + call tlb_patch_one + mov 11, %o2 + sethi %hi(__flush_tlb_pending), %o0 or %o0, %lo(__flush_tlb_pending), %o0 sethi %hi(__hypervisor_flush_tlb_pending), %o1 @@ -728,12 +859,12 @@ hypervisor_patch_cachetlbops: call tlb_patch_one mov 21, %o2 - sethi %hi(xcall_flush_tlb_pending), %o0 - or %o0, %lo(xcall_flush_tlb_pending), %o0 - sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1 - or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1 + sethi %hi(xcall_flush_tlb_page), %o0 + or %o0, %lo(xcall_flush_tlb_page), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 call tlb_patch_one - mov 21, %o2 + mov 17, %o2 sethi %hi(xcall_flush_tlb_kernel_range), %o0 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S index 6dfcc13d310..852257fcc82 100644 --- a/arch/sparc/mm/viking.S +++ b/arch/sparc/mm/viking.S @@ -14,7 +14,6 @@ #include <asm/page.h> #include <asm/pgtsrmmu.h> #include <asm/viking.h> -#include <asm/btfixup.h> #ifdef CONFIG_SMP .data @@ -109,7 +108,7 @@ viking_mxcc_flush_page: viking_flush_cache_page: viking_flush_cache_range: #ifndef CONFIG_SMP - ld [%o0 + 0x0], %o0 /* XXX vma->vm_mm, GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 #endif viking_flush_cache_mm: #ifndef CONFIG_SMP @@ -149,7 +148,7 @@ viking_flush_tlb_mm: #endif viking_flush_tlb_range: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -174,7 +173,7 @@ viking_flush_tlb_range: #endif viking_flush_tlb_page: - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 mov SRMMU_CTX_REG, %g1 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 @@ -240,7 +239,7 @@ sun4dsmp_flush_tlb_range: tst %g5 bne 3f mov SRMMU_CTX_REG, %g1 - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 sethi %hi(~((1 << SRMMU_PGDIR_SHIFT) - 1)), %o4 @@ -266,7 +265,7 @@ sun4dsmp_flush_tlb_page: tst %g5 bne 2f mov SRMMU_CTX_REG, %g1 - ld [%o0 + 0x00], %o0 /* XXX vma->vm_mm GROSS XXX */ + ld [%o0 + VMA_VM_MM], %o0 ld [%o0 + AOFF_mm_context], %o3 lda [%g1] ASI_M_MMUREGS, %g5 and %o1, PAGE_MASK, %o1 diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile new file mode 100644 index 00000000000..1306a58ac54 --- /dev/null +++ b/arch/sparc/net/Makefile @@ -0,0 +1,4 @@ +# +# Arch-specific network modules +# +obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o diff --git a/arch/sparc/net/bpf_jit.h b/arch/sparc/net/bpf_jit.h new file mode 100644 index 00000000000..33d6b375ff1 --- /dev/null +++ b/arch/sparc/net/bpf_jit.h @@ -0,0 +1,68 @@ +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +/* Conventions: + * %g1 : temporary + * %g2 : Secondary temporary used by SKB data helper stubs. + * %g3 : packet offset passed into SKB data helper stubs. + * %o0 : pointer to skb (first argument given to JIT function) + * %o1 : BPF A accumulator + * %o2 : BPF X accumulator + * %o3 : Holds saved %o7 so we can call helper functions without needing + * to allocate a register window. + * %o4 : skb->len - skb->data_len + * %o5 : skb->data + */ + +#ifndef __ASSEMBLER__ +#define G0 0x00 +#define G1 0x01 +#define G3 0x03 +#define G6 0x06 +#define O0 0x08 +#define O1 0x09 +#define O2 0x0a +#define O3 0x0b +#define O4 0x0c +#define O5 0x0d +#define SP 0x0e +#define O7 0x0f +#define FP 0x1e + +#define r_SKB O0 +#define r_A O1 +#define r_X O2 +#define r_saved_O7 O3 +#define r_HEADLEN O4 +#define r_SKB_DATA O5 +#define r_TMP G1 +#define r_TMP2 G2 +#define r_OFF G3 + +/* assembly code in arch/sparc/net/bpf_jit_asm.S */ +extern u32 bpf_jit_load_word[]; +extern u32 bpf_jit_load_half[]; +extern u32 bpf_jit_load_byte[]; +extern u32 bpf_jit_load_byte_msh[]; +extern u32 bpf_jit_load_word_positive_offset[]; +extern u32 bpf_jit_load_half_positive_offset[]; +extern u32 bpf_jit_load_byte_positive_offset[]; +extern u32 bpf_jit_load_byte_msh_positive_offset[]; +extern u32 bpf_jit_load_word_negative_offset[]; +extern u32 bpf_jit_load_half_negative_offset[]; +extern u32 bpf_jit_load_byte_negative_offset[]; +extern u32 bpf_jit_load_byte_msh_negative_offset[]; + +#else +#define r_SKB %o0 +#define r_A %o1 +#define r_X %o2 +#define r_saved_O7 %o3 +#define r_HEADLEN %o4 +#define r_SKB_DATA %o5 +#define r_TMP %g1 +#define r_TMP2 %g2 +#define r_OFF %g3 +#endif + +#endif /* _BPF_JIT_H */ diff --git a/arch/sparc/net/bpf_jit_asm.S b/arch/sparc/net/bpf_jit_asm.S new file mode 100644 index 00000000000..9d016c7017f --- /dev/null +++ b/arch/sparc/net/bpf_jit_asm.S @@ -0,0 +1,205 @@ +#include <asm/ptrace.h> + +#include "bpf_jit.h" + +#ifdef CONFIG_SPARC64 +#define SAVE_SZ 176 +#define SCRATCH_OFF STACK_BIAS + 128 +#define BE_PTR(label) be,pn %xcc, label +#else +#define SAVE_SZ 96 +#define SCRATCH_OFF 72 +#define BE_PTR(label) be label +#endif + +#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ + + .text + .globl bpf_jit_load_word +bpf_jit_load_word: + cmp r_OFF, 0 + bl bpf_slow_path_word_neg + nop + .globl bpf_jit_load_word_positive_offset +bpf_jit_load_word_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 3 + ble bpf_slow_path_word + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_A +load_word_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x2], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x3], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_A + + .globl bpf_jit_load_half +bpf_jit_load_half: + cmp r_OFF, 0 + bl bpf_slow_path_half_neg + nop + .globl bpf_jit_load_half_positive_offset +bpf_jit_load_half_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 1 + ble bpf_slow_path_half + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_A +load_half_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_A + + .globl bpf_jit_load_byte +bpf_jit_load_byte: + cmp r_OFF, 0 + bl bpf_slow_path_byte_neg + nop + .globl bpf_jit_load_byte_positive_offset +bpf_jit_load_byte_positive_offset: + cmp r_OFF, r_HEADLEN + bge bpf_slow_path_byte + nop + retl + ldub [r_SKB_DATA + r_OFF], r_A + + .globl bpf_jit_load_byte_msh +bpf_jit_load_byte_msh: + cmp r_OFF, 0 + bl bpf_slow_path_byte_msh_neg + nop + .globl bpf_jit_load_byte_msh_positive_offset +bpf_jit_load_byte_msh_positive_offset: + cmp r_OFF, r_HEADLEN + bge bpf_slow_path_byte_msh + nop + ldub [r_SKB_DATA + r_OFF], r_OFF + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +#define bpf_slow_path_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov r_OFF, %o1; \ + add %fp, SCRATCH_OFF, %o2; \ + call skb_copy_bits; \ + mov (LEN), %o3; \ + cmp %o0, 0; \ + restore; + +bpf_slow_path_word: + bpf_slow_path_common(4) + bl bpf_error + ld [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_half: + bpf_slow_path_common(2) + bl bpf_error + lduh [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_byte: + bpf_slow_path_common(1) + bl bpf_error + ldub [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_byte_msh: + bpf_slow_path_common(1) + bl bpf_error + ldub [%sp + SCRATCH_OFF], r_A + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +#define bpf_negative_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov r_OFF, %o1; \ + call bpf_internal_load_pointer_neg_helper; \ + mov (LEN), %o2; \ + mov %o0, r_TMP; \ + cmp %o0, 0; \ + BE_PTR(bpf_error); \ + restore; + +bpf_slow_path_word_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_word_negative_offset +bpf_jit_load_word_negative_offset: + bpf_negative_common(4) + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_A + +bpf_slow_path_half_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_half_negative_offset +bpf_jit_load_half_negative_offset: + bpf_negative_common(2) + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_A + +bpf_slow_path_byte_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_byte_negative_offset +bpf_jit_load_byte_negative_offset: + bpf_negative_common(1) + retl + ldub [r_TMP], r_A + +bpf_slow_path_byte_msh_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_byte_msh_negative_offset +bpf_jit_load_byte_msh_negative_offset: + bpf_negative_common(1) + ldub [r_TMP], r_OFF + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +bpf_error: + /* Make the JIT program return zero. The JIT epilogue + * stores away the original %o7 into r_saved_O7. The + * normal leaf function return is to use "retl" which + * would evalute to "jmpl %o7 + 8, %g0" but we want to + * use the saved value thus the sequence you see here. + */ + jmpl r_saved_O7 + 8, %g0 + clr %o0 diff --git a/arch/sparc/net/bpf_jit_comp.c b/arch/sparc/net/bpf_jit_comp.c new file mode 100644 index 00000000000..892a102671a --- /dev/null +++ b/arch/sparc/net/bpf_jit_comp.c @@ -0,0 +1,816 @@ +#include <linux/moduleloader.h> +#include <linux/workqueue.h> +#include <linux/netdevice.h> +#include <linux/filter.h> +#include <linux/cache.h> +#include <linux/if_vlan.h> + +#include <asm/cacheflush.h> +#include <asm/ptrace.h> + +#include "bpf_jit.h" + +int bpf_jit_enable __read_mostly; + +static inline bool is_simm13(unsigned int value) +{ + return value + 0x1000 < 0x2000; +} + +static void bpf_flush_icache(void *start_, void *end_) +{ +#ifdef CONFIG_SPARC64 + /* Cheetah's I-cache is fully coherent. */ + if (tlb_type == spitfire) { + unsigned long start = (unsigned long) start_; + unsigned long end = (unsigned long) end_; + + start &= ~7UL; + end = (end + 7UL) & ~7UL; + while (start < end) { + flushi(start); + start += 32; + } + } +#endif +} + +#define SEEN_DATAREF 1 /* might call external helpers */ +#define SEEN_XREG 2 /* ebx is used */ +#define SEEN_MEM 4 /* use mem[] for temporary storage */ + +#define S13(X) ((X) & 0x1fff) +#define IMMED 0x00002000 +#define RD(X) ((X) << 25) +#define RS1(X) ((X) << 14) +#define RS2(X) ((X)) +#define OP(X) ((X) << 30) +#define OP2(X) ((X) << 22) +#define OP3(X) ((X) << 19) +#define COND(X) ((X) << 25) +#define F1(X) OP(X) +#define F2(X, Y) (OP(X) | OP2(Y)) +#define F3(X, Y) (OP(X) | OP3(Y)) + +#define CONDN COND(0x0) +#define CONDE COND(0x1) +#define CONDLE COND(0x2) +#define CONDL COND(0x3) +#define CONDLEU COND(0x4) +#define CONDCS COND(0x5) +#define CONDNEG COND(0x6) +#define CONDVC COND(0x7) +#define CONDA COND(0x8) +#define CONDNE COND(0x9) +#define CONDG COND(0xa) +#define CONDGE COND(0xb) +#define CONDGU COND(0xc) +#define CONDCC COND(0xd) +#define CONDPOS COND(0xe) +#define CONDVS COND(0xf) + +#define CONDGEU CONDCC +#define CONDLU CONDCS + +#define WDISP22(X) (((X) >> 2) & 0x3fffff) + +#define BA (F2(0, 2) | CONDA) +#define BGU (F2(0, 2) | CONDGU) +#define BLEU (F2(0, 2) | CONDLEU) +#define BGEU (F2(0, 2) | CONDGEU) +#define BLU (F2(0, 2) | CONDLU) +#define BE (F2(0, 2) | CONDE) +#define BNE (F2(0, 2) | CONDNE) + +#ifdef CONFIG_SPARC64 +#define BE_PTR (F2(0, 1) | CONDE | (2 << 20)) +#else +#define BE_PTR BE +#endif + +#define SETHI(K, REG) \ + (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) +#define OR_LO(K, REG) \ + (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) + +#define ADD F3(2, 0x00) +#define AND F3(2, 0x01) +#define ANDCC F3(2, 0x11) +#define OR F3(2, 0x02) +#define XOR F3(2, 0x03) +#define SUB F3(2, 0x04) +#define SUBCC F3(2, 0x14) +#define MUL F3(2, 0x0a) /* umul */ +#define DIV F3(2, 0x0e) /* udiv */ +#define SLL F3(2, 0x25) +#define SRL F3(2, 0x26) +#define JMPL F3(2, 0x38) +#define CALL F1(1) +#define BR F2(0, 0x01) +#define RD_Y F3(2, 0x28) +#define WR_Y F3(2, 0x30) + +#define LD32 F3(3, 0x00) +#define LD8 F3(3, 0x01) +#define LD16 F3(3, 0x02) +#define LD64 F3(3, 0x0b) +#define ST32 F3(3, 0x04) + +#ifdef CONFIG_SPARC64 +#define LDPTR LD64 +#define BASE_STACKFRAME 176 +#else +#define LDPTR LD32 +#define BASE_STACKFRAME 96 +#endif + +#define LD32I (LD32 | IMMED) +#define LD8I (LD8 | IMMED) +#define LD16I (LD16 | IMMED) +#define LD64I (LD64 | IMMED) +#define LDPTRI (LDPTR | IMMED) +#define ST32I (ST32 | IMMED) + +#define emit_nop() \ +do { \ + *prog++ = SETHI(0, G0); \ +} while (0) + +#define emit_neg() \ +do { /* sub %g0, r_A, r_A */ \ + *prog++ = SUB | RS1(G0) | RS2(r_A) | RD(r_A); \ +} while (0) + +#define emit_reg_move(FROM, TO) \ +do { /* or %g0, FROM, TO */ \ + *prog++ = OR | RS1(G0) | RS2(FROM) | RD(TO); \ +} while (0) + +#define emit_clear(REG) \ +do { /* or %g0, %g0, REG */ \ + *prog++ = OR | RS1(G0) | RS2(G0) | RD(REG); \ +} while (0) + +#define emit_set_const(K, REG) \ +do { /* sethi %hi(K), REG */ \ + *prog++ = SETHI(K, REG); \ + /* or REG, %lo(K), REG */ \ + *prog++ = OR_LO(K, REG); \ +} while (0) + + /* Emit + * + * OP r_A, r_X, r_A + */ +#define emit_alu_X(OPCODE) \ +do { \ + seen |= SEEN_XREG; \ + *prog++ = OPCODE | RS1(r_A) | RS2(r_X) | RD(r_A); \ +} while (0) + + /* Emit either: + * + * OP r_A, K, r_A + * + * or + * + * sethi %hi(K), r_TMP + * or r_TMP, %lo(K), r_TMP + * OP r_A, r_TMP, r_A + * + * depending upon whether K fits in a signed 13-bit + * immediate instruction field. Emit nothing if K + * is zero. + */ +#define emit_alu_K(OPCODE, K) \ +do { \ + if (K) { \ + unsigned int _insn = OPCODE; \ + _insn |= RS1(r_A) | RD(r_A); \ + if (is_simm13(K)) { \ + *prog++ = _insn | IMMED | S13(K); \ + } else { \ + emit_set_const(K, r_TMP); \ + *prog++ = _insn | RS2(r_TMP); \ + } \ + } \ +} while (0) + +#define emit_loadimm(K, DEST) \ +do { \ + if (is_simm13(K)) { \ + /* or %g0, K, DEST */ \ + *prog++ = OR | IMMED | RS1(G0) | S13(K) | RD(DEST); \ + } else { \ + emit_set_const(K, DEST); \ + } \ +} while (0) + +#define emit_loadptr(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(void *)); \ + *prog++ = LDPTRI | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load32(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u32)); \ + *prog++ = LD32I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load16(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u16)); \ + *prog++ = LD16I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define __emit_load8(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + *prog++ = LD8I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load8(BASE, STRUCT, FIELD, DEST) \ +do { BUILD_BUG_ON(FIELD_SIZEOF(STRUCT, FIELD) != sizeof(u8)); \ + __emit_load8(BASE, STRUCT, FIELD, DEST); \ +} while (0) + +#define emit_ldmem(OFF, DEST) \ +do { *prog++ = LD32I | RS1(FP) | S13(-(OFF)) | RD(DEST); \ +} while (0) + +#define emit_stmem(OFF, SRC) \ +do { *prog++ = LD32I | RS1(FP) | S13(-(OFF)) | RD(SRC); \ +} while (0) + +#ifdef CONFIG_SMP +#ifdef CONFIG_SPARC64 +#define emit_load_cpu(REG) \ + emit_load16(G6, struct thread_info, cpu, REG) +#else +#define emit_load_cpu(REG) \ + emit_load32(G6, struct thread_info, cpu, REG) +#endif +#else +#define emit_load_cpu(REG) emit_clear(REG) +#endif + +#define emit_skb_loadptr(FIELD, DEST) \ + emit_loadptr(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load32(FIELD, DEST) \ + emit_load32(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load16(FIELD, DEST) \ + emit_load16(r_SKB, struct sk_buff, FIELD, DEST) +#define __emit_skb_load8(FIELD, DEST) \ + __emit_load8(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load8(FIELD, DEST) \ + emit_load8(r_SKB, struct sk_buff, FIELD, DEST) + +#define emit_jmpl(BASE, IMM_OFF, LREG) \ + *prog++ = (JMPL | IMMED | RS1(BASE) | S13(IMM_OFF) | RD(LREG)) + +#define emit_call(FUNC) \ +do { void *_here = image + addrs[i] - 8; \ + unsigned int _off = (void *)(FUNC) - _here; \ + *prog++ = CALL | (((_off) >> 2) & 0x3fffffff); \ + emit_nop(); \ +} while (0) + +#define emit_branch(BR_OPC, DEST) \ +do { unsigned int _here = addrs[i] - 8; \ + *prog++ = BR_OPC | WDISP22((DEST) - _here); \ +} while (0) + +#define emit_branch_off(BR_OPC, OFF) \ +do { *prog++ = BR_OPC | WDISP22(OFF); \ +} while (0) + +#define emit_jump(DEST) emit_branch(BA, DEST) + +#define emit_read_y(REG) *prog++ = RD_Y | RD(REG) +#define emit_write_y(REG) *prog++ = WR_Y | IMMED | RS1(REG) | S13(0) + +#define emit_cmp(R1, R2) \ + *prog++ = (SUBCC | RS1(R1) | RS2(R2) | RD(G0)) + +#define emit_cmpi(R1, IMM) \ + *prog++ = (SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); + +#define emit_btst(R1, R2) \ + *prog++ = (ANDCC | RS1(R1) | RS2(R2) | RD(G0)) + +#define emit_btsti(R1, IMM) \ + *prog++ = (ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); + +#define emit_sub(R1, R2, R3) \ + *prog++ = (SUB | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_subi(R1, IMM, R3) \ + *prog++ = (SUB | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_add(R1, R2, R3) \ + *prog++ = (ADD | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_addi(R1, IMM, R3) \ + *prog++ = (ADD | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_and(R1, R2, R3) \ + *prog++ = (AND | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_andi(R1, IMM, R3) \ + *prog++ = (AND | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_alloc_stack(SZ) \ + *prog++ = (SUB | IMMED | RS1(SP) | S13(SZ) | RD(SP)) + +#define emit_release_stack(SZ) \ + *prog++ = (ADD | IMMED | RS1(SP) | S13(SZ) | RD(SP)) + +/* A note about branch offset calculations. The addrs[] array, + * indexed by BPF instruction, records the address after all the + * sparc instructions emitted for that BPF instruction. + * + * The most common case is to emit a branch at the end of such + * a code sequence. So this would be two instructions, the + * branch and it's delay slot. + * + * Therefore by default the branch emitters calculate the branch + * offset field as: + * + * destination - (addrs[i] - 8) + * + * This "addrs[i] - 8" is the address of the branch itself or + * what "." would be in assembler notation. The "8" part is + * how we take into consideration the branch and it's delay + * slot mentioned above. + * + * Sometimes we need to emit a branch earlier in the code + * sequence. And in these situations we adjust "destination" + * to accomodate this difference. For example, if we needed + * to emit a branch (and it's delay slot) right before the + * final instruction emitted for a BPF opcode, we'd use + * "destination + 4" instead of just plain "destination" above. + * + * This is why you see all of these funny emit_branch() and + * emit_jump() calls with adjusted offsets. + */ + +void bpf_jit_compile(struct sk_filter *fp) +{ + unsigned int cleanup_addr, proglen, oldproglen = 0; + u32 temp[8], *prog, *func, seen = 0, pass; + const struct sock_filter *filter = fp->insns; + int i, flen = fp->len, pc_ret0 = -1; + unsigned int *addrs; + void *image; + + if (!bpf_jit_enable) + return; + + addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL); + if (addrs == NULL) + return; + + /* Before first pass, make a rough estimation of addrs[] + * each bpf instruction is translated to less than 64 bytes + */ + for (proglen = 0, i = 0; i < flen; i++) { + proglen += 64; + addrs[i] = proglen; + } + cleanup_addr = proglen; /* epilogue address */ + image = NULL; + for (pass = 0; pass < 10; pass++) { + u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen; + + /* no prologue/epilogue for trivial filters (RET something) */ + proglen = 0; + prog = temp; + + /* Prologue */ + if (seen_or_pass0) { + if (seen_or_pass0 & SEEN_MEM) { + unsigned int sz = BASE_STACKFRAME; + sz += BPF_MEMWORDS * sizeof(u32); + emit_alloc_stack(sz); + } + + /* Make sure we dont leek kernel memory. */ + if (seen_or_pass0 & SEEN_XREG) + emit_clear(r_X); + + /* If this filter needs to access skb data, + * load %o4 and %o5 with: + * %o4 = skb->len - skb->data_len + * %o5 = skb->data + * And also back up %o7 into r_saved_O7 so we can + * invoke the stubs using 'call'. + */ + if (seen_or_pass0 & SEEN_DATAREF) { + emit_load32(r_SKB, struct sk_buff, len, r_HEADLEN); + emit_load32(r_SKB, struct sk_buff, data_len, r_TMP); + emit_sub(r_HEADLEN, r_TMP, r_HEADLEN); + emit_loadptr(r_SKB, struct sk_buff, data, r_SKB_DATA); + } + } + emit_reg_move(O7, r_saved_O7); + + switch (filter[0].code) { + case BPF_RET | BPF_K: + case BPF_LD | BPF_W | BPF_LEN: + case BPF_LD | BPF_W | BPF_ABS: + case BPF_LD | BPF_H | BPF_ABS: + case BPF_LD | BPF_B | BPF_ABS: + /* The first instruction sets the A register (or is + * a "RET 'constant'") + */ + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + emit_clear(r_A); /* A = 0 */ + } + + for (i = 0; i < flen; i++) { + unsigned int K = filter[i].k; + unsigned int t_offset; + unsigned int f_offset; + u32 t_op, f_op; + u16 code = bpf_anc_helper(&filter[i]); + int ilen; + + switch (code) { + case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ + emit_alu_X(ADD); + break; + case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ + emit_alu_K(ADD, K); + break; + case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ + emit_alu_X(SUB); + break; + case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ + emit_alu_K(SUB, K); + break; + case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ + emit_alu_X(AND); + break; + case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ + emit_alu_K(AND, K); + break; + case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ + emit_alu_X(OR); + break; + case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ + emit_alu_K(OR, K); + break; + case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ + case BPF_ALU | BPF_XOR | BPF_X: + emit_alu_X(XOR); + break; + case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ + emit_alu_K(XOR, K); + break; + case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X */ + emit_alu_X(SLL); + break; + case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ + emit_alu_K(SLL, K); + break; + case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X */ + emit_alu_X(SRL); + break; + case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K */ + emit_alu_K(SRL, K); + break; + case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ + emit_alu_X(MUL); + break; + case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ + emit_alu_K(MUL, K); + break; + case BPF_ALU | BPF_DIV | BPF_K: /* A /= K with K != 0*/ + if (K == 1) + break; + emit_write_y(G0); +#ifdef CONFIG_SPARC32 + /* The Sparc v8 architecture requires + * three instructions between a %y + * register write and the first use. + */ + emit_nop(); + emit_nop(); + emit_nop(); +#endif + emit_alu_K(DIV, K); + break; + case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ + emit_cmpi(r_X, 0); + if (pc_ret0 > 0) { + t_offset = addrs[pc_ret0 - 1]; +#ifdef CONFIG_SPARC32 + emit_branch(BE, t_offset + 20); +#else + emit_branch(BE, t_offset + 8); +#endif + emit_nop(); /* delay slot */ + } else { + emit_branch_off(BNE, 16); + emit_nop(); +#ifdef CONFIG_SPARC32 + emit_jump(cleanup_addr + 20); +#else + emit_jump(cleanup_addr + 8); +#endif + emit_clear(r_A); + } + emit_write_y(G0); +#ifdef CONFIG_SPARC32 + /* The Sparc v8 architecture requires + * three instructions between a %y + * register write and the first use. + */ + emit_nop(); + emit_nop(); + emit_nop(); +#endif + emit_alu_X(DIV); + break; + case BPF_ALU | BPF_NEG: + emit_neg(); + break; + case BPF_RET | BPF_K: + if (!K) { + if (pc_ret0 == -1) + pc_ret0 = i; + emit_clear(r_A); + } else { + emit_loadimm(K, r_A); + } + /* Fallthrough */ + case BPF_RET | BPF_A: + if (seen_or_pass0) { + if (i != flen - 1) { + emit_jump(cleanup_addr); + emit_nop(); + break; + } + if (seen_or_pass0 & SEEN_MEM) { + unsigned int sz = BASE_STACKFRAME; + sz += BPF_MEMWORDS * sizeof(u32); + emit_release_stack(sz); + } + } + /* jmpl %r_saved_O7 + 8, %g0 */ + emit_jmpl(r_saved_O7, 8, G0); + emit_reg_move(r_A, O0); /* delay slot */ + break; + case BPF_MISC | BPF_TAX: + seen |= SEEN_XREG; + emit_reg_move(r_A, r_X); + break; + case BPF_MISC | BPF_TXA: + seen |= SEEN_XREG; + emit_reg_move(r_X, r_A); + break; + case BPF_ANC | SKF_AD_CPU: + emit_load_cpu(r_A); + break; + case BPF_ANC | SKF_AD_PROTOCOL: + emit_skb_load16(protocol, r_A); + break; +#if 0 + /* GCC won't let us take the address of + * a bit field even though we very much + * know what we are doing here. + */ + case BPF_ANC | SKF_AD_PKTTYPE: + __emit_skb_load8(pkt_type, r_A); + emit_alu_K(SRL, 5); + break; +#endif + case BPF_ANC | SKF_AD_IFINDEX: + emit_skb_loadptr(dev, r_A); + emit_cmpi(r_A, 0); + emit_branch(BE_PTR, cleanup_addr + 4); + emit_nop(); + emit_load32(r_A, struct net_device, ifindex, r_A); + break; + case BPF_ANC | SKF_AD_MARK: + emit_skb_load32(mark, r_A); + break; + case BPF_ANC | SKF_AD_QUEUE: + emit_skb_load16(queue_mapping, r_A); + break; + case BPF_ANC | SKF_AD_HATYPE: + emit_skb_loadptr(dev, r_A); + emit_cmpi(r_A, 0); + emit_branch(BE_PTR, cleanup_addr + 4); + emit_nop(); + emit_load16(r_A, struct net_device, type, r_A); + break; + case BPF_ANC | SKF_AD_RXHASH: + emit_skb_load32(hash, r_A); + break; + case BPF_ANC | SKF_AD_VLAN_TAG: + case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: + emit_skb_load16(vlan_tci, r_A); + if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) { + emit_andi(r_A, VLAN_VID_MASK, r_A); + } else { + emit_loadimm(VLAN_TAG_PRESENT, r_TMP); + emit_and(r_A, r_TMP, r_A); + } + break; + + case BPF_LD | BPF_IMM: + emit_loadimm(K, r_A); + break; + case BPF_LDX | BPF_IMM: + emit_loadimm(K, r_X); + break; + case BPF_LD | BPF_MEM: + emit_ldmem(K * 4, r_A); + break; + case BPF_LDX | BPF_MEM: + emit_ldmem(K * 4, r_X); + break; + case BPF_ST: + emit_stmem(K * 4, r_A); + break; + case BPF_STX: + emit_stmem(K * 4, r_X); + break; + +#define CHOOSE_LOAD_FUNC(K, func) \ + ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) + + case BPF_LD | BPF_W | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_word); +common_load: seen |= SEEN_DATAREF; + emit_loadimm(K, r_OFF); + emit_call(func); + break; + case BPF_LD | BPF_H | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_half); + goto common_load; + case BPF_LD | BPF_B | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte); + goto common_load; + case BPF_LDX | BPF_B | BPF_MSH: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte_msh); + goto common_load; + case BPF_LD | BPF_W | BPF_IND: + func = bpf_jit_load_word; +common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; + if (K) { + if (is_simm13(K)) { + emit_addi(r_X, K, r_OFF); + } else { + emit_loadimm(K, r_TMP); + emit_add(r_X, r_TMP, r_OFF); + } + } else { + emit_reg_move(r_X, r_OFF); + } + emit_call(func); + break; + case BPF_LD | BPF_H | BPF_IND: + func = bpf_jit_load_half; + goto common_load_ind; + case BPF_LD | BPF_B | BPF_IND: + func = bpf_jit_load_byte; + goto common_load_ind; + case BPF_JMP | BPF_JA: + emit_jump(addrs[i + K]); + emit_nop(); + break; + +#define COND_SEL(CODE, TOP, FOP) \ + case CODE: \ + t_op = TOP; \ + f_op = FOP; \ + goto cond_branch + + COND_SEL(BPF_JMP | BPF_JGT | BPF_K, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_K, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_K, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_K, BNE, BE); + COND_SEL(BPF_JMP | BPF_JGT | BPF_X, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_X, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_X, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_X, BNE, BE); + +cond_branch: f_offset = addrs[i + filter[i].jf]; + t_offset = addrs[i + filter[i].jt]; + + /* same targets, can avoid doing the test :) */ + if (filter[i].jt == filter[i].jf) { + emit_jump(t_offset); + emit_nop(); + break; + } + + switch (code) { + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JEQ | BPF_X: + seen |= SEEN_XREG; + emit_cmp(r_A, r_X); + break; + case BPF_JMP | BPF_JSET | BPF_X: + seen |= SEEN_XREG; + emit_btst(r_A, r_X); + break; + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + if (is_simm13(K)) { + emit_cmpi(r_A, K); + } else { + emit_loadimm(K, r_TMP); + emit_cmp(r_A, r_TMP); + } + break; + case BPF_JMP | BPF_JSET | BPF_K: + if (is_simm13(K)) { + emit_btsti(r_A, K); + } else { + emit_loadimm(K, r_TMP); + emit_btst(r_A, r_TMP); + } + break; + } + if (filter[i].jt != 0) { + if (filter[i].jf) + t_offset += 8; + emit_branch(t_op, t_offset); + emit_nop(); /* delay slot */ + if (filter[i].jf) { + emit_jump(f_offset); + emit_nop(); + } + break; + } + emit_branch(f_op, f_offset); + emit_nop(); /* delay slot */ + break; + + default: + /* hmm, too complex filter, give up with jit compiler */ + goto out; + } + ilen = (void *) prog - (void *) temp; + if (image) { + if (unlikely(proglen + ilen > oldproglen)) { + pr_err("bpb_jit_compile fatal error\n"); + kfree(addrs); + module_free(NULL, image); + return; + } + memcpy(image + proglen, temp, ilen); + } + proglen += ilen; + addrs[i] = proglen; + prog = temp; + } + /* last bpf instruction is always a RET : + * use it to give the cleanup instruction(s) addr + */ + cleanup_addr = proglen - 8; /* jmpl; mov r_A,%o0; */ + if (seen_or_pass0 & SEEN_MEM) + cleanup_addr -= 4; /* add %sp, X, %sp; */ + + if (image) { + if (proglen != oldproglen) + pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", + proglen, oldproglen); + break; + } + if (proglen == oldproglen) { + image = module_alloc(proglen); + if (!image) + goto out; + } + oldproglen = proglen; + } + + if (bpf_jit_enable > 1) + bpf_jit_dump(flen, proglen, pass, image); + + if (image) { + bpf_flush_icache(image, image + proglen); + fp->bpf_func = (void *)image; + fp->jited = 1; + } +out: + kfree(addrs); + return; +} + +void bpf_jit_free(struct sk_filter *fp) +{ + if (fp->jited) + module_free(NULL, fp->bpf_func); + kfree(fp); +} diff --git a/arch/sparc/power/Makefile b/arch/sparc/power/Makefile new file mode 100644 index 00000000000..3201ace0ddb --- /dev/null +++ b/arch/sparc/power/Makefile @@ -0,0 +1,3 @@ +# Makefile for Sparc-specific hibernate files. + +obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate_asm.o diff --git a/arch/sparc/power/hibernate.c b/arch/sparc/power/hibernate.c new file mode 100644 index 00000000000..42b0b8ce699 --- /dev/null +++ b/arch/sparc/power/hibernate.c @@ -0,0 +1,42 @@ +/* + * hibernate.c: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#include <linux/mm.h> + +#include <asm/hibernate.h> +#include <asm/visasm.h> +#include <asm/page.h> +#include <asm/tlb.h> + +/* References to section boundaries */ +extern const void __nosave_begin, __nosave_end; + +struct saved_context saved_context; + +/* + * pfn_is_nosave - check if given pfn is in the 'nosave' section + */ + +int pfn_is_nosave(unsigned long pfn) +{ + unsigned long nosave_begin_pfn = PFN_DOWN((unsigned long)&__nosave_begin); + unsigned long nosave_end_pfn = PFN_DOWN((unsigned long)&__nosave_end); + + return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); +} + +void save_processor_state(void) +{ + save_and_clear_fpu(); +} + +void restore_processor_state(void) +{ + struct mm_struct *mm = current->active_mm; + + load_secondary_context(mm); + tsb_context_switch(mm); +} diff --git a/arch/sparc/power/hibernate_asm.S b/arch/sparc/power/hibernate_asm.S new file mode 100644 index 00000000000..79942166df8 --- /dev/null +++ b/arch/sparc/power/hibernate_asm.S @@ -0,0 +1,131 @@ +/* + * hibernate_asm.S: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#include <linux/linkage.h> + +#include <asm/asm-offsets.h> +#include <asm/cpudata.h> +#include <asm/page.h> + +ENTRY(swsusp_arch_suspend) + save %sp, -128, %sp + save %sp, -128, %sp + flushw + + setuw saved_context, %g3 + + /* Save window regs */ + rdpr %cwp, %g2 + stx %g2, [%g3 + SC_REG_CWP] + rdpr %wstate, %g2 + stx %g2, [%g3 + SC_REG_WSTATE] + stx %fp, [%g3 + SC_REG_FP] + + /* Save state regs */ + rdpr %tick, %g2 + stx %g2, [%g3 + SC_REG_TICK] + rdpr %pstate, %g2 + stx %g2, [%g3 + SC_REG_PSTATE] + + /* Save global regs */ + stx %g4, [%g3 + SC_REG_G4] + stx %g5, [%g3 + SC_REG_G5] + stx %g6, [%g3 + SC_REG_G6] + + call swsusp_save + nop + + mov %o0, %i0 + restore + + mov %o0, %i0 + ret + restore + +ENTRY(swsusp_arch_resume) + /* Write restore_pblist to %l0 */ + sethi %hi(restore_pblist), %l0 + ldx [%l0 + %lo(restore_pblist)], %l0 + + call __flush_tlb_all + nop + + /* Write PAGE_OFFSET to %g7 */ + sethi %uhi(PAGE_OFFSET), %g7 + sllx %g7, 32, %g7 + + setuw (PAGE_SIZE-8), %g3 + + /* Use MMU Bypass */ + rd %asi, %g1 + wr %g0, ASI_PHYS_USE_EC, %asi + + ba fill_itlb + nop + +pbe_loop: + cmp %l0, %g0 + be restore_ctx + sub %l0, %g7, %l0 + + ldxa [%l0 ] %asi, %l1 /* address */ + ldxa [%l0 + 8] %asi, %l2 /* orig_address */ + + /* phys addr */ + sub %l1, %g7, %l1 + sub %l2, %g7, %l2 + + mov %g3, %l3 /* PAGE_SIZE-8 */ +copy_loop: + ldxa [%l1 + %l3] ASI_PHYS_USE_EC, %g2 + stxa %g2, [%l2 + %l3] ASI_PHYS_USE_EC + cmp %l3, %g0 + bne copy_loop + sub %l3, 8, %l3 + + /* next pbe */ + ba pbe_loop + ldxa [%l0 + 16] %asi, %l0 + +restore_ctx: + setuw saved_context, %g3 + + /* Restore window regs */ + wrpr %g0, 0, %canrestore + wrpr %g0, 0, %otherwin + wrpr %g0, 6, %cansave + wrpr %g0, 0, %cleanwin + + ldxa [%g3 + SC_REG_CWP] %asi, %g2 + wrpr %g2, %cwp + ldxa [%g3 + SC_REG_WSTATE] %asi, %g2 + wrpr %g2, %wstate + ldxa [%g3 + SC_REG_FP] %asi, %fp + + /* Restore state regs */ + ldxa [%g3 + SC_REG_PSTATE] %asi, %g2 + wrpr %g2, %pstate + ldxa [%g3 + SC_REG_TICK] %asi, %g2 + wrpr %g2, %tick + + /* Restore global regs */ + ldxa [%g3 + SC_REG_G4] %asi, %g4 + ldxa [%g3 + SC_REG_G5] %asi, %g5 + ldxa [%g3 + SC_REG_G6] %asi, %g6 + + wr %g1, %g0, %asi + + restore + restore + + wrpr %g0, 14, %pil + + retl + mov %g0, %o0 + +fill_itlb: + ba pbe_loop + wrpr %g0, 15, %pil diff --git a/arch/sparc/prom/Makefile b/arch/sparc/prom/Makefile index 8287bbe8876..020300b18c0 100644 --- a/arch/sparc/prom/Makefile +++ b/arch/sparc/prom/Makefile @@ -10,7 +10,6 @@ lib-$(CONFIG_SPARC32) += memory.o lib-y += misc_$(BITS).o lib-$(CONFIG_SPARC32) += mp.o lib-$(CONFIG_SPARC32) += ranges.o -lib-$(CONFIG_SPARC32) += segment.o lib-y += console_$(BITS).o lib-y += printf.o lib-y += tree_$(BITS).o diff --git a/arch/sparc/prom/bootstr_32.c b/arch/sparc/prom/bootstr_32.c index f5ec32e0d41..d2b49d2365e 100644 --- a/arch/sparc/prom/bootstr_32.c +++ b/arch/sparc/prom/bootstr_32.c @@ -23,23 +23,25 @@ prom_getbootargs(void) return barg_buf; } - switch(prom_vers) { + switch (prom_vers) { case PROM_V0: cp = barg_buf; /* Start from 1 and go over fd(0,0,0)kernel */ - for(iter = 1; iter < 8; iter++) { + for (iter = 1; iter < 8; iter++) { arg = (*(romvec->pv_v0bootargs))->argv[iter]; if (arg == NULL) break; - while(*arg != 0) { + while (*arg != 0) { /* Leave place for space and null. */ - if(cp >= barg_buf + BARG_LEN-2){ + if (cp >= barg_buf + BARG_LEN - 2) /* We might issue a warning here. */ break; - } *cp++ = *arg++; } *cp++ = ' '; + if (cp >= barg_buf + BARG_LEN - 1) + /* We might issue a warning here. */ + break; } *cp = 0; break; diff --git a/arch/sparc/prom/console_32.c b/arch/sparc/prom/console_32.c index b05e3db5fa6..1cfb50f4cb9 100644 --- a/arch/sparc/prom/console_32.c +++ b/arch/sparc/prom/console_32.c @@ -11,7 +11,6 @@ #include <linux/sched.h> #include <asm/openprom.h> #include <asm/oplib.h> -#include <asm/system.h> #include <linux/string.h> extern void restore_current(void); @@ -38,7 +37,7 @@ static int prom_nbputchar(const char *buf) break; default: break; - }; + } restore_current(); spin_unlock_irqrestore(&prom_lock, flags); return i; /* Ugh, we could spin forever on unsupported proms ;( */ diff --git a/arch/sparc/prom/console_64.c b/arch/sparc/prom/console_64.c index 9de6c8cfe04..f95edcc54fd 100644 --- a/arch/sparc/prom/console_64.c +++ b/arch/sparc/prom/console_64.c @@ -10,7 +10,6 @@ #include <linux/sched.h> #include <asm/openprom.h> #include <asm/oplib.h> -#include <asm/system.h> #include <linux/string.h> static int __prom_console_write_buf(const char *buf, int len) diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c index 0a601b30063..9ac30c2b7db 100644 --- a/arch/sparc/prom/init_32.c +++ b/arch/sparc/prom/init_32.c @@ -27,13 +27,10 @@ EXPORT_SYMBOL(prom_root_node); struct linux_nodeops *prom_nodeops; /* You must call prom_init() before you attempt to use any of the - * routines in the prom library. It returns 0 on success, 1 on - * failure. It gets passed the pointer to the PROM vector. + * routines in the prom library. + * It gets passed the pointer to the PROM vector. */ -extern void prom_meminit(void); -extern void prom_ranges_init(void); - void __init prom_init(struct linux_romvec *rp) { romvec = rp; @@ -53,7 +50,7 @@ void __init prom_init(struct linux_romvec *rp) romvec->pv_romvers); prom_halt(); break; - }; + } prom_rev = romvec->pv_plugin_revision; prom_prev = romvec->pv_printrev; diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c index 5016c5e2057..d95db755828 100644 --- a/arch/sparc/prom/init_64.c +++ b/arch/sparc/prom/init_64.c @@ -22,8 +22,8 @@ int prom_stdout; phandle prom_chosen_node; /* You must call prom_init() before you attempt to use any of the - * routines in the prom library. It returns 0 on success, 1 on - * failure. It gets passed the pointer to the PROM vector. + * routines in the prom library. + * It gets passed the pointer to the PROM vector. */ extern void prom_cif_init(void *, void *); diff --git a/arch/sparc/prom/misc_32.c b/arch/sparc/prom/misc_32.c index 8c278c311ba..8dc0b6b271e 100644 --- a/arch/sparc/prom/misc_32.c +++ b/arch/sparc/prom/misc_32.c @@ -13,7 +13,6 @@ #include <asm/openprom.h> #include <asm/oplib.h> #include <asm/auxio.h> -#include <asm/system.h> extern void restore_current(void); @@ -54,15 +53,11 @@ EXPORT_SYMBOL(prom_feval); void prom_cmdline(void) { - extern void install_obp_ticker(void); - extern void install_linux_ticker(void); unsigned long flags; spin_lock_irqsave(&prom_lock, flags); - install_obp_ticker(); (*(romvec->pv_abort))(); restore_current(); - install_linux_ticker(); spin_unlock_irqrestore(&prom_lock, flags); set_auxio(AUXIO_LED, 0); } diff --git a/arch/sparc/prom/misc_64.c b/arch/sparc/prom/misc_64.c index e4f31d4d371..53a696d3eb3 100644 --- a/arch/sparc/prom/misc_64.c +++ b/arch/sparc/prom/misc_64.c @@ -15,7 +15,6 @@ #include <asm/openprom.h> #include <asm/oplib.h> -#include <asm/system.h> #include <asm/ldc.h> static int prom_service_exists(const char *service_name) @@ -82,11 +81,6 @@ void prom_feval(const char *fstring) } EXPORT_SYMBOL(prom_feval); -#ifdef CONFIG_SMP -extern void smp_capture(void); -extern void smp_release(void); -#endif - /* Drop into the prom, with the chance to continue with the 'go' * prom command. */ diff --git a/arch/sparc/prom/mp.c b/arch/sparc/prom/mp.c index 97c44c9ddbc..0da8256cf76 100644 --- a/arch/sparc/prom/mp.c +++ b/arch/sparc/prom/mp.c @@ -35,7 +35,7 @@ prom_startcpu(int cpunode, struct linux_prom_registers *ctable_reg, int ctx, cha case PROM_V3: ret = (*(romvec->v3_cpustart))(cpunode, (int) ctable_reg, ctx, pc); break; - }; + } restore_current(); spin_unlock_irqrestore(&prom_lock, flags); diff --git a/arch/sparc/prom/p1275.c b/arch/sparc/prom/p1275.c index d9850c2b9bf..e58b8172631 100644 --- a/arch/sparc/prom/p1275.c +++ b/arch/sparc/prom/p1275.c @@ -5,7 +5,6 @@ */ #include <linux/kernel.h> -#include <linux/init.h> #include <linux/sched.h> #include <linux/smp.h> #include <linux/string.h> @@ -13,7 +12,6 @@ #include <asm/openprom.h> #include <asm/oplib.h> -#include <asm/system.h> #include <asm/spitfire.h> #include <asm/pstate.h> #include <asm/ldc.h> diff --git a/arch/sparc/prom/ranges.c b/arch/sparc/prom/ranges.c index 0857aa9e839..ad143c13bdc 100644 --- a/arch/sparc/prom/ranges.c +++ b/arch/sparc/prom/ranges.c @@ -11,7 +11,6 @@ #include <asm/openprom.h> #include <asm/oplib.h> #include <asm/types.h> -#include <asm/system.h> static struct linux_prom_ranges promlib_obio_ranges[PROMREG_MAX]; static int num_obio_ranges; diff --git a/arch/sparc/prom/segment.c b/arch/sparc/prom/segment.c deleted file mode 100644 index 86a663f1d3c..00000000000 --- a/arch/sparc/prom/segment.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * segment.c: Prom routine to map segments in other contexts before - * a standalone is completely mapped. This is for sun4 and - * sun4c architectures only. - * - * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) - */ - -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/sched.h> -#include <asm/openprom.h> -#include <asm/oplib.h> - -extern void restore_current(void); - -/* Set physical segment 'segment' at virtual address 'vaddr' in - * context 'ctx'. - */ -void -prom_putsegment(int ctx, unsigned long vaddr, int segment) -{ - unsigned long flags; - spin_lock_irqsave(&prom_lock, flags); - (*(romvec->pv_setctxt))(ctx, (char *) vaddr, segment); - restore_current(); - spin_unlock_irqrestore(&prom_lock, flags); -} diff --git a/arch/sparc/prom/tree_64.c b/arch/sparc/prom/tree_64.c index 92204c3800b..bd1b2a3ac34 100644 --- a/arch/sparc/prom/tree_64.c +++ b/arch/sparc/prom/tree_64.c @@ -39,7 +39,7 @@ inline phandle __prom_getchild(phandle node) return prom_node_to_node("child", node); } -inline phandle prom_getchild(phandle node) +phandle prom_getchild(phandle node) { phandle cnode; @@ -72,7 +72,7 @@ inline phandle __prom_getsibling(phandle node) return prom_node_to_node(prom_peer_name, node); } -inline phandle prom_getsibling(phandle node) +phandle prom_getsibling(phandle node) { phandle sibnode; @@ -89,7 +89,7 @@ EXPORT_SYMBOL(prom_getsibling); /* Return the length in bytes of property 'prop' at node 'node'. * Return -1 on error. */ -inline int prom_getproplen(phandle node, const char *prop) +int prom_getproplen(phandle node, const char *prop) { unsigned long args[6]; @@ -113,8 +113,8 @@ EXPORT_SYMBOL(prom_getproplen); * 'buffer' which has a size of 'bufsize'. If the acquisition * was successful the length will be returned, else -1 is returned. */ -inline int prom_getproperty(phandle node, const char *prop, - char *buffer, int bufsize) +int prom_getproperty(phandle node, const char *prop, + char *buffer, int bufsize) { unsigned long args[8]; int plen; @@ -141,7 +141,7 @@ EXPORT_SYMBOL(prom_getproperty); /* Acquire an integer property and return its value. Returns -1 * on failure. */ -inline int prom_getint(phandle node, const char *prop) +int prom_getint(phandle node, const char *prop) { int intprop; @@ -235,7 +235,7 @@ static const char *prom_nextprop_name = "nextprop"; /* Return the first property type for node 'node'. * buffer should be at least 32B in length */ -inline char *prom_firstprop(phandle node, char *buffer) +char *prom_firstprop(phandle node, char *buffer) { unsigned long args[7]; @@ -261,7 +261,7 @@ EXPORT_SYMBOL(prom_firstprop); * at node 'node' . Returns NULL string if no more * property types for this node. */ -inline char *prom_nextprop(phandle node, const char *oprop, char *buffer) +char *prom_nextprop(phandle node, const char *oprop, char *buffer) { unsigned long args[7]; char buf[32]; |