Merge branch 'linus' into perf/core

Merge reason: pick up perf fixes

Signed-off-by: Ingo Molnar <mingo@elte.hu>

25 files changed, 631 insertions, 534 deletions

diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h
index 2050ca02c42..bdb2ff880bd 100644
--- a/arch/sparc/include/asm/atomic_64.h
+++ b/arch/sparc/include/asm/atomic_64.h
@@ -20,14 +20,14 @@
 #define atomic64_set(v, i)	(((v)->counter) = i)
 
 extern void atomic_add(int, atomic_t *);
-extern void atomic64_add(int, atomic64_t *);
+extern void atomic64_add(long, atomic64_t *);
 extern void atomic_sub(int, atomic_t *);
-extern void atomic64_sub(int, atomic64_t *);
+extern void atomic64_sub(long, atomic64_t *);
 
 extern int atomic_add_ret(int, atomic_t *);
-extern int atomic64_add_ret(int, atomic64_t *);
+extern long atomic64_add_ret(long, atomic64_t *);
 extern int atomic_sub_ret(int, atomic_t *);
-extern int atomic64_sub_ret(int, atomic64_t *);
+extern long atomic64_sub_ret(long, atomic64_t *);
 
 #define atomic_dec_return(v) atomic_sub_ret(1, v)
 #define atomic64_dec_return(v) atomic64_sub_ret(1, v)
@@ -91,7 +91,7 @@ static inline int atomic_add_unless(atomic_t *v, int a, int u)
 	((__typeof__((v)->counter))cmpxchg(&((v)->counter), (o), (n)))
 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
 
-static inline int atomic64_add_unless(atomic64_t *v, long a, long u)
+static inline long atomic64_add_unless(atomic64_t *v, long a, long u)
 {
 	long c, old;
 	c = atomic64_read(v);
diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h
index fa1fdf67e35..db3af0d30fb 100644
--- a/arch/sparc/include/asm/backoff.h
+++ b/arch/sparc/include/asm/backoff.h
@@ -8,6 +8,9 @@
 #define BACKOFF_SETUP(reg)	\
 	mov	1, reg
 
+#define BACKOFF_LABEL(spin_label, continue_label) \
+	spin_label
+
 #define BACKOFF_SPIN(reg, tmp, label)	\
 	mov	reg, tmp; \
 88:	brnz,pt	tmp, 88b; \
@@ -22,9 +25,11 @@
 #else
 
 #define BACKOFF_SETUP(reg)
-#define BACKOFF_SPIN(reg, tmp, label) \
-	ba,pt	%xcc, label; \
-	 nop;
+
+#define BACKOFF_LABEL(spin_label, continue_label) \
+	continue_label
+
+#define BACKOFF_SPIN(reg, tmp, label)
 
 #endif
 
diff --git a/arch/sparc/include/asm/fb.h b/arch/sparc/include/asm/fb.h
index e834880be20..2173432ad7f 100644
--- a/arch/sparc/include/asm/fb.h
+++ b/arch/sparc/include/asm/fb.h
@@ -1,5 +1,6 @@
 #ifndef _SPARC_FB_H_
 #define _SPARC_FB_H_
+#include <linux/console.h>
 #include <linux/fb.h>
 #include <linux/fs.h>
 #include <asm/page.h>
@@ -18,6 +19,9 @@ static inline int fb_is_primary_device(struct fb_info *info)
 	struct device *dev = info->device;
 	struct device_node *node;
 
+	if (console_set_on_cmdline)
+		return 0;
+
 	node = dev->of_node;
 	if (node &&
 	    node == of_console_device)
diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h
index a5db0317b5f..3e0b2d62303 100644
--- a/arch/sparc/include/asm/oplib_64.h
+++ b/arch/sparc/include/asm/oplib_64.h
@@ -185,9 +185,8 @@ extern int prom_getunumber(int syndrome_code,
 			   char *buf, int buflen);
 
 /* Retain physical memory to the caller across soft resets. */
-extern unsigned long prom_retain(const char *name,
-				 unsigned long pa_low, unsigned long pa_high,
-				 long size, long align);
+extern int prom_retain(const char *name, unsigned long size,
+		       unsigned long align, unsigned long *paddr);
 
 /* Load explicit I/D TLB entries into the calling processor. */
 extern long prom_itlb_load(unsigned long index,
@@ -287,26 +286,6 @@ extern void prom_sun4v_guest_soft_state(void);
 extern int prom_ihandle2path(int handle, char *buffer, int bufsize);
 
 /* Client interface level routines. */
-extern long p1275_cmd(const char *, long, ...);
-
-#if 0
-#define P1275_SIZE(x) ((((long)((x) / 32)) << 32) | (x))
-#else
-#define P1275_SIZE(x) x
-#endif
-
-/* We support at most 16 input and 1 output argument */
-#define P1275_ARG_NUMBER		0
-#define P1275_ARG_IN_STRING		1
-#define P1275_ARG_OUT_BUF		2
-#define P1275_ARG_OUT_32B		3
-#define P1275_ARG_IN_FUNCTION		4
-#define P1275_ARG_IN_BUF		5
-#define P1275_ARG_IN_64B		6
-
-#define P1275_IN(x) ((x) & 0xf)
-#define P1275_OUT(x) (((x) << 4) & 0xf0)
-#define P1275_INOUT(i,o) (P1275_IN(i)|P1275_OUT(o))
-#define P1275_ARG(n,x) ((x) << ((n)*3 + 8))
+extern void p1275_cmd_direct(unsigned long *);
 
 #endif /* !(__SPARC64_OPLIB_H) */
diff --git a/arch/sparc/include/asm/rwsem-const.h b/arch/sparc/include/asm/rwsem-const.h
deleted file mode 100644
index a303c9d64d8..00000000000
--- a/arch/sparc/include/asm/rwsem-const.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* rwsem-const.h: RW semaphore counter constants.  */
-#ifndef _SPARC64_RWSEM_CONST_H
-#define _SPARC64_RWSEM_CONST_H
-
-#define RWSEM_UNLOCKED_VALUE		0x00000000
-#define RWSEM_ACTIVE_BIAS		0x00000001
-#define RWSEM_ACTIVE_MASK		0x0000ffff
-#define RWSEM_WAITING_BIAS		0xffff0000
-#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS
-#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
-
-#endif /* _SPARC64_RWSEM_CONST_H */
diff --git a/arch/sparc/include/asm/rwsem.h b/arch/sparc/include/asm/rwsem.h
index 6e5621006f8..a2b4302869b 100644
--- a/arch/sparc/include/asm/rwsem.h
+++ b/arch/sparc/include/asm/rwsem.h
@@ -15,16 +15,21 @@
 
 #include <linux/list.h>
 #include <linux/spinlock.h>
-#include <asm/rwsem-const.h>
 
 struct rwsem_waiter;
 
 struct rw_semaphore {
-	signed int count;
-	spinlock_t		wait_lock;
-	struct list_head	wait_list;
+	signed long			count;
+#define RWSEM_UNLOCKED_VALUE		0x00000000L
+#define RWSEM_ACTIVE_BIAS		0x00000001L
+#define RWSEM_ACTIVE_MASK		0xffffffffL
+#define RWSEM_WAITING_BIAS		(-RWSEM_ACTIVE_MASK-1)
+#define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS
+#define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
+	spinlock_t			wait_lock;
+	struct list_head		wait_list;
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
-	struct lockdep_map	dep_map;
+	struct lockdep_map		dep_map;
 #endif
 };
 
@@ -41,6 +46,11 @@ struct rw_semaphore {
 #define DECLARE_RWSEM(name) \
 	struct rw_semaphore name = __RWSEM_INITIALIZER(name)
 
+extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
+extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
+extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);
+extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
+
 extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
 			 struct lock_class_key *key);
 
@@ -51,27 +61,103 @@ do {								\
 	__init_rwsem((sem), #sem, &__key);			\
 } while (0)
 
-extern void __down_read(struct rw_semaphore *sem);
-extern int __down_read_trylock(struct rw_semaphore *sem);
-extern void __down_write(struct rw_semaphore *sem);
-extern int __down_write_trylock(struct rw_semaphore *sem);
-extern void __up_read(struct rw_semaphore *sem);
-extern void __up_write(struct rw_semaphore *sem);
-extern void __downgrade_write(struct rw_semaphore *sem);
+/*
+ * lock for reading
+ */
+static inline void __down_read(struct rw_semaphore *sem)
+{
+	if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
+		rwsem_down_read_failed(sem);
+}
+
+static inline int __down_read_trylock(struct rw_semaphore *sem)
+{
+	long tmp;
+
+	while ((tmp = sem->count) >= 0L) {
+		if (tmp == cmpxchg(&sem->count, tmp,
+				   tmp + RWSEM_ACTIVE_READ_BIAS)) {
+			return 1;
+		}
+	}
+	return 0;
+}
 
+/*
+ * lock for writing
+ */
 static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
 {
-	__down_write(sem);
+	long tmp;
+
+	tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
+				  (atomic64_t *)(&sem->count));
+	if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
+		rwsem_down_write_failed(sem);
 }
 
-static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
+static inline void __down_write(struct rw_semaphore *sem)
 {
-	return atomic_add_return(delta, (atomic_t *)(&sem->count));
+	__down_write_nested(sem, 0);
+}
+
+static inline int __down_write_trylock(struct rw_semaphore *sem)
+{
+	long tmp;
+
+	tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
+		      RWSEM_ACTIVE_WRITE_BIAS);
+	return tmp == RWSEM_UNLOCKED_VALUE;
 }
 
-static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
+/*
+ * unlock after reading
+ */
+static inline void __up_read(struct rw_semaphore *sem)
+{
+	long tmp;
+
+	tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
+	if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
+		rwsem_wake(sem);
+}
+
+/*
+ * unlock after writing
+ */
+static inline void __up_write(struct rw_semaphore *sem)
+{
+	if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
+					 (atomic64_t *)(&sem->count)) < 0L))
+		rwsem_wake(sem);
+}
+
+/*
+ * implement atomic add functionality
+ */
+static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
+{
+	atomic64_add(delta, (atomic64_t *)(&sem->count));
+}
+
+/*
+ * downgrade write lock to read lock
+ */
+static inline void __downgrade_write(struct rw_semaphore *sem)
+{
+	long tmp;
+
+	tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
+	if (tmp < 0L)
+		rwsem_downgrade_wake(sem);
+}
+
+/*
+ * implement exchange and add functionality
+ */
+static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
 {
-	atomic_add(delta, (atomic_t *)(&sem->count));
+	return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
 }
 
 static inline int rwsem_is_locked(struct rw_semaphore *sem)
diff --git a/arch/sparc/include/asm/system_64.h b/arch/sparc/include/asm/system_64.h
index d24cfe16afc..e3b65d8cf41 100644
--- a/arch/sparc/include/asm/system_64.h
+++ b/arch/sparc/include/asm/system_64.h
@@ -106,6 +106,7 @@ do {	__asm__ __volatile__("ba,pt	%%xcc, 1f\n\t" \
  */
 #define write_pic(__p)  					\
 	__asm__ __volatile__("ba,pt	%%xcc, 99f\n\t"		\
+			     " nop\n\t"				\
 			     ".align	64\n"			\
 			  "99:wr	%0, 0x0, %%pic\n\t"	\
 			     "rd	%%pic, %%g0" : : "r" (__p))
diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h
index d0b3b01ac9d..03eb5a8f6f9 100644
--- a/arch/sparc/include/asm/unistd.h
+++ b/arch/sparc/include/asm/unistd.h
@@ -397,8 +397,11 @@
 #define __NR_rt_tgsigqueueinfo	326
 #define __NR_perf_event_open	327
 #define __NR_recvmmsg		328
+#define __NR_fanotify_init	329
+#define __NR_fanotify_mark	330
+#define __NR_prlimit64		331
 
-#define NR_syscalls		329
+#define NR_syscalls		332
 
 #ifdef __32bit_syscall_numbers__
 /* Sparc 32-bit only has the "setresuid32", "getresuid32" variants,
diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c
index 40e29fc8a4d..17529298c50 100644
--- a/arch/sparc/kernel/process_32.c
+++ b/arch/sparc/kernel/process_32.c
@@ -633,8 +633,10 @@ asmlinkage int sparc_execve(struct pt_regs *regs)
 	if(IS_ERR(filename))
 		goto out;
 	error = do_execve(filename,
-			  (char __user * __user *)regs->u_regs[base + UREG_I1],
-			  (char __user * __user *)regs->u_regs[base + UREG_I2],
+			  (const char __user *const  __user *)
+			  regs->u_regs[base + UREG_I1],
+			  (const char __user *const  __user *)
+			  regs->u_regs[base + UREG_I2],
 			  regs);
 	putname(filename);
 out:
diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c
index dbe81a368b4..485f5474838 100644
--- a/arch/sparc/kernel/process_64.c
+++ b/arch/sparc/kernel/process_64.c
@@ -739,9 +739,9 @@ asmlinkage int sparc_execve(struct pt_regs *regs)
 	if (IS_ERR(filename))
 		goto out;
 	error = do_execve(filename,
-			  (char __user * __user *)
+			  (const char __user *const __user *)
 			  regs->u_regs[base + UREG_I1],
-			  (char __user * __user *)
+			  (const char __user *const __user *)
 			  regs->u_regs[base + UREG_I2], regs);
 	putname(filename);
 	if (!error) {
diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S
index 46a76ba3fb4..44e5faf1ad5 100644
--- a/arch/sparc/kernel/sys32.S
+++ b/arch/sparc/kernel/sys32.S
@@ -330,6 +330,15 @@ do_sys_accept4: /* sys_accept4(int, struct sockaddr *, int *, int) */
 	nop
 	nop
 
+	.globl		sys32_fanotify_mark
+sys32_fanotify_mark:
+	sethi		%hi(sys_fanotify_mark), %g1
+	sllx		%o2, 32, %o2
+	or		%o2, %o3, %o2
+	mov		%o4, %o3
+	jmpl		%g1 + %lo(sys_fanotify_mark), %g0
+	 mov		%o5, %o4
+
 	.section	__ex_table,"a"
 	.align		4
 	.word		1b, __retl_efault, 2b, __retl_efault
diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c
index ee995b7dae7..50794137d71 100644
--- a/arch/sparc/kernel/sys_sparc_32.c
+++ b/arch/sparc/kernel/sys_sparc_32.c
@@ -282,7 +282,9 @@ out:
  * Do a system call from kernel instead of calling sys_execve so we
  * end up with proper pt_regs.
  */
-int kernel_execve(const char *filename, char *const argv[], char *const envp[])
+int kernel_execve(const char *filename,
+		  const char *const argv[],
+		  const char *const envp[])
 {
 	long __res;
 	register long __g1 __asm__ ("g1") = __NR_execve;
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c
index 3d435c42e6d..f836f4e93af 100644
--- a/arch/sparc/kernel/sys_sparc_64.c
+++ b/arch/sparc/kernel/sys_sparc_64.c
@@ -758,7 +758,9 @@ SYSCALL_DEFINE5(rt_sigaction, int, sig, const struct sigaction __user *, act,
  * Do a system call from kernel instead of calling sys_execve so we
  * end up with proper pt_regs.
  */
-int kernel_execve(const char *filename, char *const argv[], char *const envp[])
+int kernel_execve(const char *filename,
+		  const char *const argv[],
+		  const char *const envp[])
 {
 	long __res;
 	register long __g1 __asm__ ("g1") = __NR_execve;
diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S
index 801fc8e5a0e..ec396e1916b 100644
--- a/arch/sparc/kernel/systbls_32.S
+++ b/arch/sparc/kernel/systbls_32.S
@@ -82,5 +82,6 @@ sys_call_table:
 /*310*/	.long sys_utimensat, sys_signalfd, sys_timerfd_create, sys_eventfd, sys_fallocate
 /*315*/	.long sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1
 /*320*/	.long sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv
-/*325*/	.long sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg
+/*325*/	.long sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init
+/*330*/	.long sys_fanotify_mark, sys_prlimit64
 
diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S
index 9db058dd039..8cfcaa54958 100644
--- a/arch/sparc/kernel/systbls_64.S
+++ b/arch/sparc/kernel/systbls_64.S
@@ -83,7 +83,8 @@ sys_call_table32:
 /*310*/	.word compat_sys_utimensat, compat_sys_signalfd, sys_timerfd_create, sys_eventfd, compat_sys_fallocate
 	.word compat_sys_timerfd_settime, compat_sys_timerfd_gettime, compat_sys_signalfd4, sys_eventfd2, sys_epoll_create1
 /*320*/	.word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, compat_sys_preadv
-	.word compat_sys_pwritev, compat_sys_rt_tgsigqueueinfo, sys_perf_event_open, compat_sys_recvmmsg
+	.word compat_sys_pwritev, compat_sys_rt_tgsigqueueinfo, sys_perf_event_open, compat_sys_recvmmsg, sys_fanotify_init
+/*330*/	.word sys32_fanotify_mark, sys_prlimit64
 
 #endif /* CONFIG_COMPAT */
 
@@ -158,4 +159,5 @@ sys_call_table:
 /*310*/	.word sys_utimensat, sys_signalfd, sys_timerfd_create, sys_eventfd, sys_fallocate
 	.word sys_timerfd_settime, sys_timerfd_gettime, sys_signalfd4, sys_eventfd2, sys_epoll_create1
 /*320*/	.word sys_dup3, sys_pipe2, sys_inotify_init1, sys_accept4, sys_preadv
-	.word sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg
+	.word sys_pwritev, sys_rt_tgsigqueueinfo, sys_perf_event_open, sys_recvmmsg, sys_fanotify_init
+/*330*/	.word sys_fanotify_mark, sys_prlimit64
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index c4b5e03af11..846d1c4374e 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -15,7 +15,7 @@ lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o
 lib-$(CONFIG_SPARC32) += copy_user.o locks.o
 lib-y                 += atomic_$(BITS).o
 lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o
-lib-y                 += rwsem_$(BITS).o
+lib-$(CONFIG_SPARC32) += rwsem_32.o
 lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o
 
 lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o
diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S
index 0268210ca16..59186e0fcf3 100644
--- a/arch/sparc/lib/atomic_64.S
+++ b/arch/sparc/lib/atomic_64.S
@@ -21,7 +21,7 @@ atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
 	add	%g1, %o0, %g7
 	cas	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%icc, 2f
+	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -36,7 +36,7 @@ atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
 	sub	%g1, %o0, %g7
 	cas	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%icc, 2f
+	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -51,11 +51,10 @@ atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
 	add	%g1, %o0, %g7
 	cas	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%icc, 2f
-	 add	%g7, %o0, %g7
-	sra	%g7, 0, %o0
+	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)
+	 add	%g1, %o0, %g1
 	retl
-	 nop
+	 sra	%g1, 0, %o0
 2:	BACKOFF_SPIN(%o2, %o3, 1b)
 	.size	atomic_add_ret, .-atomic_add_ret
 
@@ -67,11 +66,10 @@ atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
 	sub	%g1, %o0, %g7
 	cas	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%icc, 2f
-	 sub	%g7, %o0, %g7
-	sra	%g7, 0, %o0
+	bne,pn	%icc, BACKOFF_LABEL(2f, 1b)
+	 sub	%g1, %o0, %g1
 	retl
-	 nop
+	 sra	%g1, 0, %o0
 2:	BACKOFF_SPIN(%o2, %o3, 1b)
 	.size	atomic_sub_ret, .-atomic_sub_ret
 
@@ -83,7 +81,7 @@ atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
 	add	%g1, %o0, %g7
 	casx	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -98,7 +96,7 @@ atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
 	sub	%g1, %o0, %g7
 	casx	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -113,11 +111,10 @@ atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
 	add	%g1, %o0, %g7
 	casx	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%xcc, 2f
-	 add	%g7, %o0, %g7
-	mov	%g7, %o0
-	retl
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
+	retl
+	 add	%g1, %o0, %o0
 2:	BACKOFF_SPIN(%o2, %o3, 1b)
 	.size	atomic64_add_ret, .-atomic64_add_ret
 
@@ -129,10 +126,9 @@ atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
 	sub	%g1, %o0, %g7
 	casx	[%o1], %g1, %g7
 	cmp	%g1, %g7
-	bne,pn	%xcc, 2f
-	 sub	%g7, %o0, %g7
-	mov	%g7, %o0
-	retl
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
+	retl
+	 sub	%g1, %o0, %o0
 2:	BACKOFF_SPIN(%o2, %o3, 1b)
 	.size	atomic64_sub_ret, .-atomic64_sub_ret
diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S
index 2b7228cb8c2..3dc61d5537c 100644
--- a/arch/sparc/lib/bitops.S
+++ b/arch/sparc/lib/bitops.S
@@ -22,7 +22,7 @@ test_and_set_bit:	/* %o0=nr, %o1=addr */
 	or	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 and	%g7, %o2, %g2
 	clr	%o0
 	movrne	%g2, 1, %o0
@@ -45,7 +45,7 @@ test_and_clear_bit:	/* %o0=nr, %o1=addr */
 	andn	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 and	%g7, %o2, %g2
 	clr	%o0
 	movrne	%g2, 1, %o0
@@ -68,7 +68,7 @@ test_and_change_bit:	/* %o0=nr, %o1=addr */
 	xor	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 and	%g7, %o2, %g2
 	clr	%o0
 	movrne	%g2, 1, %o0
@@ -91,7 +91,7 @@ set_bit:		/* %o0=nr, %o1=addr */
 	or	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -112,7 +112,7 @@ clear_bit:		/* %o0=nr, %o1=addr */
 	andn	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
@@ -133,7 +133,7 @@ change_bit:		/* %o0=nr, %o1=addr */
 	xor	%g7, %o2, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,pn	%xcc, 2f
+	bne,pn	%xcc, BACKOFF_LABEL(2f, 1b)
 	 nop
 	retl
 	 nop
diff --git a/arch/sparc/lib/rwsem_64.S b/arch/sparc/lib/rwsem_64.S
deleted file mode 100644
index 91a7d29a79d..00000000000
--- a/arch/sparc/lib/rwsem_64.S
+++ /dev/null
@@ -1,163 +0,0 @@
-/* rwsem.S: RW semaphore assembler.
- *
- * Written by David S. Miller (davem@redhat.com), 2001.
- * Derived from asm-i386/rwsem.h
- */
-
-#include <asm/rwsem-const.h>
-
-	.section	.sched.text, "ax"
-
-	.globl		__down_read
-__down_read:
-1:	lduw		[%o0], %g1
-	add		%g1, 1, %g7
-	cas		[%o0], %g1, %g7
-	cmp		%g1, %g7
-	bne,pn		%icc, 1b
-	 add		%g7, 1, %g7
-	cmp		%g7, 0
-	bl,pn		%icc, 3f
-	 nop
-2:
-	retl
-	 nop
-3:
-	save		%sp, -192, %sp
-	call		rwsem_down_read_failed
-	 mov		%i0, %o0
-	ret
-	 restore
-	.size		__down_read, .-__down_read
-
-	.globl		__down_read_trylock
-__down_read_trylock:
-1:	lduw		[%o0], %g1
-	add		%g1, 1, %g7
-	cmp		%g7, 0
-	bl,pn		%icc, 2f
-	 mov		0, %o1
-	cas		[%o0], %g1, %g7
-	cmp		%g1, %g7
-	bne,pn		%icc, 1b
-	 mov		1, %o1
-2:	retl
-	 mov		%o1, %o0
-	.size		__down_read_trylock, .-__down_read_trylock
-
-	.globl		__down_write
-__down_write:
-	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
-	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
-1:
-	lduw		[%o0], %g3
-	add		%g3, %g1, %g7
-	cas		[%o0], %g3, %g7
-	cmp		%g3, %g7
-	bne,pn		%icc, 1b
-	 cmp		%g7, 0
-	bne,pn		%icc, 3f
-	 nop
-2:	retl
-	 nop
-3:
-	save		%sp, -192, %sp
-	call		rwsem_down_write_failed
-	 mov		%i0, %o0
-	ret
-	 restore
-	.size		__down_write, .-__down_write
-
-	.globl		__down_write_trylock
-__down_write_trylock:
-	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
-	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
-1:
-	lduw		[%o0], %g3
-	cmp		%g3, 0
-	bne,pn		%icc, 2f
-	 mov		0, %o1
-	add		%g3, %g1, %g7
-	cas		[%o0], %g3, %g7
-	cmp		%g3, %g7
-	bne,pn		%icc, 1b
-	 mov		1, %o1
-2:	retl
-	 mov		%o1, %o0
-	.size		__down_write_trylock, .-__down_write_trylock
-
-	.globl		__up_read
-__up_read:
-1:
-	lduw		[%o0], %g1
-	sub		%g1, 1, %g7
-	cas		[%o0], %g1, %g7
-	cmp		%g1, %g7
-	bne,pn		%icc, 1b
-	 cmp		%g7, 0
-	bl,pn		%icc, 3f
-	 nop
-2:	retl
-	 nop
-3:	sethi		%hi(RWSEM_ACTIVE_MASK), %g1
-	sub		%g7, 1, %g7
-	or		%g1, %lo(RWSEM_ACTIVE_MASK), %g1
-	andcc		%g7, %g1, %g0
-	bne,pn		%icc, 2b
-	 nop
-	save		%sp, -192, %sp
-	call		rwsem_wake
-	 mov		%i0, %o0
-	ret
-	 restore
-	.size		__up_read, .-__up_read
-
-	.globl		__up_write
-__up_write:
-	sethi		%hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
-	or		%g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
-1:
-	lduw		[%o0], %g3
-	sub		%g3, %g1, %g7
-	cas		[%o0], %g3, %g7
-	cmp		%g3, %g7
-	bne,pn		%icc, 1b
-	 sub		%g7, %g1, %g7
-	cmp		%g7, 0
-	bl,pn		%icc, 3f
-	 nop
-2:
-	retl
-	 nop
-3:
-	save		%sp, -192, %sp
-	call		rwsem_wake
-	 mov		%i0, %o0
-	ret
-	 restore
-	.size		__up_write, .-__up_write
-
-	.globl		__downgrade_write
-__downgrade_write:
-	sethi		%hi(RWSEM_WAITING_BIAS), %g1
-	or		%g1, %lo(RWSEM_WAITING_BIAS), %g1
-1:
-	lduw		[%o0], %g3
-	sub		%g3, %g1, %g7
-	cas		[%o0], %g3, %g7
-	cmp		%g3, %g7
-	bne,pn		%icc, 1b
-	 sub		%g7, %g1, %g7
-	cmp		%g7, 0
-	bl,pn		%icc, 3f
-	 nop
-2:
-	retl
-	 nop
-3:
-	save		%sp, -192, %sp
-	call		rwsem_downgrade_wake
-	 mov		%i0, %o0
-	ret
-	 restore
-	.size		__downgrade_write, .-__downgrade_write
diff --git a/arch/sparc/prom/cif.S b/arch/sparc/prom/cif.S
index 5f27ad779c0..9c86b4b7d42 100644
--- a/arch/sparc/prom/cif.S
+++ b/arch/sparc/prom/cif.S
@@ -9,18 +9,18 @@
 #include <asm/thread_info.h>
 
 	.text
-	.globl	prom_cif_interface
-prom_cif_interface:
-	sethi	%hi(p1275buf), %o0
-	or	%o0, %lo(p1275buf), %o0
-	ldx	[%o0 + 0x010], %o1	! prom_cif_stack
-	save	%o1, -192, %sp
-	ldx	[%i0 + 0x008], %l2	! prom_cif_handler
+	.globl	prom_cif_direct
+prom_cif_direct:
+	sethi	%hi(p1275buf), %o1
+	or	%o1, %lo(p1275buf), %o1
+	ldx	[%o1 + 0x0010], %o2	! prom_cif_stack
+	save	%o2, -192, %sp
+	ldx	[%i1 + 0x0008], %l2	! prom_cif_handler
 	mov	%g4, %l0
 	mov	%g5, %l1
 	mov	%g6, %l3
 	call	%l2
-	 add	%i0, 0x018, %o0		! prom_args
+	 mov	%i0, %o0		! prom_args
 	mov	%l0, %g4
 	mov	%l1, %g5
 	mov	%l3, %g6
diff --git a/arch/sparc/prom/console_64.c b/arch/sparc/prom/console_64.c
index f55d58a8a15..10322dc2f55 100644
--- a/arch/sparc/prom/console_64.c
+++ b/arch/sparc/prom/console_64.c
@@ -21,14 +21,22 @@ extern int prom_stdin, prom_stdout;
 inline int
 prom_nbgetchar(void)
 {
+	unsigned long args[7];
 	char inc;
 
-	if (p1275_cmd("read", P1275_ARG(1,P1275_ARG_OUT_BUF)|
-			      P1275_INOUT(3,1),
-			      prom_stdin, &inc, P1275_SIZE(1)) == 1)
+	args[0] = (unsigned long) "read";
+	args[1] = 3;
+	args[2] = 1;
+	args[3] = (unsigned int) prom_stdin;
+	args[4] = (unsigned long) &inc;
+	args[5] = 1;
+	args[6] = (unsigned long) -1;
+
+	p1275_cmd_direct(args);
+
+	if (args[6] == 1)
 		return inc;
-	else
-		return -1;
+	return -1;
 }
 
 /* Non blocking put character to console device, returns -1 if
@@ -37,12 +45,22 @@ prom_nbgetchar(void)
 inline int
 prom_nbputchar(char c)
 {
+	unsigned long args[7];
 	char outc;
 	
 	outc = c;
-	if (p1275_cmd("write", P1275_ARG(1,P1275_ARG_IN_BUF)|
-			       P1275_INOUT(3,1),
-			       prom_stdout, &outc, P1275_SIZE(1)) == 1)
+
+	args[0] = (unsigned long) "write";
+	args[1] = 3;
+	args[2] = 1;
+	args[3] = (unsigned int) prom_stdout;
+	args[4] = (unsigned long) &outc;
+	args[5] = 1;
+	args[6] = (unsigned long) -1;
+
+	p1275_cmd_direct(args);
+
+	if (args[6] == 1)
 		return 0;
 	else
 		return -1;
@@ -67,7 +85,15 @@ prom_putchar(char c)
 void
 prom_puts(const char *s, int len)
 {
-	p1275_cmd("write", P1275_ARG(1,P1275_ARG_IN_BUF)|
-			   P1275_INOUT(3,1),
-			   prom_stdout, s, P1275_SIZE(len));
+	unsigned long args[7];