1 files changed, 75 insertions, 185 deletions

diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index ac8f8a43158..ba6c30d8534 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -46,12 +46,15 @@ struct corelock_slot corelock __attribute__ ((__section__(".l2.bss")));
 unsigned long blackfin_iflush_l1_entry[NR_CPUS];
 #endif
 
-struct blackfin_initial_pda __cpuinitdata initial_pda_coreb;
-
-#define BFIN_IPI_TIMER	      0
-#define BFIN_IPI_RESCHEDULE   1
-#define BFIN_IPI_CALL_FUNC    2
-#define BFIN_IPI_CPU_STOP     3
+struct blackfin_initial_pda initial_pda_coreb;
+
+enum ipi_message_type {
+	BFIN_IPI_NONE,
+	BFIN_IPI_TIMER,
+	BFIN_IPI_RESCHEDULE,
+	BFIN_IPI_CALL_FUNC,
+	BFIN_IPI_CPU_STOP,
+};
 
 struct blackfin_flush_data {
 	unsigned long start;
@@ -60,35 +63,20 @@ struct blackfin_flush_data {
 
 void *secondary_stack;
 
-
-struct smp_call_struct {
-	void (*func)(void *info);
-	void *info;
-	int wait;
-	cpumask_t *waitmask;
-};
-
 static struct blackfin_flush_data smp_flush_data;
 
 static DEFINE_SPINLOCK(stop_lock);
 
-struct ipi_message {
-	unsigned long type;
-	struct smp_call_struct call_struct;
-};
-
 /* A magic number - stress test shows this is safe for common cases */
 #define BFIN_IPI_MSGQ_LEN 5
 
 /* Simple FIFO buffer, overflow leads to panic */
-struct ipi_message_queue {
-	spinlock_t lock;
-	unsigned long count;
-	unsigned long head; /* head of the queue */
-	struct ipi_message ipi_message[BFIN_IPI_MSGQ_LEN];
+struct ipi_data {
+	atomic_t count;
+	atomic_t bits;
 };
 
-static DEFINE_PER_CPU(struct ipi_message_queue, ipi_msg_queue);
+static DEFINE_PER_CPU(struct ipi_data, bfin_ipi);
 
 static void ipi_cpu_stop(unsigned int cpu)
 {
@@ -129,28 +117,6 @@ static void ipi_flush_icache(void *info)
 	blackfin_icache_flush_range(fdata->start, fdata->end);
 }
 
-static void ipi_call_function(unsigned int cpu, struct ipi_message *msg)
-{
-	int wait;
-	void (*func)(void *info);
-	void *info;
-	func = msg->call_struct.func;
-	info = msg->call_struct.info;
-	wait = msg->call_struct.wait;
-	func(info);
-	if (wait) {
-#ifdef __ARCH_SYNC_CORE_DCACHE
-		/*
-		 * 'wait' usually means synchronization between CPUs.
-		 * Invalidate D cache in case shared data was changed
-		 * by func() to ensure cache coherence.
-		 */
-		resync_core_dcache();
-#endif
-		cpumask_clear_cpu(cpu, msg->call_struct.waitmask);
-	}
-}
-
 /* Use IRQ_SUPPLE_0 to request reschedule.
  * When returning from interrupt to user space,
  * there is chance to reschedule */
@@ -172,152 +138,92 @@ void ipi_timer(void)
 
 static irqreturn_t ipi_handler_int1(int irq, void *dev_instance)
 {
-	struct ipi_message *msg;
-	struct ipi_message_queue *msg_queue;
+	struct ipi_data *bfin_ipi_data;
 	unsigned int cpu = smp_processor_id();
-	unsigned long flags;
+	unsigned long pending;
+	unsigned long msg;
 
 	platform_clear_ipi(cpu, IRQ_SUPPLE_1);
 
-	msg_queue = &__get_cpu_var(ipi_msg_queue);
-
-	spin_lock_irqsave(&msg_queue->lock, flags);
-
-	while (msg_queue->count) {
-		msg = &msg_queue->ipi_message[msg_queue->head];
-		switch (msg->type) {
-		case BFIN_IPI_TIMER:
-			ipi_timer();
-			break;
-		case BFIN_IPI_RESCHEDULE:
-			scheduler_ipi();
-			break;
-		case BFIN_IPI_CALL_FUNC:
-			ipi_call_function(cpu, msg);
-			break;
-		case BFIN_IPI_CPU_STOP:
-			ipi_cpu_stop(cpu);
-			break;
-		default:
-			printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%lx\n",
-			       cpu, msg->type);
-			break;
-		}
-		msg_queue->head++;
-		msg_queue->head %= BFIN_IPI_MSGQ_LEN;
-		msg_queue->count--;
+	smp_rmb();
+	bfin_ipi_data = &__get_cpu_var(bfin_ipi);
+	while ((pending = atomic_xchg(&bfin_ipi_data->bits, 0)) != 0) {
+		msg = 0;
+		do {
+			msg = find_next_bit(&pending, BITS_PER_LONG, msg + 1);
+			switch (msg) {
+			case BFIN_IPI_TIMER:
+				ipi_timer();
+				break;
+			case BFIN_IPI_RESCHEDULE:
+				scheduler_ipi();
+				break;
+			case BFIN_IPI_CALL_FUNC:
+				generic_smp_call_function_interrupt();
+				break;
+			case BFIN_IPI_CPU_STOP:
+				ipi_cpu_stop(cpu);
+				break;
+			default:
+				goto out;
+			}
+			atomic_dec(&bfin_ipi_data->count);
+		} while (msg < BITS_PER_LONG);
+
 	}
-	spin_unlock_irqrestore(&msg_queue->lock, flags);
+out:
 	return IRQ_HANDLED;
 }
 
-static void ipi_queue_init(void)
+static void bfin_ipi_init(void)
 {
 	unsigned int cpu;
-	struct ipi_message_queue *msg_queue;
+	struct ipi_data *bfin_ipi_data;
 	for_each_possible_cpu(cpu) {
-		msg_queue = &per_cpu(ipi_msg_queue, cpu);
-		spin_lock_init(&msg_queue->lock);
-		msg_queue->count = 0;
-		msg_queue->head = 0;
+		bfin_ipi_data = &per_cpu(bfin_ipi, cpu);
+		atomic_set(&bfin_ipi_data->bits, 0);
+		atomic_set(&bfin_ipi_data->count, 0);
 	}
 }
 
-static inline void smp_send_message(cpumask_t callmap, unsigned long type,
-					void (*func) (void *info), void *info, int wait)
+void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
 {
 	unsigned int cpu;
-	struct ipi_message_queue *msg_queue;
-	struct ipi_message *msg;
-	unsigned long flags, next_msg;
-	cpumask_t waitmask; /* waitmask is shared by all cpus */
-
-	cpumask_copy(&waitmask, &callmap);
-	for_each_cpu(cpu, &callmap) {
-		msg_queue = &per_cpu(ipi_msg_queue, cpu);
-		spin_lock_irqsave(&msg_queue->lock, flags);
-		if (msg_queue->count < BFIN_IPI_MSGQ_LEN) {
-			next_msg = (msg_queue->head + msg_queue->count)
-					% BFIN_IPI_MSGQ_LEN;
-			msg = &msg_queue->ipi_message[next_msg];
-			msg->type = type;
-			if (type == BFIN_IPI_CALL_FUNC) {
-				msg->call_struct.func = func;
-				msg->call_struct.info = info;
-				msg->call_struct.wait = wait;
-				msg->call_struct.waitmask = &waitmask;
-			}
-			msg_queue->count++;
-		} else
-			panic("IPI message queue overflow\n");
-		spin_unlock_irqrestore(&msg_queue->lock, flags);
-		platform_send_ipi_cpu(cpu, IRQ_SUPPLE_1);
-	}
+	struct ipi_data *bfin_ipi_data;
+	unsigned long flags;
 
-	if (wait) {
-		while (!cpumask_empty(&waitmask))
-			blackfin_dcache_invalidate_range(
-				(unsigned long)(&waitmask),
-				(unsigned long)(&waitmask));
-#ifdef __ARCH_SYNC_CORE_DCACHE
-		/*
-		 * Invalidate D cache in case shared data was changed by
-		 * other processors to ensure cache coherence.
-		 */
-		resync_core_dcache();
-#endif
+	local_irq_save(flags);
+	for_each_cpu(cpu, cpumask) {
+		bfin_ipi_data = &per_cpu(bfin_ipi, cpu);
+		atomic_set_mask((1 << msg), &bfin_ipi_data->bits);
+		atomic_inc(&bfin_ipi_data->count);
 	}
+	local_irq_restore(flags);
+	smp_wmb();
+	for_each_cpu(cpu, cpumask)
+		platform_send_ipi_cpu(cpu, IRQ_SUPPLE_1);
 }
 
-int smp_call_function(void (*func)(void *info), void *info, int wait)
+void arch_send_call_function_single_ipi(int cpu)
 {
-	cpumask_t callmap;
-
-	preempt_disable();
-	cpumask_copy(&callmap, cpu_online_mask);
-	cpumask_clear_cpu(smp_processor_id(), &callmap);
-	if (!cpumask_empty(&callmap))
-		smp_send_message(callmap, BFIN_IPI_CALL_FUNC, func, info, wait);
-
-	preempt_enable();
-
-	return 0;
+	send_ipi(cpumask_of(cpu), BFIN_IPI_CALL_FUNC);
 }
-EXPORT_SYMBOL_GPL(smp_call_function);
 
-int smp_call_function_single(int cpuid, void (*func) (void *info), void *info,
-				int wait)
+void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 {
-	unsigned int cpu = cpuid;
-	cpumask_t callmap;
-
-	if (cpu_is_offline(cpu))
-		return 0;
-	cpumask_clear(&callmap);
-	cpumask_set_cpu(cpu, &callmap);
-
-	smp_send_message(callmap, BFIN_IPI_CALL_FUNC, func, info, wait);
-
-	return 0;
+	send_ipi(mask, BFIN_IPI_CALL_FUNC);
 }
-EXPORT_SYMBOL_GPL(smp_call_function_single);
 
 void smp_send_reschedule(int cpu)
 {
-	cpumask_t callmap;
-	/* simply trigger an ipi */
-
-	cpumask_clear(&callmap);
-	cpumask_set_cpu(cpu, &callmap);
-
-	smp_send_message(callmap, BFIN_IPI_RESCHEDULE, NULL, NULL, 0);
+	send_ipi(cpumask_of(cpu), BFIN_IPI_RESCHEDULE);
 
 	return;
 }
 
 void smp_send_msg(const struct cpumask *mask, unsigned long type)
 {
-	smp_send_message(*mask, type, NULL, NULL, 0);
+	send_ipi(mask, type);
 }
 
 void smp_timer_broadcast(const struct cpumask *mask)
@@ -333,34 +239,17 @@ void smp_send_stop(void)
 	cpumask_copy(&callmap, cpu_online_mask);
 	cpumask_clear_cpu(smp_processor_id(), &callmap);
 	if (!cpumask_empty(&callmap))
-		smp_send_message(callmap, BFIN_IPI_CPU_STOP, NULL, NULL, 0);
+		send_ipi(&callmap, BFIN_IPI_CPU_STOP);
 
 	preempt_enable();
 
 	return;
 }
 
-int __cpuinit __cpu_up(unsigned int cpu)
+int __cpu_up(unsigned int cpu, struct task_struct *idle)
 {
 	int ret;
-	struct blackfin_cpudata *ci = &per_cpu(cpu_data, cpu);
-	struct task_struct *idle = ci->idle;
-
-	if (idle) {
-		free_task(idle);
-		idle = NULL;
-	}
 
-	if (!idle) {
-		idle = fork_idle(cpu);
-		if (IS_ERR(idle)) {
-			printk(KERN_ERR "CPU%u: fork() failed\n", cpu);
-			return PTR_ERR(idle);
-		}
-		ci->idle = idle;
-	} else {
-		init_idle(idle, cpu);
-	}
 	secondary_stack = task_stack_page(idle) + THREAD_SIZE;
 
 	ret = platform_boot_secondary(cpu, idle);
@@ -370,7 +259,7 @@ int __cpuinit __cpu_up(unsigned int cpu)
 	return ret;
 }
 
-static void __cpuinit setup_secondary(unsigned int cpu)
+static void setup_secondary(unsigned int cpu)
 {
 	unsigned long ilat;
 
@@ -388,7 +277,7 @@ static void __cpuinit setup_secondary(unsigned int cpu)
 	    IMASK_IVG10 | IMASK_IVG9 | IMASK_IVG8 | IMASK_IVG7 | IMASK_IVGHW;
 }
 
-void __cpuinit secondary_start_kernel(void)
+void secondary_start_kernel(void)
 {
 	unsigned int cpu = smp_processor_id();
 	struct mm_struct *mm = &init_mm;
@@ -427,7 +316,6 @@ void __cpuinit secondary_start_kernel(void)
 	setup_secondary(cpu);
 
 	platform_secondary_init(cpu);
-
 	/* setup local core timer */
 	bfin_local_timer_setup();
 
@@ -443,7 +331,9 @@ void __cpuinit secondary_start_kernel(void)
 	 */
 	calibrate_delay();
 
-	cpu_idle();
+	/* We are done with local CPU inits, unblock the boot CPU. */
+	set_cpu_online(cpu, true);
+	cpu_startup_entry(CPUHP_ONLINE);
 }
 
 void __init smp_prepare_boot_cpu(void)
@@ -453,7 +343,7 @@ void __init smp_prepare_boot_cpu(void)
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
 	platform_prepare_cpus(max_cpus);
-	ipi_queue_init();
+	bfin_ipi_init();
 	platform_request_ipi(IRQ_SUPPLE_0, ipi_handler_int0);
 	platform_request_ipi(IRQ_SUPPLE_1, ipi_handler_int1);
 }
@@ -512,7 +402,7 @@ EXPORT_SYMBOL(resync_core_dcache);
 #endif
 
 #ifdef CONFIG_HOTPLUG_CPU
-int __cpuexit __cpu_disable(void)
+int __cpu_disable(void)
 {
 	unsigned int cpu = smp_processor_id();
 
@@ -525,7 +415,7 @@ int __cpuexit __cpu_disable(void)
 
 static DECLARE_COMPLETION(cpu_killed);
 
-int __cpuexit __cpu_die(unsigned int cpu)
+int __cpu_die(unsigned int cpu)
 {
 	return wait_for_completion_timeout(&cpu_killed, 5000);
 }