aboutsummaryrefslogtreecommitdiff
path: root/arch/sparc/kernel/smp_64.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc/kernel/smp_64.c')
-rw-r--r--arch/sparc/kernel/smp_64.c190
1 files changed, 110 insertions, 80 deletions
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 555a76d1f4a..41aa2478f3c 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -3,7 +3,7 @@
* Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
*/
-#include <linux/module.h>
+#include <linux/export.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/mm.h>
@@ -25,16 +25,18 @@
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/slab.h>
+#include <linux/kgdb.h>
#include <asm/head.h>
#include <asm/ptrace.h>
-#include <asm/atomic.h>
+#include <linux/atomic.h>
#include <asm/tlbflush.h>
#include <asm/mmu_context.h>
#include <asm/cpudata.h>
#include <asm/hvtramp.h>
#include <asm/io.h>
#include <asm/timer.h>
+#include <asm/setup.h>
#include <asm/irq.h>
#include <asm/irq_regs.h>
@@ -52,8 +54,7 @@
#include <asm/pcr.h>
#include "cpumap.h"
-
-int sparc64_multi_core __read_mostly;
+#include "kernel.h"
DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
cpumask_t cpu_core_map[NR_CPUS] __read_mostly =
@@ -87,7 +88,7 @@ extern void setup_sparc64_timer(void);
static volatile unsigned long callin_flag = 0;
-void __cpuinit smp_callin(void)
+void smp_callin(void)
{
int cpuid = hard_smp_processor_id();
@@ -103,8 +104,6 @@ void __cpuinit smp_callin(void)
if (cheetah_pcache_forced_on)
cheetah_enable_pcache();
- local_irq_enable();
-
callin_flag = 1;
__asm__ __volatile__("membar #Sync\n\t"
"flush %%g6" : : : "memory");
@@ -121,15 +120,17 @@ void __cpuinit smp_callin(void)
/* inform the notifiers about the new cpu */
notify_cpu_starting(cpuid);
- while (!cpu_isset(cpuid, smp_commenced_mask))
+ while (!cpumask_test_cpu(cpuid, &smp_commenced_mask))
rmb();
- ipi_call_lock_irq();
- cpu_set(cpuid, cpu_online_map);
- ipi_call_unlock_irq();
+ set_cpu_online(cpuid, true);
/* idle thread is expected to have preempt disabled */
preempt_disable();
+
+ local_irq_enable();
+
+ cpu_startup_entry(CPUHP_ONLINE);
}
void cpu_panic(void)
@@ -151,7 +152,7 @@ void cpu_panic(void)
#define NUM_ROUNDS 64 /* magic value */
#define NUM_ITERS 5 /* likewise */
-static DEFINE_SPINLOCK(itc_sync_lock);
+static DEFINE_RAW_SPINLOCK(itc_sync_lock);
static unsigned long go[SLAVE + 1];
#define DEBUG_TICK_SYNC 0
@@ -189,7 +190,7 @@ static inline long get_delta (long *rt, long *master)
void smp_synchronize_tick_client(void)
{
long i, delta, adj, adjust_latency = 0, done = 0;
- unsigned long flags, rt, master_time_stamp, bound;
+ unsigned long flags, rt, master_time_stamp;
#if DEBUG_TICK_SYNC
struct {
long rt; /* roundtrip time */
@@ -208,10 +209,8 @@ void smp_synchronize_tick_client(void)
{
for (i = 0; i < NUM_ROUNDS; i++) {
delta = get_delta(&rt, &master_time_stamp);
- if (delta == 0) {
+ if (delta == 0)
done = 1; /* let's lock on to this... */
- bound = rt;
- }
if (!done) {
if (i > 0) {
@@ -261,7 +260,7 @@ static void smp_synchronize_one_tick(int cpu)
go[MASTER] = 0;
membar_safe("#StoreLoad");
- spin_lock_irqsave(&itc_sync_lock, flags);
+ raw_spin_lock_irqsave(&itc_sync_lock, flags);
{
for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
while (!go[MASTER])
@@ -272,19 +271,12 @@ static void smp_synchronize_one_tick(int cpu)
membar_safe("#StoreLoad");
}
}
- spin_unlock_irqrestore(&itc_sync_lock, flags);
+ raw_spin_unlock_irqrestore(&itc_sync_lock, flags);
}
#if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU)
-/* XXX Put this in some common place. XXX */
-static unsigned long kimage_addr_to_ra(void *p)
-{
- unsigned long val = (unsigned long) p;
-
- return kern_base + (val - KERNBASE);
-}
-
-static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, void **descrp)
+static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg,
+ void **descrp)
{
extern unsigned long sparc64_ttable_tl0;
extern unsigned long kern_locked_tte_data;
@@ -345,21 +337,17 @@ extern unsigned long sparc64_cpu_startup;
*/
static struct thread_info *cpu_new_thread = NULL;
-static int __cpuinit smp_boot_one_cpu(unsigned int cpu)
+static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle)
{
unsigned long entry =
(unsigned long)(&sparc64_cpu_startup);
unsigned long cookie =
(unsigned long)(&cpu_new_thread);
- struct task_struct *p;
void *descr = NULL;
int timeout, ret;
- p = fork_idle(cpu);
- if (IS_ERR(p))
- return PTR_ERR(p);
callin_flag = 0;
- cpu_new_thread = task_thread_info(p);
+ cpu_new_thread = task_thread_info(idle);
if (tlb_type == hypervisor) {
#if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU)
@@ -787,7 +775,7 @@ static void xcall_deliver(u64 data0, u64 data1, u64 data2, const cpumask_t *mask
/* Send cross call to all processors mentioned in MASK_P
* except self. Really, there are only two cases currently,
- * "&cpu_online_map" and "&mm->cpu_vm_mask".
+ * "cpu_online_mask" and "mm_cpumask(mm)".
*/
static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask)
{
@@ -799,7 +787,7 @@ static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 d
/* Send cross call to all processors except self. */
static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2)
{
- smp_cross_call_masked(func, ctx, data1, data2, &cpu_online_map);
+ smp_cross_call_masked(func, ctx, data1, data2, cpu_online_mask);
}
extern unsigned long xcall_sync_tick;
@@ -807,7 +795,7 @@ extern unsigned long xcall_sync_tick;
static void smp_start_sync_tick_client(int cpu)
{
xcall_deliver((u64) &xcall_sync_tick, 0, 0,
- &cpumask_of_cpu(cpu));
+ cpumask_of(cpu));
}
extern unsigned long xcall_call_function;
@@ -822,7 +810,7 @@ extern unsigned long xcall_call_function_single;
void arch_send_call_function_single_ipi(int cpu)
{
xcall_deliver((u64) &xcall_call_function_single, 0, 0,
- &cpumask_of_cpu(cpu));
+ cpumask_of(cpu));
}
void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs)
@@ -842,7 +830,7 @@ static void tsb_sync(void *info)
struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()];
struct mm_struct *mm = info;
- /* It is not valid to test "currrent->active_mm == mm" here.
+ /* It is not valid to test "current->active_mm == mm" here.
*
* The value of "current" is not changed atomically with
* switch_mm(). But that's OK, we just need to check the
@@ -858,9 +846,11 @@ void smp_tsb_sync(struct mm_struct *mm)
}
extern unsigned long xcall_flush_tlb_mm;
-extern unsigned long xcall_flush_tlb_pending;
+extern unsigned long xcall_flush_tlb_page;
extern unsigned long xcall_flush_tlb_kernel_range;
extern unsigned long xcall_fetch_glob_regs;
+extern unsigned long xcall_fetch_glob_pmu;
+extern unsigned long xcall_fetch_glob_pmu_n4;
extern unsigned long xcall_receive_signal;
extern unsigned long xcall_new_mmu_context_version;
#ifdef CONFIG_KGDB
@@ -872,11 +862,6 @@ extern unsigned long xcall_flush_dcache_page_cheetah;
#endif
extern unsigned long xcall_flush_dcache_page_spitfire;
-#ifdef CONFIG_DEBUG_DCFLUSH
-extern atomic_t dcpage_flushes;
-extern atomic_t dcpage_flushes_xcall;
-#endif
-
static inline void __local_flush_dcache_page(struct page *page)
{
#ifdef DCACHE_ALIASING_POSSIBLE
@@ -920,7 +905,7 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
}
if (data0) {
xcall_deliver(data0, __pa(pg_addr),
- (u64) pg_addr, &cpumask_of_cpu(cpu));
+ (u64) pg_addr, cpumask_of(cpu));
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall);
#endif
@@ -933,13 +918,12 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
{
void *pg_addr;
- int this_cpu;
u64 data0;
if (tlb_type == hypervisor)
return;
- this_cpu = get_cpu();
+ preempt_disable();
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes);
@@ -957,14 +941,14 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
}
if (data0) {
xcall_deliver(data0, __pa(pg_addr),
- (u64) pg_addr, &cpu_online_map);
+ (u64) pg_addr, cpu_online_mask);
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall);
#endif
}
__local_flush_dcache_page(page);
- put_cpu();
+ preempt_enable();
}
void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs)
@@ -1010,6 +994,15 @@ void smp_fetch_global_regs(void)
smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0);
}
+void smp_fetch_global_pmu(void)
+{
+ if (tlb_type == hypervisor &&
+ sun4v_chip_type >= SUN4V_CHIP_NIAGARA4)
+ smp_cross_call(&xcall_fetch_glob_pmu_n4, 0, 0, 0);
+ else
+ smp_cross_call(&xcall_fetch_glob_pmu, 0, 0, 0);
+}
+
/* We know that the window frames of the user have been flushed
* to the stack before we get here because all callers of us
* are flush_tlb_*() routines, and these run after flush_cache_*()
@@ -1073,23 +1066,56 @@ local_flush_and_out:
put_cpu();
}
+struct tlb_pending_info {
+ unsigned long ctx;
+ unsigned long nr;
+ unsigned long *vaddrs;
+};
+
+static void tlb_pending_func(void *info)
+{
+ struct tlb_pending_info *t = info;
+
+ __flush_tlb_pending(t->ctx, t->nr, t->vaddrs);
+}
+
void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs)
{
u32 ctx = CTX_HWBITS(mm->context);
+ struct tlb_pending_info info;
int cpu = get_cpu();
+ info.ctx = ctx;
+ info.nr = nr;
+ info.vaddrs = vaddrs;
+
if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
else
- smp_cross_call_masked(&xcall_flush_tlb_pending,
- ctx, nr, (unsigned long) vaddrs,
- mm_cpumask(mm));
+ smp_call_function_many(mm_cpumask(mm), tlb_pending_func,
+ &info, 1);
__flush_tlb_pending(ctx, nr, vaddrs);
put_cpu();
}
+void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr)
+{
+ unsigned long context = CTX_HWBITS(mm->context);
+ int cpu = get_cpu();
+
+ if (mm == current->mm && atomic_read(&mm->mm_users) == 1)
+ cpumask_copy(mm_cpumask(mm), cpumask_of(cpu));
+ else
+ smp_cross_call_masked(&xcall_flush_tlb_page,
+ context, vaddr, 0,
+ mm_cpumask(mm));
+ __flush_tlb_page(context, vaddr);
+
+ put_cpu();
+}
+
void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
start &= PAGE_MASK;
@@ -1179,7 +1205,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus)
{
}
-void __devinit smp_prepare_boot_cpu(void)
+void smp_prepare_boot_cpu(void)
{
}
@@ -1193,52 +1219,52 @@ void __init smp_setup_processor_id(void)
xcall_deliver_impl = hypervisor_xcall_deliver;
}
-void __devinit smp_fill_in_sib_core_maps(void)
+void smp_fill_in_sib_core_maps(void)
{
unsigned int i;
for_each_present_cpu(i) {
unsigned int j;
- cpus_clear(cpu_core_map[i]);
+ cpumask_clear(&cpu_core_map[i]);
if (cpu_data(i).core_id == 0) {
- cpu_set(i, cpu_core_map[i]);
+ cpumask_set_cpu(i, &cpu_core_map[i]);
continue;
}
for_each_present_cpu(j) {
if (cpu_data(i).core_id ==
cpu_data(j).core_id)
- cpu_set(j, cpu_core_map[i]);
+ cpumask_set_cpu(j, &cpu_core_map[i]);
}
}
for_each_present_cpu(i) {
unsigned int j;
- cpus_clear(per_cpu(cpu_sibling_map, i));
+ cpumask_clear(&per_cpu(cpu_sibling_map, i));
if (cpu_data(i).proc_id == -1) {
- cpu_set(i, per_cpu(cpu_sibling_map, i));
+ cpumask_set_cpu(i, &per_cpu(cpu_sibling_map, i));
continue;
}
for_each_present_cpu(j) {
if (cpu_data(i).proc_id ==
cpu_data(j).proc_id)
- cpu_set(j, per_cpu(cpu_sibling_map, i));
+ cpumask_set_cpu(j, &per_cpu(cpu_sibling_map, i));
}
}
}
-int __cpuinit __cpu_up(unsigned int cpu)
+int __cpu_up(unsigned int cpu, struct task_struct *tidle)
{
- int ret = smp_boot_one_cpu(cpu);
+ int ret = smp_boot_one_cpu(cpu, tidle);
if (!ret) {
- cpu_set(cpu, smp_commenced_mask);
- while (!cpu_isset(cpu, cpu_online_map))
+ cpumask_set_cpu(cpu, &smp_commenced_mask);
+ while (!cpu_online(cpu))
mb();
- if (!cpu_isset(cpu, cpu_online_map)) {
+ if (!cpu_online(cpu)) {
ret = -ENODEV;
} else {
/* On SUN4V, writes to %tick and %stick are
@@ -1272,7 +1298,7 @@ void cpu_play_dead(void)
tb->nonresum_mondo_pa, 0);
}
- cpu_clear(cpu, smp_commenced_mask);
+ cpumask_clear_cpu(cpu, &smp_commenced_mask);
membar_safe("#Sync");
local_irq_disable();
@@ -1293,13 +1319,13 @@ int __cpu_disable(void)
cpuinfo_sparc *c;
int i;
- for_each_cpu_mask(i, cpu_core_map[cpu])
- cpu_clear(cpu, cpu_core_map[i]);
- cpus_clear(cpu_core_map[cpu]);
+ for_each_cpu(i, &cpu_core_map[cpu])
+ cpumask_clear_cpu(cpu, &cpu_core_map[i]);
+ cpumask_clear(&cpu_core_map[cpu]);
- for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu))
- cpu_clear(cpu, per_cpu(cpu_sibling_map, i));
- cpus_clear(per_cpu(cpu_sibling_map, cpu));
+ for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu))
+ cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
+ cpumask_clear(&per_cpu(cpu_sibling_map, cpu));
c = &cpu_data(cpu);
@@ -1315,9 +1341,7 @@ int __cpu_disable(void)
mdelay(1);
local_irq_disable();
- ipi_call_lock();
- cpu_clear(cpu, cpu_online_map);
- ipi_call_unlock();
+ set_cpu_online(cpu, false);
cpu_map_rebuild();
@@ -1330,11 +1354,11 @@ void __cpu_die(unsigned int cpu)
for (i = 0; i < 100; i++) {
smp_rmb();
- if (!cpu_isset(cpu, smp_commenced_mask))
+ if (!cpumask_test_cpu(cpu, &smp_commenced_mask))
break;
msleep(100);
}
- if (cpu_isset(cpu, smp_commenced_mask)) {
+ if (cpumask_test_cpu(cpu, &smp_commenced_mask)) {
printk(KERN_ERR "CPU %u didn't die...\n", cpu);
} else {
#if defined(CONFIG_SUN_LDOMS)
@@ -1344,7 +1368,7 @@ void __cpu_die(unsigned int cpu)
do {
hv_err = sun4v_cpu_stop(cpu);
if (hv_err == HV_EOK) {
- cpu_clear(cpu, cpu_present_map);
+ set_cpu_present(cpu, false);
break;
}
} while (--limit > 0);
@@ -1364,13 +1388,19 @@ void __init smp_cpus_done(unsigned int max_cpus)
void smp_send_reschedule(int cpu)
{
- xcall_deliver((u64) &xcall_receive_signal, 0, 0,
- &cpumask_of_cpu(cpu));
+ if (cpu == smp_processor_id()) {
+ WARN_ON_ONCE(preemptible());
+ set_softint(1 << PIL_SMP_RECEIVE_SIGNAL);
+ } else {
+ xcall_deliver((u64) &xcall_receive_signal,
+ 0, 0, cpumask_of(cpu));
+ }
}
void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs)
{
clear_softint(1 << irq);
+ scheduler_ipi();
}
/* This is a nop because we capture all other cpus
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-11-09 18:41:10 +0000