diff options
Diffstat (limited to 'arch/powerpc/kernel/irq.c')
| -rw-r--r-- | arch/powerpc/kernel/irq.c | 1221 |
1 files changed, 342 insertions, 879 deletions
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index ce557f6f00f..248ee7e5beb 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -30,7 +30,7 @@ #undef DEBUG -#include <linux/module.h> +#include <linux/export.h> #include <linux/threads.h> #include <linux/kernel_stat.h> #include <linux/signal.h> @@ -57,7 +57,6 @@ #include <linux/of_irq.h> #include <asm/uaccess.h> -#include <asm/system.h> #include <asm/io.h> #include <asm/pgtable.h> #include <asm/irq.h> @@ -66,8 +65,8 @@ #include <asm/ptrace.h> #include <asm/machdep.h> #include <asm/udbg.h> -#include <asm/dbell.h> #include <asm/smp.h> +#include <asm/debug.h> #ifdef CONFIG_PPC64 #include <asm/paca.h> @@ -94,20 +93,16 @@ extern int tau_interrupts(int); #ifdef CONFIG_PPC64 -#ifndef CONFIG_SPARSE_IRQ -EXPORT_SYMBOL(irq_desc); -#endif - int distribute_irqs = 1; -static inline notrace unsigned long get_hard_enabled(void) +static inline notrace unsigned long get_irq_happened(void) { - unsigned long enabled; + unsigned long happened; __asm__ __volatile__("lbz %0,%1(13)" - : "=r" (enabled) : "i" (offsetof(struct paca_struct, hard_enabled))); + : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened))); - return enabled; + return happened; } static inline notrace void set_soft_enabled(unsigned long enable) @@ -116,86 +111,235 @@ static inline notrace void set_soft_enabled(unsigned long enable) : : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled))); } -notrace void arch_local_irq_restore(unsigned long en) +static inline notrace int decrementer_check_overflow(void) +{ + u64 now = get_tb_or_rtc(); + u64 *next_tb = &__get_cpu_var(decrementers_next_tb); + + return now >= *next_tb; +} + +/* This is called whenever we are re-enabling interrupts + * and returns either 0 (nothing to do) or 500/900/280/a00/e80 if + * there's an EE, DEC or DBELL to generate. + * + * This is called in two contexts: From arch_local_irq_restore() + * before soft-enabling interrupts, and from the exception exit + * path when returning from an interrupt from a soft-disabled to + * a soft enabled context. In both case we have interrupts hard + * disabled. + * + * We take care of only clearing the bits we handled in the + * PACA irq_happened field since we can only re-emit one at a + * time and we don't want to "lose" one. + */ +notrace unsigned int __check_irq_replay(void) { /* - * get_paca()->soft_enabled = en; - * Is it ever valid to use local_irq_restore(0) when soft_enabled is 1? - * That was allowed before, and in such a case we do need to take care - * that gcc will set soft_enabled directly via r13, not choose to use - * an intermediate register, lest we're preempted to a different cpu. + * We use local_paca rather than get_paca() to avoid all + * the debug_smp_processor_id() business in this low level + * function + */ + unsigned char happened = local_paca->irq_happened; + + /* Clear bit 0 which we wouldn't clear otherwise */ + local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; + + /* + * Force the delivery of pending soft-disabled interrupts on PS3. + * Any HV call will have this side effect. + */ + if (firmware_has_feature(FW_FEATURE_PS3_LV1)) { + u64 tmp, tmp2; + lv1_get_version_info(&tmp, &tmp2); + } + + /* + * We may have missed a decrementer interrupt. We check the + * decrementer itself rather than the paca irq_happened field + * in case we also had a rollover while hard disabled + */ + local_paca->irq_happened &= ~PACA_IRQ_DEC; + if ((happened & PACA_IRQ_DEC) || decrementer_check_overflow()) + return 0x900; + + /* Finally check if an external interrupt happened */ + local_paca->irq_happened &= ~PACA_IRQ_EE; + if (happened & PACA_IRQ_EE) + return 0x500; + +#ifdef CONFIG_PPC_BOOK3E + /* Finally check if an EPR external interrupt happened + * this bit is typically set if we need to handle another + * "edge" interrupt from within the MPIC "EPR" handler */ + local_paca->irq_happened &= ~PACA_IRQ_EE_EDGE; + if (happened & PACA_IRQ_EE_EDGE) + return 0x500; + + local_paca->irq_happened &= ~PACA_IRQ_DBELL; + if (happened & PACA_IRQ_DBELL) + return 0x280; +#else + local_paca->irq_happened &= ~PACA_IRQ_DBELL; + if (happened & PACA_IRQ_DBELL) { + if (cpu_has_feature(CPU_FTR_HVMODE)) + return 0xe80; + return 0xa00; + } +#endif /* CONFIG_PPC_BOOK3E */ + + /* There should be nothing left ! */ + BUG_ON(local_paca->irq_happened != 0); + + return 0; +} + +notrace void arch_local_irq_restore(unsigned long en) +{ + unsigned char irq_happened; + unsigned int replay; + + /* Write the new soft-enabled value */ set_soft_enabled(en); if (!en) return; + /* + * From this point onward, we can take interrupts, preempt, + * etc... unless we got hard-disabled. We check if an event + * happened. If none happened, we know we can just return. + * + * We may have preempted before the check below, in which case + * we are checking the "new" CPU instead of the old one. This + * is only a problem if an event happened on the "old" CPU. + * + * External interrupt events will have caused interrupts to + * be hard-disabled, so there is no problem, we + * cannot have preempted. + */ + irq_happened = get_irq_happened(); + if (!irq_happened) + return; -#ifdef CONFIG_PPC_STD_MMU_64 - if (firmware_has_feature(FW_FEATURE_ISERIES)) { + /* + * We need to hard disable to get a trusted value from + * __check_irq_replay(). We also need to soft-disable + * again to avoid warnings in there due to the use of + * per-cpu variables. + * + * We know that if the value in irq_happened is exactly 0x01 + * then we are already hard disabled (there are other less + * common cases that we'll ignore for now), so we skip the + * (expensive) mtmsrd. + */ + if (unlikely(irq_happened != PACA_IRQ_HARD_DIS)) + __hard_irq_disable(); +#ifdef CONFIG_TRACE_IRQFLAGS + else { /* - * Do we need to disable preemption here? Not really: in the - * unlikely event that we're preempted to a different cpu in - * between getting r13, loading its lppaca_ptr, and loading - * its any_int, we might call iseries_handle_interrupts without - * an interrupt pending on the new cpu, but that's no disaster, - * is it? And the business of preempting us off the old cpu - * would itself involve a local_irq_restore which handles the - * interrupt to that cpu. - * - * But use "local_paca->lppaca_ptr" instead of "get_lppaca()" - * to avoid any preemption checking added into get_paca(). + * We should already be hard disabled here. We had bugs + * where that wasn't the case so let's dbl check it and + * warn if we are wrong. Only do that when IRQ tracing + * is enabled as mfmsr() can be costly. */ - if (local_paca->lppaca_ptr->int_dword.any_int) - iseries_handle_interrupts(); + if (WARN_ON(mfmsr() & MSR_EE)) + __hard_irq_disable(); } -#endif /* CONFIG_PPC_STD_MMU_64 */ +#endif /* CONFIG_TRACE_IRQFLAG */ + + set_soft_enabled(0); + + /* + * Check if anything needs to be re-emitted. We haven't + * soft-enabled yet to avoid warnings in decrementer_check_overflow + * accessing per-cpu variables + */ + replay = __check_irq_replay(); + + /* We can soft-enable now */ + set_soft_enabled(1); /* - * if (get_paca()->hard_enabled) return; - * But again we need to take care that gcc gets hard_enabled directly - * via r13, not choose to use an intermediate register, lest we're - * preempted to a different cpu in between the two instructions. + * And replay if we have to. This will return with interrupts + * hard-enabled. */ - if (get_hard_enabled()) + if (replay) { + __replay_interrupt(replay); return; + } -#if defined(CONFIG_BOOKE) && defined(CONFIG_SMP) - /* Check for pending doorbell interrupts and resend to ourself */ - doorbell_check_self(); -#endif + /* Finally, let's ensure we are hard enabled */ + __hard_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_restore); + +/* + * This is specifically called by assembly code to re-enable interrupts + * if they are currently disabled. This is typically called before + * schedule() or do_signal() when returning to userspace. We do it + * in C to avoid the burden of dealing with lockdep etc... + * + * NOTE: This is called with interrupts hard disabled but not marked + * as such in paca->irq_happened, so we need to resync this. + */ +void notrace restore_interrupts(void) +{ + if (irqs_disabled()) { + local_paca->irq_happened |= PACA_IRQ_HARD_DIS; + local_irq_enable(); + } else + __hard_irq_enable(); +} +/* + * This is a helper to use when about to go into idle low-power + * when the latter has the side effect of re-enabling interrupts + * (such as calling H_CEDE under pHyp). + * + * You call this function with interrupts soft-disabled (this is + * already the case when ppc_md.power_save is called). The function + * will return whether to enter power save or just return. + * + * In the former case, it will have notified lockdep of interrupts + * being re-enabled and generally sanitized the lazy irq state, + * and in the latter case it will leave with interrupts hard + * disabled and marked as such, so the local_irq_enable() call + * in arch_cpu_idle() will properly re-enable everything. + */ +bool prep_irq_for_idle(void) +{ /* - * Need to hard-enable interrupts here. Since currently disabled, - * no need to take further asm precautions against preemption; but - * use local_paca instead of get_paca() to avoid preemption checking. + * First we need to hard disable to ensure no interrupt + * occurs before we effectively enter the low power state */ - local_paca->hard_enabled = en; + hard_irq_disable(); -#ifndef CONFIG_BOOKE - /* On server, re-trigger the decrementer if it went negative since - * some processors only trigger on edge transitions of the sign bit. - * - * BookE has a level sensitive decrementer (latches in TSR) so we - * don't need that + /* + * If anything happened while we were soft-disabled, + * we return now and do not enter the low power state. */ - if ((int)mfspr(SPRN_DEC) < 0) - mtspr(SPRN_DEC, 1); -#endif /* CONFIG_BOOKE */ + if (lazy_irq_pending()) + return false; + + /* Tell lockdep we are about to re-enable */ + trace_hardirqs_on(); /* - * Force the delivery of pending soft-disabled interrupts on PS3. - * Any HV call will have this side effect. + * Mark interrupts as soft-enabled and clear the + * PACA_IRQ_HARD_DIS from the pending mask since we + * are about to hard enable as well as a side effect + * of entering the low power state. */ - if (firmware_has_feature(FW_FEATURE_PS3_LV1)) { - u64 tmp; - lv1_get_version_info(&tmp); - } + local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; + local_paca->soft_enabled = 1; - __hard_irq_enable(); + /* Tell the caller to enter the low power state */ + return true; } -EXPORT_SYMBOL(arch_local_irq_restore); + #endif /* CONFIG_PPC64 */ -static int show_other_interrupts(struct seq_file *p, int prec) +int arch_show_interrupts(struct seq_file *p, int prec) { int j; @@ -210,15 +354,20 @@ static int show_other_interrupts(struct seq_file *p, int prec) seq_printf(p, "%*s: ", prec, "LOC"); for_each_online_cpu(j) - seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs); - seq_printf(p, " Local timer interrupts\n"); + seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event); + seq_printf(p, " Local timer interrupts for timer event device\n"); + + seq_printf(p, "%*s: ", prec, "LOC"); + for_each_online_cpu(j) + seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others); + seq_printf(p, " Local timer interrupts for others\n"); seq_printf(p, "%*s: ", prec, "SPU"); for_each_online_cpu(j) seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs); seq_printf(p, " Spurious interrupts\n"); - seq_printf(p, "%*s: ", prec, "CNT"); + seq_printf(p, "%*s: ", prec, "PMI"); for_each_online_cpu(j) seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs); seq_printf(p, " Performance monitoring interrupts\n"); @@ -228,63 +377,15 @@ static int show_other_interrupts(struct seq_file *p, int prec) seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions); seq_printf(p, " Machine check exceptions\n"); - return 0; -} - -int show_interrupts(struct seq_file *p, void *v) -{ - unsigned long flags, any_count = 0; - int i = *(loff_t *) v, j, prec; - struct irqaction *action; - struct irq_desc *desc; - - if (i > nr_irqs) - return 0; - - for (prec = 3, j = 1000; prec < 10 && j <= nr_irqs; ++prec) - j *= 10; - - if (i == nr_irqs) - return show_other_interrupts(p, prec); - - /* print header */ - if (i == 0) { - seq_printf(p, "%*s", prec + 8, ""); +#ifdef CONFIG_PPC_DOORBELL + if (cpu_has_feature(CPU_FTR_DBELL)) { + seq_printf(p, "%*s: ", prec, "DBL"); for_each_online_cpu(j) - seq_printf(p, "CPU%-8d", j); - seq_putc(p, '\n'); - } - - desc = irq_to_desc(i); - if (!desc) - return 0; - - raw_spin_lock_irqsave(&desc->lock, flags); - for_each_online_cpu(j) - any_count |= kstat_irqs_cpu(i, j); - action = desc->action; - if (!action && !any_count) - goto out; - - seq_printf(p, "%*d: ", prec, i); - for_each_online_cpu(j) - seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); - - if (desc->chip) - seq_printf(p, " %-16s", desc->chip->name); - else - seq_printf(p, " %-16s", "None"); - seq_printf(p, " %-8s", (desc->status & IRQ_LEVEL) ? "Level" : "Edge"); - - if (action) { - seq_printf(p, " %s", action->name); - while ((action = action->next) != NULL) - seq_printf(p, ", %s", action->name); + seq_printf(p, "%10u ", per_cpu(irq_stat, j).doorbell_irqs); + seq_printf(p, " Doorbell interrupts\n"); } +#endif - seq_putc(p, '\n'); -out: - raw_spin_unlock_irqrestore(&desc->lock, flags); return 0; } @@ -293,40 +394,47 @@ out: */ u64 arch_irq_stat_cpu(unsigned int cpu) { - u64 sum = per_cpu(irq_stat, cpu).timer_irqs; + u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event; sum += per_cpu(irq_stat, cpu).pmu_irqs; sum += per_cpu(irq_stat, cpu).mce_exceptions; sum += per_cpu(irq_stat, cpu).spurious_irqs; + sum += per_cpu(irq_stat, cpu).timer_irqs_others; +#ifdef CONFIG_PPC_DOORBELL + sum += per_cpu(irq_stat, cpu).doorbell_irqs; +#endif return sum; } #ifdef CONFIG_HOTPLUG_CPU -void fixup_irqs(const struct cpumask *map) +void migrate_irqs(void) { struct irq_desc *desc; unsigned int irq; static int warned; cpumask_var_t mask; + const struct cpumask *map = cpu_online_mask; alloc_cpumask_var(&mask, GFP_KERNEL); - for_each_irq(irq) { - desc = irq_to_desc(irq); - if (!desc) - continue; + for_each_irq_desc(irq, desc) { + struct irq_data *data; + struct irq_chip *chip; - if (desc->status & IRQ_PER_CPU) + data = irq_desc_get_irq_data(desc); + if (irqd_is_per_cpu(data)) continue; - cpumask_and(mask, desc->affinity, map); + chip = irq_data_get_irq_chip(data); + + cpumask_and(mask, data->affinity, map); if (cpumask_any(mask) >= nr_cpu_ids) { printk("Breaking affinity for irq %i\n", irq); cpumask_copy(mask, map); } - if (desc->chip->set_affinity) - desc->chip->set_affinity(irq, mask); + if (chip->irq_set_affinity) + chip->irq_set_affinity(data, mask, true); else if (desc->action && !(warned++)) printk("Cannot set affinity for irq %i\n", irq); } @@ -339,47 +447,6 @@ void fixup_irqs(const struct cpumask *map) } #endif -static inline void handle_one_irq(unsigned int irq) -{ - struct thread_info *curtp, *irqtp; - unsigned long saved_sp_limit; - struct irq_desc *desc; - - /* Switch to the irq stack to handle this */ - curtp = current_thread_info(); - irqtp = hardirq_ctx[smp_processor_id()]; - - if (curtp == irqtp) { - /* We're already on the irq stack, just handle it */ - generic_handle_irq(irq); - return; - } - - desc = irq_to_desc(irq); - saved_sp_limit = current->thread.ksp_limit; - - irqtp->task = curtp->task; - irqtp->flags = 0; - - /* Copy the softirq bits in preempt_count so that the - * softirq checks work in the hardirq context. */ - irqtp->preempt_count = (irqtp->preempt_count & ~SOFTIRQ_MASK) | - (curtp->preempt_count & SOFTIRQ_MASK); - - current->thread.ksp_limit = (unsigned long)irqtp + - _ALIGN_UP(sizeof(struct thread_info), 16); - - call_handle_irq(irq, desc, irqtp, desc->handle_irq); - current->thread.ksp_limit = saved_sp_limit; - irqtp->task = NULL; - - /* Set any flag that may have been set on the - * alternate stack - */ - if (irqtp->flags) - set_bits(irqtp->flags, &curtp->flags); -} - static inline void check_stack_overflow(void) { #ifdef CONFIG_DEBUG_STACKOVERFLOW @@ -396,37 +463,72 @@ static inline void check_stack_overflow(void) #endif } -void do_IRQ(struct pt_regs *regs) +void __do_irq(struct pt_regs *regs) { - struct pt_regs *old_regs = set_irq_regs(regs); unsigned int irq; - trace_irq_entry(regs); - irq_enter(); + trace_irq_entry(regs); + check_stack_overflow(); + /* + * Query the platform PIC for the interrupt & ack it. + * + * This will typically lower the interrupt line to the CPU + */ irq = ppc_md.get_irq(); - if (irq != NO_IRQ && irq != NO_IRQ_IGNORE) - handle_one_irq(irq); - else if (irq != NO_IRQ_IGNORE) + /* We can hard enable interrupts now to allow perf interrupts */ + may_hard_irq_enable(); + + /* And finally process it */ + if (unlikely(irq == NO_IRQ)) __get_cpu_var(irq_stat).spurious_irqs++; + else + generic_handle_irq(irq); + + trace_irq_exit(regs); irq_exit(); - set_irq_regs(old_regs); +} + +void do_IRQ(struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + struct thread_info *curtp, *irqtp, *sirqtp; -#ifdef CONFIG_PPC_ISERIES - if (firmware_has_feature(FW_FEATURE_ISERIES) && - get_lppaca()->int_dword.fields.decr_int) { - get_lppaca()->int_dword.fields.decr_int = 0; - /* Signal a fake decrementer interrupt */ - timer_interrupt(regs); + /* Switch to the irq stack to handle this */ + curtp = current_thread_info(); + irqtp = hardirq_ctx[raw_smp_processor_id()]; + sirqtp = softirq_ctx[raw_smp_processor_id()]; + + /* Already there ? */ + if (unlikely(curtp == irqtp || curtp == sirqtp)) { + __do_irq(regs); + set_irq_regs(old_regs); + return; } -#endif - trace_irq_exit(regs); + /* Prepare the thread_info in the irq stack */ + irqtp->task = curtp->task; + irqtp->flags = 0; + + /* Copy the preempt_count so that the [soft]irq checks work. */ + irqtp->preempt_count = curtp->preempt_count; + + /* Switch stack and call */ + call_do_irq(regs, irqtp); + + /* Restore stack limit */ + irqtp->task = NULL; + + /* Copy back updates to the thread_info */ + if (irqtp->flags) + set_bits(irqtp->flags, &curtp->flags); + + set_irq_regs(old_regs); } void __init init_IRQ(void) @@ -447,24 +549,33 @@ struct thread_info *mcheckirq_ctx[NR_CPUS] __read_mostly; void exc_lvl_ctx_init(void) { struct thread_info *tp; - int i, hw_cpu; + int i, cpu_nr; for_each_possible_cpu(i) { - hw_cpu = get_hard_smp_processor_id(i); - memset((void *)critirq_ctx[hw_cpu], 0, THREAD_SIZE); - tp = critirq_ctx[hw_cpu]; - tp->cpu = i; +#ifdef CONFIG_PPC64 + cpu_nr = i; +#else +#ifdef CONFIG_SMP + cpu_nr = get_hard_smp_processor_id(i); +#else + cpu_nr = 0; +#endif +#endif + + memset((void *)critirq_ctx[cpu_nr], 0, THREAD_SIZE); + tp = critirq_ctx[cpu_nr]; + tp->cpu = cpu_nr; tp->preempt_count = 0; #ifdef CONFIG_BOOKE - memset((void *)dbgirq_ctx[hw_cpu], 0, THREAD_SIZE); - tp = dbgirq_ctx[hw_cpu]; - tp->cpu = i; + memset((void *)dbgirq_ctx[cpu_nr], 0, THREAD_SIZE); + tp = dbgirq_ctx[cpu_nr]; + tp->cpu = cpu_nr; tp->preempt_count = 0; - memset((void *)mcheckirq_ctx[hw_cpu], 0, THREAD_SIZE); - tp = mcheckirq_ctx[hw_cpu]; - tp->cpu = i; + memset((void *)mcheckirq_ctx[cpu_nr], 0, THREAD_SIZE); + tp = mcheckirq_ctx[cpu_nr]; + tp->cpu = cpu_nr; tp->preempt_count = HARDIRQ_OFFSET; #endif } @@ -483,726 +594,78 @@ void irq_ctx_init(void) memset((void *)softirq_ctx[i], 0, THREAD_SIZE); tp = softirq_ctx[i]; tp->cpu = i; - tp->preempt_count = 0; memset((void *)hardirq_ctx[i], 0, THREAD_SIZE); tp = hardirq_ctx[i]; tp->cpu = i; - tp->preempt_count = HARDIRQ_OFFSET; } } -static inline void do_softirq_onstack(void) +void do_softirq_own_stack(void) { struct thread_info *curtp, *irqtp; - unsigned long saved_sp_limit = current->thread.ksp_limit; curtp = current_thread_info(); irqtp = softirq_ctx[smp_processor_id()]; irqtp->task = curtp->task; - current->thread.ksp_limit = (unsigned long)irqtp + - _ALIGN_UP(sizeof(struct thread_info), 16); + irqtp->flags = 0; call_do_softirq(irqtp); - current->thread.ksp_limit = saved_sp_limit; irqtp->task = NULL; -} - -void do_softirq(void) -{ - unsigned long flags; - - if (in_interrupt()) - return; - - local_irq_save(flags); - - if (local_softirq_pending()) - do_softirq_onstack(); - local_irq_restore(flags); + /* Set any flag that may have been set on the + * alternate stack + */ + if (irqtp->flags) + set_bits(irqtp->flags, &curtp->flags); } - -/* - * IRQ controller and virtual interrupts - */ - -static LIST_HEAD(irq_hosts); -static DEFINE_RAW_SPINLOCK(irq_big_lock); -static unsigned int revmap_trees_allocated; -static DEFINE_MUTEX(revmap_trees_mutex); -struct irq_map_entry irq_map[NR_IRQS]; -static unsigned int irq_virq_count = NR_IRQS; -static struct irq_host *irq_default_host; - irq_hw_number_t virq_to_hw(unsigned int virq) { - return irq_map[virq].hwirq; + struct irq_data *irq_data = irq_get_irq_data(virq); + return WARN_ON(!irq_data) ? 0 : irq_data->hwirq; } EXPORT_SYMBOL_GPL(virq_to_hw); -static int default_irq_host_match(struct irq_host *h, struct device_node *np) +#ifdef CONFIG_SMP +int irq_choose_cpu(const struct cpumask *mask) { - return h->of_node != NULL && h->of_node == np; -} + int cpuid; -struct irq_host *irq_alloc_host(struct device_node *of_node, - unsigned int revmap_type, - unsigned int revmap_arg, - struct irq_host_ops *ops, - irq_hw_number_t inval_irq) -{ - struct irq_host *host; - unsigned int size = sizeof(struct irq_host); - unsigned int i; - unsigned int *rmap; - unsigned long flags; - - /* Allocate structure and revmap table if using linear mapping */ - if (revmap_type == IRQ_HOST_MAP_LINEAR) - size += revmap_arg * sizeof(unsigned int); - host = zalloc_maybe_bootmem(size, GFP_KERNEL); - if (host == NULL) - return NULL; - - /* Fill structure */ - host->revmap_type = revmap_type; - host->inval_irq = inval_irq; - host->ops = ops; - host->of_node = of_node_get(of_node); - - if (host->ops->match == NULL) - host->ops->match = default_irq_host_match; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* If it's a legacy controller, check for duplicates and - * mark it as allocated (we use irq 0 host pointer for that - */ - if (revmap_type == IRQ_HOST_MAP_LEGACY) { - if (irq_map[0].host != NULL) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - /* If we are early boot, we can't free the structure, - * too bad... - * this will be fixed once slab is made available early - * instead of the current cruft - */ - if (mem_init_done) { - of_node_put(host->of_node); - kfree(host); - } - return NULL; - } - irq_map[0].host = host; - } - - list_add(&host->link, &irq_hosts); - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - - /* Additional setups per revmap type */ - switch(revmap_type) { - case IRQ_HOST_MAP_LEGACY: - /* 0 is always the invalid number for legacy */ - host->inval_irq = 0; - /* setup us as the host for all legacy interrupts */ - for (i = 1; i < NUM_ISA_INTERRUPTS; i++) { - irq_map[i].hwirq = i; - smp_wmb(); - irq_map[i].host = host; - smp_wmb(); - - /* Clear norequest flags */ - irq_to_desc(i)->status &= ~IRQ_NOREQUEST; - - /* Legacy flags are left to default at this point, - * one can then use irq_create_mapping() to - * explicitly change them - */ - ops->map(host, i, i); - } - break; - case IRQ_HOST_MAP_LINEAR: - rmap = (unsigned int *)(host + 1); - for (i = 0; i < revmap_arg; i++) - rmap[i] = NO_IRQ; - host->revmap_data.linear.size = revmap_arg; - smp_wmb(); - host->revmap_data.linear.revmap = rmap; - break; - default: - break; - } - - pr_debug("irq: Allocated host of type %d @0x%p\n", revmap_type, host); - - return host; -} - -struct irq_host *irq_find_host(struct device_node *node) -{ - struct irq_host *h, *found = NULL; - unsigned long flags; - - /* We might want to match the legacy controller last since - * it might potentially be set to match all interrupts in - * the absence of a device node. This isn't a problem so far - * yet though... - */ - raw_spin_lock_irqsave(&irq_big_lock, flags); - list_for_each_entry(h, &irq_hosts, link) - if (h->ops->match(h, node)) { - found = h; - break; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} -EXPORT_SYMBOL_GPL(irq_find_host); - -void irq_set_default_host(struct irq_host *host) -{ - pr_debug("irq: Default host set to @0x%p\n", host); - - irq_default_host = host; -} - -void irq_set_virq_count(unsigned int count) -{ - pr_debug("irq: Trying to set virq count to %d\n", count); - - BUG_ON(count < NUM_ISA_INTERRUPTS); - if (count < NR_IRQS) - irq_virq_count = count; -} - -static int irq_setup_virq(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - struct irq_desc *desc; + if (cpumask_equal(mask, cpu_online_mask)) { + static int irq_rover; + static DEFINE_RAW_SPINLOCK(irq_rover_lock); + unsigned long flags; - desc = irq_to_desc_alloc_node(virq, 0); - if (!desc) { - pr_debug("irq: -> allocating desc failed\n"); - goto error; - } - - /* Clear IRQ_NOREQUEST flag */ - desc->status &= ~IRQ_NOREQUEST; - - /* map it */ - smp_wmb(); - irq_map[virq].hwirq = hwirq; - smp_mb(); - - if (host->ops->map(host, virq, hwirq)) { - pr_debug("irq: -> mapping failed, freeing\n"); - goto error; - } - - return 0; - -error: - irq_free_virt(virq, 1); - return -1; -} - -unsigned int irq_create_direct_mapping(struct irq_host *host) -{ - unsigned int virq; - - if (host == NULL) - host = irq_default_host; + /* Round-robin distribution... */ +do_round_robin: + raw_spin_lock_irqsave(&irq_rover_lock, flags); - BUG_ON(host == NULL); - WARN_ON(host->revmap_type != IRQ_HOST_MAP_NOMAP); + irq_rover = cpumask_next(irq_rover, cpu_online_mask); + if (irq_rover >= nr_cpu_ids) + irq_rover = cpumask_first(cpu_online_mask); - virq = irq_alloc_virt(host, 1, 0); - if (virq == NO_IRQ) { - pr_debug("irq: create_direct virq allocation failed\n"); - return NO_IRQ; - } - - pr_debug("irq: create_direct obtained virq %d\n", virq); - - if (irq_setup_virq(host, virq, virq)) - return NO_IRQ; - - return virq; -} - -unsigned int irq_create_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int virq, hint; - - pr_debug("irq: irq_create_mapping(0x%p, 0x%lx)\n", host, hwirq); - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) { - printk(KERN_WARNING "irq_create_mapping called for" - " NULL host, hwirq=%lx\n", hwirq); - WARN_ON(1); - return NO_IRQ; - } - pr_debug("irq: -> using host @%p\n", host); - - /* Check if mapping already exist, if it does, call - * host->ops->map() to update the flags - */ - virq = irq_find_mapping(host, hwirq); - if (virq != NO_IRQ) { - if (host->ops->remap) - host->ops->remap(host, virq, hwirq); - pr_debug("irq: -> existing mapping on virq %d\n", virq); - return virq; - } + cpuid = irq_rover; - /* Get a virtual interrupt number */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) { - /* Handle legacy */ - virq = (unsigned int)hwirq; - if (virq == 0 || virq >= NUM_ISA_INTERRUPTS) - return NO_IRQ; - return virq; + raw_spin_unlock_irqrestore(&irq_rover_lock, flags); } else { - /* Allocate a virtual interrupt number */ - hint = hwirq % irq_virq_count; - virq = irq_alloc_virt(host, 1, hint); - if (virq == NO_IRQ) { - pr_debug("irq: -> virq allocation failed\n"); - return NO_IRQ; - } + cpuid = cpumask_first_and(mask, cpu_online_mask); + if (cpuid >= nr_cpu_ids) + goto do_round_robin; } - if (irq_setup_virq(host, virq, hwirq)) - return NO_IRQ; - - printk(KERN_DEBUG "irq: irq %lu on host %s mapped to virtual irq %u\n", - hwirq, host->of_node ? host->of_node->full_name : "null", virq); - - return virq; + return get_hard_smp_processor_id(cpuid); } -EXPORT_SYMBOL_GPL(irq_create_mapping); - -unsigned int irq_create_of_mapping(struct device_node *controller, - const u32 *intspec, unsigned int intsize) +#else +int irq_choose_cpu(const struct cpumask *mask) { - struct irq_host *host; - irq_hw_number_t hwirq; - unsigned int type = IRQ_TYPE_NONE; - unsigned int virq; - - if (controller == NULL) - host = irq_default_host; - else - host = irq_find_host(controller); - if (host == NULL) { - printk(KERN_WARNING "irq: no irq host found for %s !\n", - controller->full_name); - return NO_IRQ; - } - - /* If host has no translation, then we assume interrupt line */ - if (host->ops->xlate == NULL) - hwirq = intspec[0]; - else { - if (host->ops->xlate(host, controller, intspec, intsize, - &hwirq, &type)) - return NO_IRQ; - } - - /* Create mapping */ - virq = irq_create_mapping(host, hwirq); - if (virq == NO_IRQ) - return virq; - - /* Set type if specified and different than the current one */ - if (type != IRQ_TYPE_NONE && - type != (irq_to_desc(virq)->status & IRQF_TRIGGER_MASK)) - set_irq_type(virq, type); - return virq; -} -EXPORT_SYMBOL_GPL(irq_create_of_mapping); - -void irq_dispose_mapping(unsigned int virq) -{ - struct irq_host *host; - irq_hw_number_t hwirq; - - if (virq == NO_IRQ) - return; - - host = irq_map[virq].host; - WARN_ON (host == NULL); - if (host == NULL) - return; - - /* Never unmap legacy interrupts */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) - return; - - /* remove chip and handler */ - set_irq_chip_and_handler(virq, NULL, NULL); - - /* Make sure it's completed */ - synchronize_irq(virq); - - /* Tell the PIC about it */ - if (host->ops->unmap) - host->ops->unmap(host, virq); - smp_mb(); - - /* Clear reverse map */ - hwirq = irq_map[virq].hwirq; - switch(host->revmap_type) { - case IRQ_HOST_MAP_LINEAR: - if (hwirq < host->revmap_data.linear.size) - host->revmap_data.linear.revmap[hwirq] = NO_IRQ; - break; - case IRQ_HOST_MAP_TREE: - /* - * Check if radix tree allocated yet, if not then nothing to - * remove. - */ - smp_rmb(); - if (revmap_trees_allocated < 1) - break; - mutex_lock(&revmap_trees_mutex); - radix_tree_delete(&host->revmap_data.tree, hwirq); - mutex_unlock(&revmap_trees_mutex); - break; - } - - /* Destroy map */ - smp_mb(); - irq_map[virq].hwirq = host->inval_irq; - - /* Set some flags */ - irq_to_desc(virq)->status |= IRQ_NOREQUEST; - - /* Free it */ - irq_free_virt(virq, 1); -} -EXPORT_SYMBOL_GPL(irq_dispose_mapping); - -unsigned int irq_find_mapping(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int i; - unsigned int hint = hwirq % irq_virq_count; - - /* Look for default host if nececssary */ - if (host == NULL) - host = irq_default_host; - if (host == NULL) - return NO_IRQ; - - /* legacy -> bail early */ - if (host->revmap_type == IRQ_HOST_MAP_LEGACY) - return hwirq; - - /* Slow path does a linear search of the map */ - if (hint < NUM_ISA_INTERRUPTS) - hint = NUM_ISA_INTERRUPTS; - i = hint; - do { - if (irq_map[i].host == host && - irq_map[i].hwirq == hwirq) - return i; - i++; - if (i >= irq_virq_count) - i = NUM_ISA_INTERRUPTS; - } while(i != hint); - return NO_IRQ; -} -EXPORT_SYMBOL_GPL(irq_find_mapping); - - -unsigned int irq_radix_revmap_lookup(struct irq_host *host, - irq_hw_number_t hwirq) -{ - struct irq_map_entry *ptr; - unsigned int virq; - - WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE); - - /* - * Check if the radix tree exists and has bee initialized. - * If not, we fallback to slow mode - */ - if (revmap_trees_allocated < 2) - return irq_find_mapping(host, hwirq); - - /* Now try to resolve */ - /* - * No rcu_read_lock(ing) needed, the ptr returned can't go under us - * as it's referencing an entry in the static irq_map table. - */ - ptr = radix_tree_lookup(&host->revmap_data.tree, hwirq); - - /* - * If found in radix tree, then fine. - * Else fallback to linear lookup - this should not happen in practice - * as it means that we failed to insert the node in the radix tree. - */ - if (ptr) - virq = ptr - irq_map; - else - virq = irq_find_mapping(host, hwirq); - - return virq; -} - -void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq, - irq_hw_number_t hwirq) -{ - - WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE); - - /* - * Check if the radix tree exists yet. - * If not, then the irq will be inserted into the tree when it gets - * initialized. - */ - smp_rmb(); - if (revmap_trees_allocated < 1) - return; - - if (virq != NO_IRQ) { - mutex_lock(&revmap_trees_mutex); - radix_tree_insert(&host->revmap_data.tree, hwirq, - &irq_map[virq]); - mutex_unlock(&revmap_trees_mutex); - } -} - -unsigned int irq_linear_revmap(struct irq_host *host, - irq_hw_number_t hwirq) -{ - unsigned int *revmap; - - WARN_ON(host->revmap_type != IRQ_HOST_MAP_LINEAR); - - /* Check revmap bounds */ - if (unlikely(hwirq >= host->revmap_data.linear.size)) - return irq_find_mapping(host, hwirq); - - /* Check if revmap was allocated */ - revmap = host->revmap_data.linear.revmap; - if (unlikely(revmap == NULL)) - return irq_find_mapping(host, hwirq); - - /* Fill up revmap with slow path if no mapping found */ - if (unlikely(revmap[hwirq] == NO_IRQ)) - revmap[hwirq] = irq_find_mapping(host, hwirq); - - return revmap[hwirq]; -} - -unsigned int irq_alloc_virt(struct irq_host *host, - unsigned int count, - unsigned int hint) -{ - unsigned long flags; - unsigned int i, j, found = NO_IRQ; - - if (count == 0 || count > (irq_virq_count - NUM_ISA_INTERRUPTS)) - return NO_IRQ; - - raw_spin_lock_irqsave(&irq_big_lock, flags); - - /* Use hint for 1 interrupt if any */ - if (count == 1 && hint >= NUM_ISA_INTERRUPTS && - hint < irq_virq_count && irq_map[hint].host == NULL) { - found = hint; - goto hint_found; - } - - /* Look for count consecutive numbers in the allocatable - * (non-legacy) space - */ - for (i = NUM_ISA_INTERRUPTS, j = 0; i < irq_virq_count; i++) { - if (irq_map[i].host != NULL) - j = 0; - else - j++; - - if (j == count) { - found = i - count + 1; - break; - } - } - if (found == NO_IRQ) { - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return NO_IRQ; - } - hint_found: - for (i = found; i < (found + count); i++) { - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = host; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); - return found; -} - -void irq_free_virt(unsigned int virq, unsigned int count) -{ - unsigned long flags; - unsigned int i; - - WARN_ON (virq < NUM_ISA_INTERRUPTS); - WARN_ON (count == 0 || (virq + count) > irq_virq_count); - - raw_spin_lock_irqsave(&irq_big_lock, flags); - for (i = virq; i < (virq + count); i++) { - struct irq_host *host; - - if (i < NUM_ISA_INTERRUPTS || - (virq + count) > irq_virq_count) - continue; - - host = irq_map[i].host; - irq_map[i].hwirq = host->inval_irq; - smp_wmb(); - irq_map[i].host = NULL; - } - raw_spin_unlock_irqrestore(&irq_big_lock, flags); + return hard_smp_processor_id(); } +#endif int arch_early_irq_init(void) { - struct irq_desc *desc; - int i; - - for (i = 0; i < NR_IRQS; i++) { - desc = irq_to_desc(i); - if (desc) - desc->status |= IRQ_NOREQUEST; - } - - return 0; -} - -int arch_init_chip_data(struct irq_desc *desc, int node) -{ - desc->status |= IRQ_NOREQUEST; - return 0; -} - -/* We need to create the radix trees late */ -static int irq_late_init(void) -{ - struct irq_host *h; - unsigned int i; - - /* - * No mutual exclusion with respect to accessors of the tree is needed - * here as the synchronization is done via the state variable - * revmap_trees_allocated. - */ - list_for_each_entry(h, &irq_hosts, link) { - if (h->revmap_type == IRQ_HOST_MAP_TREE) - INIT_RADIX_TREE(&h->revmap_data.tree, GFP_KERNEL); - } - - /* - * Make sure the radix trees inits are visible before setting - * the flag - */ - smp_wmb(); - revmap_trees_allocated = 1; - - /* - * Insert the reverse mapping for those interrupts already present - * in irq_map[]. - */ - mutex_lock(&revmap_trees_mutex); - for (i = 0; i < irq_virq_count; i++) { - if (irq_map[i].host && - (irq_map[i].host->revmap_type == IRQ_HOST_MAP_TREE)) - radix_tree_insert(&irq_map[i].host->revmap_data.tree, - irq_map[i].hwirq, &irq_map[i]); - } - mutex_unlock(&revmap_trees_mutex); - - /* - * Make sure the radix trees insertions are visible before setting - * the flag - */ - smp_wmb(); - revmap_trees_allocated = 2; - - return 0; -} -arch_initcall(irq_late_init); - -#ifdef CONFIG_VIRQ_DEBUG -static int virq_debug_show(struct seq_file *m, void *private) -{ - unsigned long flags; - struct irq_desc *desc; - const char *p; - static const char none[] = "none"; - int i; - - seq_printf(m, "%-5s %-7s %-15s %s\n", "virq", "hwirq", - "chip name", "host name"); - - for (i = 1; i < nr_irqs; i++) { - desc = irq_to_desc(i); - if (!desc) - continue; - - raw_spin_lock_irqsave(&desc->lock, flags); - - if (desc->action && desc->action->handler) { - seq_printf(m, "%5d ", i); - seq_printf(m, "0x%05lx ", virq_to_hw(i)); - - if (desc->chip && desc->chip->name) - p = desc->chip->name; - else - p = none; - seq_printf(m, "%-15s ", p); - - if (irq_map[i].host && irq_map[i].host->of_node) - p = irq_map[i].host->of_node->full_name; - else - p = none; - seq_printf(m, "%s\n", p); - } - - raw_spin_unlock_irqrestore(&desc->lock, flags); - } - - return 0; -} - -static int virq_debug_open(struct inode *inode, struct file *file) -{ - return single_open(file, virq_debug_show, inode->i_private); -} - -static const struct file_operations virq_debug_fops = { - .open = virq_debug_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int __init irq_debugfs_init(void) -{ - if (debugfs_create_file("virq_mapping", S_IRUGO, powerpc_debugfs_root, - NULL, &virq_debug_fops) == NULL) - return -ENOMEM; - return 0; } -__initcall(irq_debugfs_init); -#endif /* CONFIG_VIRQ_DEBUG */ #ifdef CONFIG_PPC64 static int __init setup_noirqdistrib(char *str) |