diff options
Diffstat (limited to 'drivers/pci/msi.c')
| -rw-r--r-- | drivers/pci/msi.c | 1777 |
1 files changed, 915 insertions, 862 deletions
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index a2033552423..13f3d303727 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -6,572 +6,726 @@ * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com) */ +#include <linux/err.h> #include <linux/mm.h> #include <linux/irq.h> #include <linux/interrupt.h> -#include <linux/init.h> -#include <linux/config.h> +#include <linux/export.h> #include <linux/ioport.h> -#include <linux/smp_lock.h> #include <linux/pci.h> #include <linux/proc_fs.h> - -#include <asm/errno.h> -#include <asm/io.h> -#include <asm/smp.h> +#include <linux/msi.h> +#include <linux/smp.h> +#include <linux/errno.h> +#include <linux/io.h> +#include <linux/slab.h> #include "pci.h" -#include "msi.h" - -static DEFINE_SPINLOCK(msi_lock); -static struct msi_desc* msi_desc[NR_IRQS] = { [0 ... NR_IRQS-1] = NULL }; -static kmem_cache_t* msi_cachep; static int pci_msi_enable = 1; -static int last_alloc_vector; -static int nr_released_vectors; -static int nr_reserved_vectors = NR_HP_RESERVED_VECTORS; -static int nr_msix_devices; -#ifndef CONFIG_X86_IO_APIC -int vector_irq[NR_VECTORS] = { [0 ... NR_VECTORS - 1] = -1}; -u8 irq_vector[NR_IRQ_VECTORS] = { FIRST_DEVICE_VECTOR , 0 }; -#endif +#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) -static void msi_cache_ctor(void *p, kmem_cache_t *cache, unsigned long flags) -{ - memset(p, 0, NR_IRQS * sizeof(struct msi_desc)); -} -static int msi_cache_init(void) +/* Arch hooks */ + +int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) { - msi_cachep = kmem_cache_create("msi_cache", - NR_IRQS * sizeof(struct msi_desc), - 0, SLAB_HWCACHE_ALIGN, msi_cache_ctor, NULL); - if (!msi_cachep) - return -ENOMEM; + struct msi_chip *chip = dev->bus->msi; + int err; + + if (!chip || !chip->setup_irq) + return -EINVAL; + + err = chip->setup_irq(chip, dev, desc); + if (err < 0) + return err; + + irq_set_chip_data(desc->irq, chip); return 0; } -static void msi_set_mask_bit(unsigned int vector, int flag) +void __weak arch_teardown_msi_irq(unsigned int irq) { - struct msi_desc *entry; + struct msi_chip *chip = irq_get_chip_data(irq); - entry = (struct msi_desc *)msi_desc[vector]; - if (!entry || !entry->dev || !entry->mask_base) + if (!chip || !chip->teardown_irq) return; - switch (entry->msi_attrib.type) { - case PCI_CAP_ID_MSI: - { - int pos; - u32 mask_bits; - - pos = (long)entry->mask_base; - pci_read_config_dword(entry->dev, pos, &mask_bits); - mask_bits &= ~(1); - mask_bits |= flag; - pci_write_config_dword(entry->dev, pos, mask_bits); - break; - } - case PCI_CAP_ID_MSIX: - { - int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET; - writel(flag, entry->mask_base + offset); - break; - } - default: - break; - } + + chip->teardown_irq(chip, irq); } -#ifdef CONFIG_SMP -static void set_msi_affinity(unsigned int vector, cpumask_t cpu_mask) +int __weak arch_msi_check_device(struct pci_dev *dev, int nvec, int type) { - struct msi_desc *entry; - struct msg_address address; - unsigned int irq = vector; + struct msi_chip *chip = dev->bus->msi; - entry = (struct msi_desc *)msi_desc[vector]; - if (!entry || !entry->dev) - return; + if (!chip || !chip->check_device) + return 0; - switch (entry->msi_attrib.type) { - case PCI_CAP_ID_MSI: - { - int pos; - - if (!(pos = pci_find_capability(entry->dev, PCI_CAP_ID_MSI))) - return; - - pci_read_config_dword(entry->dev, msi_lower_address_reg(pos), - &address.lo_address.value); - address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; - address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << - MSI_TARGET_CPU_SHIFT); - entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); - pci_write_config_dword(entry->dev, msi_lower_address_reg(pos), - address.lo_address.value); - set_native_irq_info(irq, cpu_mask); - break; - } - case PCI_CAP_ID_MSIX: - { - int offset = entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET; - - address.lo_address.value = readl(entry->mask_base + offset); - address.lo_address.value &= MSI_ADDRESS_DEST_ID_MASK; - address.lo_address.value |= (cpu_mask_to_apicid(cpu_mask) << - MSI_TARGET_CPU_SHIFT); - entry->msi_attrib.current_cpu = cpu_mask_to_apicid(cpu_mask); - writel(address.lo_address.value, entry->mask_base + offset); - set_native_irq_info(irq, cpu_mask); - break; - } - default: - break; + return chip->check_device(chip, dev, nvec, type); +} + +int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + struct msi_desc *entry; + int ret; + + /* + * If an architecture wants to support multiple MSI, it needs to + * override arch_setup_msi_irqs() + */ + if (type == PCI_CAP_ID_MSI && nvec > 1) + return 1; + + list_for_each_entry(entry, &dev->msi_list, list) { + ret = arch_setup_msi_irq(dev, entry); + if (ret < 0) + return ret; + if (ret > 0) + return -ENOSPC; } + + return 0; } -#endif /* CONFIG_SMP */ -static void mask_MSI_irq(unsigned int vector) +/* + * We have a default implementation available as a separate non-weak + * function, as it is used by the Xen x86 PCI code + */ +void default_teardown_msi_irqs(struct pci_dev *dev) { - msi_set_mask_bit(vector, 1); + struct msi_desc *entry; + + list_for_each_entry(entry, &dev->msi_list, list) { + int i, nvec; + if (entry->irq == 0) + continue; + if (entry->nvec_used) + nvec = entry->nvec_used; + else + nvec = 1 << entry->msi_attrib.multiple; + for (i = 0; i < nvec; i++) + arch_teardown_msi_irq(entry->irq + i); + } } -static void unmask_MSI_irq(unsigned int vector) +void __weak arch_teardown_msi_irqs(struct pci_dev *dev) { - msi_set_mask_bit(vector, 0); + return default_teardown_msi_irqs(dev); } -static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector) +static void default_restore_msi_irq(struct pci_dev *dev, int irq) { struct msi_desc *entry; - unsigned long flags; - spin_lock_irqsave(&msi_lock, flags); - entry = msi_desc[vector]; - if (!entry || !entry->dev) { - spin_unlock_irqrestore(&msi_lock, flags); - return 0; + entry = NULL; + if (dev->msix_enabled) { + list_for_each_entry(entry, &dev->msi_list, list) { + if (irq == entry->irq) + break; + } + } else if (dev->msi_enabled) { + entry = irq_get_msi_desc(irq); } - entry->msi_attrib.state = 1; /* Mark it active */ - spin_unlock_irqrestore(&msi_lock, flags); - return 0; /* never anything pending */ + if (entry) + write_msi_msg(irq, &entry->msg); } -static unsigned int startup_msi_irq_w_maskbit(unsigned int vector) +void __weak arch_restore_msi_irqs(struct pci_dev *dev) { - startup_msi_irq_wo_maskbit(vector); - unmask_MSI_irq(vector); - return 0; /* never anything pending */ + return default_restore_msi_irqs(dev); } -static void shutdown_msi_irq(unsigned int vector) +static void msi_set_enable(struct pci_dev *dev, int enable) { - struct msi_desc *entry; - unsigned long flags; + u16 control; - spin_lock_irqsave(&msi_lock, flags); - entry = msi_desc[vector]; - if (entry && entry->dev) - entry->msi_attrib.state = 0; /* Mark it not active */ - spin_unlock_irqrestore(&msi_lock, flags); + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + control &= ~PCI_MSI_FLAGS_ENABLE; + if (enable) + control |= PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } -static void end_msi_irq_wo_maskbit(unsigned int vector) +static void msix_set_enable(struct pci_dev *dev, int enable) { - move_native_irq(vector); - ack_APIC_irq(); + u16 control; + + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); + control &= ~PCI_MSIX_FLAGS_ENABLE; + if (enable) + control |= PCI_MSIX_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); } -static void end_msi_irq_w_maskbit(unsigned int vector) +static inline __attribute_const__ u32 msi_mask(unsigned x) { - move_native_irq(vector); - unmask_MSI_irq(vector); - ack_APIC_irq(); + /* Don't shift by >= width of type */ + if (x >= 5) + return 0xffffffff; + return (1 << (1 << x)) - 1; } -static void do_nothing(unsigned int vector) +static inline __attribute_const__ u32 msi_capable_mask(u16 control) { + return msi_mask((control >> 1) & 7); } -/* - * Interrupt Type for MSI-X PCI/PCI-X/PCI-Express Devices, - * which implement the MSI-X Capability Structure. - */ -static struct hw_interrupt_type msix_irq_type = { - .typename = "PCI-MSI-X", - .startup = startup_msi_irq_w_maskbit, - .shutdown = shutdown_msi_irq, - .enable = unmask_MSI_irq, - .disable = mask_MSI_irq, - .ack = mask_MSI_irq, - .end = end_msi_irq_w_maskbit, - .set_affinity = set_msi_irq_affinity -}; +static inline __attribute_const__ u32 msi_enabled_mask(u16 control) +{ + return msi_mask((control >> 4) & 7); +} /* - * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, - * which implement the MSI Capability Structure with - * Mask-and-Pending Bits. + * PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to + * mask all MSI interrupts by clearing the MSI enable bit does not work + * reliably as devices without an INTx disable bit will then generate a + * level IRQ which will never be cleared. */ -static struct hw_interrupt_type msi_irq_w_maskbit_type = { - .typename = "PCI-MSI", - .startup = startup_msi_irq_w_maskbit, - .shutdown = shutdown_msi_irq, - .enable = unmask_MSI_irq, - .disable = mask_MSI_irq, - .ack = mask_MSI_irq, - .end = end_msi_irq_w_maskbit, - .set_affinity = set_msi_irq_affinity -}; +u32 default_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) +{ + u32 mask_bits = desc->masked; -/* - * Interrupt Type for MSI PCI/PCI-X/PCI-Express Devices, - * which implement the MSI Capability Structure without - * Mask-and-Pending Bits. - */ -static struct hw_interrupt_type msi_irq_wo_maskbit_type = { - .typename = "PCI-MSI", - .startup = startup_msi_irq_wo_maskbit, - .shutdown = shutdown_msi_irq, - .enable = do_nothing, - .disable = do_nothing, - .ack = do_nothing, - .end = end_msi_irq_wo_maskbit, - .set_affinity = set_msi_irq_affinity -}; - -static void msi_data_init(struct msg_data *msi_data, - unsigned int vector) + if (!desc->msi_attrib.maskbit) + return 0; + + mask_bits &= ~mask; + mask_bits |= flag; + pci_write_config_dword(desc->dev, desc->mask_pos, mask_bits); + + return mask_bits; +} + +__weak u32 arch_msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) { - memset(msi_data, 0, sizeof(struct msg_data)); - msi_data->vector = (u8)vector; - msi_data->delivery_mode = MSI_DELIVERY_MODE; - msi_data->level = MSI_LEVEL_MODE; - msi_data->trigger = MSI_TRIGGER_MODE; + return default_msi_mask_irq(desc, mask, flag); } -static void msi_address_init(struct msg_address *msi_address) +static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag) { - unsigned int dest_id; - - memset(msi_address, 0, sizeof(struct msg_address)); - msi_address->hi_address = (u32)0; - dest_id = (MSI_ADDRESS_HEADER << MSI_ADDRESS_HEADER_SHIFT); - msi_address->lo_address.u.dest_mode = MSI_DEST_MODE; - msi_address->lo_address.u.redirection_hint = MSI_REDIRECTION_HINT_MODE; - msi_address->lo_address.u.dest_id = dest_id; - msi_address->lo_address.value |= (MSI_TARGET_CPU << MSI_TARGET_CPU_SHIFT); + desc->masked = arch_msi_mask_irq(desc, mask, flag); } -static int msi_free_vector(struct pci_dev* dev, int vector, int reassign); -static int assign_msi_vector(void) +/* + * This internal function does not flush PCI writes to the device. + * All users must ensure that they read from the device before either + * assuming that the device state is up to date, or returning out of this + * file. This saves a few milliseconds when initialising devices with lots + * of MSI-X interrupts. + */ +u32 default_msix_mask_irq(struct msi_desc *desc, u32 flag) { - static int new_vector_avail = 1; - int vector; - unsigned long flags; + u32 mask_bits = desc->masked; + unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE + + PCI_MSIX_ENTRY_VECTOR_CTRL; + mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; + if (flag) + mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT; + writel(mask_bits, desc->mask_base + offset); + + return mask_bits; +} - /* - * msi_lock is provided to ensure that successful allocation of MSI - * vector is assigned unique among drivers. - */ - spin_lock_irqsave(&msi_lock, flags); +__weak u32 arch_msix_mask_irq(struct msi_desc *desc, u32 flag) +{ + return default_msix_mask_irq(desc, flag); +} - if (!new_vector_avail) { - int free_vector = 0; +static void msix_mask_irq(struct msi_desc *desc, u32 flag) +{ + desc->masked = arch_msix_mask_irq(desc, flag); +} - /* - * vector_irq[] = -1 indicates that this specific vector is: - * - assigned for MSI (since MSI have no associated IRQ) or - * - assigned for legacy if less than 16, or - * - having no corresponding 1:1 vector-to-IOxAPIC IRQ mapping - * vector_irq[] = 0 indicates that this vector, previously - * assigned for MSI, is freed by hotplug removed operations. - * This vector will be reused for any subsequent hotplug added - * operations. - * vector_irq[] > 0 indicates that this vector is assigned for - * IOxAPIC IRQs. This vector and its value provides a 1-to-1 - * vector-to-IOxAPIC IRQ mapping. - */ - for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { - if (vector_irq[vector] != 0) - continue; - free_vector = vector; - if (!msi_desc[vector]) - break; - else - continue; - } - if (!free_vector) { - spin_unlock_irqrestore(&msi_lock, flags); - return -EBUSY; - } - vector_irq[free_vector] = -1; - nr_released_vectors--; - spin_unlock_irqrestore(&msi_lock, flags); - if (msi_desc[free_vector] != NULL) { - struct pci_dev *dev; - int tail; - - /* free all linked vectors before re-assign */ - do { - spin_lock_irqsave(&msi_lock, flags); - dev = msi_desc[free_vector]->dev; - tail = msi_desc[free_vector]->link.tail; - spin_unlock_irqrestore(&msi_lock, flags); - msi_free_vector(dev, tail, 1); - } while (free_vector != tail); - } +static void msi_set_mask_bit(struct irq_data *data, u32 flag) +{ + struct msi_desc *desc = irq_data_get_msi(data); - return free_vector; + if (desc->msi_attrib.is_msix) { + msix_mask_irq(desc, flag); + readl(desc->mask_base); /* Flush write to device */ + } else { + unsigned offset = data->irq - desc->dev->irq; + msi_mask_irq(desc, 1 << offset, flag << offset); } - vector = assign_irq_vector(AUTO_ASSIGN); - last_alloc_vector = vector; - if (vector == LAST_DEVICE_VECTOR) - new_vector_avail = 0; +} - spin_unlock_irqrestore(&msi_lock, flags); - return vector; +void mask_msi_irq(struct irq_data *data) +{ + msi_set_mask_bit(data, 1); } -static int get_new_vector(void) +void unmask_msi_irq(struct irq_data *data) { - int vector; + msi_set_mask_bit(data, 0); +} - if ((vector = assign_msi_vector()) > 0) - set_intr_gate(vector, interrupt[vector]); +void default_restore_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *entry; - return vector; + list_for_each_entry(entry, &dev->msi_list, list) { + default_restore_msi_irq(dev, entry->irq); + } } -static int msi_init(void) +void __read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { - static int status = -ENOMEM; + BUG_ON(entry->dev->current_state != PCI_D0); - if (!status) - return status; + if (entry->msi_attrib.is_msix) { + void __iomem *base = entry->mask_base + + entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; - if (pci_msi_quirk) { - pci_msi_enable = 0; - printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n"); - status = -EINVAL; - return status; + msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); + msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); + msg->data = readl(base + PCI_MSIX_ENTRY_DATA); + } else { + struct pci_dev *dev = entry->dev; + int pos = dev->msi_cap; + u16 data; + + pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, + &msg->address_lo); + if (entry->msi_attrib.is_64) { + pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, + &msg->address_hi); + pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data); + } else { + msg->address_hi = 0; + pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data); + } + msg->data = data; } +} - if ((status = msi_cache_init()) < 0) { - pci_msi_enable = 0; - printk(KERN_WARNING "PCI: MSI cache init failed\n"); - return status; - } - last_alloc_vector = assign_irq_vector(AUTO_ASSIGN); - if (last_alloc_vector < 0) { - pci_msi_enable = 0; - printk(KERN_WARNING "PCI: No interrupt vectors available for MSI\n"); - status = -EBUSY; - return status; - } - vector_irq[last_alloc_vector] = 0; - nr_released_vectors++; +void read_msi_msg(unsigned int irq, struct msi_msg *msg) +{ + struct msi_desc *entry = irq_get_msi_desc(irq); - return status; + __read_msi_msg(entry, msg); } -static int get_msi_vector(struct pci_dev *dev) +void __get_cached_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { - return get_new_vector(); + /* Assert that the cache is valid, assuming that + * valid messages are not all-zeroes. */ + BUG_ON(!(entry->msg.address_hi | entry->msg.address_lo | + entry->msg.data)); + + *msg = entry->msg; } -static struct msi_desc* alloc_msi_entry(void) +void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg) { - struct msi_desc *entry; - - entry = kmem_cache_alloc(msi_cachep, SLAB_KERNEL); - if (!entry) - return NULL; + struct msi_desc *entry = irq_get_msi_desc(irq); - memset(entry, 0, sizeof(struct msi_desc)); - entry->link.tail = entry->link.head = 0; /* single message */ - entry->dev = NULL; + __get_cached_msi_msg(entry, msg); +} - return entry; +void __write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) +{ + if (entry->dev->current_state != PCI_D0) { + /* Don't touch the hardware now */ + } else if (entry->msi_attrib.is_msix) { + void __iomem *base; + base = entry->mask_base + + entry->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE; + + writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); + writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); + writel(msg->data, base + PCI_MSIX_ENTRY_DATA); + } else { + struct pci_dev *dev = entry->dev; + int pos = dev->msi_cap; + u16 msgctl; + + pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); + msgctl &= ~PCI_MSI_FLAGS_QSIZE; + msgctl |= entry->msi_attrib.multiple << 4; + pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); + + pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, + msg->address_lo); + if (entry->msi_attrib.is_64) { + pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, + msg->address_hi); + pci_write_config_word(dev, pos + PCI_MSI_DATA_64, + msg->data); + } else { + pci_write_config_word(dev, pos + PCI_MSI_DATA_32, + msg->data); + } + } + entry->msg = *msg; } -static void attach_msi_entry(struct msi_desc *entry, int vector) +void write_msi_msg(unsigned int irq, struct msi_msg *msg) { - unsigned long flags; + struct msi_desc *entry = irq_get_msi_desc(irq); - spin_lock_irqsave(&msi_lock, flags); - msi_desc[vector] = entry; - spin_unlock_irqrestore(&msi_lock, flags); + __write_msi_msg(entry, msg); } -static void irq_handler_init(int cap_id, int pos, int mask) +static void free_msi_irqs(struct pci_dev *dev) { - spin_lock(&irq_desc[pos].lock); - if (cap_id == PCI_CAP_ID_MSIX) - irq_desc[pos].handler = &msix_irq_type; - else { - if (!mask) - irq_desc[pos].handler = &msi_irq_wo_maskbit_type; + struct msi_desc *entry, *tmp; + struct attribute **msi_attrs; + struct device_attribute *dev_attr; + int count = 0; + + list_for_each_entry(entry, &dev->msi_list, list) { + int i, nvec; + if (!entry->irq) + continue; + if (entry->nvec_used) + nvec = entry->nvec_used; else - irq_desc[pos].handler = &msi_irq_w_maskbit_type; + nvec = 1 << entry->msi_attrib.multiple; + for (i = 0; i < nvec; i++) + BUG_ON(irq_has_action(entry->irq + i)); } - spin_unlock(&irq_desc[pos].lock); + + arch_teardown_msi_irqs(dev); + + list_for_each_entry_safe(entry, tmp, &dev->msi_list, list) { + if (entry->msi_attrib.is_msix) { + if (list_is_last(&entry->list, &dev->msi_list)) + iounmap(entry->mask_base); + } + + /* + * Its possible that we get into this path + * When populate_msi_sysfs fails, which means the entries + * were not registered with sysfs. In that case don't + * unregister them. + */ + if (entry->kobj.parent) { + kobject_del(&entry->kobj); + kobject_put(&entry->kobj); + } + + list_del(&entry->list); + kfree(entry); + } + + if (dev->msi_irq_groups) { + sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups); + msi_attrs = dev->msi_irq_groups[0]->attrs; + while (msi_attrs[count]) { + dev_attr = container_of(msi_attrs[count], + struct device_attribute, attr); + kfree(dev_attr->attr.name); + kfree(dev_attr); + ++count; + } + kfree(msi_attrs); + kfree(dev->msi_irq_groups[0]); + kfree(dev->msi_irq_groups); + dev->msi_irq_groups = NULL; + } +} + +static struct msi_desc *alloc_msi_entry(struct pci_dev *dev) +{ + struct msi_desc *desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + return NULL; + + INIT_LIST_HEAD(&desc->list); + desc->dev = dev; + + return desc; +} + +static void pci_intx_for_msi(struct pci_dev *dev, int enable) +{ + if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG)) + pci_intx(dev, enable); } -static void enable_msi_mode(struct pci_dev *dev, int pos, int type) +static void __pci_restore_msi_state(struct pci_dev *dev) { u16 control; + struct msi_desc *entry; - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (type == PCI_CAP_ID_MSI) { - /* Set enabled bits to single MSI & enable MSI_enable bit */ - msi_enable(control, 1); - pci_write_config_word(dev, msi_control_reg(pos), control); - } else { - msix_enable(control); - pci_write_config_word(dev, msi_control_reg(pos), control); - } - if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { - /* PCI Express Endpoint device detected */ - pci_intx(dev, 0); /* disable intx */ - } + if (!dev->msi_enabled) + return; + + entry = irq_get_msi_desc(dev->irq); + + pci_intx_for_msi(dev, 0); + msi_set_enable(dev, 0); + arch_restore_msi_irqs(dev); + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + msi_mask_irq(entry, msi_capable_mask(control), entry->masked); + control &= ~PCI_MSI_FLAGS_QSIZE; + control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } -void disable_msi_mode(struct pci_dev *dev, int pos, int type) +static void __pci_restore_msix_state(struct pci_dev *dev) { + struct msi_desc *entry; u16 control; - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (type == PCI_CAP_ID_MSI) { - /* Set enabled bits to single MSI & enable MSI_enable bit */ - msi_disable(control); - pci_write_config_word(dev, msi_control_reg(pos), control); - } else { - msix_disable(control); - pci_write_config_word(dev, msi_control_reg(pos), control); + if (!dev->msix_enabled) + return; + BUG_ON(list_empty(&dev->msi_list)); + entry = list_first_entry(&dev->msi_list, struct msi_desc, list); + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); + + /* route the table */ + pci_intx_for_msi(dev, 0); + control |= PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + + arch_restore_msi_irqs(dev); + list_for_each_entry(entry, &dev->msi_list, list) { + msix_mask_irq(entry, entry->masked); } - if (pci_find_capability(dev, PCI_CAP_ID_EXP)) { - /* PCI Express Endpoint device detected */ - pci_intx(dev, 1); /* enable intx */ + + control &= ~PCI_MSIX_FLAGS_MASKALL; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); +} + +void pci_restore_msi_state(struct pci_dev *dev) +{ + __pci_restore_msi_state(dev); + __pci_restore_msix_state(dev); +} +EXPORT_SYMBOL_GPL(pci_restore_msi_state); + +static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct msi_desc *entry; + unsigned long irq; + int retval; + + retval = kstrtoul(attr->attr.name, 10, &irq); + if (retval) + return retval; + + list_for_each_entry(entry, &pdev->msi_list, list) { + if (entry->irq == irq) { + return sprintf(buf, "%s\n", + entry->msi_attrib.is_msix ? "msix" : "msi"); + } } + return -ENODEV; } -static int msi_lookup_vector(struct pci_dev *dev, int type) +static int populate_msi_sysfs(struct pci_dev *pdev) { - int vector; - unsigned long flags; - - spin_lock_irqsave(&msi_lock, flags); - for (vector = FIRST_DEVICE_VECTOR; vector < NR_IRQS; vector++) { - if (!msi_desc[vector] || msi_desc[vector]->dev != dev || - msi_desc[vector]->msi_attrib.type != type || - msi_desc[vector]->msi_attrib.default_vector != dev->irq) - continue; - spin_unlock_irqrestore(&msi_lock, flags); - /* This pre-assigned MSI vector for this device - already exits. Override dev->irq with this vector */ - dev->irq = vector; + struct attribute **msi_attrs; + struct attribute *msi_attr; + struct device_attribute *msi_dev_attr; + struct attribute_group *msi_irq_group; + const struct attribute_group **msi_irq_groups; + struct msi_desc *entry; + int ret = -ENOMEM; + int num_msi = 0; + int count = 0; + + /* Determine how many msi entries we have */ + list_for_each_entry(entry, &pdev->msi_list, list) { + ++num_msi; + } + if (!num_msi) return 0; + + /* Dynamically create the MSI attributes for the PCI device */ + msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL); + if (!msi_attrs) + return -ENOMEM; + list_for_each_entry(entry, &pdev->msi_list, list) { + msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL); + if (!msi_dev_attr) + goto error_attrs; + msi_attrs[count] = &msi_dev_attr->attr; + + sysfs_attr_init(&msi_dev_attr->attr); + msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d", + entry->irq); + if (!msi_dev_attr->attr.name) + goto error_attrs; + msi_dev_attr->attr.mode = S_IRUGO; + msi_dev_attr->show = msi_mode_show; + ++count; } - spin_unlock_irqrestore(&msi_lock, flags); - return -EACCES; -} + msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL); + if (!msi_irq_group) + goto error_attrs; + msi_irq_group->name = "msi_irqs"; + msi_irq_group->attrs = msi_attrs; -void pci_scan_msi_device(struct pci_dev *dev) -{ - if (!dev) - return; + msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL); + if (!msi_irq_groups) + goto error_irq_group; + msi_irq_groups[0] = msi_irq_group; + + ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups); + if (ret) + goto error_irq_groups; + pdev->msi_irq_groups = msi_irq_groups; - if (pci_find_capability(dev, PCI_CAP_ID_MSIX) > 0) - nr_msix_devices++; - else if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0) - nr_reserved_vectors++; + return 0; + +error_irq_groups: + kfree(msi_irq_groups); +error_irq_group: + kfree(msi_irq_group); +error_attrs: + count = 0; + msi_attr = msi_attrs[count]; + while (msi_attr) { + msi_dev_attr = container_of(msi_attr, struct device_attribute, attr); + kfree(msi_attr->name); + kfree(msi_dev_attr); + ++count; + msi_attr = msi_attrs[count]; + } + kfree(msi_attrs); + return ret; } /** * msi_capability_init - configure device's MSI capability structure * @dev: pointer to the pci_dev data structure of MSI device function + * @nvec: number of interrupts to allocate * - * Setup the MSI capability structure of device function with a single - * MSI vector, regardless of device function is capable of handling - * multiple messages. A return of zero indicates the successful setup - * of an entry zero with the new MSI vector or non-zero for otherwise. - **/ -static int msi_capability_init(struct pci_dev *dev) + * Setup the MSI capability structure of the device with the requested + * number of interrupts. A return value of zero indicates the successful + * setup of an entry with the new MSI irq. A negative return value indicates + * an error, and a positive return value indicates the number of interrupts + * which could have been allocated. + */ +static int msi_capability_init(struct pci_dev *dev, int nvec) { struct msi_desc *entry; - struct msg_address address; - struct msg_data data; - int pos, vector; + int ret; u16 control; + unsigned mask; + + msi_set_enable(dev, 0); /* Disable MSI during set up */ - pos = pci_find_capability(dev, PCI_CAP_ID_MSI); - pci_read_config_word(dev, msi_control_reg(pos), &control); + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); /* MSI Entry Initialization */ - if (!(entry = alloc_msi_entry())) + entry = alloc_msi_entry(dev); + if (!entry) return -ENOMEM; - if ((vector = get_msi_vector(dev)) < 0) { - kmem_cache_free(msi_cachep, entry); - return -EBUSY; - } - entry->link.head = vector; - entry->link.tail = vector; - entry->msi_attrib.type = PCI_CAP_ID_MSI; - entry->msi_attrib.state = 0; /* Mark it not active */ - entry->msi_attrib.entry_nr = 0; - entry->msi_attrib.maskbit = is_mask_bit_support(control); - entry->msi_attrib.default_vector = dev->irq; /* Save IOAPIC IRQ */ - dev->irq = vector; - entry->dev = dev; - if (is_mask_bit_support(control)) { - entry->mask_base = (void __iomem *)(long)msi_mask_bits_reg(pos, - is_64bit_address(control)); - } - /* Replace with MSI handler */ - irq_handler_init(PCI_CAP_ID_MSI, vector, entry->msi_attrib.maskbit); + entry->msi_attrib.is_msix = 0; + entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT); + entry->msi_attrib.entry_nr = 0; + entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT); + entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */ + entry->msi_attrib.pos = dev->msi_cap; + + if (control & PCI_MSI_FLAGS_64BIT) + entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64; + else + entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32; + /* All MSIs are unmasked by default, Mask them all */ + if (entry->msi_attrib.maskbit) + pci_read_config_dword(dev, entry->mask_pos, &entry->masked); + mask = msi_capable_mask(control); + msi_mask_irq(entry, mask, mask); + + list_add_tail(&entry->list, &dev->msi_list); + /* Configure MSI capability structure */ - msi_address_init(&address); - msi_data_init(&data, vector); - entry->msi_attrib.current_cpu = ((address.lo_address.u.dest_id >> - MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); - pci_write_config_dword(dev, msi_lower_address_reg(pos), - address.lo_address.value); - if (is_64bit_address(control)) { - pci_write_config_dword(dev, - msi_upper_address_reg(pos), address.hi_address); - pci_write_config_word(dev, - msi_data_reg(pos, 1), *((u32*)&data)); - } else - pci_write_config_word(dev, - msi_data_reg(pos, 0), *((u32*)&data)); - if (entry->msi_attrib.maskbit) { - unsigned int maskbits, temp; - /* All MSIs are unmasked by default, Mask them all */ - pci_read_config_dword(dev, - msi_mask_bits_reg(pos, is_64bit_address(control)), - &maskbits); - temp = (1 << multi_msi_capable(control)); - temp = ((temp - 1) & ~temp); - maskbits |= temp; - pci_write_config_dword(dev, - msi_mask_bits_reg(pos, is_64bit_address(control)), - maskbits); + ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI); + if (ret) { + msi_mask_irq(entry, mask, ~mask); + free_msi_irqs(dev); + return ret; } - attach_msi_entry(entry, vector); + + ret = populate_msi_sysfs(dev); + if (ret) { + msi_mask_irq(entry, mask, ~mask); + free_msi_irqs(dev); + return ret; + } + /* Set MSI enabled bits */ - enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); + pci_intx_for_msi(dev, 0); + msi_set_enable(dev, 1); + dev->msi_enabled = 1; + dev->irq = entry->irq; return 0; } +static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries) +{ + resource_size_t phys_addr; + u32 table_offset; + u8 bir; + + pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE, + &table_offset); + bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR); + table_offset &= PCI_MSIX_TABLE_OFFSET; + phys_addr = pci_resource_start(dev, bir) + table_offset; + + return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); +} + +static int msix_setup_entries(struct pci_dev *dev, void __iomem *base, + struct msix_entry *entries, int nvec) +{ + struct msi_desc *entry; + int i; + + for (i = 0; i < nvec; i++) { + entry = alloc_msi_entry(dev); + if (!entry) { + if (!i) + iounmap(base); + else + free_msi_irqs(dev); + /* No enough memory. Don't try again */ + return -ENOMEM; + } + + entry->msi_attrib.is_msix = 1; + entry->msi_attrib.is_64 = 1; + entry->msi_attrib.entry_nr = entries[i].entry; + entry->msi_attrib.default_irq = dev->irq; + entry->msi_attrib.pos = dev->msix_cap; + entry->mask_base = base; + + list_add_tail(&entry->list, &dev->msi_list); + } + + return 0; +} + +static void msix_program_entries(struct pci_dev *dev, + struct msix_entry *entries) +{ + struct msi_desc *entry; + int i = 0; + + list_for_each_entry(entry, &dev->msi_list, list) { + int offset = entries[i].entry * PCI_MSIX_ENTRY_SIZE + + PCI_MSIX_ENTRY_VECTOR_CTRL; + + entries[i].vector = entry->irq; + irq_set_msi_desc(entry->irq, entry); + entry->masked = readl(entry->mask_base + offset); + msix_mask_irq(entry, 1); + i++; + } +} + /** * msix_capability_init - configure device's MSI-X capability * @dev: pointer to the pci_dev data structure of MSI-X device function @@ -579,356 +733,239 @@ static int msi_capability_init(struct pci_dev *dev) * @nvec: number of @entries * * Setup the MSI-X capability structure of device function with a - * single MSI-X vector. A return of zero indicates the successful setup of - * requested MSI-X entries with allocated vectors or non-zero for otherwise. + * single MSI-X irq. A return of zero indicates the successful setup of + * requested MSI-X entries with allocated irqs or non-zero for otherwise. **/ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec) { - struct msi_desc *head = NULL, *tail = NULL, *entry = NULL; - struct msg_address address; - struct msg_data data; - int vector, pos, i, j, nr_entries, temp = 0; - u32 phys_addr, table_offset; - u16 control; - u8 bir; + int ret; + u16 control; void __iomem *base; - pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); + + /* Ensure MSI-X is disabled while it is set up */ + control &= ~PCI_MSIX_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + /* Request & Map MSI-X table region */ - pci_read_config_word(dev, msi_control_reg(pos), &control); - nr_entries = multi_msix_capable(control); - pci_read_config_dword(dev, msix_table_offset_reg(pos), - &table_offset); - bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); - phys_addr = pci_resource_start (dev, bir); - phys_addr += (u32)(table_offset & ~PCI_MSIX_FLAGS_BIRMASK); - base = ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); - if (base == NULL) + base = msix_map_region(dev, msix_table_size(control)); + if (!base) return -ENOMEM; - /* MSI-X Table Initialization */ - for (i = 0; i < nvec; i++) { - entry = alloc_msi_entry(); - if (!entry) - break; - if ((vector = get_msi_vector(dev)) < 0) - break; - - j = entries[i].entry; - entries[i].vector = vector; - entry->msi_attrib.type = PCI_CAP_ID_MSIX; - entry->msi_attrib.state = 0; /* Mark it not active */ - entry->msi_attrib.entry_nr = j; - entry->msi_attrib.maskbit = 1; - entry->msi_attrib.default_vector = dev->irq; - entry->dev = dev; - entry->mask_base = base; - if (!head) { - entry->link.head = vector; - entry->link.tail = vector; - head = entry; - } else { - entry->link.head = temp; - entry->link.tail = tail->link.tail; - tail->link.tail = vector; - head->link.head = vector; - } - temp = vector; - tail = entry; - /* Replace with MSI-X handler */ - irq_handler_init(PCI_CAP_ID_MSIX, vector, 1); - /* Configure MSI-X capability structure */ - msi_address_init(&address); - msi_data_init(&data, vector); - entry->msi_attrib.current_cpu = - ((address.lo_address.u.dest_id >> - MSI_TARGET_CPU_SHIFT) & MSI_TARGET_CPU_MASK); - writel(address.lo_address.value, - base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); - writel(address.hi_address, - base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); - writel(*(u32*)&data, - base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_DATA_OFFSET); - attach_msi_entry(entry, vector); - } - if (i != nvec) { - i--; - for (; i >= 0; i--) { - vector = (entries + i)->vector; - msi_free_vector(dev, vector, 0); - (entries + i)->vector = 0; + ret = msix_setup_entries(dev, base, entries, nvec); + if (ret) + return ret; + + ret = arch_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX); + if (ret) + goto out_avail; + + /* + * Some devices require MSI-X to be enabled before we can touch the + * MSI-X registers. We need to mask all the vectors to prevent + * interrupts coming in before they're fully set up. + */ + control |= PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + + msix_program_entries(dev, entries); + + ret = populate_msi_sysfs(dev); + if (ret) + goto out_free; + + /* Set MSI-X enabled bits and unmask the function */ + pci_intx_for_msi(dev, 0); + dev->msix_enabled = 1; + + control &= ~PCI_MSIX_FLAGS_MASKALL; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, control); + + return 0; + +out_avail: + if (ret < 0) { + /* + * If we had some success, report the number of irqs + * we succeeded in setting up. + */ + struct msi_desc *entry; + int avail = 0; + + list_for_each_entry(entry, &dev->msi_list, list) { + if (entry->irq != 0) + avail++; } - return -EBUSY; + if (avail != 0) + ret = avail; } - /* Set MSI-X enabled bits */ - enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); - return 0; +out_free: + free_msi_irqs(dev); + + return ret; } /** - * pci_enable_msi - configure device's MSI capability structure + * pci_msi_check_device - check whether MSI may be enabled on a device * @dev: pointer to the pci_dev data structure of MSI device function + * @nvec: how many MSIs have been requested ? + * @type: are we checking for MSI or MSI-X ? * - * Setup the MSI capability structure of device function with - * a single MSI vector upon its software driver call to request for - * MSI mode enabled on its hardware device function. A return of zero - * indicates the successful setup of an entry zero with the new MSI - * vector or non-zero for otherwise. + * Look at global flags, the device itself, and its parent buses + * to determine if MSI/-X are supported for the device. If MSI/-X is + * supported return 0, else return an error code. **/ -int pci_enable_msi(struct pci_dev* dev) +static int pci_msi_check_device(struct pci_dev *dev, int nvec, int type) { - int pos, temp, status = -EINVAL; - u16 control; + struct pci_bus *bus; + int ret; - if (!pci_msi_enable || !dev) - return status; + /* MSI must be globally enabled and supported by the device */ + if (!pci_msi_enable || !dev || dev->no_msi) + return -EINVAL; - if (dev->no_msi) - return status; + /* + * You can't ask to have 0 or less MSIs configured. + * a) it's stupid .. + * b) the list manipulation code assumes nvec >= 1. + */ + if (nvec < 1) + return -ERANGE; - temp = dev->irq; + /* + * Any bridge which does NOT route MSI transactions from its + * secondary bus to its primary bus must set NO_MSI flag on + * the secondary pci_bus. + * We expect only arch-specific PCI host bus controller driver + * or quirks for specific PCI bridges to be setting NO_MSI. + */ + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return -EINVAL; - if ((status = msi_init()) < 0) - return status; + ret = arch_msi_check_device(dev, nvec, type); + if (ret) + return ret; + + return 0; +} + +/** + * pci_msi_vec_count - Return the number of MSI vectors a device can send + * @dev: device to report about + * + * This function returns the number of MSI vectors a device requested via + * Multiple Message Capable register. It returns a negative errno if the + * device is not capable sending MSI interrupts. Otherwise, the call succeeds + * and returns a power of two, up to a maximum of 2^5 (32), according to the + * MSI specification. + **/ +int pci_msi_vec_count(struct pci_dev *dev) +{ + int ret; + u16 msgctl; - if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) + if (!dev->msi_cap) return -EINVAL; - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (control & PCI_MSI_FLAGS_ENABLE) - return 0; /* Already in MSI mode */ - - if (!msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { - /* Lookup Sucess */ - unsigned long flags; - - spin_lock_irqsave(&msi_lock, flags); - if (!vector_irq[dev->irq]) { - msi_desc[dev->irq]->msi_attrib.state = 0; - vector_irq[dev->irq] = -1; - nr_released_vectors--; - spin_unlock_irqrestore(&msi_lock, flags); - enable_msi_mode(dev, pos, PCI_CAP_ID_MSI); - return 0; - } - spin_unlock_irqrestore(&msi_lock, flags); - dev->irq = temp; - } - /* Check whether driver already requested for MSI-X vectors */ - if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && - !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { - printk(KERN_INFO "PCI: %s: Can't enable MSI. " - "Device already has MSI-X vectors assigned\n", - pci_name(dev)); - dev->irq = temp; - return -EINVAL; - } - status = msi_capability_init(dev); - if (!status) { - if (!pos) - nr_reserved_vectors--; /* Only MSI capable */ - else if (nr_msix_devices > 0) - nr_msix_devices--; /* Both MSI and MSI-X capable, - but choose enabling MSI */ - } + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl); + ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); - return status; + return ret; } +EXPORT_SYMBOL(pci_msi_vec_count); -void pci_disable_msi(struct pci_dev* dev) +void pci_msi_shutdown(struct pci_dev *dev) { - struct msi_desc *entry; - int pos, default_vector; - u16 control; - unsigned long flags; + struct msi_desc *desc; + u32 mask; + u16 ctrl; - if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSI))) + if (!pci_msi_enable || !dev || !dev->msi_enabled) return; - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (!(control & PCI_MSI_FLAGS_ENABLE)) - return; + BUG_ON(list_empty(&dev->msi_list)); + desc = list_first_entry(&dev->msi_list, struct msi_desc, list); - spin_lock_irqsave(&msi_lock, flags); - entry = msi_desc[dev->irq]; - if (!entry || !entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) { - spin_unlock_irqrestore(&msi_lock, flags); - return; - } - if (entry->msi_attrib.state) { - spin_unlock_irqrestore(&msi_lock, flags); - printk(KERN_WARNING "PCI: %s: pci_disable_msi() called without " - "free_irq() on MSI vector %d\n", - pci_name(dev), dev->irq); - BUG_ON(entry->msi_attrib.state > 0); - } else { - vector_irq[dev->irq] = 0; /* free it */ - nr_released_vectors++; - default_vector = entry->msi_attrib.default_vector; - spin_unlock_irqrestore(&msi_lock, flags); - /* Restore dev->irq to its default pin-assertion vector */ - dev->irq = default_vector; - disable_msi_mode(dev, pci_find_capability(dev, PCI_CAP_ID_MSI), - PCI_CAP_ID_MSI); - } + msi_set_enable(dev, 0); + pci_intx_for_msi(dev, 1); + dev->msi_enabled = 0; + + /* Return the device with MSI unmasked as initial states */ + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &ctrl); + mask = msi_capable_mask(ctrl); + /* Keep cached state to be restored */ + arch_msi_mask_irq(desc, mask, ~mask); + + /* Restore dev->irq to its default pin-assertion irq */ + dev->irq = desc->msi_attrib.default_irq; } -static int msi_free_vector(struct pci_dev* dev, int vector, int reassign) +void pci_disable_msi(struct pci_dev *dev) { - struct msi_desc *entry; - int head, entry_nr, type; - void __iomem *base; - unsigned long flags; - - spin_lock_irqsave(&msi_lock, flags); - entry = msi_desc[vector]; - if (!entry || entry->dev != dev) { - spin_unlock_irqrestore(&msi_lock, flags); - return -EINVAL; - } - type = entry->msi_attrib.type; - entry_nr = entry->msi_attrib.entry_nr; - head = entry->link.head; - base = entry->mask_base; - msi_desc[entry->link.head]->link.tail = entry->link.tail; - msi_desc[entry->link.tail]->link.head = entry->link.head; - entry->dev = NULL; - if (!reassign) { - vector_irq[vector] = 0; - nr_released_vectors++; - } - msi_desc[vector] = NULL; - spin_unlock_irqrestore(&msi_lock, flags); - - kmem_cache_free(msi_cachep, entry); - - if (type == PCI_CAP_ID_MSIX) { - if (!reassign) - writel(1, base + - entry_nr * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET); - - if (head == vector) { - /* - * Detect last MSI-X vector to be released. - * Release the MSI-X memory-mapped table. - */ - int pos, nr_entries; - u32 phys_addr, table_offset; - u16 control; - u8 bir; - - pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); - pci_read_config_word(dev, msi_control_reg(pos), - &control); - nr_entries = multi_msix_capable(control); - pci_read_config_dword(dev, msix_table_offset_reg(pos), - &table_offset); - bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); - phys_addr = pci_resource_start (dev, bir); - phys_addr += (u32)(table_offset & - ~PCI_MSIX_FLAGS_BIRMASK); - iounmap(base); - } - } + if (!pci_msi_enable || !dev || !dev->msi_enabled) + return; - return 0; + pci_msi_shutdown(dev); + free_msi_irqs(dev); } +EXPORT_SYMBOL(pci_disable_msi); -static int reroute_msix_table(int head, struct msix_entry *entries, int *nvec) +/** + * pci_msix_vec_count - return the number of device's MSI-X table entries + * @dev: pointer to the pci_dev data structure of MSI-X device function + * This function returns the number of device's MSI-X table entries and + * therefore the number of MSI-X vectors device is capable of sending. + * It returns a negative errno if the device is not capable of sending MSI-X + * interrupts. + **/ +int pci_msix_vec_count(struct pci_dev *dev) { - int vector = head, tail = 0; - int i, j = 0, nr_entries = 0; - void __iomem *base; - unsigned long flags; - - spin_lock_irqsave(&msi_lock, flags); - while (head != tail) { - nr_entries++; - tail = msi_desc[vector]->link.tail; - if (entries[0].entry == msi_desc[vector]->msi_attrib.entry_nr) - j = vector; - vector = tail; - } - if (*nvec > nr_entries) { - spin_unlock_irqrestore(&msi_lock, flags); - *nvec = nr_entries; + u16 control; + + if (!dev->msix_cap) return -EINVAL; - } - vector = ((j > 0) ? j : head); - for (i = 0; i < *nvec; i++) { - j = msi_desc[vector]->msi_attrib.entry_nr; - msi_desc[vector]->msi_attrib.state = 0; /* Mark it not active */ - vector_irq[vector] = -1; /* Mark it busy */ - nr_released_vectors--; - entries[i].vector = vector; - if (j != (entries + i)->entry) { - base = msi_desc[vector]->mask_base; - msi_desc[vector]->msi_attrib.entry_nr = - (entries + i)->entry; - writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET), base + - (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_LOWER_ADDR_OFFSET); - writel( readl(base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET), base + - (entries + i)->entry * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_UPPER_ADDR_OFFSET); - writel( (readl(base + j * PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_DATA_OFFSET) & 0xff00) | vector, - base + (entries+i)->entry*PCI_MSIX_ENTRY_SIZE + - PCI_MSIX_ENTRY_DATA_OFFSET); - } - vector = msi_desc[vector]->link.tail; - } - spin_unlock_irqrestore(&msi_lock, flags); - return 0; + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); + return msix_table_size(control); } +EXPORT_SYMBOL(pci_msix_vec_count); /** * pci_enable_msix - configure device's MSI-X capability structure * @dev: pointer to the pci_dev data structure of MSI-X device function * @entries: pointer to an array of MSI-X entries - * @nvec: number of MSI-X vectors requested for allocation by device driver + * @nvec: number of MSI-X irqs requested for allocation by device driver * * Setup the MSI-X capability structure of device function with the number - * of requested vectors upon its software driver call to request for + * of requested irqs upon its software driver call to request for * MSI-X mode enabled on its hardware device function. A return of zero * indicates the successful configuration of MSI-X capability structure - * with new allocated MSI-X vectors. A return of < 0 indicates a failure. + * with new allocated MSI-X irqs. A return of < 0 indicates a failure. * Or a return of > 0 indicates that driver request is exceeding the number - * of vectors available. Driver should use the returned value to re-send - * its request. + * of irqs or MSI-X vectors available. Driver should use the returned value to + * re-send its request. **/ -int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) +int pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, int nvec) { - int status, pos, nr_entries, free_vectors; - int i, j, temp; - u16 control; - unsigned long flags; + int status, nr_entries; + int i, j; - if (!pci_msi_enable || !dev || !entries) - return -EINVAL; + if (!entries || !dev->msix_cap || dev->current_state != PCI_D0) + return -EINVAL; - if ((status = msi_init()) < 0) + status = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSIX); + if (status) return status; - if (!(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) - return -EINVAL; - - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (control & PCI_MSIX_FLAGS_ENABLE) - return -EINVAL; /* Already in MSI-X mode */ - - nr_entries = multi_msix_capable(control); + nr_entries = pci_msix_vec_count(dev); + if (nr_entries < 0) + return nr_entries; if (nvec > nr_entries) - return -EINVAL; + return nr_entries; /* Check for any invalid entries */ for (i = 0; i < nvec; i++) { @@ -939,183 +976,199 @@ int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) return -EINVAL; /* duplicate entry */ } } - temp = dev->irq; - if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { - /* Lookup Sucess */ - nr_entries = nvec; - /* Reroute MSI-X table */ - if (reroute_msix_table(dev->irq, entries, &nr_entries)) { - /* #requested > #previous-assigned */ - dev->irq = temp; - return nr_entries; - } - dev->irq = temp; - enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX); - return 0; - } - /* Check whether driver already requested for MSI vector */ - if (pci_find_capability(dev, PCI_CAP_ID_MSI) > 0 && - !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { - printk(KERN_INFO "PCI: %s: Can't enable MSI-X. " - "Device already has an MSI vector assigned\n", - pci_name(dev)); - dev->irq = temp; - return -EINVAL; - } + WARN_ON(!!dev->msix_enabled); - spin_lock_irqsave(&msi_lock, flags); - /* - * msi_lock is provided to ensure that enough vectors resources are - * available before granting. - */ - free_vectors = pci_vector_resources(last_alloc_vector, - nr_released_vectors); - /* Ensure that each MSI/MSI-X device has one vector reserved by - default to avoid any MSI-X driver to take all available - resources */ - free_vectors -= nr_reserved_vectors; - /* Find the average of free vectors among MSI-X devices */ - if (nr_msix_devices > 0) - free_vectors /= nr_msix_devices; - spin_unlock_irqrestore(&msi_lock, flags); - - if (nvec > free_vectors) { - if (free_vectors > 0) - return free_vectors; - else - return -EBUSY; + /* Check whether driver already requested for MSI irq */ + if (dev->msi_enabled) { + dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n"); + return -EINVAL; } - status = msix_capability_init(dev, entries, nvec); - if (!status && nr_msix_devices > 0) - nr_msix_devices--; - return status; } +EXPORT_SYMBOL(pci_enable_msix); -void pci_disable_msix(struct pci_dev* dev) +void pci_msix_shutdown(struct pci_dev *dev) { - int pos, temp; - u16 control; + struct msi_desc *entry; - if (!dev || !(pos = pci_find_capability(dev, PCI_CAP_ID_MSIX))) + if (!pci_msi_enable || !dev || !dev->msix_enabled) return; - pci_read_config_word(dev, msi_control_reg(pos), &control); - if (!(control & PCI_MSIX_FLAGS_ENABLE)) - return; + /* Return the device with MSI-X masked as initial states */ + list_for_each_entry(entry, &dev->msi_list, list) { + /* Keep cached states to be restored */ + arch_msix_mask_irq(entry, 1); + } - temp = dev->irq; - if (!msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { - int state, vector, head, tail = 0, warning = 0; - unsigned long flags; - - vector = head = dev->irq; - spin_lock_irqsave(&msi_lock, flags); - while (head != tail) { - state = msi_desc[vector]->msi_attrib.state; - if (state) - warning = 1; - else { - vector_irq[vector] = 0; /* free it */ - nr_released_vectors++; - } - tail = msi_desc[vector]->link.tail; - vector = tail; - } - spin_unlock_irqrestore(&msi_lock, flags); - if (warning) { - dev->irq = temp; - printk(KERN_WARNING "PCI: %s: pci_disable_msix() called without " - "free_irq() on all MSI-X vectors\n", - pci_name(dev)); - BUG_ON(warning > 0); - } else { - dev->irq = temp; - disable_msi_mode(dev, - pci_find_capability(dev, PCI_CAP_ID_MSIX), - PCI_CAP_ID_MSIX); + msix_set_enable(dev, 0); + pci_intx_for_msi(dev, 1); + dev->msix_enabled = 0; +} - } - } +void pci_disable_msix(struct pci_dev *dev) +{ + if (!pci_msi_enable || !dev || !dev->msix_enabled) + return; + + pci_msix_shutdown(dev); + free_msi_irqs(dev); } +EXPORT_SYMBOL(pci_disable_msix); /** - * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state + * msi_remove_pci_irq_vectors - reclaim MSI(X) irqs to unused state * @dev: pointer to the pci_dev data structure of MSI(X) device function * * Being called during hotplug remove, from which the device function - * is hot-removed. All previous assigned MSI/MSI-X vectors, if + * is hot-removed. All previous assigned MSI/MSI-X irqs, if * allocated for this device function, are reclaimed to unused state, * which may be used later on. **/ -void msi_remove_pci_irq_vectors(struct pci_dev* dev) +void msi_remove_pci_irq_vectors(struct pci_dev *dev) { - int state, pos, temp; - unsigned long flags; - if (!pci_msi_enable || !dev) - return; - - temp = dev->irq; /* Save IOAPIC IRQ */ - if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSI)) > 0 && - !msi_lookup_vector(dev, PCI_CAP_ID_MSI)) { - spin_lock_irqsave(&msi_lock, flags); - state = msi_desc[dev->irq]->msi_attrib.state; - spin_unlock_irqrestore(&msi_lock, flags); - if (state) { - printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " - "called without free_irq() on MSI vector %d\n", - pci_name(dev), dev->irq); - BUG_ON(state > 0); - } else /* Release MSI vector assigned to this device */ - msi_free_vector(dev, dev->irq, 0); - dev->irq = temp; /* Restore IOAPIC IRQ */ + return; + + if (dev->msi_enabled || dev->msix_enabled) + free_msi_irqs(dev); +} + +void pci_no_msi(void) +{ + pci_msi_enable = 0; +} + +/** + * pci_msi_enabled - is MSI enabled? + * + * Returns true if MSI has not been disabled by the command-line option + * pci=nomsi. + **/ +int pci_msi_enabled(void) +{ + return pci_msi_enable; +} +EXPORT_SYMBOL(pci_msi_enabled); + +void pci_msi_init_pci_dev(struct pci_dev *dev) +{ + INIT_LIST_HEAD(&dev->msi_list); + + /* Disable the msi hardware to avoid screaming interrupts + * during boot. This is the power on reset default so + * usually this should be a noop. + */ + dev->msi_cap = pci_find_capability(dev, PCI_CAP_ID_MSI); + if (dev->msi_cap) + msi_set_enable(dev, 0); + + dev->msix_cap = pci_find_capability(dev, PCI_CAP_ID_MSIX); + if (dev->msix_cap) + msix_set_enable(dev, 0); +} + +/** + * pci_enable_msi_range - configure device's MSI capability structure + * @dev: device to configure + * @minvec: minimal number of interrupts to configure + * @maxvec: maximum number of interrupts to configure + * + * This function tries to allocate a maximum possible number of interrupts in a + * range between @minvec and @maxvec. It returns a negative errno if an error + * occurs. If it succeeds, it returns the actual number of interrupts allocated + * and updates the @dev's irq member to the lowest new interrupt number; + * the other interrupt numbers allocated to this device are consecutive. + **/ +int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec) +{ + int nvec; + int rc; + + if (dev->current_state != PCI_D0) + return -EINVAL; + + WARN_ON(!!dev->msi_enabled); + + /* Check whether driver already requested MSI-X irqs */ + if (dev->msix_enabled) { + dev_info(&dev->dev, + "can't enable MSI (MSI-X already enabled)\n"); + return -EINVAL; } - if ((pos = pci_find_capability(dev, PCI_CAP_ID_MSIX)) > 0 && - !msi_lookup_vector(dev, PCI_CAP_ID_MSIX)) { - int vector, head, tail = 0, warning = 0; - void __iomem *base = NULL; - - vector = head = dev->irq; - while (head != tail) { - spin_lock_irqsave(&msi_lock, flags); - state = msi_desc[vector]->msi_attrib.state; - tail = msi_desc[vector]->link.tail; - base = msi_desc[vector]->mask_base; - spin_unlock_irqrestore(&msi_lock, flags); - if (state) - warning = 1; - else if (vector != head) /* Release MSI-X vector */ - msi_free_vector(dev, vector, 0); - vector = tail; + + if (maxvec < minvec) + return -ERANGE; + + nvec = pci_msi_vec_count(dev); + if (nvec < 0) + return nvec; + else if (nvec < minvec) + return -EINVAL; + else if (nvec > maxvec) + nvec = maxvec; + + do { + rc = pci_msi_check_device(dev, nvec, PCI_CAP_ID_MSI); + if (rc < 0) { + return rc; + } else if (rc > 0) { + if (rc < minvec) + return -ENOSPC; + nvec = rc; } - msi_free_vector(dev, vector, 0); - if (warning) { - /* Force to release the MSI-X memory-mapped table */ - u32 phys_addr, table_offset; - u16 control; - u8 bir; - - pci_read_config_word(dev, msi_control_reg(pos), - &control); - pci_read_config_dword(dev, msix_table_offset_reg(pos), - &table_offset); - bir = (u8)(table_offset & PCI_MSIX_FLAGS_BIRMASK); - phys_addr = pci_resource_start (dev, bir); - phys_addr += (u32)(table_offset & - ~PCI_MSIX_FLAGS_BIRMASK); - iounmap(base); - printk(KERN_WARNING "PCI: %s: msi_remove_pci_irq_vectors() " - "called without free_irq() on all MSI-X vectors\n", - pci_name(dev)); - BUG_ON(warning > 0); + } while (rc); + + do { + rc = msi_capability_init(dev, nvec); + if (rc < 0) { + return rc; + } else if (rc > 0) { + if (rc < minvec) + return -ENOSPC; + nvec = rc; } - dev->irq = temp; /* Restore IOAPIC IRQ */ - } + } while (rc); + + return nvec; } +EXPORT_SYMBOL(pci_enable_msi_range); -EXPORT_SYMBOL(pci_enable_msi); -EXPORT_SYMBOL(pci_disable_msi); -EXPORT_SYMBOL(pci_enable_msix); -EXPORT_SYMBOL(pci_disable_msix); +/** + * pci_enable_msix_range - configure device's MSI-X capability structure + * @dev: pointer to the pci_dev data structure of MSI-X device function + * @entries: pointer to an array of MSI-X entries + * @minvec: minimum number of MSI-X irqs requested + * @maxvec: maximum number of MSI-X irqs requested + * + * Setup the MSI-X capability structure of device function with a maximum + * possible number of interrupts in the range between @minvec and @maxvec + * upon its software driver call to request for MSI-X mode enabled on its + * hardware device function. It returns a negative errno if an error occurs. + * If it succeeds, it returns the actual number of interrupts allocated and + * indicates the successful configuration of MSI-X capability structure + * with new allocated MSI-X interrupts. + **/ +int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, + int minvec, int maxvec) +{ + int nvec = maxvec; + int rc; + + if (maxvec < minvec) + return -ERANGE; + + do { + rc = pci_enable_msix(dev, entries, nvec); + if (rc < 0) { + return rc; + } else if (rc > 0) { + if (rc < minvec) + return -ENOSPC; + nvec = rc; + } + } while (rc); + + return nvec; +} +EXPORT_SYMBOL(pci_enable_msix_range); |