diff options
Diffstat (limited to 'drivers/pci/pcie')
| -rw-r--r-- | drivers/pci/pcie/Kconfig | 64 | ||||
| -rw-r--r-- | drivers/pci/pcie/Makefile | 6 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/Kconfig | 15 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/Kconfig.debug | 18 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/Makefile | 3 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aer_inject.c | 536 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv.c | 253 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv.h | 78 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_acpi.c | 147 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_core.c | 814 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_errprint.c | 321 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/ecrc.c | 131 | ||||
| -rw-r--r-- | drivers/pci/pcie/aspm.c | 991 | ||||
| -rw-r--r-- | drivers/pci/pcie/pme.c | 438 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv.h | 88 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_acpi.c | 63 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_bus.c | 52 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_core.c | 737 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_pci.c | 273 |
19 files changed, 3898 insertions, 1130 deletions
diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 287a9311716..7958e59d607 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -2,7 +2,7 @@ # PCI Express Port Bus Configuration # config PCIEPORTBUS - bool "PCI Express support" + bool "PCI Express Port Bus support" depends on PCI help This automatically enables PCI Express Port Bus support. Users can @@ -14,15 +14,69 @@ config PCIEPORTBUS # Include service Kconfig here # config HOTPLUG_PCI_PCIE - tristate "PCI Express Hotplug driver" + bool "PCI Express Hotplug driver" depends on HOTPLUG_PCI && PCIEPORTBUS help Say Y here if you have a motherboard that supports PCI Express Native Hotplug - To compile this driver as a module, choose M here: the - module will be called pciehp. - When in doubt, say N. source "drivers/pci/pcie/aer/Kconfig" + +# +# PCI Express ASPM +# +config PCIEASPM + bool "PCI Express ASPM control" if EXPERT + depends on PCI && PCIEPORTBUS + default y + help + This enables OS control over PCI Express ASPM (Active State + Power Management) and Clock Power Management. ASPM supports + state L0/L0s/L1. + + ASPM is initially set up by the firmware. With this option enabled, + Linux can modify this state in order to disable ASPM on known-bad + hardware or configurations and enable it when known-safe. + + ASPM can be disabled or enabled at runtime via + /sys/module/pcie_aspm/parameters/policy + + When in doubt, say Y. +config PCIEASPM_DEBUG + bool "Debug PCI Express ASPM" + depends on PCIEASPM + default n + help + This enables PCI Express ASPM debug support. It will add per-device + interface to control ASPM. + +choice + prompt "Default ASPM policy" + default PCIEASPM_DEFAULT + depends on PCIEASPM + +config PCIEASPM_DEFAULT + bool "BIOS default" + depends on PCIEASPM + help + Use the BIOS defaults for PCI Express ASPM. + +config PCIEASPM_POWERSAVE + bool "Powersave" + depends on PCIEASPM + help + Enable PCI Express ASPM L0s and L1 where possible, even if the + BIOS did not. + +config PCIEASPM_PERFORMANCE + bool "Performance" + depends on PCIEASPM + help + Disable PCI Express ASPM L0s and L1, even if the BIOS enabled them. +endchoice + +config PCIE_PME + def_bool y + depends on PCIEPORTBUS && PM_RUNTIME diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index e00fb99acf4..00c62df5a9f 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -2,9 +2,15 @@ # Makefile for PCI-Express PORT Driver # +# Build PCI Express ASPM if needed +obj-$(CONFIG_PCIEASPM) += aspm.o + pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o +pcieportdrv-$(CONFIG_ACPI) += portdrv_acpi.o obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o # Build PCI Express AER if needed obj-$(CONFIG_PCIEAER) += aer/ + +obj-$(CONFIG_PCIE_PME) += pme.o diff --git a/drivers/pci/pcie/aer/Kconfig b/drivers/pci/pcie/aer/Kconfig index c3bde588aa1..50e94e02378 100644 --- a/drivers/pci/pcie/aer/Kconfig +++ b/drivers/pci/pcie/aer/Kconfig @@ -10,3 +10,18 @@ config PCIEAER This enables PCI Express Root Port Advanced Error Reporting (AER) driver support. Error reporting messages sent to Root Port will be handled by PCI Express AER driver. + + +# +# PCI Express ECRC +# +config PCIE_ECRC + bool "PCI Express ECRC settings control" + depends on PCIEAER + help + Used to override firmware/bios settings for PCI Express ECRC + (transaction layer end-to-end CRC checking). + + When in doubt, say N. + +source "drivers/pci/pcie/aer/Kconfig.debug" diff --git a/drivers/pci/pcie/aer/Kconfig.debug b/drivers/pci/pcie/aer/Kconfig.debug new file mode 100644 index 00000000000..9142949734f --- /dev/null +++ b/drivers/pci/pcie/aer/Kconfig.debug @@ -0,0 +1,18 @@ +# +# PCI Express Root Port Device AER Debug Configuration +# + +config PCIEAER_INJECT + tristate "PCIe AER error injector support" + depends on PCIEAER + default n + help + This enables PCI Express Root Port Advanced Error Reporting + (AER) software error injector. + + Debugging PCIe AER code is quite difficult because it is hard + to trigger various real hardware errors. Software based + error injection can fake almost all kinds of errors with the + help of a user space helper tool aer-inject, which can be + gotten from: + http://www.kernel.org/pub/linux/utils/pci/aer-inject/ diff --git a/drivers/pci/pcie/aer/Makefile b/drivers/pci/pcie/aer/Makefile index 8da3bd8455a..2cba67510dc 100644 --- a/drivers/pci/pcie/aer/Makefile +++ b/drivers/pci/pcie/aer/Makefile @@ -4,6 +4,9 @@ obj-$(CONFIG_PCIEAER) += aerdriver.o +obj-$(CONFIG_PCIE_ECRC) += ecrc.o + aerdriver-objs := aerdrv_errprint.o aerdrv_core.o aerdrv.o aerdriver-$(CONFIG_ACPI) += aerdrv_acpi.o +obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c new file mode 100644 index 00000000000..182224acedb --- /dev/null +++ b/drivers/pci/pcie/aer/aer_inject.c @@ -0,0 +1,536 @@ +/* + * PCIe AER software error injection support. + * + * Debuging PCIe AER code is quite difficult because it is hard to + * trigger various real hardware errors. Software based error + * injection can fake almost all kinds of errors with the help of a + * user space helper tool aer-inject, which can be gotten from: + * http://www.kernel.org/pub/linux/utils/pci/aer-inject/ + * + * Copyright 2009 Intel Corporation. + * Huang Ying <ying.huang@intel.com> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; version 2 + * of the License. + * + */ + +#include <linux/module.h> +#include <linux/init.h> +#include <linux/miscdevice.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/fs.h> +#include <linux/uaccess.h> +#include <linux/stddef.h> +#include "aerdrv.h" + +/* Override the existing corrected and uncorrected error masks */ +static bool aer_mask_override; +module_param(aer_mask_override, bool, 0); + +struct aer_error_inj { + u8 bus; + u8 dev; + u8 fn; + u32 uncor_status; + u32 cor_status; + u32 header_log0; + u32 header_log1; + u32 header_log2; + u32 header_log3; + u16 domain; +}; + +struct aer_error { + struct list_head list; + u16 domain; + unsigned int bus; + unsigned int devfn; + int pos_cap_err; + + u32 uncor_status; + u32 cor_status; + u32 header_log0; + u32 header_log1; + u32 header_log2; + u32 header_log3; + u32 root_status; + u32 source_id; +}; + +struct pci_bus_ops { + struct list_head list; + struct pci_bus *bus; + struct pci_ops *ops; +}; + +static LIST_HEAD(einjected); + +static LIST_HEAD(pci_bus_ops_list); + +/* Protect einjected and pci_bus_ops_list */ +static DEFINE_SPINLOCK(inject_lock); + +static void aer_error_init(struct aer_error *err, u16 domain, + unsigned int bus, unsigned int devfn, + int pos_cap_err) +{ + INIT_LIST_HEAD(&err->list); + err->domain = domain; + err->bus = bus; + err->devfn = devfn; + err->pos_cap_err = pos_cap_err; +} + +/* inject_lock must be held before calling */ +static struct aer_error *__find_aer_error(u16 domain, unsigned int bus, + unsigned int devfn) +{ + struct aer_error *err; + + list_for_each_entry(err, &einjected, list) { + if (domain == err->domain && + bus == err->bus && + devfn == err->devfn) + return err; + } + return NULL; +} + +/* inject_lock must be held before calling */ +static struct aer_error *__find_aer_error_by_dev(struct pci_dev *dev) +{ + int domain = pci_domain_nr(dev->bus); + if (domain < 0) + return NULL; + return __find_aer_error((u16)domain, dev->bus->number, dev->devfn); +} + +/* inject_lock must be held before calling */ +static struct pci_ops *__find_pci_bus_ops(struct pci_bus *bus) +{ + struct pci_bus_ops *bus_ops; + + list_for_each_entry(bus_ops, &pci_bus_ops_list, list) { + if (bus_ops->bus == bus) + return bus_ops->ops; + } + return NULL; +} + +static struct pci_bus_ops *pci_bus_ops_pop(void) +{ + unsigned long flags; + struct pci_bus_ops *bus_ops = NULL; + + spin_lock_irqsave(&inject_lock, flags); + if (list_empty(&pci_bus_ops_list)) + bus_ops = NULL; + else { + struct list_head *lh = pci_bus_ops_list.next; + list_del(lh); + bus_ops = list_entry(lh, struct pci_bus_ops, list); + } + spin_unlock_irqrestore(&inject_lock, flags); + return bus_ops; +} + +static u32 *find_pci_config_dword(struct aer_error *err, int where, + int *prw1cs) +{ + int rw1cs = 0; + u32 *target = NULL; + + if (err->pos_cap_err == -1) + return NULL; + + switch (where - err->pos_cap_err) { + case PCI_ERR_UNCOR_STATUS: + target = &err->uncor_status; + rw1cs = 1; + break; + case PCI_ERR_COR_STATUS: + target = &err->cor_status; + rw1cs = 1; + break; + case PCI_ERR_HEADER_LOG: + target = &err->header_log0; + break; + case PCI_ERR_HEADER_LOG+4: + target = &err->header_log1; + break; + case PCI_ERR_HEADER_LOG+8: + target = &err->header_log2; + break; + case PCI_ERR_HEADER_LOG+12: + target = &err->header_log3; + break; + case PCI_ERR_ROOT_STATUS: + target = &err->root_status; + rw1cs = 1; + break; + case PCI_ERR_ROOT_ERR_SRC: + target = &err->source_id; + break; + } + if (prw1cs) + *prw1cs = rw1cs; + return target; +} + +static int pci_read_aer(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 *val) +{ + u32 *sim; + struct aer_error *err; + unsigned long flags; + struct pci_ops *ops; + int domain; + + spin_lock_irqsave(&inject_lock, flags); + if (size != sizeof(u32)) + goto out; + domain = pci_domain_nr(bus); + if (domain < 0) + goto out; + err = __find_aer_error((u16)domain, bus->number, devfn); + if (!err) + goto out; + + sim = find_pci_config_dword(err, where, NULL); + if (sim) { + *val = *sim; + spin_unlock_irqrestore(&inject_lock, flags); + return 0; + } +out: + ops = __find_pci_bus_ops(bus); + spin_unlock_irqrestore(&inject_lock, flags); + return ops->read(bus, devfn, where, size, val); +} + +static int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 val) +{ + u32 *sim; + struct aer_error *err; + unsigned long flags; + int rw1cs; + struct pci_ops *ops; + int domain; + + spin_lock_irqsave(&inject_lock, flags); + if (size != sizeof(u32)) + goto out; + domain = pci_domain_nr(bus); + if (domain < 0) + goto out; + err = __find_aer_error((u16)domain, bus->number, devfn); + if (!err) + goto out; + + sim = find_pci_config_dword(err, where, &rw1cs); + if (sim) { + if (rw1cs) + *sim ^= val; + else + *sim = val; + spin_unlock_irqrestore(&inject_lock, flags); + return 0; + } +out: + ops = __find_pci_bus_ops(bus); + spin_unlock_irqrestore(&inject_lock, flags); + return ops->write(bus, devfn, where, size, val); +} + +static struct pci_ops pci_ops_aer = { + .read = pci_read_aer, + .write = pci_write_aer, +}; + +static void pci_bus_ops_init(struct pci_bus_ops *bus_ops, + struct pci_bus *bus, + struct pci_ops *ops) +{ + INIT_LIST_HEAD(&bus_ops->list); + bus_ops->bus = bus; + bus_ops->ops = ops; +} + +static int pci_bus_set_aer_ops(struct pci_bus *bus) +{ + struct pci_ops *ops; + struct pci_bus_ops *bus_ops; + unsigned long flags; + + bus_ops = kmalloc(sizeof(*bus_ops), GFP_KERNEL); + if (!bus_ops) + return -ENOMEM; + ops = pci_bus_set_ops(bus, &pci_ops_aer); + spin_lock_irqsave(&inject_lock, flags); + if (ops == &pci_ops_aer) + goto out; + pci_bus_ops_init(bus_ops, bus, ops); + list_add(&bus_ops->list, &pci_bus_ops_list); + bus_ops = NULL; +out: + spin_unlock_irqrestore(&inject_lock, flags); + kfree(bus_ops); + return 0; +} + +static struct pci_dev *pcie_find_root_port(struct pci_dev *dev) +{ + while (1) { + if (!pci_is_pcie(dev)) + break; + if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) + return dev; + if (!dev->bus->self) + break; + dev = dev->bus->self; + } + return NULL; +} + +static int find_aer_device_iter(struct device *device, void *data) +{ + struct pcie_device **result = data; + struct pcie_device *pcie_dev; + + if (device->bus == &pcie_port_bus_type) { + pcie_dev = to_pcie_device(device); + if (pcie_dev->service & PCIE_PORT_SERVICE_AER) { + *result = pcie_dev; + return 1; + } + } + return 0; +} + +static int find_aer_device(struct pci_dev *dev, struct pcie_device **result) +{ + return device_for_each_child(&dev->dev, result, find_aer_device_iter); +} + +static int aer_inject(struct aer_error_inj *einj) +{ + struct aer_error *err, *rperr; + struct aer_error *err_alloc = NULL, *rperr_alloc = NULL; + struct pci_dev *dev, *rpdev; + struct pcie_device *edev; + unsigned long flags; + unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn); + int pos_cap_err, rp_pos_cap_err; + u32 sever, cor_mask, uncor_mask, cor_mask_orig = 0, uncor_mask_orig = 0; + int ret = 0; + + dev = pci_get_domain_bus_and_slot((int)einj->domain, einj->bus, devfn); + if (!dev) + return -ENODEV; + rpdev = pcie_find_root_port(dev); + if (!rpdev) { + ret = -ENODEV; + goto out_put; + } + + pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); + if (!pos_cap_err) { + ret = -EPERM; + goto out_put; + } + pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever); + pci_read_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, &cor_mask); + pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + &uncor_mask); + + rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR); + if (!rp_pos_cap_err) { + ret = -EPERM; + goto out_put; + } + + err_alloc = kzalloc(sizeof(struct aer_error), GFP_KERNEL); + if (!err_alloc) { + ret = -ENOMEM; + goto out_put; + } + rperr_alloc = kzalloc(sizeof(struct aer_error), GFP_KERNEL); + if (!rperr_alloc) { + ret = -ENOMEM; + goto out_put; + } + + if (aer_mask_override) { + cor_mask_orig = cor_mask; + cor_mask &= !(einj->cor_status); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, + cor_mask); + + uncor_mask_orig = uncor_mask; + uncor_mask &= !(einj->uncor_status); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + uncor_mask); + } + + spin_lock_irqsave(&inject_lock, flags); + + err = __find_aer_error_by_dev(dev); + if (!err) { + err = err_alloc; + err_alloc = NULL; + aer_error_init(err, einj->domain, einj->bus, devfn, + pos_cap_err); + list_add(&err->list, &einjected); + } + err->uncor_status |= einj->uncor_status; + err->cor_status |= einj->cor_status; + err->header_log0 = einj->header_log0; + err->header_log1 = einj->header_log1; + err->header_log2 = einj->header_log2; + err->header_log3 = einj->header_log3; + + if (!aer_mask_override && einj->cor_status && + !(einj->cor_status & ~cor_mask)) { + ret = -EINVAL; + printk(KERN_WARNING "The correctable error(s) is masked by device\n"); + spin_unlock_irqrestore(&inject_lock, flags); + goto out_put; + } + if (!aer_mask_override && einj->uncor_status && + !(einj->uncor_status & ~uncor_mask)) { + ret = -EINVAL; + printk(KERN_WARNING "The uncorrectable error(s) is masked by device\n"); + spin_unlock_irqrestore(&inject_lock, flags); + goto out_put; + } + + rperr = __find_aer_error_by_dev(rpdev); + if (!rperr) { + rperr = rperr_alloc; + rperr_alloc = NULL; + aer_error_init(rperr, pci_domain_nr(rpdev->bus), + rpdev->bus->number, rpdev->devfn, + rp_pos_cap_err); + list_add(&rperr->list, &einjected); + } + if (einj->cor_status) { + if (rperr->root_status & PCI_ERR_ROOT_COR_RCV) + rperr->root_status |= PCI_ERR_ROOT_MULTI_COR_RCV; + else + rperr->root_status |= PCI_ERR_ROOT_COR_RCV; + rperr->source_id &= 0xffff0000; + rperr->source_id |= (einj->bus << 8) | devfn; + } + if (einj->uncor_status) { + if (rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV) + rperr->root_status |= PCI_ERR_ROOT_MULTI_UNCOR_RCV; + if (sever & einj->uncor_status) { + rperr->root_status |= PCI_ERR_ROOT_FATAL_RCV; + if (!(rperr->root_status & PCI_ERR_ROOT_UNCOR_RCV)) + rperr->root_status |= PCI_ERR_ROOT_FIRST_FATAL; + } else + rperr->root_status |= PCI_ERR_ROOT_NONFATAL_RCV; + rperr->root_status |= PCI_ERR_ROOT_UNCOR_RCV; + rperr->source_id &= 0x0000ffff; + rperr->source_id |= ((einj->bus << 8) | devfn) << 16; + } + spin_unlock_irqrestore(&inject_lock, flags); + + if (aer_mask_override) { + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, + cor_mask_orig); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + uncor_mask_orig); + } + + ret = pci_bus_set_aer_ops(dev->bus); + if (ret) + goto out_put; + ret = pci_bus_set_aer_ops(rpdev->bus); + if (ret) + goto out_put; + + if (find_aer_device(rpdev, &edev)) { + if (!get_service_data(edev)) { + printk(KERN_WARNING "AER service is not initialized\n"); + ret = -EINVAL; + goto out_put; + } + aer_irq(-1, edev); + } else + ret = -EINVAL; +out_put: + kfree(err_alloc); + kfree(rperr_alloc); + pci_dev_put(dev); + return ret; +} + +static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf, + size_t usize, loff_t *off) +{ + struct aer_error_inj einj; + int ret; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (usize < offsetof(struct aer_error_inj, domain) || + usize > sizeof(einj)) + return -EINVAL; + + memset(&einj, 0, sizeof(einj)); + if (copy_from_user(&einj, ubuf, usize)) + return -EFAULT; + + ret = aer_inject(&einj); + return ret ? ret : usize; +} + +static const struct file_operations aer_inject_fops = { + .write = aer_inject_write, + .owner = THIS_MODULE, + .llseek = noop_llseek, +}; + +static struct miscdevice aer_inject_device = { + .minor = MISC_DYNAMIC_MINOR, + .name = "aer_inject", + .fops = &aer_inject_fops, +}; + +static int __init aer_inject_init(void) +{ + return misc_register(&aer_inject_device); +} + +static void __exit aer_inject_exit(void) +{ + struct aer_error *err, *err_next; + unsigned long flags; + struct pci_bus_ops *bus_ops; + + misc_deregister(&aer_inject_device); + + while ((bus_ops = pci_bus_ops_pop())) { + pci_bus_set_ops(bus_ops->bus, bus_ops->ops); + kfree(bus_ops); + } + + spin_lock_irqsave(&inject_lock, flags); + list_for_each_entry_safe(err, err_next, &einjected, list) { + list_del(&err->list); + kfree(err); + } + spin_unlock_irqrestore(&inject_lock, flags); +} + +module_init(aer_inject_init); +module_exit(aer_inject_exit); + +MODULE_DESCRIPTION("PCIe AER software error injector"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c index 7a62f7dd900..0bf82a20a0f 100644 --- a/drivers/pci/pcie/aer/aerdrv.c +++ b/drivers/pci/pcie/aer/aerdrv.c @@ -17,6 +17,8 @@ #include <linux/module.h> #include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/sched.h> #include <linux/kernel.h> #include <linux/errno.h> #include <linux/pm.h> @@ -24,8 +26,10 @@ #include <linux/interrupt.h> #include <linux/delay.h> #include <linux/pcieport_if.h> +#include <linux/slab.h> #include "aerdrv.h" +#include "../../pci.h" /* * Version Information @@ -37,45 +41,26 @@ MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL"); -static int __devinit aer_probe (struct pcie_device *dev, - const struct pcie_port_service_id *id ); +static int aer_probe(struct pcie_device *dev); static void aer_remove(struct pcie_device *dev); -static int aer_suspend(struct pcie_device *dev, pm_message_t state) -{return 0;} -static int aer_resume(struct pcie_device *dev) {return 0;} static pci_ers_result_t aer_error_detected(struct pci_dev *dev, enum pci_channel_state error); static void aer_error_resume(struct pci_dev *dev); static pci_ers_result_t aer_root_reset(struct pci_dev *dev); -/* - * PCI Express bus's AER Root service driver data structure - */ -static struct pcie_port_service_id aer_id[] = { - { - .vendor = PCI_ANY_ID, - .device = PCI_ANY_ID, - .port_type = PCIE_RC_PORT, - .service_type = PCIE_PORT_SERVICE_AER, - }, - { /* end: all zeroes */ } -}; - -static struct pci_error_handlers aer_error_handlers = { +static const struct pci_error_handlers aer_error_handlers = { .error_detected = aer_error_detected, - .resume = aer_error_resume, + .resume = aer_error_resume, }; static struct pcie_port_service_driver aerdriver = { .name = "aer", - .id_table = &aer_id[0], + .port_type = PCI_EXP_TYPE_ROOT_PORT, + .service = PCIE_PORT_SERVICE_AER, .probe = aer_probe, .remove = aer_remove, - .suspend = aer_suspend, - .resume = aer_resume, - .err_handler = &aer_error_handlers, .reset_link = aer_root_reset, @@ -88,14 +73,125 @@ void pci_no_aer(void) pcie_aer_disable = 1; /* has priority over 'forceload' */ } +bool pci_aer_available(void) +{ + return !pcie_aer_disable && pci_msi_enabled(); +} + +static int set_device_error_reporting(struct pci_dev *dev, void *data) +{ + bool enable = *((bool *)data); + int type = pci_pcie_type(dev); + + if ((type == PCI_EXP_TYPE_ROOT_PORT) || + (type == PCI_EXP_TYPE_UPSTREAM) || + (type == PCI_EXP_TYPE_DOWNSTREAM)) { + if (enable) + pci_enable_pcie_error_reporting(dev); + else + pci_disable_pcie_error_reporting(dev); + } + + if (enable) + pcie_set_ecrc_checking(dev); + + return 0; +} + +/** + * set_downstream_devices_error_reporting - enable/disable the error reporting bits on the root port and its downstream ports. + * @dev: pointer to root port's pci_dev data structure + * @enable: true = enable error reporting, false = disable error reporting. + */ +static void set_downstream_devices_error_reporting(struct pci_dev *dev, + bool enable) +{ + set_device_error_reporting(dev, &enable); + + if (!dev->subordinate) + return; + pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable); +} + +/** + * aer_enable_rootport - enable Root Port's interrupts when receiving messages + * @rpc: pointer to a Root Port data structure + * + * Invoked when PCIe bus loads AER service driver. + */ +static void aer_enable_rootport(struct aer_rpc *rpc) +{ + struct pci_dev *pdev = rpc->rpd->port; + int aer_pos; + u16 reg16; + u32 reg32; + + /* Clear PCIe Capability's Device Status */ + pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); + pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); + + /* Disable system error generation in response to error messages */ + pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, + SYSTEM_ERROR_INTR_ON_MESG_MASK); + + aer_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + /* Clear error status */ + pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32); + pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32); + pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32); + + /* + * Enable error reporting for the root port device and downstream port + * devices. + */ + set_downstream_devices_error_reporting(pdev, true); + + /* Enable Root Port's interrupt in response to error messages */ + pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, reg32); +} + +/** + * aer_disable_rootport - disable Root Port's interrupts when receiving messages + * @rpc: pointer to a Root Port data structure + * + * Invoked when PCIe bus unloads AER service driver. + */ +static void aer_disable_rootport(struct aer_rpc *rpc) +{ + struct pci_dev *pdev = rpc->rpd->port; + u32 reg32; + int pos; + + /* + * Disable error reporting for the root port device and downstream port + * devices. + */ + set_downstream_devices_error_reporting(pdev, false); + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + /* Disable Root's interrupt in response to error messages */ + pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, reg32); + + /* Clear Root's error status reg */ + pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32); +} + /** * aer_irq - Root Port's ISR * @irq: IRQ assigned to Root Port * @context: pointer to Root Port data structure * * Invoked when Root Port detects AER messages. - **/ -static irqreturn_t aer_irq(int irq, void *context) + */ +irqreturn_t aer_irq(int irq, void *context) { unsigned int status, id; struct pcie_device *pdev = (struct pcie_device *)context; @@ -104,7 +200,7 @@ static irqreturn_t aer_irq(int irq, void *context) unsigned long flags; int pos; - pos = pci_find_aer_capability(pdev->port); + pos = pci_find_ext_capability(pdev->port, PCI_EXT_CAP_ID_ERR); /* * Must lock access to Root Error Status Reg, Root Error ID Reg, * and Root error producer/consumer index @@ -113,13 +209,13 @@ static irqreturn_t aer_irq(int irq, void *context) /* Read error status */ pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, &status); - if (!(status & ROOT_ERR_STATUS_MASKS)) { + if (!(status & (PCI_ERR_ROOT_UNCOR_RCV|PCI_ERR_ROOT_COR_RCV))) { spin_unlock_irqrestore(&rpc->e_lock, flags); return IRQ_NONE; } /* Read error source and clear error status */ - pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_COR_SRC, &id); + pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_ERR_SRC, &id); pci_write_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, status); /* Store error source for later DPC handler */ @@ -144,34 +240,31 @@ static irqreturn_t aer_irq(int irq, void *context) return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(aer_irq); /** * aer_alloc_rpc - allocate Root Port data structure * @dev: pointer to the pcie_dev data structure * * Invoked when Root Port's AER service is loaded. - **/ -static struct aer_rpc* aer_alloc_rpc(struct pcie_device *dev) + */ +static struct aer_rpc *aer_alloc_rpc(struct pcie_device *dev) { struct aer_rpc *rpc; - if (!(rpc = kzalloc(sizeof(struct aer_rpc), - GFP_KERNEL))) + rpc = kzalloc(sizeof(struct aer_rpc), GFP_KERNEL); + if (!rpc) return NULL; - /* - * Initialize Root lock access, e_lock, to Root Error Status Reg, - * Root Error ID Reg, and Root error producer/consumer index. - */ + /* Initialize Root lock access, e_lock, to Root Error Status Reg */ spin_lock_init(&rpc->e_lock); rpc->rpd = dev; INIT_WORK(&rpc->dpc_handler, aer_isr); - rpc->prod_idx = rpc->cons_idx = 0; mutex_init(&rpc->rpc_mutex); init_waitqueue_head(&rpc->wait_release); - /* Use PCIE bus function to store rpc into PCIE device */ + /* Use PCIe bus function to store rpc into PCIe device */ set_service_data(dev, rpc); return rpc; @@ -182,7 +275,7 @@ static struct aer_rpc* aer_alloc_rpc(struct pcie_device *dev) * @dev: pointer to the pcie_dev data structure * * Invoked when PCI Express bus unloads or AER probe fails. - **/ + */ static void aer_remove(struct pcie_device *dev) { struct aer_rpc *rpc = get_service_data(dev); @@ -194,7 +287,8 @@ static void aer_remove(struct pcie_device *dev) wait_event(rpc->wait_release, rpc->prod_idx == rpc->cons_idx); - aer_delete_rootport(rpc); + aer_disable_rootport(rpc); + kfree(rpc); set_service_data(dev, NULL); } } @@ -205,31 +299,30 @@ static void aer_remove(struct pcie_device *dev) * @id: pointer to the service id data structure * * Invoked when PCI Express bus loads AER service driver. - **/ -static int __devinit aer_probe (struct pcie_device *dev, - const struct pcie_port_service_id *id ) + */ +static int aer_probe(struct pcie_device *dev) { int status; struct aer_rpc *rpc; struct device *device = &dev->device; /* Init */ - if ((status = aer_init(dev))) + status = aer_init(dev); + if (status) return status; /* Alloc rpc data structure */ - if (!(rpc = aer_alloc_rpc(dev))) { - printk(KERN_DEBUG "%s: Alloc rpc fails on PCIE device[%s]\n", - __FUNCTION__, device->bus_id); + rpc = aer_alloc_rpc(dev); + if (!rpc) { + dev_printk(KERN_DEBUG, device, "alloc rpc failed\n"); aer_remove(dev); return -ENOMEM; } /* Request IRQ ISR */ - if ((status = request_irq(dev->irq, aer_irq, IRQF_SHARED, "aerdrv", - dev))) { - printk(KERN_DEBUG "%s: Request ISR fails on PCIE device[%s]\n", - __FUNCTION__, device->bus_id); + status = request_irq(dev->irq, aer_irq, IRQF_SHARED, "aerdrv", dev); + if (status) { + dev_printk(KERN_DEBUG, device, "request IRQ failed\n"); aer_remove(dev); return status; } @@ -246,41 +339,30 @@ static int __devinit aer_probe (struct pcie_device *dev, * @dev: pointer to Root Port's pci_dev data structure * * Invoked by Port Bus driver when performing link reset at Root Port. - **/ + */ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) { - u16 p2p_ctrl; - u32 status; + u32 reg32; int pos; - pos = pci_find_aer_capability(dev); + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); /* Disable Root's interrupt in response to error messages */ - pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, 0); + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32); - /* Assert Secondary Bus Reset */ - pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &p2p_ctrl); - p2p_ctrl |= PCI_CB_BRIDGE_CTL_CB_RESET; - pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl); + pci_reset_bridge_secondary_bus(dev); + dev_printk(KERN_DEBUG, &dev->dev, "Root Port link has been reset\n"); - /* De-assert Secondary Bus Reset */ - p2p_ctrl &= ~PCI_CB_BRIDGE_CTL_CB_RESET; - pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl); - - /* - * System software must wait for at least 100ms from the end - * of a reset of one or more device before it is permitted - * to issue Configuration Requests to those devices. - */ - msleep(200); - printk(KERN_DEBUG "Complete link reset at Root[%s]\n", dev->dev.bus_id); + /* Clear Root Error Status */ + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, reg32); /* Enable Root Port's interrupt in response to error messages */ - pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &status); - pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, status); - pci_write_config_dword(dev, - pos + PCI_ERR_ROOT_COMMAND, - ROOT_PORT_INTR_ON_MESG_MASK); + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32); return PCI_ERS_RESULT_RECOVERED; } @@ -291,7 +373,7 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) * @error: error severity being notified by port bus * * Invoked by Port Bus driver during error recovery. - **/ + */ static pci_ers_result_t aer_error_detected(struct pci_dev *dev, enum pci_channel_state error) { @@ -304,7 +386,7 @@ static pci_ers_result_t aer_error_detected(struct pci_dev *dev, * @dev: pointer to Root Port's pci_dev data structure * * Invoked by Port Bus driver during nonfatal recovery. - **/ + */ static void aer_error_resume(struct pci_dev *dev) { int pos; @@ -312,12 +394,11 @@ static void aer_error_resume(struct pci_dev *dev) u16 reg16; /* Clean up Root device status */ - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - pci_read_config_word(dev, pos + PCI_EXP_DEVSTA, ®16); - pci_write_config_word(dev, pos + PCI_EXP_DEVSTA, reg16); + pcie_capability_read_word(dev, PCI_EXP_DEVSTA, ®16); + pcie_capability_write_word(dev, PCI_EXP_DEVSTA, reg16); /* Clean AER Root Error Status */ - pos = pci_find_aer_capability(dev); + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); if (dev->error_state == pci_channel_io_normal) @@ -331,10 +412,10 @@ static void aer_error_resume(struct pci_dev *dev) * aer_service_init - register AER root service driver * * Invoked when AER root service driver is loaded. - **/ + */ static int __init aer_service_init(void) { - if (pcie_aer_disable) + if (!pci_aer_available() || aer_acpi_firmware_first()) return -ENXIO; return pcie_port_service_register(&aerdriver); } @@ -343,7 +424,7 @@ static int __init aer_service_init(void) * aer_service_exit - unregister AER root service driver * * Invoked when AER root service driver is unloaded. - **/ + */ static void __exit aer_service_exit(void) { pcie_port_service_unregister(&aerdriver); diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h index c7ad68b6c6d..84420b7c945 100644 --- a/drivers/pci/pcie/aer/aerdrv.h +++ b/drivers/pci/pcie/aer/aerdrv.h @@ -11,16 +11,7 @@ #include <linux/workqueue.h> #include <linux/pcieport_if.h> #include <linux/aer.h> - -#define AER_NONFATAL 0 -#define AER_FATAL 1 -#define AER_CORRECTABLE 2 -#define AER_UNCORRECTABLE 4 -#define AER_ERROR_MASK 0x001fffff -#define AER_ERROR(d) (d & AER_ERROR_MASK) - -/* Root Error Status Register Bits */ -#define ROOT_ERR_STATUS_MASKS 0x0f +#include <linux/interrupt.h> #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ PCI_EXP_RTCTL_SENFEE| \ @@ -31,8 +22,6 @@ #define ERR_COR_ID(d) (d & 0xffff) #define ERR_UNCOR_ID(d) (d >> 16) -#define AER_SUCCESS 0 -#define AER_UNSUCCESS 1 #define AER_ERROR_SOURCES_MAX 100 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ @@ -42,25 +31,24 @@ PCI_ERR_UNC_UNX_COMP| \ PCI_ERR_UNC_MALF_TLP) -/* AER Error Info Flags */ -#define AER_TLP_HEADER_VALID_FLAG 0x00000001 -#define AER_MULTI_ERROR_VALID_FLAG 0x00000002 +#define AER_MAX_MULTI_ERR_DEVICES 5 /* Not likely to have more */ +struct aer_err_info { + struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; + int error_dev_num; -#define ERR_CORRECTABLE_ERROR_MASK 0x000031c1 -#define ERR_UNCORRECTABLE_ERROR_MASK 0x001ff010 + unsigned int id:16; -struct header_log_regs { - unsigned int dw0; - unsigned int dw1; - unsigned int dw2; - unsigned int dw3; -}; + unsigned int severity:2; /* 0:NONFATAL | 1:FATAL | 2:COR */ + unsigned int __pad1:5; + unsigned int multi_error_valid:1; + + unsigned int first_error:5; + unsigned int __pad2:2; + unsigned int tlp_header_valid:1; -struct aer_err_info { - int severity; /* 0:NONFATAL | 1:FATAL | 2:COR */ - int flags; unsigned int status; /* COR/UNCOR Error Status */ - struct header_log_regs tlp; /* TLP Header */ + unsigned int mask; /* COR/UNCOR Error Mask */ + struct aer_header_log_regs tlp; /* TLP Header */ }; struct aer_err_source { @@ -95,6 +83,12 @@ struct aer_broadcast_data { static inline pci_ers_result_t merge_result(enum pci_ers_result orig, enum pci_ers_result new) { + if (new == PCI_ERS_RESULT_NO_AER_DRIVER) + return PCI_ERS_RESULT_NO_AER_DRIVER; + + if (new == PCI_ERS_RESULT_NONE) + return orig; + switch (orig) { case PCI_ERS_RESULT_CAN_RECOVER: case PCI_ERS_RESULT_RECOVERED: @@ -102,7 +96,7 @@ static inline pci_ers_result_t merge_result(enum pci_ers_result orig, break; case PCI_ERS_RESULT_DISCONNECT: if (new == PCI_ERS_RESULT_NEED_RESET) - orig = new; + orig = PCI_ERS_RESULT_NEED_RESET; break; default: break; @@ -112,19 +106,27 @@ static inline pci_ers_result_t merge_result(enum pci_ers_result orig, } extern struct bus_type pcie_port_bus_type; -extern void aer_enable_rootport(struct aer_rpc *rpc); -extern void aer_delete_rootport(struct aer_rpc *rpc); -extern int aer_init(struct pcie_device *dev); -extern void aer_isr(struct work_struct *work); -extern void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); - -#ifdef CONFIG_ACPI -extern int aer_osc_setup(struct pcie_device *pciedev); +int aer_init(struct pcie_device *dev); +void aer_isr(struct work_struct *work); +void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); +void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info); +irqreturn_t aer_irq(int irq, void *context); + +#ifdef CONFIG_ACPI_APEI +int pcie_aer_get_firmware_first(struct pci_dev *pci_dev); #else -static inline int aer_osc_setup(struct pcie_device *pciedev) +static inline int pcie_aer_get_firmware_first(struct pci_dev *pci_dev) { + if (pci_dev->__aer_firmware_first_valid) + return pci_dev->__aer_firmware_first; return 0; } #endif -#endif //_AERDRV_H_ +static inline void pcie_aer_force_firmware_first(struct pci_dev *pci_dev, + int enable) +{ + pci_dev->__aer_firmware_first = !!enable; + pci_dev->__aer_firmware_first_valid = 1; +} +#endif /* _AERDRV_H_ */ diff --git a/drivers/pci/pcie/aer/aerdrv_acpi.c b/drivers/pci/pcie/aer/aerdrv_acpi.c index 8c199ae84f6..01906576ab9 100644 --- a/drivers/pci/pcie/aer/aerdrv_acpi.c +++ b/drivers/pci/pcie/aer/aerdrv_acpi.c @@ -16,43 +16,126 @@ #include <linux/acpi.h> #include <linux/pci-acpi.h> #include <linux/delay.h> +#include <acpi/apei.h> #include "aerdrv.h" -/** - * aer_osc_setup - run ACPI _OSC method - * @pciedev: pcie_device which AER is being enabled on - * - * @return: Zero on success. Nonzero otherwise. - * - * Invoked when PCIE bus loads AER service driver. To avoid conflict with - * BIOS AER support requires BIOS to yield AER control to OS native driver. - **/ -int aer_osc_setup(struct pcie_device *pciedev) +#ifdef CONFIG_ACPI_APEI +static inline int hest_match_pci(struct acpi_hest_aer_common *p, + struct pci_dev *pci) { - acpi_status status = AE_NOT_FOUND; - struct pci_dev *pdev = pciedev->port; - acpi_handle handle = 0; - - /* Find root host bridge */ - while (pdev->bus && pdev->bus->self) - pdev = pdev->bus->self; - handle = acpi_get_pci_rootbridge_handle( - pci_domain_nr(pdev->bus), pdev->bus->number); - - if (handle) { - pcie_osc_support_set(OSC_EXT_PCI_CONFIG_SUPPORT); - status = pci_osc_control_set(handle, - OSC_PCI_EXPRESS_AER_CONTROL | - OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL); - } + return ACPI_HEST_SEGMENT(p->bus) == pci_domain_nr(pci->bus) && + ACPI_HEST_BUS(p->bus) == pci->bus->number && + p->device == PCI_SLOT(pci->devfn) && + p->function == PCI_FUNC(pci->devfn); +} + +static inline bool hest_match_type(struct acpi_hest_header *hest_hdr, + struct pci_dev *dev) +{ + u16 hest_type = hest_hdr->type; + u8 pcie_type = pci_pcie_type(dev); + + if ((hest_type == ACPI_HEST_TYPE_AER_ROOT_PORT && + pcie_type == PCI_EXP_TYPE_ROOT_PORT) || + (hest_type == ACPI_HEST_TYPE_AER_ENDPOINT && + pcie_type == PCI_EXP_TYPE_ENDPOINT) || + (hest_type == ACPI_HEST_TYPE_AER_BRIDGE && + (dev->class >> 16) == PCI_BASE_CLASS_BRIDGE)) + return true; + return false; +} + +struct aer_hest_parse_info { + struct pci_dev *pci_dev; + int firmware_first; +}; + +static int hest_source_is_pcie_aer(struct acpi_hest_header *hest_hdr) +{ + if (hest_hdr->type == ACPI_HEST_TYPE_AER_ROOT_PORT || + hest_hdr->type == ACPI_HEST_TYPE_AER_ENDPOINT || + hest_hdr->type == ACPI_HEST_TYPE_AER_BRIDGE) + return 1; + return 0; +} + +static int aer_hest_parse(struct acpi_hest_header *hest_hdr, void *data) +{ + struct aer_hest_parse_info *info = data; + struct acpi_hest_aer_common *p; + int ff; + + if (!hest_source_is_pcie_aer(hest_hdr)) + return 0; + + p = (struct acpi_hest_aer_common *)(hest_hdr + 1); + ff = !!(p->flags & ACPI_HEST_FIRMWARE_FIRST); - if (ACPI_FAILURE(status)) { - printk(KERN_DEBUG "AER service couldn't init device %s - %s\n", - pciedev->device.bus_id, - (status == AE_SUPPORT || status == AE_NOT_FOUND) ? - "no _OSC support" : "Run ACPI _OSC fails"); - return -1; + /* + * If no specific device is supplied, determine whether + * FIRMWARE_FIRST is set for *any* PCIe device. + */ + if (!info->pci_dev) { + info->firmware_first |= ff; + return 0; } + /* Otherwise, check the specific device */ + if (p->flags & ACPI_HEST_GLOBAL) { + if (hest_match_type(hest_hdr, info->pci_dev)) + info->firmware_first = ff; + } else + if (hest_match_pci(p, info->pci_dev)) + info->firmware_first = ff; + return 0; } + +static void aer_set_firmware_first(struct pci_dev *pci_dev) +{ + int rc; + struct aer_hest_parse_info info = { + .pci_dev = pci_dev, + .firmware_first = 0, + }; + + rc = apei_hest_parse(aer_hest_parse, &info); + + if (rc) + pci_dev->__aer_firmware_first = 0; + else + pci_dev->__aer_firmware_first = info.firmware_first; + pci_dev->__aer_firmware_first_valid = 1; +} + +int pcie_aer_get_firmware_first(struct pci_dev *dev) +{ + if (!pci_is_pcie(dev)) + return 0; + + if (!dev->__aer_firmware_first_valid) + aer_set_firmware_first(dev); + return dev->__aer_firmware_first; +} + +static bool aer_firmware_first; + +/** + * aer_acpi_firmware_first - Check if APEI should control AER. + */ +bool aer_acpi_firmware_first(void) +{ + static bool parsed = false; + struct aer_hest_parse_info info = { + .pci_dev = NULL, /* Check all PCIe devices */ + .firmware_first = 0, + }; + + if (!parsed) { + apei_hest_parse(aer_hest_parse, &info); + aer_firmware_first = info.firmware_first; + parsed = true; + } + return aer_firmware_first; +} +#endif diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c index 3c0d8d138f5..5653ea94547 100644 --- a/drivers/pci/pcie/aer/aerdrv_core.c +++ b/drivers/pci/pcie/aer/aerdrv_core.c @@ -23,188 +23,195 @@ #include <linux/pm.h> #include <linux/suspend.h> #include <linux/delay.h> +#include <linux/slab.h> +#include <linux/kfifo.h> #include "aerdrv.h" -static int forceload; +static bool forceload; +static bool nosourceid; module_param(forceload, bool, 0); +module_param(nosourceid, bool, 0); -#define PCI_CFG_SPACE_SIZE (0x100) -int pci_find_aer_capability(struct pci_dev *dev) -{ - int pos; - u32 reg32 = 0; - - /* Check if it's a pci-express device */ - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - if (!pos) - return 0; - - /* Check if it supports pci-express AER */ - pos = PCI_CFG_SPACE_SIZE; - while (pos) { - if (pci_read_config_dword(dev, pos, ®32)) - return 0; - - /* some broken boards return ~0 */ - if (reg32 == 0xffffffff) - return 0; - - if (PCI_EXT_CAP_ID(reg32) == PCI_EXT_CAP_ID_ERR) - break; - - pos = reg32 >> 20; - } - - return pos; -} +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) int pci_enable_pcie_error_reporting(struct pci_dev *dev) { - u16 reg16 = 0; - int pos; + if (pcie_aer_get_firmware_first(dev)) + return -EIO; - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - if (!pos) + if (!pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) return -EIO; - pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16); - reg16 = reg16 | - PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE; - pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, - reg16); - return 0; + return pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); } +EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting); int pci_disable_pcie_error_reporting(struct pci_dev *dev) { - u16 reg16 = 0; - int pos; - - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - if (!pos) + if (pcie_aer_get_firmware_first(dev)) return -EIO; - pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16); - reg16 = reg16 & ~(PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE); - pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, - reg16); - return 0; + return pcie_capability_clear_word(dev, PCI_EXP_DEVCTL, + PCI_EXP_AER_FLAGS); } +EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting); int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev) { int pos; - u32 status, mask; + u32 status; - pos = pci_find_aer_capability(dev); + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos) return -EIO; pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); - pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask); - if (dev->error_state == pci_channel_io_normal) - status &= ~mask; /* Clear corresponding nonfatal bits */ - else - status &= mask; /* Clear corresponding fatal bits */ - pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); + if (status) + pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status); return 0; } +EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status); + +/** + * add_error_device - list device to be handled + * @e_info: pointer to error info + * @dev: pointer to pci_dev to be added + */ +static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) +{ + if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) { + e_info->dev[e_info->error_dev_num] = dev; + e_info->error_dev_num++; + return 0; + } + return -ENOSPC; +} -int pci_cleanup_aer_correct_error_status(struct pci_dev *dev) +/** + * is_error_source - check whether the device is source of reported error + * @dev: pointer to pci_dev to be checked + * @e_info: pointer to reported error info + */ +static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) { int pos; - u32 status; + u32 status, mask; + u16 reg16; + + /* + * When bus id is equal to 0, it might be a bad id + * reported by root port. + */ + if (!nosourceid && (PCI_BUS_NUM(e_info->id) != 0)) { + /* Device ID match? */ + if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) + return true; + + /* Continue id comparing if there is no multiple error */ + if (!e_info->multi_error_valid) + return false; + } - pos = pci_find_aer_capability(dev); + /* + * When either + * 1) nosourceid==y; + * 2) bus id is equal to 0. Some ports might lose the bus + * id of error source id; + * 3) There are multiple errors and prior id comparing fails; + * We check AER status registers to find possible reporter. + */ + if (atomic_read(&dev->enable_cnt) == 0) + return false; + + /* Check if AER is enabled */ + pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); + if (!(reg16 & PCI_EXP_AER_FLAGS)) + return false; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos) - return -EIO; + return false; - pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status); - pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status); + /* Check if error is recorded */ + if (e_info->severity == AER_CORRECTABLE) { + pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status); + pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask); + } else { + pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); + pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask); + } + if (status & ~mask) + return true; - return 0; + return false; } -static int find_device_iter(struct device *device, void *data) +static int find_device_iter(struct pci_dev *dev, void *data) { - struct pci_dev *dev; - u16 id = *(unsigned long *)data; - u8 secondary, subordinate, d_bus = id >> 8; + struct aer_err_info *e_info = (struct aer_err_info *)data; - if (device->bus == &pci_bus_type) { - dev = to_pci_dev(device); - if (id == ((dev->bus->number << 8) | dev->devfn)) { - /* - * Device ID match - */ - *(unsigned long*)data = (unsigned long)device; + if (is_error_source(dev, e_info)) { + /* List this device */ + if (add_error_device(e_info, dev)) { + /* We cannot handle more... Stop iteration */ + /* TODO: Should print error message here? */ return 1; } - /* - * If device is P2P, check if it is an upstream? - */ - if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) { - pci_read_config_byte(dev, PCI_SECONDARY_BUS, - &secondary); - pci_read_config_byte(dev, PCI_SUBORDINATE_BUS, - &subordinate); - if (d_bus >= secondary && d_bus <= subordinate) { - *(unsigned long*)data = (unsigned long)device; - return 1; - } - } + /* If there is only a single error, stop iteration */ + if (!e_info->multi_error_valid) + return 1; } - return 0; } /** * find_source_device - search through device hierarchy for source device * @parent: pointer to Root Port pci_dev data structure - * @id: device ID of agent who sends an error message to this Root Port + * @e_info: including detailed error information such like id + * + * Return true if found. * - * Invoked when error is detected at the Root Port. + * Invoked by DPC when error is detected at the Root Port. + * Caller of this function must set id, severity, and multi_error_valid of + * struct aer_err_info pointed by @e_info properly. This function must fill + * e_info->error_dev_num and e_info->dev[], based on the given information. */ -static struct device* find_source_device(struct pci_dev *parent, u16 id) +static bool find_source_device(struct pci_dev *parent, + struct aer_err_info *e_info) { struct pci_dev *dev = parent; - struct device *device; - unsigned long device_addr; - int status; + int result; + + /* Must reset in this function */ + e_info->error_dev_num = 0; /* Is Root Port an agent that sends error message? */ - if (id == ((dev->bus->number << 8) | dev->devfn)) - return &dev->dev; - - do { - device_addr = id; - if ((status = device_for_each_child(&dev->dev, - &device_addr, find_device_iter))) { - device = (struct device*)device_addr; - dev = to_pci_dev(device); - if (id == ((dev->bus->number << 8) | dev->devfn)) - return device; - } - }while (status); + result = find_device_iter(dev, e_info); + if (result) + return true; + + pci_walk_bus(parent->subordinate, find_device_iter, e_info); - return NULL; + if (!e_info->error_dev_num) { + dev_printk(KERN_DEBUG, &parent->dev, + "can't find device of ID%04x\n", + e_info->id); + return false; + } + return true; } -static void report_error_detected(struct pci_dev *dev, void *data) +static int report_error_detected(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); dev->error_state = result_data->state; if (!dev->driver || @@ -219,69 +226,94 @@ static void report_error_detected(struct pci_dev *dev, void *data) * of a driver for this device is unaware of * its hw state. */ - printk(KERN_DEBUG "Device ID[%s] has %s\n", - dev->dev.bus_id, (dev->driver) ? - "no AER-aware driver" : "no driver"); + dev_printk(KERN_DEBUG, &dev->dev, "device has %s\n", + dev->driver ? + "no AER-aware driver" : "no driver"); } - return; + + /* + * If there's any device in the subtree that does not + * have an error_detected callback, returning + * PCI_ERS_RESULT_NO_AER_DRIVER prevents calling of + * the subsequent mmio_enabled/slot_reset/resume + * callbacks of "any" device in the subtree. All the + * devices in the subtree are left in the error state + * without recovery. + */ + + if (!(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) + vote = PCI_ERS_RESULT_NO_AER_DRIVER; + else + vote = PCI_ERS_RESULT_NONE; + } else { + err_handler = dev->driver->err_handler; + vote = err_handler->error_detected(dev, result_data->state); } - err_handler = dev->driver->err_handler; - vote = err_handler->error_detected(dev, result_data->state); result_data->result = merge_result(result_data->result, vote); - return; + device_unlock(&dev->dev); + return 0; } -static void report_mmio_enabled(struct pci_dev *dev, void *data) +static int report_mmio_enabled(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); if (!dev->driver || !dev->driver->err_handler || !dev->driver->err_handler->mmio_enabled) - return; + goto out; err_handler = dev->driver->err_handler; vote = err_handler->mmio_enabled(dev); result_data->result = merge_result(result_data->result, vote); - return; +out: + device_unlock(&dev->dev); + return 0; } -static void report_slot_reset(struct pci_dev *dev, void *data) +static int report_slot_reset(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); if (!dev->driver || !dev->driver->err_handler || !dev->driver->err_handler->slot_reset) - return; + goto out; err_handler = dev->driver->err_handler; vote = err_handler->slot_reset(dev); result_data->result = merge_result(result_data->result, vote); - return; +out: + device_unlock(&dev->dev); + return 0; } -static void report_resume(struct pci_dev *dev, void *data) +static int report_resume(struct pci_dev *dev, void *data) { - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; + device_lock(&dev->dev); dev->error_state = pci_channel_io_normal; if (!dev->driver || !dev->driver->err_handler || - !dev->driver->err_handler->slot_reset) - return; + !dev->driver->err_handler->resume) + goto out; err_handler = dev->driver->err_handler; err_handler->resume(dev); - return; +out: + device_unlock(&dev->dev); + return 0; } /** @@ -298,11 +330,11 @@ static void report_resume(struct pci_dev *dev, void *data) static pci_ers_result_t broadcast_error_message(struct pci_dev *dev, enum pci_channel_state state, char *error_mesg, - void (*cb)(struct pci_dev *, void *)) + int (*cb)(struct pci_dev *, void *)) { struct aer_broadcast_data result_data; - printk(KERN_DEBUG "Broadcast %s message\n", error_mesg); + dev_printk(KERN_DEBUG, &dev->dev, "broadcast %s message\n", error_mesg); result_data.state = state; if (cb == report_error_detected) result_data.result = PCI_ERS_RESULT_CAN_RECOVER; @@ -323,8 +355,7 @@ static pci_ers_result_t broadcast_error_message(struct pci_dev *dev, pci_cleanup_aer_uncorrect_error_status(dev); dev->error_state = pci_channel_io_normal; } - } - else { + } else { /* * If the error is reported by an end point, we think this * error is related to the upstream link of the end point. @@ -335,85 +366,79 @@ static pci_ers_result_t broadcast_error_message(struct pci_dev *dev, return result_data.result; } -struct find_aer_service_data { - struct pcie_port_service_driver *aer_driver; - int is_downstream; -}; +/** + * default_reset_link - default reset function + * @dev: pointer to pci_dev data structure + * + * Invoked when performing link reset on a Downstream Port or a + * Root Port with no aer driver. + */ +static pci_ers_result_t default_reset_link(struct pci_dev *dev) +{ + pci_reset_bridge_secondary_bus(dev); + dev_printk(KERN_DEBUG, &dev->dev, "downstream link has been reset\n"); + return PCI_ERS_RESULT_RECOVERED; +} static int find_aer_service_iter(struct device *device, void *data) { - struct device_driver *driver; - struct pcie_port_service_driver *service_driver; - struct pcie_device *pcie_dev; - struct find_aer_service_data *result; - - result = (struct find_aer_service_data *) data; - - if (device->bus == &pcie_port_bus_type) { - pcie_dev = to_pcie_device(device); - if (pcie_dev->id.port_type == PCIE_SW_DOWNSTREAM_PORT) - result->is_downstream = 1; - - driver = device->driver; - if (driver) { - service_driver = to_service_driver(driver); - if (service_driver->id_table->service_type == - PCIE_PORT_SERVICE_AER) { - result->aer_driver = service_driver; - return 1; - } + struct pcie_port_service_driver *service_driver, **drv; + + drv = (struct pcie_port_service_driver **) data; + + if (device->bus == &pcie_port_bus_type && device->driver) { + service_driver = to_service_driver(device->driver); + if (service_driver->service == PCIE_PORT_SERVICE_AER) { + *drv = service_driver; + return 1; } } return 0; } -static void find_aer_service(struct pci_dev *dev, - struct find_aer_service_data *data) +static struct pcie_port_service_driver *find_aer_service(struct pci_dev *dev) { - int retval; - retval = device_for_each_child(&dev->dev, data, find_aer_service_iter); + struct pcie_port_service_driver *drv = NULL; + + device_for_each_child(&dev->dev, &drv, find_aer_service_iter); + + return drv; } -static pci_ers_result_t reset_link(struct pcie_device *aerdev, - struct pci_dev *dev) +static pci_ers_result_t reset_link(struct pci_dev *dev) { struct pci_dev *udev; pci_ers_result_t status; - struct find_aer_service_data data; + struct pcie_port_service_driver *driver; - if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) + if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) { + /* Reset this port for all subordinates */ udev = dev; - else - udev= dev->bus->self; + } else { + /* Reset the upstream component (likely downstream port) */ + udev = dev->bus->self; + } - data.is_downstream = 0; - data.aer_driver = NULL; - find_aer_service(udev, &data); + /* Use the aer driver of the component firstly */ + driver = find_aer_service(udev); - /* - * Use the aer driver of the error agent firstly. - * If it hasn't the aer driver, use the root port's - */ - if (!data.aer_driver || !data.aer_driver->reset_link) { - if (data.is_downstream && - aerdev->device.driver && - to_service_driver(aerdev->device.driver)->reset_link) { - data.aer_driver = - to_service_driver(aerdev->device.driver); - } else { - printk(KERN_DEBUG "No link-reset support to Device ID" - "[%s]\n", - dev->dev.bus_id); - return PCI_ERS_RESULT_DISCONNECT; - } + if (driver && driver->reset_link) { + status = driver->reset_link(udev); + } else if (pci_pcie_type(udev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(udev) == PCI_EXP_TYPE_ROOT_PORT) { + status = default_reset_link(udev); + } else { + dev_printk(KERN_DEBUG, &dev->dev, + "no link-reset support at upstream device %s\n", + pci_name(udev)); + return PCI_ERS_RESULT_DISCONNECT; } - status = data.aer_driver->reset_link(udev); if (status != PCI_ERS_RESULT_RECOVERED) { - printk(KERN_DEBUG "Link reset at upstream Device ID" - "[%s] failed\n", - udev->dev.bus_id); + dev_printk(KERN_DEBUG, &dev->dev, + "link reset at upstream device %s failed\n", + pci_name(udev)); return PCI_ERS_RESULT_DISCONNECT; } @@ -422,7 +447,6 @@ static pci_ers_result_t reset_link(struct pcie_device *aerdev, /** * do_recovery - handle nonfatal/fatal error recovery process - * @aerdev: pointer to a pcie_device data structure of root port * @dev: pointer to a pci_dev data structure of agent detecting an error * @severity: error severity type * @@ -430,9 +454,7 @@ static pci_ers_result_t reset_link(struct pcie_device *aerdev, * error detected message to all downstream drivers within a hierarchy in * question and return the returned code. */ -static pci_ers_result_t do_recovery(struct pcie_device *aerdev, - struct pci_dev *dev, - int severity) +static void do_recovery(struct pci_dev *dev, int severity) { pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED; enum pci_channel_state state; @@ -448,11 +470,9 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev, report_error_detected); if (severity == AER_FATAL) { - result = reset_link(aerdev, dev); - if (result != PCI_ERS_RESULT_RECOVERED) { - /* TODO: Should panic here? */ - return result; - } + result = reset_link(dev); + if (result != PCI_ERS_RESULT_RECOVERED) + goto failed; } if (status == PCI_ERS_RESULT_CAN_RECOVER) @@ -473,13 +493,20 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev, report_slot_reset); } - if (status == PCI_ERS_RESULT_RECOVERED) - broadcast_error_message(dev, + if (status != PCI_ERS_RESULT_RECOVERED) + goto failed; + + broadcast_error_message(dev, state, "resume", report_resume); - return status; + dev_info(&dev->dev, "AER: Device recovery successful\n"); + return; + +failed: + /* TODO: Should kernel panic here? */ + dev_info(&dev->dev, "AER: Device recovery failed\n"); } /** @@ -490,154 +517,139 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev, * * Invoked when an error being detected by Root Port. */ -static void handle_error_source(struct pcie_device * aerdev, +static void handle_error_source(struct pcie_device *aerdev, struct pci_dev *dev, - struct aer_err_info info) + struct aer_err_info *info) { - pci_ers_result_t status = 0; int pos; - if (info.severity == AER_CORRECTABLE) { + if (info->severity == AER_CORRECTABLE) { /* - * Correctable error does not need software intevention. + * Correctable error does not need software intervention. * No need to go through error recovery process. */ - pos = pci_find_aer_capability(dev); + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (pos) pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, - info.status); - } else { - status = do_recovery(aerdev, dev, info.severity); - if (status == PCI_ERS_RESULT_RECOVERED) { - printk(KERN_DEBUG "AER driver successfully recovered\n"); - } else { - /* TODO: Should kernel panic here? */ - printk(KERN_DEBUG "AER driver didn't recover\n"); - } - } -} - -/** - * aer_enable_rootport - enable Root Port's interrupts when receiving messages - * @rpc: pointer to a Root Port data structure - * - * Invoked when PCIE bus loads AER service driver. - */ -void aer_enable_rootport(struct aer_rpc *rpc) -{ - struct pci_dev *pdev = rpc->rpd->port; - int pos, aer_pos; - u16 reg16; - u32 reg32; - - pos = pci_find_capability(pdev, PCI_CAP_ID_EXP); - /* Clear PCIE Capability's Device Status */ - pci_read_config_word(pdev, pos+PCI_EXP_DEVSTA, ®16); - pci_write_config_word(pdev, pos+PCI_EXP_DEVSTA, reg16); - - /* Disable system error generation in response to error messages */ - pci_read_config_word(pdev, pos + PCI_EXP_RTCTL, ®16); - reg16 &= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK); - pci_write_config_word(pdev, pos + PCI_EXP_RTCTL, reg16); - - aer_pos = pci_find_aer_capability(pdev); - /* Clear error status */ - pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32); - pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32); - pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32); - - /* Enable Root Port device reporting error itself */ - pci_read_config_word(pdev, pos+PCI_EXP_DEVCTL, ®16); - reg16 = reg16 | - PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE; - pci_write_config_word(pdev, pos+PCI_EXP_DEVCTL, - reg16); - - /* Enable Root Port's interrupt in response to error messages */ - pci_write_config_dword(pdev, - aer_pos + PCI_ERR_ROOT_COMMAND, - ROOT_PORT_INTR_ON_MESG_MASK); + info->status); + } else + do_recovery(dev, info->severity); } -/** - * disable_root_aer - disable Root Port's interrupts when receiving messages - * @rpc: pointer to a Root Port data structure - * - * Invoked when PCIE bus unloads AER service driver. - */ -static void disable_root_aer(struct aer_rpc *rpc) -{ - struct pci_dev *pdev = rpc->rpd->port; - u32 reg32; - int pos; +#ifdef CONFIG_ACPI_APEI_PCIEAER +static void aer_recover_work_func(struct work_struct *work); - pos = pci_find_aer_capability(pdev); - /* Disable Root's interrupt in response to error messages */ - pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, 0); +#define AER_RECOVER_RING_ORDER 4 +#define AER_RECOVER_RING_SIZE (1 << AER_RECOVER_RING_ORDER) - /* Clear Root's error status reg */ - pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, ®32); - pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32); -} +struct aer_recover_entry { + u8 bus; + u8 devfn; + u16 domain; + int severity; + struct aer_capability_regs *regs; +}; -/** - * get_e_source - retrieve an error source - * @rpc: pointer to the root port which holds an error - * - * Invoked by DPC handler to consume an error. +static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, + AER_RECOVER_RING_SIZE); +/* + * Mutual exclusion for writers of aer_recover_ring, reader side don't + * need lock, because there is only one reader and lock is not needed + * between reader and writer. */ -static struct aer_err_source* get_e_source(struct aer_rpc *rpc) +static DEFINE_SPINLOCK(aer_recover_ring_lock); +static DECLARE_WORK(aer_recover_work, aer_recover_work_func); + +void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, + int severity, struct aer_capability_regs *aer_regs) { - struct aer_err_source *e_source; unsigned long flags; + struct aer_recover_entry entry = { + .bus = bus, + .devfn = devfn, + .domain = domain, + .severity = severity, + .regs = aer_regs, + }; + + spin_lock_irqsave(&aer_recover_ring_lock, flags); + if (kfifo_put(&aer_recover_ring, entry)) + schedule_work(&aer_recover_work); + else + pr_err("AER recover: Buffer overflow when recovering AER for %04x:%02x:%02x:%x\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + spin_unlock_irqrestore(&aer_recover_ring_lock, flags); +} +EXPORT_SYMBOL_GPL(aer_recover_queue); - /* Lock access to Root error producer/consumer index */ - spin_lock_irqsave(&rpc->e_lock, flags); - if (rpc->prod_idx == rpc->cons_idx) { - spin_unlock_irqrestore(&rpc->e_lock, flags); - return NULL; +static void aer_recover_work_func(struct work_struct *work) +{ + struct aer_recover_entry entry; + struct pci_dev *pdev; + + while (kfifo_get(&aer_recover_ring, &entry)) { + pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, + entry.devfn); + if (!pdev) { + pr_err("AER recover: Can not find pci_dev for %04x:%02x:%02x:%x\n", + entry.domain, entry.bus, + PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); + continue; + } + cper_print_aer(pdev, entry.severity, entry.regs); + do_recovery(pdev, entry.severity); + pci_dev_put(pdev); } - e_source = &rpc->e_sources[rpc->cons_idx]; - rpc->cons_idx++; - if (rpc->cons_idx == AER_ERROR_SOURCES_MAX) - rpc->cons_idx = 0; - spin_unlock_irqrestore(&rpc->e_lock, flags); - - return e_source; } +#endif +/** + * get_device_error_info - read error status from dev and store it to info + * @dev: pointer to the device expected to have a error record + * @info: pointer to structure to store the error record + * + * Return 1 on success, 0 on error. + * + * Note that @info is reused among all error devices. Clear fields properly. + */ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) { - int pos; + int pos, temp; - pos = pci_find_aer_capability(dev); + /* Must reset in this function */ + info->status = 0; + info->tlp_header_valid = 0; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); /* The device might not support AER */ if (!pos) - return AER_SUCCESS; + return 1; if (info->severity == AER_CORRECTABLE) { pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &info->status); - if (!(info->status & ERR_CORRECTABLE_ERROR_MASK)) - return AER_UNSUCCESS; + pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, + &info->mask); + if (!(info->status & ~info->mask)) + return 0; } else if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE || info->severity == AER_NONFATAL) { /* Link is still healthy for IO reads */ pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &info->status); - if (!(info->status & ERR_UNCORRECTABLE_ERROR_MASK)) - return AER_UNSUCCESS; + pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, + &info->mask); + if (!(info->status & ~info->mask)) + return 0; + + /* Get First Error Pointer */ + pci_read_config_dword(dev, pos + PCI_ERR_CAP, &temp); + info->first_error = PCI_ERR_CAP_FEP(temp); if (info->status & AER_LOG_TLP_MASKS) { - info->flags |= AER_TLP_HEADER_VALID_FLAG; + info->tlp_header_valid = 1; pci_read_config_dword(dev, pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0); pci_read_config_dword(dev, @@ -649,7 +661,23 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) } } - return AER_SUCCESS; + return 1; +} + +static inline void aer_process_err_devices(struct pcie_device *p_device, + struct aer_err_info *e_info) +{ + int i; + + /* Report all before handle them, not to lost records by reset etc. */ + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { + if (get_device_error_info(e_info->dev[i], e_info)) + aer_print_error(e_info->dev[i], e_info); + } + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { + if (get_device_error_info(e_info->dev[i], e_info)) + handle_error_source(p_device, e_info->dev[i], e_info); + } } /** @@ -660,46 +688,84 @@ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) static void aer_isr_one_error(struct pcie_device *p_device, struct aer_err_source *e_src) { - struct device *s_device; - struct aer_err_info e_info = {0, 0, 0,}; - int i; - u16 id; + struct aer_err_info *e_info; + + /* struct aer_err_info might be big, so we allocate it with slab */ + e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL); + if (!e_info) { + dev_printk(KERN_DEBUG, &p_device->port->dev, + "Can't allocate mem when processing AER errors\n"); + return; + } /* * There is a possibility that both correctable error and * uncorrectable error being logged. Report correctable error first. */ - for (i = 1; i & ROOT_ERR_STATUS_MASKS ; i <<= 2) { - if (i > 4) - break; - if (!(e_src->status & i)) - continue; + if (e_src->status & PCI_ERR_ROOT_COR_RCV) { + e_info->id = ERR_COR_ID(e_src->id); + e_info->severity = AER_CORRECTABLE; - /* Init comprehensive error information */ - if (i & PCI_ERR_ROOT_COR_RCV) { - id = ERR_COR_ID(e_src->id); - e_info.severity = AER_CORRECTABLE; - } else { - id = ERR_UNCOR_ID(e_src->id); - e_info.severity = ((e_src->status >> 6) & 1); - } - if (e_src->status & - (PCI_ERR_ROOT_MULTI_COR_RCV | - PCI_ERR_ROOT_MULTI_UNCOR_RCV)) - e_info.flags |= AER_MULTI_ERROR_VALID_FLAG; - if (!(s_device = find_source_device(p_device->port, id))) { - printk(KERN_DEBUG "%s->can't find device of ID%04x\n", - __FUNCTION__, id); - continue; - } - if (get_device_error_info(to_pci_dev(s_device), &e_info) == - AER_SUCCESS) { - aer_print_error(to_pci_dev(s_device), &e_info); - handle_error_source(p_device, - to_pci_dev(s_device), - e_info); - } + if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV) + e_info->multi_error_valid = 1; + else + e_info->multi_error_valid = 0; + + aer_print_port_info(p_device->port, e_info); + + if (find_source_device(p_device->port, e_info)) + aer_process_err_devices(p_device, e_info); + } + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { + e_info->id = ERR_UNCOR_ID(e_src->id); + + if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) + e_info->severity = AER_FATAL; + else + e_info->severity = AER_NONFATAL; + + if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV) + e_info->multi_error_valid = 1; + else + e_info->multi_error_valid = 0; + + aer_print_port_info(p_device->port, e_info); + + if (find_source_device(p_device->port, e_info)) + aer_process_err_devices(p_device, e_info); } + + kfree(e_info); +} + +/** + * get_e_source - retrieve an error source + * @rpc: pointer to the root port which holds an error + * @e_src: pointer to store retrieved error source + * + * Return 1 if an error source is retrieved, otherwise 0. + * + * Invoked by DPC handler to consume an error. + */ +static int get_e_source(struct aer_rpc *rpc, struct aer_err_source *e_src) +{ + unsigned long flags; + + /* Lock access to Root error producer/consumer index */ + spin_lock_irqsave(&rpc->e_lock, flags); + if (rpc->prod_idx == rpc->cons_idx) { + spin_unlock_irqrestore(&rpc->e_lock, flags); + return 0; + } + + *e_src = rpc->e_sources[rpc->cons_idx]; + rpc->cons_idx++; + if (rpc->cons_idx == AER_ERROR_SOURCES_MAX) + rpc->cons_idx = 0; + spin_unlock_irqrestore(&rpc->e_lock, flags); + + return 1; } /** @@ -712,34 +778,17 @@ void aer_isr(struct work_struct *work) { struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler); struct pcie_device *p_device = rpc->rpd; - struct aer_err_source *e_src; + struct aer_err_source uninitialized_var(e_src); mutex_lock(&rpc->rpc_mutex); - e_src = get_e_source(rpc); - while (e_src) { - aer_isr_one_error(p_device, e_src); - e_src = get_e_source(rpc); - } + while (get_e_source(rpc, &e_src)) + aer_isr_one_error(p_device, &e_src); mutex_unlock(&rpc->rpc_mutex); wake_up(&rpc->wait_release); } /** - * aer_delete_rootport - disable root port aer and delete service data - * @rpc: pointer to a root port device being deleted - * - * Invoked when AER service unloaded on a specific Root Port - */ -void aer_delete_rootport(struct aer_rpc *rpc) -{ - /* Disable root port AER itself */ - disable_root_aer(rpc); - - kfree(rpc); -} - -/** * aer_init - provide AER initialization * @dev: pointer to AER pcie device * @@ -747,15 +796,10 @@ void aer_delete_rootport(struct aer_rpc *rpc) */ int aer_init(struct pcie_device *dev) { - if (aer_osc_setup(dev) && !forceload) - return -ENXIO; - - return AER_SUCCESS; + if (forceload) { + dev_printk(KERN_DEBUG, &dev->device, + "aerdrv forceload requested.\n"); + pcie_aer_force_firmware_first(dev->port, 0); + } + return 0; } - -EXPORT_SYMBOL_GPL(pci_find_aer_capability); -EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting); -EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting); -EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status); -EXPORT_SYMBOL_GPL(pci_cleanup_aer_correct_error_status); - diff --git a/drivers/pci/pcie/aer/aerdrv_errprint.c b/drivers/pci/pcie/aer/aerdrv_errprint.c index 3933d4f30e8..36ed31b5219 100644 --- a/drivers/pci/pcie/aer/aerdrv_errprint.c +++ b/drivers/pci/pcie/aer/aerdrv_errprint.c @@ -19,123 +19,86 @@ #include <linux/errno.h> #include <linux/pm.h> #include <linux/suspend.h> +#include <linux/cper.h> #include "aerdrv.h" +#define CREATE_TRACE_POINTS +#include <trace/events/ras.h> + #define AER_AGENT_RECEIVER 0 #define AER_AGENT_REQUESTER 1 #define AER_AGENT_COMPLETER 2 #define AER_AGENT_TRANSMITTER 3 -#define AER_AGENT_REQUESTER_MASK (PCI_ERR_UNC_COMP_TIME| \ - PCI_ERR_UNC_UNSUP) - -#define AER_AGENT_COMPLETER_MASK PCI_ERR_UNC_COMP_ABORT - -#define AER_AGENT_TRANSMITTER_MASK(t, e) (e & (PCI_ERR_COR_REP_ROLL| \ - ((t == AER_CORRECTABLE) ? PCI_ERR_COR_REP_TIMER: 0))) +#define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ + 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) +#define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ + 0 : PCI_ERR_UNC_COMP_ABORT) +#define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) #define AER_GET_AGENT(t, e) \ - ((e & AER_AGENT_COMPLETER_MASK) ? AER_AGENT_COMPLETER : \ - (e & AER_AGENT_REQUESTER_MASK) ? AER_AGENT_REQUESTER : \ - (AER_AGENT_TRANSMITTER_MASK(t, e)) ? AER_AGENT_TRANSMITTER : \ + ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ + (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ + (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ AER_AGENT_RECEIVER) -#define AER_PHYSICAL_LAYER_ERROR_MASK PCI_ERR_COR_RCVR -#define AER_DATA_LINK_LAYER_ERROR_MASK(t, e) \ - (PCI_ERR_UNC_DLP| \ - PCI_ERR_COR_BAD_TLP| \ - PCI_ERR_COR_BAD_DLLP| \ - PCI_ERR_COR_REP_ROLL| \ - ((t == AER_CORRECTABLE) ? \ - PCI_ERR_COR_REP_TIMER: 0)) - #define AER_PHYSICAL_LAYER_ERROR 0 #define AER_DATA_LINK_LAYER_ERROR 1 #define AER_TRANSACTION_LAYER_ERROR 2 -#define AER_GET_LAYER_ERROR(t, e) \ - ((e & AER_PHYSICAL_LAYER_ERROR_MASK) ? \ - AER_PHYSICAL_LAYER_ERROR : \ - (e & AER_DATA_LINK_LAYER_ERROR_MASK(t, e)) ? \ - AER_DATA_LINK_LAYER_ERROR : \ - AER_TRANSACTION_LAYER_ERROR) +#define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ + PCI_ERR_COR_RCVR : 0) +#define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_BAD_TLP| \ + PCI_ERR_COR_BAD_DLLP| \ + PCI_ERR_COR_REP_ROLL| \ + PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) + +#define AER_GET_LAYER_ERROR(t, e) \ + ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ + (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ + AER_TRANSACTION_LAYER_ERROR) /* * AER error strings */ -static char* aer_error_severity_string[] = { +static const char *aer_error_severity_string[] = { "Uncorrected (Non-Fatal)", "Uncorrected (Fatal)", "Corrected" }; -static char* aer_error_layer[] = { +static const char *aer_error_layer[] = { "Physical Layer", "Data Link Layer", "Transaction Layer" }; -static char* aer_correctable_error_string[] = { - "Receiver Error ", /* Bit Position 0 */ - NULL, - NULL, - NULL, - NULL, - NULL, - "Bad TLP ", /* Bit Position 6 */ - "Bad DLLP ", /* Bit Position 7 */ - "RELAY_NUM Rollover ", /* Bit Position 8 */ - NULL, - NULL, - NULL, - "Replay Timer Timeout ", /* Bit Position 12 */ - "Advisory Non-Fatal ", /* Bit Position 13 */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + +static const char *aer_correctable_error_string[] = { + "Receiver Error", /* Bit Position 0 */ NULL, NULL, NULL, NULL, NULL, + "Bad TLP", /* Bit Position 6 */ + "Bad DLLP", /* Bit Position 7 */ + "RELAY_NUM Rollover", /* Bit Position 8 */ NULL, NULL, NULL, + "Replay Timer Timeout", /* Bit Position 12 */ + "Advisory Non-Fatal", /* Bit Position 13 */ }; -static char* aer_uncorrectable_error_string[] = { - NULL, - NULL, - NULL, - NULL, - "Data Link Protocol ", /* Bit Position 4 */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - "Poisoned TLP ", /* Bit Position 12 */ - "Flow Control Protocol ", /* Bit Position 13 */ - "Completion Timeout ", /* Bit Position 14 */ - "Completer Abort ", /* Bit Position 15 */ - "Unexpected Completion ", /* Bit Position 16 */ - "Receiver Overflow ", /* Bit Position 17 */ - "Malformed TLP ", /* Bit Position 18 */ - "ECRC ", /* Bit Position 19 */ - "Unsupported Request ", /* Bit Position 20 */ +static const char *aer_uncorrectable_error_string[] = { NULL, NULL, NULL, NULL, + "Data Link Protocol", /* Bit Position 4 */ NULL, NULL, NULL, @@ -143,106 +106,160 @@ static char* aer_uncorrectable_error_string[] = { NULL, NULL, NULL, + "Poisoned TLP", /* Bit Position 12 */ + "Flow Control Protocol", /* Bit Position 13 */ + "Completion Timeout", /* Bit Position 14 */ + "Completer Abort", /* Bit Position 15 */ + "Unexpected Completion", /* Bit Position 16 */ + "Receiver Overflow", /* Bit Position 17 */ + "Malformed TLP", /* Bit Position 18 */ + "ECRC", /* Bit Position 19 */ + "Unsupported Request", /* Bit Position 20 */ }; -static char* aer_agent_string[] = { +static const char *aer_agent_string[] = { "Receiver ID", "Requester ID", "Completer ID", "Transmitter ID" }; -static char * aer_get_error_source_name(int severity, - unsigned int status, - char errmsg_buff[]) +static void __print_tlp_header(struct pci_dev *dev, + struct aer_header_log_regs *t) +{ + unsigned char *tlp = (unsigned char *)&t; + + dev_err(&dev->dev, " TLP Header:" + " %02x%02x%02x%02x %02x%02x%02x%02x" + " %02x%02x%02x%02x %02x%02x%02x%02x\n", + *(tlp + 3), *(tlp + 2), *(tlp + 1), *tlp, + *(tlp + 7), *(tlp + 6), *(tlp + 5), *(tlp + 4), + *(tlp + 11), *(tlp + 10), *(tlp + 9), + *(tlp + 8), *(tlp + 15), *(tlp + 14), + *(tlp + 13), *(tlp + 12)); +} + +static void __aer_print_error(struct pci_dev *dev, + struct aer_err_info *info) { - int i; - char * errmsg = NULL; + int i, status; + const char *errmsg = NULL; + status = (info->status & ~info->mask); for (i = 0; i < 32; i++) { if (!(status & (1 << i))) continue; - if (severity == AER_CORRECTABLE) - errmsg = aer_correctable_error_string[i]; + if (info->severity == AER_CORRECTABLE) + errmsg = i < ARRAY_SIZE(aer_correctable_error_string) ? + aer_correctable_error_string[i] : NULL; else - errmsg = aer_uncorrectable_error_string[i]; + errmsg = i < ARRAY_SIZE(aer_uncorrectable_error_string) ? + aer_uncorrectable_error_string[i] : NULL; - if (!errmsg) { - sprintf(errmsg_buff, "Unknown Error Bit %2d ", i); - errmsg = errmsg_buff; - } + if (errmsg) + dev_err(&dev->dev, " [%2d] %-22s%s\n", i, errmsg, + info->first_error == i ? " (First)" : ""); + else + dev_err(&dev->dev, " [%2d] Unknown Error Bit%s\n", + i, info->first_error == i ? " (First)" : ""); + } +} - break; +void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) +{ + int layer, agent; + int id = ((dev->bus->number << 8) | dev->devfn); + + if (!info->status) { + dev_err(&dev->dev, "PCIe Bus Error: severity=%s, type=Unaccessible, id=%04x(Unregistered Agent ID)\n", + aer_error_severity_string[info->severity], id); + goto out; } - return errmsg; + layer = AER_GET_LAYER_ERROR(info->severity, info->status); + agent = AER_GET_AGENT(info->severity, info->status); + + dev_err(&dev->dev, "PCIe Bus Error: severity=%s, type=%s, id=%04x(%s)\n", + aer_error_severity_string[info->severity], + aer_error_layer[layer], id, aer_agent_string[agent]); + + dev_err(&dev->dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", + dev->vendor, dev->device, + info->status, info->mask); + + __aer_print_error(dev, info); + + if (info->tlp_header_valid) + __print_tlp_header(dev, &info->tlp); + +out: + if (info->id && info->error_dev_num > 1 && info->id == id) + dev_err(&dev->dev, " Error of this Agent(%04x) is reported first\n", id); + + trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask), + info->severity); } -static DEFINE_SPINLOCK(logbuf_lock); -static char errmsg_buff[100]; -void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) +void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) { - char * errmsg; - int err_layer, agent; - char * loglevel; - - if (info->severity == AER_CORRECTABLE) - loglevel = KERN_WARNING; - else - loglevel = KERN_ERR; - - printk("%s+------ PCI-Express Device Error ------+\n", loglevel); - printk("%sError Severity\t\t: %s\n", loglevel, - aer_error_severity_string[info->severity]); - - if ( info->status == 0) { - printk("%sPCIE Bus Error type\t: (Unaccessible)\n", loglevel); - printk("%sUnaccessible Received\t: %s\n", loglevel, - info->flags & AER_MULTI_ERROR_VALID_FLAG ? - "Multiple" : "First"); - printk("%sUnregistered Agent ID\t: %04x\n", loglevel, - (dev->bus->number << 8) | dev->devfn); - } else { - err_layer = AER_GET_LAYER_ERROR(info->severity, info->status); - printk("%sPCIE Bus Error type\t: %s\n", loglevel, - aer_error_layer[err_layer]); - - spin_lock(&logbuf_lock); - errmsg = aer_get_error_source_name(info->severity, - info->status, - errmsg_buff); - printk("%s%s\t: %s\n", loglevel, errmsg, - info->flags & AER_MULTI_ERROR_VALID_FLAG ? - "Multiple" : "First"); - spin_unlock(&logbuf_lock); - - agent = AER_GET_AGENT(info->severity, info->status); - printk("%s%s\t\t: %04x\n", loglevel, - aer_agent_string[agent], - (dev->bus->number << 8) | dev->devfn); - - printk("%sVendorID=%04xh, DeviceID=%04xh," - " Bus=%02xh, Device=%02xh, Function=%02xh\n", - loglevel, - dev->vendor, - dev->device, - dev->bus->number, - PCI_SLOT(dev->devfn), - PCI_FUNC(dev->devfn)); - - if (info->flags & AER_TLP_HEADER_VALID_FLAG) { - unsigned char *tlp = (unsigned char *) &info->tlp; - printk("%sTLB Header:\n", loglevel); - printk("%s%02x%02x%02x%02x %02x%02x%02x%02x" - " %02x%02x%02x%02x %02x%02x%02x%02x\n", - loglevel, - *(tlp + 3), *(tlp + 2), *(tlp + 1), *tlp, - *(tlp + 7), *(tlp + 6), *(tlp + 5), *(tlp + 4), - *(tlp + 11), *(tlp + 10), *(tlp + 9), - *(tlp + 8), *(tlp + 15), *(tlp + 14), - *(tlp + 13), *(tlp + 12)); - } + dev_info(&dev->dev, "AER: %s%s error received: id=%04x\n", + info->multi_error_valid ? "Multiple " : "", + aer_error_severity_string[info->severity], info->id); +} + +#ifdef CONFIG_ACPI_APEI_PCIEAER +int cper_severity_to_aer(int cper_severity) +{ + switch (cper_severity) { + case CPER_SEV_RECOVERABLE: + return AER_NONFATAL; + case CPER_SEV_FATAL: + return AER_FATAL; + default: + return AER_CORRECTABLE; } } +EXPORT_SYMBOL_GPL(cper_severity_to_aer); + +void cper_print_aer(struct pci_dev *dev, int cper_severity, + struct aer_capability_regs *aer) +{ + int aer_severity, layer, agent, status_strs_size, tlp_header_valid = 0; + u32 status, mask; + const char **status_strs; + + aer_severity = cper_severity_to_aer(cper_severity); + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; + mask = aer->cor_mask; + status_strs = aer_correctable_error_string; + status_strs_size = ARRAY_SIZE(aer_correctable_error_string); + } else { + status = aer->uncor_status; + mask = aer->uncor_mask; + status_strs = aer_uncorrectable_error_string; + status_strs_size = ARRAY_SIZE(aer_uncorrectable_error_string); + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + + dev_err(&dev->dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask); + cper_print_bits("", status, status_strs, status_strs_size); + dev_err(&dev->dev, "aer_layer=%s, aer_agent=%s\n", + aer_error_layer[layer], aer_agent_string[agent]); + + if (aer_severity != AER_CORRECTABLE) + dev_err(&dev->dev, "aer_uncor_severity: 0x%08x\n", + aer->uncor_severity); + + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); + + trace_aer_event(dev_name(&dev->dev), (status & ~mask), + aer_severity); +} +#endif diff --git a/drivers/pci/pcie/aer/ecrc.c b/drivers/pci/pcie/aer/ecrc.c new file mode 100644 index 00000000000..a2747a663bc --- /dev/null +++ b/drivers/pci/pcie/aer/ecrc.c @@ -0,0 +1,131 @@ +/* + * Enables/disables PCIe ECRC checking. + * + * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. + * Andrew Patterson <andrew.patterson@hp.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + * 02111-1307, USA. + * + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <linux/errno.h> +#include "../../pci.h" + +#define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ +#define ECRC_POLICY_OFF 1 /* ECRC off for performance */ +#define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ + +static int ecrc_policy = ECRC_POLICY_DEFAULT; + +static const char *ecrc_policy_str[] = { + [ECRC_POLICY_DEFAULT] = "bios", + [ECRC_POLICY_OFF] = "off", + [ECRC_POLICY_ON] = "on" +}; + +/** + * enable_ercr_checking - enable PCIe ECRC checking for a device + * @dev: the PCI device + * + * Returns 0 on success, or negative on failure. + */ +static int enable_ecrc_checking(struct pci_dev *dev) +{ + int pos; + u32 reg32; + + if (!pci_is_pcie(dev)) + return -ENODEV; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); + if (!pos) + return -ENODEV; + + pci_read_config_dword(dev, pos + PCI_ERR_CAP, ®32); + if (reg32 & PCI_ERR_CAP_ECRC_GENC) + reg32 |= PCI_ERR_CAP_ECRC_GENE; + if (reg32 & PCI_ERR_CAP_ECRC_CHKC) + reg32 |= PCI_ERR_CAP_ECRC_CHKE; + pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32); + + return 0; +} + +/** + * disable_ercr_checking - disables PCIe ECRC checking for a device + * @dev: the PCI device + * + * Returns 0 on success, or negative on failure. + */ +static int disable_ecrc_checking(struct pci_dev *dev) +{ + int pos; + u32 reg32; + + if (!pci_is_pcie(dev)) + return -ENODEV; + + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); + if (!pos) + return -ENODEV; + + pci_read_config_dword(dev, pos + PCI_ERR_CAP, ®32); + reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); + pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32); + + return 0; +} + +/** + * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based on global policy + * @dev: the PCI device + */ +void pcie_set_ecrc_checking(struct pci_dev *dev) +{ + switch (ecrc_policy) { + case ECRC_POLICY_DEFAULT: + return; + case ECRC_POLICY_OFF: + disable_ecrc_checking(dev); + break; + case ECRC_POLICY_ON: + enable_ecrc_checking(dev); + break; + default: + return; + } +} + +/** + * pcie_ecrc_get_policy - parse kernel command-line ecrc option + */ +void pcie_ecrc_get_policy(char *str) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(ecrc_policy_str); i++) + if (!strncmp(str, ecrc_policy_str[i], + strlen(ecrc_policy_str[i]))) + break; + if (i >= ARRAY_SIZE(ecrc_policy_str)) + return; + + ecrc_policy = i; +} diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c new file mode 100644 index 00000000000..e1e7026b838 --- /dev/null +++ b/drivers/pci/pcie/aspm.c @@ -0,0 +1,991 @@ +/* + * File: drivers/pci/pcie/aspm.c + * Enabling PCIe link L0s/L1 state and Clock Power Management + * + * Copyright (C) 2007 Intel + * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) + * Copyright (C) Shaohua Li (shaohua.li@intel.com) + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <linux/errno.h> +#include <linux/pm.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/jiffies.h> +#include <linux/delay.h> +#include <linux/pci-aspm.h> +#include "../pci.h" + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "pcie_aspm." + +/* Note: those are not register definitions */ +#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ +#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ +#define ASPM_STATE_L1 (4) /* L1 state */ +#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) +#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1) + +struct aspm_latency { + u32 l0s; /* L0s latency (nsec) */ + u32 l1; /* L1 latency (nsec) */ +}; + +struct pcie_link_state { + struct pci_dev *pdev; /* Upstream component of the Link */ + struct pcie_link_state *root; /* pointer to the root port link */ + struct pcie_link_state *parent; /* pointer to the parent Link state */ + struct list_head sibling; /* node in link_list */ + struct list_head children; /* list of child link states */ + struct list_head link; /* node in parent's children list */ + + /* ASPM state */ + u32 aspm_support:3; /* Supported ASPM state */ + u32 aspm_enabled:3; /* Enabled ASPM state */ + u32 aspm_capable:3; /* Capable ASPM state with latency */ + u32 aspm_default:3; /* Default ASPM state by BIOS */ + u32 aspm_disable:3; /* Disabled ASPM state */ + + /* Clock PM state */ + u32 clkpm_capable:1; /* Clock PM capable? */ + u32 clkpm_enabled:1; /* Current Clock PM state */ + u32 clkpm_default:1; /* Default Clock PM state by BIOS */ + + /* Exit latencies */ + struct aspm_latency latency_up; /* Upstream direction exit latency */ + struct aspm_latency latency_dw; /* Downstream direction exit latency */ + /* + * Endpoint acceptable latencies. A pcie downstream port only + * has one slot under it, so at most there are 8 functions. + */ + struct aspm_latency acceptable[8]; +}; + +static int aspm_disabled, aspm_force; +static bool aspm_support_enabled = true; +static DEFINE_MUTEX(aspm_lock); +static LIST_HEAD(link_list); + +#define POLICY_DEFAULT 0 /* BIOS default setting */ +#define POLICY_PERFORMANCE 1 /* high performance */ +#define POLICY_POWERSAVE 2 /* high power saving */ + +#ifdef CONFIG_PCIEASPM_PERFORMANCE +static int aspm_policy = POLICY_PERFORMANCE; +#elif defined CONFIG_PCIEASPM_POWERSAVE +static int aspm_policy = POLICY_POWERSAVE; +#else +static int aspm_policy; +#endif + +static const char *policy_str[] = { + [POLICY_DEFAULT] = "default", + [POLICY_PERFORMANCE] = "performance", + [POLICY_POWERSAVE] = "powersave" +}; + +#define LINK_RETRAIN_TIMEOUT HZ + +static int policy_to_aspm_state(struct pcie_link_state *link) +{ + switch (aspm_policy) { + case POLICY_PERFORMANCE: + /* Disable ASPM and Clock PM */ + return 0; + case POLICY_POWERSAVE: + /* Enable ASPM L0s/L1 */ + return ASPM_STATE_ALL; + case POLICY_DEFAULT: + return link->aspm_default; + } + return 0; +} + +static int policy_to_clkpm_state(struct pcie_link_state *link) +{ + switch (aspm_policy) { + case POLICY_PERFORMANCE: + /* Disable ASPM and Clock PM */ + return 0; + case POLICY_POWERSAVE: + /* Disable Clock PM */ + return 1; + case POLICY_DEFAULT: + return link->clkpm_default; + } + return 0; +} + +static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) +{ + struct pci_dev *child; + struct pci_bus *linkbus = link->pdev->subordinate; + + list_for_each_entry(child, &linkbus->devices, bus_list) { + if (enable) + pcie_capability_set_word(child, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN); + else + pcie_capability_clear_word(child, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN); + } + link->clkpm_enabled = !!enable; +} + +static void pcie_set_clkpm(struct pcie_link_state *link, int enable) +{ + /* Don't enable Clock PM if the link is not Clock PM capable */ + if (!link->clkpm_capable && enable) + enable = 0; + /* Need nothing if the specified equals to current state */ + if (link->clkpm_enabled == enable) + return; + pcie_set_clkpm_nocheck(link, enable); +} + +static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) +{ + int capable = 1, enabled = 1; + u32 reg32; + u16 reg16; + struct pci_dev *child; + struct pci_bus *linkbus = link->pdev->subordinate; + + /* All functions should have the same cap and state, take the worst */ + list_for_each_entry(child, &linkbus->devices, bus_list) { + pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); + if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { + capable = 0; + enabled = 0; + break; + } + pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); + if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) + enabled = 0; + } + link->clkpm_enabled = enabled; + link->clkpm_default = enabled; + link->clkpm_capable = (blacklist) ? 0 : capable; +} + +/* + * pcie_aspm_configure_common_clock: check if the 2 ends of a link + * could use common clock. If they are, configure them to use the + * common clock. That will reduce the ASPM state exit latency. + */ +static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) +{ + int same_clock = 1; + u16 reg16, parent_reg, child_reg[8]; + unsigned long start_jiffies; + struct pci_dev *child, *parent = link->pdev; + struct pci_bus *linkbus = parent->subordinate; + /* + * All functions of a slot should have the same Slot Clock + * Configuration, so just check one function + */ + child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); + BUG_ON(!pci_is_pcie(child)); + + /* Check downstream component if bit Slot Clock Configuration is 1 */ + pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); + if (!(reg16 & PCI_EXP_LNKSTA_SLC)) + same_clock = 0; + + /* Check upstream component if bit Slot Clock Configuration is 1 */ + pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); + if (!(reg16 & PCI_EXP_LNKSTA_SLC)) + same_clock = 0; + + /* Configure downstream component, all functions */ + list_for_each_entry(child, &linkbus->devices, bus_list) { + pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); + child_reg[PCI_FUNC(child->devfn)] = reg16; + if (same_clock) + reg16 |= PCI_EXP_LNKCTL_CCC; + else + reg16 &= ~PCI_EXP_LNKCTL_CCC; + pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); + } + + /* Configure upstream component */ + pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); + parent_reg = reg16; + if (same_clock) + reg16 |= PCI_EXP_LNKCTL_CCC; + else + reg16 &= ~PCI_EXP_LNKCTL_CCC; + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); + + /* Retrain link */ + reg16 |= PCI_EXP_LNKCTL_RL; + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); + + /* Wait for link training end. Break out after waiting for timeout */ + start_jiffies = jiffies; + for (;;) { + pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); + if (!(reg16 & PCI_EXP_LNKSTA_LT)) + break; + if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) + break; + msleep(1); + } + if (!(reg16 & PCI_EXP_LNKSTA_LT)) + return; + + /* Training failed. Restore common clock configurations */ + dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); + list_for_each_entry(child, &linkbus->devices, bus_list) + pcie_capability_write_word(child, PCI_EXP_LNKCTL, + child_reg[PCI_FUNC(child->devfn)]); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); +} + +/* Convert L0s latency encoding to ns */ +static u32 calc_l0s_latency(u32 encoding) +{ + if (encoding == 0x7) + return (5 * 1000); /* > 4us */ + return (64 << encoding); +} + +/* Convert L0s acceptable latency encoding to ns */ +static u32 calc_l0s_acceptable(u32 encoding) +{ + if (encoding == 0x7) + return -1U; + return (64 << encoding); +} + +/* Convert L1 latency encoding to ns */ +static u32 calc_l1_latency(u32 encoding) +{ + if (encoding == 0x7) + return (65 * 1000); /* > 64us */ + return (1000 << encoding); +} + +/* Convert L1 acceptable latency encoding to ns */ +static u32 calc_l1_acceptable(u32 encoding) +{ + if (encoding == 0x7) + return -1U; + return (1000 << encoding); +} + +struct aspm_register_info { + u32 support:2; + u32 enabled:2; + u32 latency_encoding_l0s; + u32 latency_encoding_l1; +}; + +static void pcie_get_aspm_reg(struct pci_dev *pdev, + struct aspm_register_info *info) +{ + u16 reg16; + u32 reg32; + + pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); + info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; + info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; + info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; + pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); + info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; +} + +static void pcie_aspm_check_latency(struct pci_dev *endpoint) +{ + u32 latency, l1_switch_latency = 0; + struct aspm_latency *acceptable; + struct pcie_link_state *link; + + /* Device not in D0 doesn't need latency check */ + if ((endpoint->current_state != PCI_D0) && + (endpoint->current_state != PCI_UNKNOWN)) + return; + + link = endpoint->bus->self->link_state; + acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; + + while (link) { + /* Check upstream direction L0s latency */ + if ((link->aspm_capable & ASPM_STATE_L0S_UP) && + (link->latency_up.l0s > acceptable->l0s)) + link->aspm_capable &= ~ASPM_STATE_L0S_UP; + + /* Check downstream direction L0s latency */ + if ((link->aspm_capable & ASPM_STATE_L0S_DW) && + (link->latency_dw.l0s > acceptable->l0s)) + link->aspm_capable &= ~ASPM_STATE_L0S_DW; + /* + * Check L1 latency. + * Every switch on the path to root complex need 1 + * more microsecond for L1. Spec doesn't mention L0s. + */ + latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); + if ((link->aspm_capable & ASPM_STATE_L1) && + (latency + l1_switch_latency > acceptable->l1)) + link->aspm_capable &= ~ASPM_STATE_L1; + l1_switch_latency += 1000; + + link = link->parent; + } +} + +static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) +{ + struct pci_dev *child, *parent = link->pdev; + struct pci_bus *linkbus = parent->subordinate; + struct aspm_register_info upreg, dwreg; + + if (blacklist) { + /* Set enabled/disable so that we will disable ASPM later */ + link->aspm_enabled = ASPM_STATE_ALL; + link->aspm_disable = ASPM_STATE_ALL; + return; + } + + /* Configure common clock before checking latencies */ + pcie_aspm_configure_common_clock(link); + + /* Get upstream/downstream components' register state */ + pcie_get_aspm_reg(parent, &upreg); + child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); + pcie_get_aspm_reg(child, &dwreg); + + /* + * Setup L0s state + * + * Note that we must not enable L0s in either direction on a + * given link unless components on both sides of the link each + * support L0s. + */ + if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) + link->aspm_support |= ASPM_STATE_L0S; + if (dwreg.enabled & PCIE_LINK_STATE_L0S) + link->aspm_enabled |= ASPM_STATE_L0S_UP; + if (upreg.enabled & PCIE_LINK_STATE_L0S) + link->aspm_enabled |= ASPM_STATE_L0S_DW; + link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); + link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); + + /* Setup L1 state */ + if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) + link->aspm_support |= ASPM_STATE_L1; + if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) + link->aspm_enabled |= ASPM_STATE_L1; + link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); + link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); + + /* Save default state */ + link->aspm_default = link->aspm_enabled; + + /* Setup initial capable state. Will be updated later */ + link->aspm_capable = link->aspm_support; + /* + * If the downstream component has pci bridge function, don't + * do ASPM for now. + */ + list_for_each_entry(child, &linkbus->devices, bus_list) { + if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { + link->aspm_disable = ASPM_STATE_ALL; + break; + } + } + + /* Get and check endpoint acceptable latencies */ + list_for_each_entry(child, &linkbus->devices, bus_list) { + u32 reg32, encoding; + struct aspm_latency *acceptable = + &link->acceptable[PCI_FUNC(child->devfn)]; + + if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && + pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) + continue; + + pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); + /* Calculate endpoint L0s acceptable latency */ + encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; + acceptable->l0s = calc_l0s_acceptable(encoding); + /* Calculate endpoint L1 acceptable latency */ + encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; + acceptable->l1 = calc_l1_acceptable(encoding); + + pcie_aspm_check_latency(child); + } +} + +static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) +{ + pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, val); +} + +static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) +{ + u32 upstream = 0, dwstream = 0; + struct pci_dev *child, *parent = link->pdev; + struct pci_bus *linkbus = parent->subordinate; + + /* Nothing to do if the link is already in the requested state */ + state &= (link->aspm_capable & ~link->aspm_disable); + if (link->aspm_enabled == state) + return; + /* Convert ASPM state to upstream/downstream ASPM register state */ + if (state & ASPM_STATE_L0S_UP) + dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; + if (state & ASPM_STATE_L0S_DW) + upstream |= PCI_EXP_LNKCTL_ASPM_L0S; + if (state & ASPM_STATE_L1) { + upstream |= PCI_EXP_LNKCTL_ASPM_L1; + dwstream |= PCI_EXP_LNKCTL_ASPM_L1; + } + /* + * Spec 2.0 suggests all functions should be configured the + * same setting for ASPM. Enabling ASPM L1 should be done in + * upstream component first and then downstream, and vice + * versa for disabling ASPM L1. Spec doesn't mention L0S. + */ + if (state & ASPM_STATE_L1) + pcie_config_aspm_dev(parent, upstream); + list_for_each_entry(child, &linkbus->devices, bus_list) + pcie_config_aspm_dev(child, dwstream); + if (!(state & ASPM_STATE_L1)) + pcie_config_aspm_dev(parent, upstream); + + link->aspm_enabled = state; +} + +static void pcie_config_aspm_path(struct pcie_link_state *link) +{ + while (link) { + pcie_config_aspm_link(link, policy_to_aspm_state(link)); + link = link->parent; + } +} + +static void free_link_state(struct pcie_link_state *link) +{ + link->pdev->link_state = NULL; + kfree(link); +} + +static int pcie_aspm_sanity_check(struct pci_dev *pdev) +{ + struct pci_dev *child; + u32 reg32; + + /* + * Some functions in a slot might not all be PCIe functions, + * very strange. Disable ASPM for the whole slot + */ + list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { + if (!pci_is_pcie(child)) + return -EINVAL; + + /* + * If ASPM is disabled then we're not going to change + * the BIOS state. It's safe to continue even if it's a + * pre-1.1 device + */ + + if (aspm_disabled) + continue; + + /* + * Disable ASPM for pre-1.1 PCIe device, we follow MS to use + * RBER bit to determine if a function is 1.1 version device + */ + pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); + if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { + dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); + return -EINVAL; + } + } + return 0; +} + +static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) +{ + struct pcie_link_state *link; + + link = kzalloc(sizeof(*link), GFP_KERNEL); + if (!link) + return NULL; + INIT_LIST_HEAD(&link->sibling); + INIT_LIST_HEAD(&link->children); + INIT_LIST_HEAD(&link->link); + link->pdev = pdev; + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) { + struct pcie_link_state *parent; + parent = pdev->bus->parent->self->link_state; + if (!parent) { + kfree(link); + return NULL; + } + link->parent = parent; + list_add(&link->link, &parent->children); + } + /* Setup a pointer to the root port link */ + if (!link->parent) + link->root = link; + else + link->root = link->parent->root; + + list_add(&link->sibling, &link_list); + pdev->link_state = link; + return link; +} + +/* + * pcie_aspm_init_link_state: Initiate PCI express link state. + * It is called after the pcie and its children devices are scanned. + * @pdev: the root port or switch downstream port + */ +void pcie_aspm_init_link_state(struct pci_dev *pdev) +{ + struct pcie_link_state *link; + int blacklist = !!pcie_aspm_sanity_check(pdev); + + if (!aspm_support_enabled) + return; + + if (!pci_is_pcie(pdev) || pdev->link_state) + return; + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) + return; + + /* VIA has a strange chipset, root port is under a bridge */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && + pdev->bus->self) + return; + + down_read(&pci_bus_sem); + if (list_empty(&pdev->subordinate->devices)) + goto out; + + mutex_lock(&aspm_lock); + link = alloc_pcie_link_state(pdev); + if (!link) + goto unlock; + /* + * Setup initial ASPM state. Note that we need to configure + * upstream links also because capable state of them can be + * update through pcie_aspm_cap_init(). + */ + pcie_aspm_cap_init(link, blacklist); + + /* Setup initial Clock PM state */ + pcie_clkpm_cap_init(link, blacklist); + + /* + * At this stage drivers haven't had an opportunity to change the + * link policy setting. Enabling ASPM on broken hardware can cripple + * it even before the driver has had a chance to disable ASPM, so + * default to a safe level right now. If we're enabling ASPM beyond + * the BIOS's expectation, we'll do so once pci_enable_device() is + * called. + */ + if (aspm_policy != POLICY_POWERSAVE) { + pcie_config_aspm_path(link); + pcie_set_clkpm(link, policy_to_clkpm_state(link)); + } + +unlock: + mutex_unlock(&aspm_lock); +out: + up_read(&pci_bus_sem); +} + +/* Recheck latencies and update aspm_capable for links under the root */ +static void pcie_update_aspm_capable(struct pcie_link_state *root) +{ + struct pcie_link_state *link; + BUG_ON(root->parent); + list_for_each_entry(link, &link_list, sibling) { + if (link->root != root) + continue; + link->aspm_capable = link->aspm_support; + } + list_for_each_entry(link, &link_list, sibling) { + struct pci_dev *child; + struct pci_bus *linkbus = link->pdev->subordinate; + if (link->root != root) + continue; + list_for_each_entry(child, &linkbus->devices, bus_list) { + if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && + (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) + continue; + pcie_aspm_check_latency(child); + } + } +} + +/* @pdev: the endpoint device */ +void pcie_aspm_exit_link_state(struct pci_dev *pdev) +{ + struct pci_dev *parent = pdev->bus->self; + struct pcie_link_state *link, *root, *parent_link; + + if (!parent || !parent->link_state) + return; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + /* + * All PCIe functions are in one slot, remove one function will remove + * the whole slot, so just wait until we are the last function left. + */ + if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) + goto out; + + link = parent->link_state; + root = link->root; + parent_link = link->parent; + + /* All functions are removed, so just disable ASPM for the link */ + pcie_config_aspm_link(link, 0); + list_del(&link->sibling); + list_del(&link->link); + /* Clock PM is for endpoint device */ + free_link_state(link); + + /* Recheck latencies and configure upstream links */ + if (parent_link) { + pcie_update_aspm_capable(root); + pcie_config_aspm_path(parent_link); + } +out: + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); +} + +/* @pdev: the root port or switch downstream port */ +void pcie_aspm_pm_state_change(struct pci_dev *pdev) +{ + struct pcie_link_state *link = pdev->link_state; + + if (aspm_disabled || !pci_is_pcie(pdev) || !link) + return; + if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) + return; + /* + * Devices changed PM state, we should recheck if latency + * meets all functions' requirement + */ + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + pcie_update_aspm_capable(link->root); + pcie_config_aspm_path(link); + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); +} + +void pcie_aspm_powersave_config_link(struct pci_dev *pdev) +{ + struct pcie_link_state *link = pdev->link_state; + + if (aspm_disabled || !pci_is_pcie(pdev) || !link) + return; + + if (aspm_policy != POLICY_POWERSAVE) + return; + + if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) + return; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + pcie_config_aspm_path(link); + pcie_set_clkpm(link, policy_to_clkpm_state(link)); + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); +} + +static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem, + bool force) +{ + struct pci_dev *parent = pdev->bus->self; + struct pcie_link_state *link; + + if (!pci_is_pcie(pdev)) + return; + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) + parent = pdev; + if (!parent || !parent->link_state) + return; + + /* + * A driver requested that ASPM be disabled on this device, but + * if we don't have permission to manage ASPM (e.g., on ACPI + * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and + * the _OSC method), we can't honor that request. Windows has + * a similar mechanism using "PciASPMOptOut", which is also + * ignored in this situation. + */ + if (aspm_disabled && !force) { + dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); + return; + } + + if (sem) + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + link = parent->link_state; + if (state & PCIE_LINK_STATE_L0S) + link->aspm_disable |= ASPM_STATE_L0S; + if (state & PCIE_LINK_STATE_L1) + link->aspm_disable |= ASPM_STATE_L1; + pcie_config_aspm_link(link, policy_to_aspm_state(link)); + + if (state & PCIE_LINK_STATE_CLKPM) { + link->clkpm_capable = 0; + pcie_set_clkpm(link, 0); + } + mutex_unlock(&aspm_lock); + if (sem) + up_read(&pci_bus_sem); +} + +void pci_disable_link_state_locked(struct pci_dev *pdev, int state) +{ + __pci_disable_link_state(pdev, state, false, false); +} +EXPORT_SYMBOL(pci_disable_link_state_locked); + +/** + * pci_disable_link_state - Disable device's link state, so the link will + * never enter specific states. Note that if the BIOS didn't grant ASPM + * control to the OS, this does nothing because we can't touch the LNKCTL + * register. + * + * @pdev: PCI device + * @state: ASPM link state to disable + */ +void pci_disable_link_state(struct pci_dev *pdev, int state) +{ + __pci_disable_link_state(pdev, state, true, false); +} +EXPORT_SYMBOL(pci_disable_link_state); + +void pcie_clear_aspm(struct pci_bus *bus) +{ + struct pci_dev *child; + + if (aspm_force) + return; + + /* + * Clear any ASPM setup that the firmware has carried out on this bus + */ + list_for_each_entry(child, &bus->devices, bus_list) { + __pci_disable_link_state(child, PCIE_LINK_STATE_L0S | + PCIE_LINK_STATE_L1 | + PCIE_LINK_STATE_CLKPM, + false, true); + } +} + +static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) +{ + int i; + struct pcie_link_state *link; + + if (aspm_disabled) + return -EPERM; + for (i = 0; i < ARRAY_SIZE(policy_str); i++) + if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) + break; + if (i >= ARRAY_SIZE(policy_str)) + return -EINVAL; + if (i == aspm_policy) + return 0; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + aspm_policy = i; + list_for_each_entry(link, &link_list, sibling) { + pcie_config_aspm_link(link, policy_to_aspm_state(link)); + pcie_set_clkpm(link, policy_to_clkpm_state(link)); + } + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); + return 0; +} + +static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) +{ + int i, cnt = 0; + for (i = 0; i < ARRAY_SIZE(policy_str); i++) + if (i == aspm_policy) + cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); + else + cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); + return cnt; +} + +module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, + NULL, 0644); + +#ifdef CONFIG_PCIEASPM_DEBUG +static ssize_t link_state_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pci_device = to_pci_dev(dev); + struct pcie_link_state *link_state = pci_device->link_state; + + return sprintf(buf, "%d\n", link_state->aspm_enabled); +} + +static ssize_t link_state_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t n) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pcie_link_state *link, *root = pdev->link_state->root; + u32 val = buf[0] - '0', state = 0; + + if (aspm_disabled) + return -EPERM; + if (n < 1 || val > 3) + return -EINVAL; + + /* Convert requested state to ASPM state */ + if (val & PCIE_LINK_STATE_L0S) + state |= ASPM_STATE_L0S; + if (val & PCIE_LINK_STATE_L1) + state |= ASPM_STATE_L1; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + list_for_each_entry(link, &link_list, sibling) { + if (link->root != root) + continue; + pcie_config_aspm_link(link, state); + } + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); + return n; +} + +static ssize_t clk_ctl_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pci_device = to_pci_dev(dev); + struct pcie_link_state *link_state = pci_device->link_state; + + return sprintf(buf, "%d\n", link_state->clkpm_enabled); +} + +static ssize_t clk_ctl_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t n) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int state; + + if (n < 1) + return -EINVAL; + state = buf[0]-'0'; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + pcie_set_clkpm_nocheck(pdev->link_state, !!state); + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); + + return n; +} + +static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store); +static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store); + +static char power_group[] = "power"; +void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) +{ + struct pcie_link_state *link_state = pdev->link_state; + + if (!pci_is_pcie(pdev) || + (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) + return; + + if (link_state->aspm_support) + sysfs_add_file_to_group(&pdev->dev.kobj, + &dev_attr_link_state.attr, power_group); + if (link_state->clkpm_capable) + sysfs_add_file_to_group(&pdev->dev.kobj, + &dev_attr_clk_ctl.attr, power_group); +} + +void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) +{ + struct pcie_link_state *link_state = pdev->link_state; + + if (!pci_is_pcie(pdev) || + (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) + return; + + if (link_state->aspm_support) + sysfs_remove_file_from_group(&pdev->dev.kobj, + &dev_attr_link_state.attr, power_group); + if (link_state->clkpm_capable) + sysfs_remove_file_from_group(&pdev->dev.kobj, + &dev_attr_clk_ctl.attr, power_group); +} +#endif + +static int __init pcie_aspm_disable(char *str) +{ + if (!strcmp(str, "off")) { + aspm_policy = POLICY_DEFAULT; + aspm_disabled = 1; + aspm_support_enabled = false; + printk(KERN_INFO "PCIe ASPM is disabled\n"); + } else if (!strcmp(str, "force")) { + aspm_force = 1; + printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); + } + return 1; +} + +__setup("pcie_aspm=", pcie_aspm_disable); + +void pcie_no_aspm(void) +{ + /* + * Disabling ASPM is intended to prevent the kernel from modifying + * existing hardware state, not to clear existing state. To that end: + * (a) set policy to POLICY_DEFAULT in order to avoid changing state + * (b) prevent userspace from changing policy + */ + if (!aspm_force) { + aspm_policy = POLICY_DEFAULT; + aspm_disabled = 1; + } +} + +bool pcie_aspm_support_enabled(void) +{ + return aspm_support_enabled; +} +EXPORT_SYMBOL(pcie_aspm_support_enabled); diff --git a/drivers/pci/pcie/pme.c b/drivers/pci/pcie/pme.c new file mode 100644 index 00000000000..82e06a86cd7 --- /dev/null +++ b/drivers/pci/pcie/pme.c @@ -0,0 +1,438 @@ +/* + * PCIe Native PME support + * + * Copyright (C) 2007 - 2009 Intel Corp + * Copyright (C) 2007 - 2009 Shaohua Li <shaohua.li@intel.com> + * Copyright (C) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. + * + * This file is subject to the terms and conditions of the GNU General Public + * License V2. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/slab.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/device.h> +#include <linux/pcieport_if.h> +#include <linux/pm_runtime.h> + +#include "../pci.h" +#include "portdrv.h" + +/* + * If this switch is set, MSI will not be used for PCIe PME signaling. This + * causes the PCIe port driver to use INTx interrupts only, but it turns out + * that using MSI for PCIe PME signaling doesn't play well with PCIe PME-based + * wake-up from system sleep states. + */ +bool pcie_pme_msi_disabled; + +static int __init pcie_pme_setup(char *str) +{ + if (!strncmp(str, "nomsi", 5)) + pcie_pme_msi_disabled = true; + + return 1; +} +__setup("pcie_pme=", pcie_pme_setup); + +struct pcie_pme_service_data { + spinlock_t lock; + struct pcie_device *srv; + struct work_struct work; + bool noirq; /* Don't enable the PME interrupt used by this service. */ +}; + +/** + * pcie_pme_interrupt_enable - Enable/disable PCIe PME interrupt generation. + * @dev: PCIe root port or event collector. + * @enable: Enable or disable the interrupt. + */ +void pcie_pme_interrupt_enable(struct pci_dev *dev, bool enable) +{ + if (enable) + pcie_capability_set_word(dev, PCI_EXP_RTCTL, + PCI_EXP_RTCTL_PMEIE); + else + pcie_capability_clear_word(dev, PCI_EXP_RTCTL, + PCI_EXP_RTCTL_PMEIE); +} + +/** + * pcie_pme_walk_bus - Scan a PCI bus for devices asserting PME#. + * @bus: PCI bus to scan. + * + * Scan given PCI bus and all buses under it for devices asserting PME#. + */ +static bool pcie_pme_walk_bus(struct pci_bus *bus) +{ + struct pci_dev *dev; + bool ret = false; + + list_for_each_entry(dev, &bus->devices, bus_list) { + /* Skip PCIe devices in case we started from a root port. */ + if (!pci_is_pcie(dev) && pci_check_pme_status(dev)) { + if (dev->pme_poll) + dev->pme_poll = false; + + pci_wakeup_event(dev); + pm_request_resume(&dev->dev); + ret = true; + } + + if (dev->subordinate && pcie_pme_walk_bus(dev->subordinate)) + ret = true; + } + + return ret; +} + +/** + * pcie_pme_from_pci_bridge - Check if PCIe-PCI bridge generated a PME. + * @bus: Secondary bus of the bridge. + * @devfn: Device/function number to check. + * + * PME from PCI devices under a PCIe-PCI bridge may be converted to an in-band + * PCIe PME message. In such that case the bridge should use the Requester ID + * of device/function number 0 on its secondary bus. + */ +static bool pcie_pme_from_pci_bridge(struct pci_bus *bus, u8 devfn) +{ + struct pci_dev *dev; + bool found = false; + + if (devfn) + return false; + + dev = pci_dev_get(bus->self); + if (!dev) + return false; + + if (pci_is_pcie(dev) && pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE) { + down_read(&pci_bus_sem); + if (pcie_pme_walk_bus(bus)) + found = true; + up_read(&pci_bus_sem); + } + + pci_dev_put(dev); + return found; +} + +/** + * pcie_pme_handle_request - Find device that generated PME and handle it. + * @port: Root port or event collector that generated the PME interrupt. + * @req_id: PCIe Requester ID of the device that generated the PME. + */ +static void pcie_pme_handle_request(struct pci_dev *port, u16 req_id) +{ + u8 busnr = req_id >> 8, devfn = req_id & 0xff; + struct pci_bus *bus; + struct pci_dev *dev; + bool found = false; + + /* First, check if the PME is from the root port itself. */ + if (port->devfn == devfn && port->bus->number == busnr) { + if (port->pme_poll) + port->pme_poll = false; + + if (pci_check_pme_status(port)) { + pm_request_resume(&port->dev); + found = true; + } else { + /* + * Apparently, the root port generated the PME on behalf + * of a non-PCIe device downstream. If this is done by + * a root port, the Requester ID field in its status + * register may contain either the root port's, or the + * source device's information (PCI Express Base + * Specification, Rev. 2.0, Section 6.1.9). + */ + down_read(&pci_bus_sem); + found = pcie_pme_walk_bus(port->subordinate); + up_read(&pci_bus_sem); + } + goto out; + } + + /* Second, find the bus the source device is on. */ + bus = pci_find_bus(pci_domain_nr(port->bus), busnr); + if (!bus) + goto out; + + /* Next, check if the PME is from a PCIe-PCI bridge. */ + found = pcie_pme_from_pci_bridge(bus, devfn); + if (found) + goto out; + + /* Finally, try to find the PME source on the bus. */ + down_read(&pci_bus_sem); + list_for_each_entry(dev, &bus->devices, bus_list) { + pci_dev_get(dev); + if (dev->devfn == devfn) { + found = true; + break; + } + pci_dev_put(dev); + } + up_read(&pci_bus_sem); + + if (found) { + /* The device is there, but we have to check its PME status. */ + found = pci_check_pme_status(dev); + if (found) { + if (dev->pme_poll) + dev->pme_poll = false; + + pci_wakeup_event(dev); + pm_request_resume(&dev->dev); + } + pci_dev_put(dev); + } else if (devfn) { + /* + * The device is not there, but we can still try to recover by + * assuming that the PME was reported by a PCIe-PCI bridge that + * used devfn different from zero. + */ + dev_dbg(&port->dev, "PME interrupt generated for non-existent device %02x:%02x.%d\n", + busnr, PCI_SLOT(devfn), PCI_FUNC(devfn)); + found = pcie_pme_from_pci_bridge(bus, 0); + } + + out: + if (!found) + dev_dbg(&port->dev, "Spurious native PME interrupt!\n"); +} + +/** + * pcie_pme_work_fn - Work handler for PCIe PME interrupt. + * @work: Work structure giving access to service data. + */ +static void pcie_pme_work_fn(struct work_struct *work) +{ + struct pcie_pme_service_data *data = + container_of(work, struct pcie_pme_service_data, work); + struct pci_dev *port = data->srv->port; + u32 rtsta; + + spin_lock_irq(&data->lock); + + for (;;) { + if (data->noirq) + break; + + pcie_capability_read_dword(port, PCI_EXP_RTSTA, &rtsta); + if (rtsta & PCI_EXP_RTSTA_PME) { + /* + * Clear PME status of the port. If there are other + * pending PMEs, the status will be set again. + */ + pcie_clear_root_pme_status(port); + + spin_unlock_irq(&data->lock); + pcie_pme_handle_request(port, rtsta & 0xffff); + spin_lock_irq(&data->lock); + + continue; + } + + /* No need to loop if there are no more PMEs pending. */ + if (!(rtsta & PCI_EXP_RTSTA_PENDING)) + break; + + spin_unlock_irq(&data->lock); + cpu_relax(); + spin_lock_irq(&data->lock); + } + + if (!data->noirq) + pcie_pme_interrupt_enable(port, true); + + spin_unlock_irq(&data->lock); +} + +/** + * pcie_pme_irq - Interrupt handler for PCIe root port PME interrupt. + * @irq: Interrupt vector. + * @context: Interrupt context pointer. + */ +static irqreturn_t pcie_pme_irq(int irq, void *context) +{ + struct pci_dev *port; + struct pcie_pme_service_data *data; + u32 rtsta; + unsigned long flags; + + port = ((struct pcie_device *)context)->port; + data = get_service_data((struct pcie_device *)context); + + spin_lock_irqsave(&data->lock, flags); + pcie_capability_read_dword(port, PCI_EXP_RTSTA, &rtsta); + + if (!(rtsta & PCI_EXP_RTSTA_PME)) { + spin_unlock_irqrestore(&data->lock, flags); + return IRQ_NONE; + } + + pcie_pme_interrupt_enable(port, false); + spin_unlock_irqrestore(&data->lock, flags); + + /* We don't use pm_wq, because it's freezable. */ + schedule_work(&data->work); + + return IRQ_HANDLED; +} + +/** + * pcie_pme_set_native - Set the PME interrupt flag for given device. + * @dev: PCI device to handle. + * @ign: Ignored. + */ +static int pcie_pme_set_native(struct pci_dev *dev, void *ign) +{ + dev_info(&dev->dev, "Signaling PME through PCIe PME interrupt\n"); + + device_set_run_wake(&dev->dev, true); + dev->pme_interrupt = true; + return 0; +} + +/** + * pcie_pme_mark_devices - Set the PME interrupt flag for devices below a port. + * @port: PCIe root port or event collector to handle. + * + * For each device below given root port, including the port itself (or for each + * root complex integrated endpoint if @port is a root complex event collector) + * set the flag indicating that it can signal run-time wake-up events via PCIe + * PME interrupts. + */ +static void pcie_pme_mark_devices(struct pci_dev *port) +{ + pcie_pme_set_native(port, NULL); + if (port->subordinate) { + pci_walk_bus(port->subordinate, pcie_pme_set_native, NULL); + } else { + struct pci_bus *bus = port->bus; + struct pci_dev *dev; + + /* Check if this is a root port event collector. */ + if (pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC || !bus) + return; + + down_read(&pci_bus_sem); + list_for_each_entry(dev, &bus->devices, bus_list) + if (pci_is_pcie(dev) + && pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) + pcie_pme_set_native(dev, NULL); + up_read(&pci_bus_sem); + } +} + +/** + * pcie_pme_probe - Initialize PCIe PME service for given root port. + * @srv: PCIe service to initialize. + */ +static int pcie_pme_probe(struct pcie_device *srv) +{ + struct pci_dev *port; + struct pcie_pme_service_data *data; + int ret; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + spin_lock_init(&data->lock); + INIT_WORK(&data->work, pcie_pme_work_fn); + data->srv = srv; + set_service_data(srv, data); + + port = srv->port; + pcie_pme_interrupt_enable(port, false); + pcie_clear_root_pme_status(port); + + ret = request_irq(srv->irq, pcie_pme_irq, IRQF_SHARED, "PCIe PME", srv); + if (ret) { + kfree(data); + } else { + pcie_pme_mark_devices(port); + pcie_pme_interrupt_enable(port, true); + } + + return ret; +} + +/** + * pcie_pme_suspend - Suspend PCIe PME service device. + * @srv: PCIe service device to suspend. + */ +static int pcie_pme_suspend(struct pcie_device *srv) +{ + struct pcie_pme_service_data *data = get_service_data(srv); + struct pci_dev *port = srv->port; + + spin_lock_irq(&data->lock); + pcie_pme_interrupt_enable(port, false); + pcie_clear_root_pme_status(port); + data->noirq = true; + spin_unlock_irq(&data->lock); + + synchronize_irq(srv->irq); + + return 0; +} + +/** + * pcie_pme_resume - Resume PCIe PME service device. + * @srv - PCIe service device to resume. + */ +static int pcie_pme_resume(struct pcie_device *srv) +{ + struct pcie_pme_service_data *data = get_service_data(srv); + struct pci_dev *port = srv->port; + + spin_lock_irq(&data->lock); + data->noirq = false; + pcie_clear_root_pme_status(port); + pcie_pme_interrupt_enable(port, true); + spin_unlock_irq(&data->lock); + + return 0; +} + +/** + * pcie_pme_remove - Prepare PCIe PME service device for removal. + * @srv - PCIe service device to remove. + */ +static void pcie_pme_remove(struct pcie_device *srv) +{ + pcie_pme_suspend(srv); + free_irq(srv->irq, srv); + kfree(get_service_data(srv)); +} + +static struct pcie_port_service_driver pcie_pme_driver = { + .name = "pcie_pme", + .port_type = PCI_EXP_TYPE_ROOT_PORT, + .service = PCIE_PORT_SERVICE_PME, + + .probe = pcie_pme_probe, + .suspend = pcie_pme_suspend, + .resume = pcie_pme_resume, + .remove = pcie_pme_remove, +}; + +/** + * pcie_pme_service_init - Register the PCIe PME service driver. + */ +static int __init pcie_pme_service_init(void) +{ + return pcie_port_service_register(&pcie_pme_driver); +} + +module_init(pcie_pme_service_init); diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index 3656e0349dd..d525548404d 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -11,37 +11,73 @@ #include <linux/compiler.h> -#if !defined(PCI_CAP_ID_PME) -#define PCI_CAP_ID_PME 1 -#endif - -#if !defined(PCI_CAP_ID_EXP) -#define PCI_CAP_ID_EXP 0x10 -#endif - -#define PORT_TYPE_MASK 0xf -#define PORT_TO_SLOT_MASK 0x100 -#define SLOT_HP_CAPABLE_MASK 0x40 -#define PCIE_CAPABILITIES_REG 0x2 -#define PCIE_SLOT_CAPABILITIES_REG 0x14 -#define PCIE_PORT_DEVICE_MAXSERVICES 4 -#define PCI_CFG_SPACE_SIZE 256 +#define PCIE_PORT_DEVICE_MAXSERVICES 4 +/* + * According to the PCI Express Base Specification 2.0, the indices of + * the MSI-X table entries used by port services must not exceed 31 + */ +#define PCIE_PORT_MAX_MSIX_ENTRIES 32 #define get_descriptor_id(type, service) (((type - 4) << 4) | service) -struct pcie_port_device_ext { - int interrupt_mode; /* [0:INTx | 1:MSI | 2:MSI-X] */ -}; - extern struct bus_type pcie_port_bus_type; -extern int pcie_port_device_probe(struct pci_dev *dev); -extern int pcie_port_device_register(struct pci_dev *dev); +int pcie_port_device_register(struct pci_dev *dev); #ifdef CONFIG_PM -extern int pcie_port_device_suspend(struct pci_dev *dev, pm_message_t state); -extern int pcie_port_device_resume(struct pci_dev *dev); +int pcie_port_device_suspend(struct device *dev); +int pcie_port_device_resume(struct device *dev); #endif -extern void pcie_port_device_remove(struct pci_dev *dev); -extern int __must_check pcie_port_bus_register(void); -extern void pcie_port_bus_unregister(void); +void pcie_port_device_remove(struct pci_dev *dev); +int __must_check pcie_port_bus_register(void); +void pcie_port_bus_unregister(void); + +struct pci_dev; + +void pcie_clear_root_pme_status(struct pci_dev *dev); + +#ifdef CONFIG_HOTPLUG_PCI_PCIE +extern bool pciehp_msi_disabled; + +static inline bool pciehp_no_msi(void) +{ + return pciehp_msi_disabled; +} + +#else /* !CONFIG_HOTPLUG_PCI_PCIE */ +static inline bool pciehp_no_msi(void) { return false; } +#endif /* !CONFIG_HOTPLUG_PCI_PCIE */ + +#ifdef CONFIG_PCIE_PME +extern bool pcie_pme_msi_disabled; + +static inline void pcie_pme_disable_msi(void) +{ + pcie_pme_msi_disabled = true; +} + +static inline bool pcie_pme_no_msi(void) +{ + return pcie_pme_msi_disabled; +} + +void pcie_pme_interrupt_enable(struct pci_dev *dev, bool enable); +#else /* !CONFIG_PCIE_PME */ +static inline void pcie_pme_disable_msi(void) {} +static inline bool pcie_pme_no_msi(void) { return false; } +static inline void pcie_pme_interrupt_enable(struct pci_dev *dev, bool en) {} +#endif /* !CONFIG_PCIE_PME */ + +#ifdef CONFIG_ACPI +int pcie_port_acpi_setup(struct pci_dev *port, int *mask); + +static inline int pcie_port_platform_notify(struct pci_dev *port, int *mask) +{ + return pcie_port_acpi_setup(port, mask); +} +#else /* !CONFIG_ACPI */ +static inline int pcie_port_platform_notify(struct pci_dev *port, int *mask) +{ + return 0; +} +#endif /* !CONFIG_ACPI */ #endif /* _PORTDRV_H_ */ diff --git a/drivers/pci/pcie/portdrv_acpi.c b/drivers/pci/pcie/portdrv_acpi.c new file mode 100644 index 00000000000..b4d2894ee3f --- /dev/null +++ b/drivers/pci/pcie/portdrv_acpi.c @@ -0,0 +1,63 @@ +/* + * PCIe Port Native Services Support, ACPI-Related Part + * + * Copyright (C) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. + * + * This file is subject to the terms and conditions of the GNU General Public + * License V2. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include <linux/pci.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/acpi.h> +#include <linux/pci-acpi.h> +#include <linux/pcieport_if.h> + +#include "aer/aerdrv.h" +#include "../pci.h" +#include "portdrv.h" + +/** + * pcie_port_acpi_setup - Request the BIOS to release control of PCIe services. + * @port: PCIe Port service for a root port or event collector. + * @srv_mask: Bit mask of services that can be enabled for @port. + * + * Invoked when @port is identified as a PCIe port device. To avoid conflicts + * with the BIOS PCIe port native services support requires the BIOS to yield + * control of these services to the kernel. The mask of services that the BIOS + * allows to be enabled for @port is written to @srv_mask. + * + * NOTE: It turns out that we cannot do that for individual port services + * separately, because that would make some systems work incorrectly. + */ +int pcie_port_acpi_setup(struct pci_dev *port, int *srv_mask) +{ + struct acpi_pci_root *root; + acpi_handle handle; + u32 flags; + + if (acpi_pci_disabled) + return 0; + + handle = acpi_find_root_bridge_handle(port); + if (!handle) + return -EINVAL; + + root = acpi_pci_find_root(handle); + if (!root) + return -ENODEV; + + flags = root->osc_control_set; + + *srv_mask = PCIE_PORT_SERVICE_VC; + if (flags & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_HP; + if (flags & OSC_PCI_EXPRESS_PME_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_PME; + if (flags & OSC_PCI_EXPRESS_AER_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_AER; + + return 0; +} diff --git a/drivers/pci/pcie/portdrv_bus.c b/drivers/pci/pcie/portdrv_bus.c index 3f0976868ed..87e79a6ffb5 100644 --- a/drivers/pci/pcie/portdrv_bus.c +++ b/drivers/pci/pcie/portdrv_bus.c @@ -13,16 +13,13 @@ #include <linux/pm.h> #include <linux/pcieport_if.h> +#include "portdrv.h" static int pcie_port_bus_match(struct device *dev, struct device_driver *drv); -static int pcie_port_bus_suspend(struct device *dev, pm_message_t state); -static int pcie_port_bus_resume(struct device *dev); struct bus_type pcie_port_bus_type = { - .name = "pci_express", - .match = pcie_port_bus_match, - .suspend = pcie_port_bus_suspend, - .resume = pcie_port_bus_resume, + .name = "pci_express", + .match = pcie_port_bus_match, }; EXPORT_SYMBOL_GPL(pcie_port_bus_type); @@ -33,47 +30,26 @@ static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type) return 0; - + pciedev = to_pcie_device(dev); driver = to_service_driver(drv); - if ( (driver->id_table->vendor != PCI_ANY_ID && - driver->id_table->vendor != pciedev->id.vendor) || - (driver->id_table->device != PCI_ANY_ID && - driver->id_table->device != pciedev->id.device) || - (driver->id_table->port_type != PCIE_ANY_PORT && - driver->id_table->port_type != pciedev->id.port_type) || - driver->id_table->service_type != pciedev->id.service_type ) + + if (driver->service != pciedev->service) + return 0; + + if ((driver->port_type != PCIE_ANY_PORT) && + (driver->port_type != pci_pcie_type(pciedev->port))) return 0; return 1; } -static int pcie_port_bus_suspend(struct device *dev, pm_message_t state) +int pcie_port_bus_register(void) { - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; - - if (!dev || !dev->driver) - return 0; - - pciedev = to_pcie_device(dev); - driver = to_service_driver(dev->driver); - if (driver && driver->suspend) - driver->suspend(pciedev, state); - return 0; + return bus_register(&pcie_port_bus_type); } -static int pcie_port_bus_resume(struct device *dev) +void pcie_port_bus_unregister(void) { - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; - - if (!dev || !dev->driver) - return 0; - - pciedev = to_pcie_device(dev); - driver = to_service_driver(dev->driver); - if (driver && driver->resume) - driver->resume(pciedev); - return 0; + bus_unregister(&pcie_port_bus_type); } diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c index 23d9eb07329..2f0ce668a77 100644 --- a/drivers/pci/pcie/portdrv_core.c +++ b/drivers/pci/pcie/portdrv_core.c @@ -14,331 +14,424 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/pcieport_if.h> +#include <linux/aer.h> +#include "../pci.h" #include "portdrv.h" -extern int pcie_mch_quirk; /* MSI-quirk Indicator */ +bool pciehp_msi_disabled; -static int pcie_port_probe_service(struct device *dev) +static int __init pciehp_setup(char *str) { - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; - int status = -ENODEV; + if (!strncmp(str, "nomsi", 5)) + pciehp_msi_disabled = true; - if (!dev || !dev->driver) - return status; + return 1; +} +__setup("pcie_hp=", pciehp_setup); - driver = to_service_driver(dev->driver); - if (!driver || !driver->probe) - return status; +/** + * release_pcie_device - free PCI Express port service device structure + * @dev: Port service device to release + * + * Invoked automatically when device is being removed in response to + * device_unregister(dev). Release all resources being claimed. + */ +static void release_pcie_device(struct device *dev) +{ + kfree(to_pcie_device(dev)); +} - pciedev = to_pcie_device(dev); - status = driver->probe(pciedev, driver->id_table); - if (!status) { - printk(KERN_DEBUG "Load service driver %s on pcie device %s\n", - driver->name, dev->bus_id); - get_device(dev); - } - return status; +/** + * pcie_port_msix_add_entry - add entry to given array of MSI-X entries + * @entries: Array of MSI-X entries + * @new_entry: Index of the entry to add to the array + * @nr_entries: Number of entries already in the array + * + * Return value: Position of the added entry in the array + */ +static int pcie_port_msix_add_entry( + struct msix_entry *entries, int new_entry, int nr_entries) +{ + int j; + + for (j = 0; j < nr_entries; j++) + if (entries[j].entry == new_entry) + return j; + + entries[j].entry = new_entry; + return j; } -static int pcie_port_remove_service(struct device *dev) +/** + * pcie_port_enable_msix - try to set up MSI-X as interrupt mode for given port + * @dev: PCI Express port to handle + * @vectors: Array of interrupt vectors to populate + * @mask: Bitmask of port capabilities returned by get_port_device_capability() + * + * Return value: 0 on success, error code on failure + */ +static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) { - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; + struct msix_entry *msix_entries; + int idx[PCIE_PORT_DEVICE_MAXSERVICES]; + int nr_entries, status, pos, i, nvec; + u16 reg16; + u32 reg32; - if (!dev || !dev->driver) - return 0; + nr_entries = pci_msix_vec_count(dev); + if (nr_entries < 0) + return nr_entries; + BUG_ON(!nr_entries); + if (nr_entries > PCIE_PORT_MAX_MSIX_ENTRIES) + nr_entries = PCIE_PORT_MAX_MSIX_ENTRIES; - pciedev = to_pcie_device(dev); - driver = to_service_driver(dev->driver); - if (driver && driver->remove) { - printk(KERN_DEBUG "Unload service driver %s on pcie device %s\n", - driver->name, dev->bus_id); - driver->remove(pciedev); - put_device(dev); + msix_entries = kzalloc(sizeof(*msix_entries) * nr_entries, GFP_KERNEL); + if (!msix_entries) + return -ENOMEM; + + /* + * Allocate as many entries as the port wants, so that we can check + * which of them will be useful. Moreover, if nr_entries is correctly + * equal to the number of entries this port actually uses, we'll happily + * go through without any tricks. + */ + for (i = 0; i < nr_entries; i++) + msix_entries[i].entry = i; + + status = pci_enable_msix_exact(dev, msix_entries, nr_entries); + if (status) + goto Exit; + + for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) + idx[i] = -1; + status = -EIO; + nvec = 0; + + if (mask & (PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP)) { + int entry; + + /* + * The code below follows the PCI Express Base Specification 2.0 + * stating in Section 6.1.6 that "PME and Hot-Plug Event + * interrupts (when both are implemented) always share the same + * MSI or MSI-X vector, as indicated by the Interrupt Message + * Number field in the PCI Express Capabilities register", where + * according to Section 7.8.2 of the specification "For MSI-X, + * the value in this field indicates which MSI-X Table entry is + * used to generate the interrupt message." + */ + pcie_capability_read_word(dev, PCI_EXP_FLAGS, ®16); + entry = (reg16 & PCI_EXP_FLAGS_IRQ) >> 9; + if (entry >= nr_entries) + goto Error; + + i = pcie_port_msix_add_entry(msix_entries, entry, nvec); + if (i == nvec) + nvec++; + + idx[PCIE_PORT_SERVICE_PME_SHIFT] = i; + idx[PCIE_PORT_SERVICE_HP_SHIFT] = i; } - return 0; -} -static void pcie_port_shutdown_service(struct device *dev) {} + if (mask & PCIE_PORT_SERVICE_AER) { + int entry; + + /* + * The code below follows Section 7.10.10 of the PCI Express + * Base Specification 2.0 stating that bits 31-27 of the Root + * Error Status Register contain a value indicating which of the + * MSI/MSI-X vectors assigned to the port is going to be used + * for AER, where "For MSI-X, the value in this register + * indicates which MSI-X Table entry is used to generate the + * interrupt message." + */ + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, ®32); + entry = reg32 >> 27; + if (entry >= nr_entries) + goto Error; + + i = pcie_port_msix_add_entry(msix_entries, entry, nvec); + if (i == nvec) + nvec++; -static int pcie_port_suspend_service(struct device *dev, pm_message_t state) -{ - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; + idx[PCIE_PORT_SERVICE_AER_SHIFT] = i; + } - if (!dev || !dev->driver) - return 0; + /* + * If nvec is equal to the allocated number of entries, we can just use + * what we have. Otherwise, the port has some extra entries not for the + * services we know and we need to work around that. + */ + if (nvec == nr_entries) { + status = 0; + } else { + /* Drop the temporary MSI-X setup */ + pci_disable_msix(dev); - pciedev = to_pcie_device(dev); - driver = to_service_driver(dev->driver); - if (driver && driver->suspend) - driver->suspend(pciedev, state); - return 0; + /* Now allocate the MSI-X vectors for real */ + status = pci_enable_msix_exact(dev, msix_entries, nvec); + if (status) + goto Exit; + } + + for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) + vectors[i] = idx[i] >= 0 ? msix_entries[idx[i]].vector : -1; + + Exit: + kfree(msix_entries); + return status; + + Error: + pci_disable_msix(dev); + goto Exit; } -static int pcie_port_resume_service(struct device *dev) +/** + * init_service_irqs - initialize irqs for PCI Express port services + * @dev: PCI Express port to handle + * @irqs: Array of irqs to populate + * @mask: Bitmask of port capabilities returned by get_port_device_capability() + * + * Return value: Interrupt mode associated with the port + */ +static int init_service_irqs(struct pci_dev *dev, int *irqs, int mask) { - struct pcie_device *pciedev; - struct pcie_port_service_driver *driver; + int i, irq = -1; + + /* + * If MSI cannot be used for PCIe PME or hotplug, we have to use + * INTx or other interrupts, e.g. system shared interrupt. + */ + if (((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi()) || + ((mask & PCIE_PORT_SERVICE_HP) && pciehp_no_msi())) { + if (dev->irq) + irq = dev->irq; + goto no_msi; + } - if (!dev || !dev->driver) + /* Try to use MSI-X if supported */ + if (!pcie_port_enable_msix(dev, irqs, mask)) return 0; - pciedev = to_pcie_device(dev); - driver = to_service_driver(dev->driver); + /* + * We're not going to use MSI-X, so try MSI and fall back to INTx. + * If neither MSI/MSI-X nor INTx available, try other interrupt. On + * some platforms, root port doesn't support MSI/MSI-X/INTx in RC mode. + */ + if (!pci_enable_msi(dev) || dev->irq) + irq = dev->irq; + + no_msi: + for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) + irqs[i] = irq; + irqs[PCIE_PORT_SERVICE_VC_SHIFT] = -1; - if (driver && driver->resume) - driver->resume(pciedev); + if (irq < 0) + return -ENODEV; return 0; } -/* - * release_pcie_device - * - * Being invoked automatically when device is being removed - * in response to device_unregister(dev) call. - * Release all resources being claimed. - */ -static void release_pcie_device(struct device *dev) +static void cleanup_service_irqs(struct pci_dev *dev) { - printk(KERN_DEBUG "Free Port Service[%s]\n", dev->bus_id); - kfree(to_pcie_device(dev)); + if (dev->msix_enabled) + pci_disable_msix(dev); + else if (dev->msi_enabled) + pci_disable_msi(dev); } -static int is_msi_quirked(struct pci_dev *dev) +/** + * get_port_device_capability - discover capabilities of a PCI Express port + * @dev: PCI Express port to examine + * + * The capabilities are read from the port's PCI Express configuration registers + * as described in PCI Express Base Specification 1.0a sections 7.8.2, 7.8.9 and + * 7.9 - 7.11. + * + * Return value: Bitmask of discovered port capabilities + */ +static int get_port_device_capability(struct pci_dev *dev) { - int port_type, quirk = 0; - u16 reg16; + int services = 0; + u32 reg32; + int cap_mask = 0; + int err; - pci_read_config_word(dev, - pci_find_capability(dev, PCI_CAP_ID_EXP) + - PCIE_CAPABILITIES_REG, ®16); - port_type = (reg16 >> 4) & PORT_TYPE_MASK; - switch(port_type) { - case PCIE_RC_PORT: - if (pcie_mch_quirk == 1) - quirk = 1; - break; - case PCIE_SW_UPSTREAM_PORT: - case PCIE_SW_DOWNSTREAM_PORT: - default: - break; - } - return quirk; -} - -static int assign_interrupt_mode(struct pci_dev *dev, int *vectors, int mask) -{ - int i, pos, nvec, status = -EINVAL; - int interrupt_mode = PCIE_PORT_INTx_MODE; + if (pcie_ports_disabled) + return 0; - /* Set INTx as default */ - for (i = 0, nvec = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) { - if (mask & (1 << i)) - nvec++; - vectors[i] = dev->irq; - } - - /* Check MSI quirk */ - if (is_msi_quirked(dev)) - return interrupt_mode; - - /* Select MSI-X over MSI if supported */ - pos = pci_find_capability(dev, PCI_CAP_ID_MSIX); - if (pos) { - struct msix_entry msix_entries[PCIE_PORT_DEVICE_MAXSERVICES] = - {{0, 0}, {0, 1}, {0, 2}, {0, 3}}; - printk("%s Found MSIX capability\n", __FUNCTION__); - status = pci_enable_msix(dev, msix_entries, nvec); - if (!status) { - int j = 0; - - interrupt_mode = PCIE_PORT_MSIX_MODE; - for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) { - if (mask & (1 << i)) - vectors[i] = msix_entries[j++].vector; - } - } - } - if (status) { - pos = pci_find_capability(dev, PCI_CAP_ID_MSI); - if (pos) { - printk("%s Found MSI capability\n", __FUNCTION__); - status = pci_enable_msi(dev); - if (!status) { - interrupt_mode = PCIE_PORT_MSI_MODE; - for (i = 0;i < PCIE_PORT_DEVICE_MAXSERVICES;i++) - vectors[i] = dev->irq; - } - } - } - return interrupt_mode; -} + cap_mask = PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP + | PCIE_PORT_SERVICE_VC; + if (pci_aer_available()) + cap_mask |= PCIE_PORT_SERVICE_AER; -static int get_port_device_capability(struct pci_dev *dev) -{ - int services = 0, pos; - u16 reg16; - u32 reg32; + if (pcie_ports_auto) { + err = pcie_port_platform_notify(dev, &cap_mask); + if (err) + return 0; + } - pos = pci_find_capability(dev, PCI_CAP_ID_EXP); - pci_read_config_word(dev, pos + PCIE_CAPABILITIES_REG, ®16); /* Hot-Plug Capable */ - if (reg16 & PORT_TO_SLOT_MASK) { - pci_read_config_dword(dev, - pos + PCIE_SLOT_CAPABILITIES_REG, ®32); - if (reg32 & SLOT_HP_CAPABLE_MASK) + if ((cap_mask & PCIE_PORT_SERVICE_HP) && + pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT) { + pcie_capability_read_dword(dev, PCI_EXP_SLTCAP, ®32); + if (reg32 & PCI_EXP_SLTCAP_HPC) { services |= PCIE_PORT_SERVICE_HP; - } - /* PME Capable - root port capability */ - if (((reg16 >> 4) & PORT_TYPE_MASK) == PCIE_RC_PORT) - services |= PCIE_PORT_SERVICE_PME; - - pos = PCI_CFG_SPACE_SIZE; - while (pos) { - pci_read_config_dword(dev, pos, ®32); - switch (reg32 & 0xffff) { - case PCI_EXT_CAP_ID_ERR: - services |= PCIE_PORT_SERVICE_AER; - pos = reg32 >> 20; - break; - case PCI_EXT_CAP_ID_VC: - services |= PCIE_PORT_SERVICE_VC; - pos = reg32 >> 20; - break; - default: - pos = 0; - break; + /* + * Disable hot-plug interrupts in case they have been + * enabled by the BIOS and the hot-plug service driver + * is not loaded. + */ + pcie_capability_clear_word(dev, PCI_EXP_SLTCTL, + PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE); } } + /* AER capable */ + if ((cap_mask & PCIE_PORT_SERVICE_AER) + && pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) { + services |= PCIE_PORT_SERVICE_AER; + /* + * Disable AER on this port in case it's been enabled by the + * BIOS (the AER service driver will enable it when necessary). + */ + pci_disable_pcie_error_reporting(dev); + } + /* VC support */ + if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_VC)) + services |= PCIE_PORT_SERVICE_VC; + /* Root ports are capable of generating PME too */ + if ((cap_mask & PCIE_PORT_SERVICE_PME) + && pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) { + services |= PCIE_PORT_SERVICE_PME; + /* + * Disable PME interrupt on this port in case it's been enabled + * by the BIOS (the PME service driver will enable it when + * necessary). + */ + pcie_pme_interrupt_enable(dev, false); + } return services; } -static void pcie_device_init(struct pci_dev *parent, struct pcie_device *dev, - int port_type, int service_type, int irq, int irq_mode) +/** + * pcie_device_init - allocate and initialize PCI Express port service device + * @pdev: PCI Express port to associate the service device with + * @service: Type of service to associate with the service device + * @irq: Interrupt vector to associate with the service device + */ +static int pcie_device_init(struct pci_dev *pdev, int service, int irq) { + int retval; + struct pcie_device *pcie; struct device *device; - dev->port = parent; - dev->interrupt_mode = irq_mode; - dev->irq = irq; - dev->id.vendor = parent->vendor; - dev->id.device = parent->device; - dev->id.port_type = port_type; - dev->id.service_type = (1 << service_type); + pcie = kzalloc(sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + pcie->port = pdev; + pcie->irq = irq; + pcie->service = service; /* Initialize generic device interface */ - device = &dev->device; - memset(device, 0, sizeof(struct device)); + device = &pcie->device; device->bus = &pcie_port_bus_type; - device->driver = NULL; - device->driver_data = NULL; device->release = release_pcie_device; /* callback to free pcie dev */ - snprintf(device->bus_id, sizeof(device->bus_id), "%s:pcie%02x", - pci_name(parent), get_descriptor_id(port_type, service_type)); - device->parent = &parent->dev; -} - -static struct pcie_device* alloc_pcie_device(struct pci_dev *parent, - int port_type, int service_type, int irq, int irq_mode) -{ - struct pcie_device *device; - - device = kzalloc(sizeof(struct pcie_device), GFP_KERNEL); - if (!device) - return NULL; - - pcie_device_init(parent, device, port_type, service_type, irq,irq_mode); - printk(KERN_DEBUG "Allocate Port Service[%s]\n", device->device.bus_id); - return device; -} - -int pcie_port_device_probe(struct pci_dev *dev) -{ - int pos, type; - u16 reg; - - if (!(pos = pci_find_capability(dev, PCI_CAP_ID_EXP))) - return -ENODEV; - - pci_read_config_word(dev, pos + PCIE_CAPABILITIES_REG, ®); - type = (reg >> 4) & PORT_TYPE_MASK; - if ( type == PCIE_RC_PORT || type == PCIE_SW_UPSTREAM_PORT || - type == PCIE_SW_DOWNSTREAM_PORT ) - return 0; + dev_set_name(device, "%s:pcie%02x", + pci_name(pdev), + get_descriptor_id(pci_pcie_type(pdev), service)); + device->parent = &pdev->dev; + device_enable_async_suspend(device); + + retval = device_register(device); + if (retval) { + put_device(device); + return retval; + } - return -ENODEV; + return 0; } +/** + * pcie_port_device_register - register PCI Express port + * @dev: PCI Express port to register + * + * Allocate the port extension structure and register services associated with + * the port. + */ int pcie_port_device_register(struct pci_dev *dev) { - struct pcie_port_device_ext *p_ext; - int status, type, capabilities, irq_mode, i; - int vectors[PCIE_PORT_DEVICE_MAXSERVICES]; - u16 reg16; - - /* Allocate port device extension */ - if (!(p_ext = kmalloc(sizeof(struct pcie_port_device_ext), GFP_KERNEL))) - return -ENOMEM; + int status, capabilities, i, nr_service; + int irqs[PCIE_PORT_DEVICE_MAXSERVICES]; - pci_set_drvdata(dev, p_ext); - - /* Get port type */ - pci_read_config_word(dev, - pci_find_capability(dev, PCI_CAP_ID_EXP) + - PCIE_CAPABILITIES_REG, ®16); - type = (reg16 >> 4) & PORT_TYPE_MASK; + /* Enable PCI Express port device */ + status = pci_enable_device(dev); + if (status) + return status; - /* Now get port services */ + /* Get and check PCI Express port services */ capabilities = get_port_device_capability(dev); - irq_mode = assign_interrupt_mode(dev, vectors, capabilities); - p_ext->interrupt_mode = irq_mode; + if (!capabilities) + return 0; + + pci_set_master(dev); + /* + * Initialize service irqs. Don't use service devices that + * require interrupts if there is no way to generate them. + * However, some drivers may have a polling mode (e.g. pciehp_poll_mode) + * that can be used in the absence of irqs. Allow them to determine + * if that is to be used. + */ + status = init_service_irqs(dev, irqs, capabilities); + if (status) { + capabilities &= PCIE_PORT_SERVICE_VC | PCIE_PORT_SERVICE_HP; + if (!capabilities) + goto error_disable; + } /* Allocate child services if any */ + status = -ENODEV; + nr_service = 0; for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) { - struct pcie_device *child; - - if (capabilities & (1 << i)) { - child = alloc_pcie_device( - dev, /* parent */ - type, /* port type */ - i, /* service type */ - vectors[i], /* irq */ - irq_mode /* interrupt mode */); - if (child) { - status = device_register(&child->device); - if (status) { - kfree(child); - continue; - } - get_device(&child->device); - } - } + int service = 1 << i; + if (!(capabilities & service)) + continue; + if (!pcie_device_init(dev, service, irqs[i])) + nr_service++; } + if (!nr_service) + goto error_cleanup_irqs; + return 0; + +error_cleanup_irqs: + cleanup_service_irqs(dev); +error_disable: + pci_disable_device(dev); + return status; } #ifdef CONFIG_PM static int suspend_iter(struct device *dev, void *data) { struct pcie_port_service_driver *service_driver; - pm_message_t state = * (pm_message_t *) data; - - if ((dev->bus == &pcie_port_bus_type) && - (dev->driver)) { - service_driver = to_service_driver(dev->driver); - if (service_driver->suspend) - service_driver->suspend(to_pcie_device(dev), state); - } + + if ((dev->bus == &pcie_port_bus_type) && dev->driver) { + service_driver = to_service_driver(dev->driver); + if (service_driver->suspend) + service_driver->suspend(to_pcie_device(dev)); + } return 0; } -int pcie_port_device_suspend(struct pci_dev *dev, pm_message_t state) +/** + * pcie_port_device_suspend - suspend port services associated with a PCIe port + * @dev: PCI Express port to handle + */ +int pcie_port_device_suspend(struct device *dev) { - return device_for_each_child(&dev->dev, &state, suspend_iter); + return device_for_each_child(dev, NULL, suspend_iter); } static int resume_iter(struct device *dev, void *data) @@ -354,78 +447,132 @@ static int resume_iter(struct device *dev, void *data) return 0; } -int pcie_port_device_resume(struct pci_dev *dev) +/** + * pcie_port_device_suspend - resume port services associated with a PCIe port + * @dev: PCI Express port to handle + */ +int pcie_port_device_resume(struct device *dev) { - return device_for_each_child(&dev->dev, NULL, resume_iter); + return device_for_each_child(dev, NULL, resume_iter); } -#endif +#endif /* PM */ static int remove_iter(struct device *dev, void *data) { - struct pcie_port_service_driver *service_driver; - - if (dev->bus == &pcie_port_bus_type) { - if (dev->driver) { - service_driver = to_service_driver(dev->driver); - if (service_driver->remove) - service_driver->remove(to_pcie_device(dev)); - } - *(unsigned long*)data = (unsigned long)dev; - return 1; - } + if (dev->bus == &pcie_port_bus_type) + device_unregister(dev); return 0; } +/** + * pcie_port_device_remove - unregister PCI Express port service devices + * @dev: PCI Express port the service devices to unregister are associated with + * + * Remove PCI Express port service devices associated with given port and + * disable MSI-X or MSI for the port. + */ void pcie_port_device_remove(struct pci_dev *dev) { - struct device *device; - unsigned long device_addr; - int interrupt_mode = PCIE_PORT_INTx_MODE; - int status; - - do { - status = device_for_each_child(&dev->dev, &device_addr, remove_iter); - if (status) { - device = (struct device*)device_addr; - interrupt_mode = (to_pcie_device(device))->interrupt_mode; - put_device(device); - device_unregister(device); - } - } while (status); - /* Switch to INTx by default if MSI enabled */ - if (interrupt_mode == PCIE_PORT_MSIX_MODE) - pci_disable_msix(dev); - else if (interrupt_mode == PCIE_PORT_MSI_MODE) - pci_disable_msi(dev); + device_for_each_child(&dev->dev, NULL, remove_iter); + cleanup_service_irqs(dev); + pci_disable_device(dev); } -int pcie_port_bus_register(void) +/** + * pcie_port_probe_service - probe driver for given PCI Express port service + * @dev: PCI Express port service device to probe against + * + * If PCI Express port service driver is registered with + * pcie_port_service_register(), this function will be called by the driver core + * whenever match is found between the driver and a port service device. + */ +static int pcie_port_probe_service(struct device *dev) { - return bus_register(&pcie_port_bus_type); + struct pcie_device *pciedev; + struct pcie_port_service_driver *driver; + int status; + + if (!dev || !dev->driver) + return -ENODEV; + + driver = to_service_driver(dev->driver); + if (!driver || !driver->probe) + return -ENODEV; + + pciedev = to_pcie_device(dev); + status = driver->probe(pciedev); + if (status) + return status; + + dev_printk(KERN_DEBUG, dev, "service driver %s loaded\n", driver->name); + get_device(dev); + return 0; } -void pcie_port_bus_unregister(void) +/** + * pcie_port_remove_service - detach driver from given PCI Express port service + * @dev: PCI Express port service device to handle + * + * If PCI Express port service driver is registered with + * pcie_port_service_register(), this function will be called by the driver core + * when device_unregister() is called for the port service device associated + * with the driver. + */ +static int pcie_port_remove_service(struct device *dev) { - bus_unregister(&pcie_port_bus_type); + struct pcie_device *pciedev; + struct pcie_port_service_driver *driver; + + if (!dev || !dev->driver) + return 0; + + pciedev = to_pcie_device(dev); + driver = to_service_driver(dev->driver); + if (driver && driver->remove) { + dev_printk(KERN_DEBUG, dev, "unloading service driver %s\n", + driver->name); + driver->remove(pciedev); + put_device(dev); + } + return 0; } +/** + * pcie_port_shutdown_service - shut down given PCI Express port service + * @dev: PCI Express port service device to handle + * + * If PCI Express port service driver is registered with + * pcie_port_service_register(), this function will be called by the driver core + * when device_shutdown() is called for the port service device associated + * with the driver. + */ +static void pcie_port_shutdown_service(struct device *dev) {} + +/** + * pcie_port_service_register - register PCI Express port service driver + * @new: PCI Express port service driver to register + */ int pcie_port_service_register(struct pcie_port_service_driver *new) { - new->driver.name = (char *)new->name; + if (pcie_ports_disabled) + return -ENODEV; + + new->driver.name = new->name; new->driver.bus = &pcie_port_bus_type; new->driver.probe = pcie_port_probe_service; new->driver.remove = pcie_port_remove_service; new->driver.shutdown = pcie_port_shutdown_service; - new->driver.suspend = pcie_port_suspend_service; - new->driver.resume = pcie_port_resume_service; return driver_register(&new->driver); } +EXPORT_SYMBOL(pcie_port_service_register); -void pcie_port_service_unregister(struct pcie_port_service_driver *new) +/** + * pcie_port_service_unregister - unregister PCI Express port service driver + * @drv: PCI Express port service driver to unregister + */ +void pcie_port_service_unregister(struct pcie_port_service_driver *drv) { - driver_unregister(&new->driver); + driver_unregister(&drv->driver); } - -EXPORT_SYMBOL(pcie_port_service_register); EXPORT_SYMBOL(pcie_port_service_unregister); diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 26057f98f72..80887eaa066 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -11,10 +11,12 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/pm.h> +#include <linux/pm_runtime.h> #include <linux/init.h> -#include <linux/slab.h> #include <linux/pcieport_if.h> #include <linux/aer.h> +#include <linux/dmi.h> +#include <linux/pci-aspm.h> #include "portdrv.h" #include "aer/aerdrv.h" @@ -24,24 +26,51 @@ */ #define DRIVER_VERSION "v1.0" #define DRIVER_AUTHOR "tom.l.nguyen@intel.com" -#define DRIVER_DESC "PCIE Port Bus Driver" +#define DRIVER_DESC "PCIe Port Bus Driver" MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL"); +/* If this switch is set, PCIe port native services should not be enabled. */ +bool pcie_ports_disabled; + +/* + * If this switch is set, ACPI _OSC will be used to determine whether or not to + * enable PCIe port native services. + */ +bool pcie_ports_auto = true; + +static int __init pcie_port_setup(char *str) +{ + if (!strncmp(str, "compat", 6)) { + pcie_ports_disabled = true; + } else if (!strncmp(str, "native", 6)) { + pcie_ports_disabled = false; + pcie_ports_auto = false; + } else if (!strncmp(str, "auto", 4)) { + pcie_ports_disabled = false; + pcie_ports_auto = true; + } + + return 1; +} +__setup("pcie_ports=", pcie_port_setup); + /* global data */ -static const char device_name[] = "pcieport-driver"; -static int pcie_portdrv_save_config(struct pci_dev *dev) +/** + * pcie_clear_root_pme_status - Clear root port PME interrupt status. + * @dev: PCIe root port or event collector. + */ +void pcie_clear_root_pme_status(struct pci_dev *dev) { - return pci_save_state(dev); + pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); } static int pcie_portdrv_restore_config(struct pci_dev *dev) { int retval; - pci_restore_state(dev); retval = pci_enable_device(dev); if (retval) return retval; @@ -50,67 +79,150 @@ static int pcie_portdrv_restore_config(struct pci_dev *dev) } #ifdef CONFIG_PM -static int pcie_portdrv_suspend(struct pci_dev *dev, pm_message_t state) +static int pcie_port_resume_noirq(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + + /* + * Some BIOSes forget to clear Root PME Status bits after system wakeup + * which breaks ACPI-based runtime wakeup on PCI Express, so clear those + * bits now just in case (shouldn't hurt). + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + pcie_clear_root_pme_status(pdev); + return 0; +} + +#ifdef CONFIG_PM_RUNTIME +struct d3cold_info { + bool no_d3cold; + unsigned int d3cold_delay; +}; + +static int pci_dev_d3cold_info(struct pci_dev *pdev, void *data) { - int ret = pcie_port_device_suspend(dev, state); + struct d3cold_info *info = data; - if (!ret) - ret = pcie_portdrv_save_config(dev); - return ret; + info->d3cold_delay = max_t(unsigned int, pdev->d3cold_delay, + info->d3cold_delay); + if (pdev->no_d3cold) + info->no_d3cold = true; + return 0; +} + +static int pcie_port_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct d3cold_info d3cold_info = { + .no_d3cold = false, + .d3cold_delay = PCI_PM_D3_WAIT, + }; + + /* + * If any subordinate device disable D3cold, we should not put + * the port into D3cold. The D3cold delay of port should be + * the max of that of all subordinate devices. + */ + pci_walk_bus(pdev->subordinate, pci_dev_d3cold_info, &d3cold_info); + pdev->no_d3cold = d3cold_info.no_d3cold; + pdev->d3cold_delay = d3cold_info.d3cold_delay; + return 0; +} + +static int pcie_port_runtime_resume(struct device *dev) +{ + return 0; +} + +static int pci_dev_pme_poll(struct pci_dev *pdev, void *data) +{ + bool *pme_poll = data; + + if (pdev->pme_poll) + *pme_poll = true; + return 0; } -static int pcie_portdrv_resume(struct pci_dev *dev) +static int pcie_port_runtime_idle(struct device *dev) { - pcie_portdrv_restore_config(dev); - return pcie_port_device_resume(dev); + struct pci_dev *pdev = to_pci_dev(dev); + bool pme_poll = false; + + /* + * If any subordinate device needs pme poll, we should keep + * the port in D0, because we need port in D0 to poll it. + */ + pci_walk_bus(pdev->subordinate, pci_dev_pme_poll, &pme_poll); + /* Delay for a short while to prevent too frequent suspend/resume */ + if (!pme_poll) + pm_schedule_suspend(dev, 10); + return -EBUSY; } #else -#define pcie_portdrv_suspend NULL -#define pcie_portdrv_resume NULL +#define pcie_port_runtime_suspend NULL +#define pcie_port_runtime_resume NULL +#define pcie_port_runtime_idle NULL #endif +static const struct dev_pm_ops pcie_portdrv_pm_ops = { + .suspend = pcie_port_device_suspend, + .resume = pcie_port_device_resume, + .freeze = pcie_port_device_suspend, + .thaw = pcie_port_device_resume, + .poweroff = pcie_port_device_suspend, + .restore = pcie_port_device_resume, + .resume_noirq = pcie_port_resume_noirq, + .runtime_suspend = pcie_port_runtime_suspend, + .runtime_resume = pcie_port_runtime_resume, + .runtime_idle = pcie_port_runtime_idle, +}; + +#define PCIE_PORTDRV_PM_OPS (&pcie_portdrv_pm_ops) + +#else /* !PM */ + +#define PCIE_PORTDRV_PM_OPS NULL +#endif /* !PM */ + /* * pcie_portdrv_probe - Probe PCI-Express port devices * @dev: PCI-Express port device being probed * - * If detected invokes the pcie_port_device_register() method for + * If detected invokes the pcie_port_device_register() method for * this port device. * */ -static int __devinit pcie_portdrv_probe (struct pci_dev *dev, - const struct pci_device_id *id ) +static int pcie_portdrv_probe(struct pci_dev *dev, + const struct pci_device_id *id) { - int status; - - status = pcie_port_device_probe(dev); - if (status) - return status; + int status; - if (pci_enable_device(dev) < 0) + if (!pci_is_pcie(dev) || + ((pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM) && + (pci_pcie_type(dev) != PCI_EXP_TYPE_DOWNSTREAM))) return -ENODEV; - - pci_set_master(dev); - if (!dev->irq && dev->pin) { - printk(KERN_WARNING - "%s->Dev[%04x:%04x] has invalid IRQ. Check vendor BIOS\n", - __FUNCTION__, dev->vendor, dev->device); - } - if (pcie_port_device_register(dev)) { - pci_disable_device(dev); - return -ENOMEM; - } - - pcie_portdrv_save_config(dev); - pci_enable_pcie_error_reporting(dev); + if (!dev->irq && dev->pin) { + dev_warn(&dev->dev, "device [%04x:%04x] has invalid IRQ; check vendor BIOS\n", + dev->vendor, dev->device); + } + status = pcie_port_device_register(dev); + if (status) + return status; + pci_save_state(dev); + /* + * D3cold may not work properly on some PCIe port, so disable + * it by default. + */ + dev->d3cold_allowed = false; return 0; } -static void pcie_portdrv_remove (struct pci_dev *dev) +static void pcie_portdrv_remove(struct pci_dev *dev) { pcie_port_device_remove(dev); - kfree(pci_get_drvdata(dev)); } static int error_detected_iter(struct device *device, void *data) @@ -145,14 +257,11 @@ static int error_detected_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, enum pci_channel_state error) { - struct aer_broadcast_data result_data = - {error, PCI_ERS_RESULT_CAN_RECOVER}; - int retval; + struct aer_broadcast_data data = {error, PCI_ERS_RESULT_CAN_RECOVER}; - /* can not fail */ - retval = device_for_each_child(&dev->dev, &result_data, error_detected_iter); - - return result_data.result; + /* get true return value from &data */ + device_for_each_child(&dev->dev, &data, error_detected_iter); + return data.result; } static int mmio_enabled_iter(struct device *device, void *data) @@ -183,10 +292,9 @@ static int mmio_enabled_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev) { pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; - int retval; /* get true return value from &status */ - retval = device_for_each_child(&dev->dev, &status, mmio_enabled_iter); + device_for_each_child(&dev->dev, &status, mmio_enabled_iter); return status; } @@ -217,18 +325,18 @@ static int slot_reset_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev) { - pci_ers_result_t status = PCI_ERS_RESULT_NONE; - int retval; + pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; /* If fatal, restore cfg space for possible link reset at upstream */ if (dev->error_state == pci_channel_io_frozen) { + dev->state_saved = true; + pci_restore_state(dev); pcie_portdrv_restore_config(dev); pci_enable_pcie_error_reporting(dev); } /* get true return value from &status */ - retval = device_for_each_child(&dev->dev, &status, slot_reset_iter); - + device_for_each_child(&dev->dev, &status, slot_reset_iter); return status; } @@ -254,9 +362,7 @@ static int resume_iter(struct device *device, void *data) static void pcie_portdrv_err_resume(struct pci_dev *dev) { - int retval; - /* nothing to do with error value, if it ever happens */ - retval = device_for_each_child(&dev->dev, NULL, resume_iter); + device_for_each_child(&dev->dev, NULL, resume_iter); } /* @@ -269,30 +375,58 @@ static const struct pci_device_id port_pci_ids[] = { { }; MODULE_DEVICE_TABLE(pci, port_pci_ids); -static struct pci_error_handlers pcie_portdrv_err_handler = { - .error_detected = pcie_portdrv_error_detected, - .mmio_enabled = pcie_portdrv_mmio_enabled, - .slot_reset = pcie_portdrv_slot_reset, - .resume = pcie_portdrv_err_resume, +static const struct pci_error_handlers pcie_portdrv_err_handler = { + .error_detected = pcie_portdrv_error_detected, + .mmio_enabled = pcie_portdrv_mmio_enabled, + .slot_reset = pcie_portdrv_slot_reset, + .resume = pcie_portdrv_err_resume, }; static struct pci_driver pcie_portdriver = { - .name = (char *)device_name, + .name = "pcieport", .id_table = &port_pci_ids[0], .probe = pcie_portdrv_probe, .remove = pcie_portdrv_remove, - .suspend = pcie_portdrv_suspend, - .resume = pcie_portdrv_resume, + .err_handler = &pcie_portdrv_err_handler, - .err_handler = &pcie_portdrv_err_handler, + .driver.pm = PCIE_PORTDRV_PM_OPS, +}; + +static int __init dmi_pcie_pme_disable_msi(const struct dmi_system_id *d) +{ + pr_notice("%s detected: will not use MSI for PCIe PME signaling\n", + d->ident); + pcie_pme_disable_msi(); + return 0; +} + +static struct dmi_system_id __initdata pcie_portdrv_dmi_table[] = { + /* + * Boxes that should not use MSI for PCIe PME signaling. + */ + { + .callback = dmi_pcie_pme_disable_msi, + .ident = "MSI Wind U-100", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, + "MICRO-STAR INTERNATIONAL CO., LTD"), + DMI_MATCH(DMI_PRODUCT_NAME, "U-100"), + }, + }, + {} }; static int __init pcie_portdrv_init(void) { int retval; + if (pcie_ports_disabled) + return pci_register_driver(&pcie_portdriver); + + dmi_check_system(pcie_portdrv_dmi_table); + retval = pcie_port_bus_register(); if (retval) { printk(KERN_WARNING "PCIE: bus_register error: %d\n", retval); @@ -305,11 +439,4 @@ static int __init pcie_portdrv_init(void) return retval; } -static void __exit pcie_portdrv_exit(void) -{ - pci_unregister_driver(&pcie_portdriver); - pcie_port_bus_unregister(); -} - module_init(pcie_portdrv_init); -module_exit(pcie_portdrv_exit); |