diff options
Diffstat (limited to 'drivers/pci/pcie')
| -rw-r--r-- | drivers/pci/pcie/Kconfig | 56 | ||||
| -rw-r--r-- | drivers/pci/pcie/Makefile | 3 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/Kconfig.debug | 4 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aer_inject.c | 74 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv.c | 196 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv.h | 49 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_acpi.c | 144 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_core.c | 678 | ||||
| -rw-r--r-- | drivers/pci/pcie/aer/aerdrv_errprint.c | 229 | ||||
| -rw-r--r-- | drivers/pci/pcie/aspm.c | 302 | ||||
| -rw-r--r-- | drivers/pci/pcie/pme.c | 438 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv.h | 64 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_acpi.c | 63 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_bus.c | 6 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_core.c | 174 | ||||
| -rw-r--r-- | drivers/pci/pcie/portdrv_pci.c | 228 |
16 files changed, 1889 insertions, 819 deletions
diff --git a/drivers/pci/pcie/Kconfig b/drivers/pci/pcie/Kconfig index 5a0c6ad53f8..7958e59d607 100644 --- a/drivers/pci/pcie/Kconfig +++ b/drivers/pci/pcie/Kconfig @@ -2,7 +2,7 @@ # PCI Express Port Bus Configuration # config PCIEPORTBUS - bool "PCI Express support" + bool "PCI Express Port Bus support" depends on PCI help This automatically enables PCI Express Port Bus support. Users can @@ -14,15 +14,12 @@ config PCIEPORTBUS # Include service Kconfig here # config HOTPLUG_PCI_PCIE - tristate "PCI Express Hotplug driver" + bool "PCI Express Hotplug driver" depends on HOTPLUG_PCI && PCIEPORTBUS help Say Y here if you have a motherboard that supports PCI Express Native Hotplug - To compile this driver as a module, choose M here: the - module will be called pciehp. - When in doubt, say N. source "drivers/pci/pcie/aer/Kconfig" @@ -31,14 +28,22 @@ source "drivers/pci/pcie/aer/Kconfig" # PCI Express ASPM # config PCIEASPM - bool "PCI Express ASPM support(Experimental)" - depends on PCI && EXPERIMENTAL && PCIEPORTBUS - default n + bool "PCI Express ASPM control" if EXPERT + depends on PCI && PCIEPORTBUS + default y help - This enables PCI Express ASPM (Active State Power Management) and - Clock Power Management. ASPM supports state L0/L0s/L1. + This enables OS control over PCI Express ASPM (Active State + Power Management) and Clock Power Management. ASPM supports + state L0/L0s/L1. - When in doubt, say N. + ASPM is initially set up by the firmware. With this option enabled, + Linux can modify this state in order to disable ASPM on known-bad + hardware or configurations and enable it when known-safe. + + ASPM can be disabled or enabled at runtime via + /sys/module/pcie_aspm/parameters/policy + + When in doubt, say Y. config PCIEASPM_DEBUG bool "Debug PCI Express ASPM" depends on PCIEASPM @@ -46,3 +51,32 @@ config PCIEASPM_DEBUG help This enables PCI Express ASPM debug support. It will add per-device interface to control ASPM. + +choice + prompt "Default ASPM policy" + default PCIEASPM_DEFAULT + depends on PCIEASPM + +config PCIEASPM_DEFAULT + bool "BIOS default" + depends on PCIEASPM + help + Use the BIOS defaults for PCI Express ASPM. + +config PCIEASPM_POWERSAVE + bool "Powersave" + depends on PCIEASPM + help + Enable PCI Express ASPM L0s and L1 where possible, even if the + BIOS did not. + +config PCIEASPM_PERFORMANCE + bool "Performance" + depends on PCIEASPM + help + Disable PCI Express ASPM L0s and L1, even if the BIOS enabled them. +endchoice + +config PCIE_PME + def_bool y + depends on PCIEPORTBUS && PM_RUNTIME diff --git a/drivers/pci/pcie/Makefile b/drivers/pci/pcie/Makefile index 11f6bb1eae2..00c62df5a9f 100644 --- a/drivers/pci/pcie/Makefile +++ b/drivers/pci/pcie/Makefile @@ -6,8 +6,11 @@ obj-$(CONFIG_PCIEASPM) += aspm.o pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o +pcieportdrv-$(CONFIG_ACPI) += portdrv_acpi.o obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o # Build PCI Express AER if needed obj-$(CONFIG_PCIEAER) += aer/ + +obj-$(CONFIG_PCIE_PME) += pme.o diff --git a/drivers/pci/pcie/aer/Kconfig.debug b/drivers/pci/pcie/aer/Kconfig.debug index b8c925c1f6a..9142949734f 100644 --- a/drivers/pci/pcie/aer/Kconfig.debug +++ b/drivers/pci/pcie/aer/Kconfig.debug @@ -3,14 +3,14 @@ # config PCIEAER_INJECT - tristate "PCIE AER error injector support" + tristate "PCIe AER error injector support" depends on PCIEAER default n help This enables PCI Express Root Port Advanced Error Reporting (AER) software error injector. - Debuging PCIE AER code is quite difficult because it is hard + Debugging PCIe AER code is quite difficult because it is hard to trigger various real hardware errors. Software based error injection can fake almost all kinds of errors with the help of a user space helper tool aer-inject, which can be diff --git a/drivers/pci/pcie/aer/aer_inject.c b/drivers/pci/pcie/aer/aer_inject.c index 7fcd5331b14..182224acedb 100644 --- a/drivers/pci/pcie/aer/aer_inject.c +++ b/drivers/pci/pcie/aer/aer_inject.c @@ -1,7 +1,7 @@ /* - * PCIE AER software error injection support. + * PCIe AER software error injection support. * - * Debuging PCIE AER code is quite difficult because it is hard to + * Debuging PCIe AER code is quite difficult because it is hard to * trigger various real hardware errors. Software based error * injection can fake almost all kinds of errors with the help of a * user space helper tool aer-inject, which can be gotten from: @@ -21,11 +21,16 @@ #include <linux/init.h> #include <linux/miscdevice.h> #include <linux/pci.h> +#include <linux/slab.h> #include <linux/fs.h> #include <linux/uaccess.h> #include <linux/stddef.h> #include "aerdrv.h" +/* Override the existing corrected and uncorrected error masks */ +static bool aer_mask_override; +module_param(aer_mask_override, bool, 0); + struct aer_error_inj { u8 bus; u8 dev; @@ -167,7 +172,7 @@ static u32 *find_pci_config_dword(struct aer_error *err, int where, target = &err->root_status; rw1cs = 1; break; - case PCI_ERR_ROOT_COR_SRC: + case PCI_ERR_ROOT_ERR_SRC: target = &err->source_id; break; } @@ -207,8 +212,8 @@ out: return ops->read(bus, devfn, where, size, val); } -int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, int size, - u32 val) +static int pci_write_aer(struct pci_bus *bus, unsigned int devfn, int where, + int size, u32 val) { u32 *sim; struct aer_error *err; @@ -283,7 +288,7 @@ static struct pci_dev *pcie_find_root_port(struct pci_dev *dev) while (1) { if (!pci_is_pcie(dev)) break; - if (dev->pcie_type == PCI_EXP_TYPE_ROOT_PORT) + if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) return dev; if (!dev->bus->self) break; @@ -321,7 +326,7 @@ static int aer_inject(struct aer_error_inj *einj) unsigned long flags; unsigned int devfn = PCI_DEVFN(einj->dev, einj->fn); int pos_cap_err, rp_pos_cap_err; - u32 sever; + u32 sever, cor_mask, uncor_mask, cor_mask_orig = 0, uncor_mask_orig = 0; int ret = 0; dev = pci_get_domain_bus_and_slot((int)einj->domain, einj->bus, devfn); @@ -329,20 +334,23 @@ static int aer_inject(struct aer_error_inj *einj) return -ENODEV; rpdev = pcie_find_root_port(dev); if (!rpdev) { - ret = -ENOTTY; + ret = -ENODEV; goto out_put; } pos_cap_err = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos_cap_err) { - ret = -ENOTTY; + ret = -EPERM; goto out_put; } pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_SEVER, &sever); + pci_read_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, &cor_mask); + pci_read_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + &uncor_mask); rp_pos_cap_err = pci_find_ext_capability(rpdev, PCI_EXT_CAP_ID_ERR); if (!rp_pos_cap_err) { - ret = -ENOTTY; + ret = -EPERM; goto out_put; } @@ -357,6 +365,18 @@ static int aer_inject(struct aer_error_inj *einj) goto out_put; } + if (aer_mask_override) { + cor_mask_orig = cor_mask; + cor_mask &= !(einj->cor_status); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, + cor_mask); + + uncor_mask_orig = uncor_mask; + uncor_mask &= !(einj->uncor_status); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + uncor_mask); + } + spin_lock_irqsave(&inject_lock, flags); err = __find_aer_error_by_dev(dev); @@ -374,6 +394,21 @@ static int aer_inject(struct aer_error_inj *einj) err->header_log2 = einj->header_log2; err->header_log3 = einj->header_log3; + if (!aer_mask_override && einj->cor_status && + !(einj->cor_status & ~cor_mask)) { + ret = -EINVAL; + printk(KERN_WARNING "The correctable error(s) is masked by device\n"); + spin_unlock_irqrestore(&inject_lock, flags); + goto out_put; + } + if (!aer_mask_override && einj->uncor_status && + !(einj->uncor_status & ~uncor_mask)) { + ret = -EINVAL; + printk(KERN_WARNING "The uncorrectable error(s) is masked by device\n"); + spin_unlock_irqrestore(&inject_lock, flags); + goto out_put; + } + rperr = __find_aer_error_by_dev(rpdev); if (!rperr) { rperr = rperr_alloc; @@ -406,6 +441,13 @@ static int aer_inject(struct aer_error_inj *einj) } spin_unlock_irqrestore(&inject_lock, flags); + if (aer_mask_override) { + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_COR_MASK, + cor_mask_orig); + pci_write_config_dword(dev, pos_cap_err + PCI_ERR_UNCOR_MASK, + uncor_mask_orig); + } + ret = pci_bus_set_aer_ops(dev->bus); if (ret) goto out_put; @@ -413,9 +455,14 @@ static int aer_inject(struct aer_error_inj *einj) if (ret) goto out_put; - if (find_aer_device(rpdev, &edev)) + if (find_aer_device(rpdev, &edev)) { + if (!get_service_data(edev)) { + printk(KERN_WARNING "AER service is not initialized\n"); + ret = -EINVAL; + goto out_put; + } aer_irq(-1, edev); - else + } else ret = -EINVAL; out_put: kfree(err_alloc); @@ -447,6 +494,7 @@ static ssize_t aer_inject_write(struct file *filp, const char __user *ubuf, static const struct file_operations aer_inject_fops = { .write = aer_inject_write, .owner = THIS_MODULE, + .llseek = noop_llseek, }; static struct miscdevice aer_inject_device = { @@ -484,5 +532,5 @@ static void __exit aer_inject_exit(void) module_init(aer_inject_init); module_exit(aer_inject_exit); -MODULE_DESCRIPTION("PCIE AER software error injector"); +MODULE_DESCRIPTION("PCIe AER software error injector"); MODULE_LICENSE("GPL"); diff --git a/drivers/pci/pcie/aer/aerdrv.c b/drivers/pci/pcie/aer/aerdrv.c index 97a345927b5..0bf82a20a0f 100644 --- a/drivers/pci/pcie/aer/aerdrv.c +++ b/drivers/pci/pcie/aer/aerdrv.c @@ -17,6 +17,7 @@ #include <linux/module.h> #include <linux/pci.h> +#include <linux/pci-acpi.h> #include <linux/sched.h> #include <linux/kernel.h> #include <linux/errno.h> @@ -25,6 +26,7 @@ #include <linux/interrupt.h> #include <linux/delay.h> #include <linux/pcieport_if.h> +#include <linux/slab.h> #include "aerdrv.h" #include "../../pci.h" @@ -39,14 +41,14 @@ MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL"); -static int __devinit aer_probe(struct pcie_device *dev); +static int aer_probe(struct pcie_device *dev); static void aer_remove(struct pcie_device *dev); static pci_ers_result_t aer_error_detected(struct pci_dev *dev, enum pci_channel_state error); static void aer_error_resume(struct pci_dev *dev); static pci_ers_result_t aer_root_reset(struct pci_dev *dev); -static struct pci_error_handlers aer_error_handlers = { +static const struct pci_error_handlers aer_error_handlers = { .error_detected = aer_error_detected, .resume = aer_error_resume, }; @@ -71,13 +73,124 @@ void pci_no_aer(void) pcie_aer_disable = 1; /* has priority over 'forceload' */ } +bool pci_aer_available(void) +{ + return !pcie_aer_disable && pci_msi_enabled(); +} + +static int set_device_error_reporting(struct pci_dev *dev, void *data) +{ + bool enable = *((bool *)data); + int type = pci_pcie_type(dev); + + if ((type == PCI_EXP_TYPE_ROOT_PORT) || + (type == PCI_EXP_TYPE_UPSTREAM) || + (type == PCI_EXP_TYPE_DOWNSTREAM)) { + if (enable) + pci_enable_pcie_error_reporting(dev); + else + pci_disable_pcie_error_reporting(dev); + } + + if (enable) + pcie_set_ecrc_checking(dev); + + return 0; +} + +/** + * set_downstream_devices_error_reporting - enable/disable the error reporting bits on the root port and its downstream ports. + * @dev: pointer to root port's pci_dev data structure + * @enable: true = enable error reporting, false = disable error reporting. + */ +static void set_downstream_devices_error_reporting(struct pci_dev *dev, + bool enable) +{ + set_device_error_reporting(dev, &enable); + + if (!dev->subordinate) + return; + pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable); +} + +/** + * aer_enable_rootport - enable Root Port's interrupts when receiving messages + * @rpc: pointer to a Root Port data structure + * + * Invoked when PCIe bus loads AER service driver. + */ +static void aer_enable_rootport(struct aer_rpc *rpc) +{ + struct pci_dev *pdev = rpc->rpd->port; + int aer_pos; + u16 reg16; + u32 reg32; + + /* Clear PCIe Capability's Device Status */ + pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); + pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); + + /* Disable system error generation in response to error messages */ + pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, + SYSTEM_ERROR_INTR_ON_MESG_MASK); + + aer_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + /* Clear error status */ + pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32); + pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32); + pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, ®32); + pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32); + + /* + * Enable error reporting for the root port device and downstream port + * devices. + */ + set_downstream_devices_error_reporting(pdev, true); + + /* Enable Root Port's interrupt in response to error messages */ + pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, reg32); +} + +/** + * aer_disable_rootport - disable Root Port's interrupts when receiving messages + * @rpc: pointer to a Root Port data structure + * + * Invoked when PCIe bus unloads AER service driver. + */ +static void aer_disable_rootport(struct aer_rpc *rpc) +{ + struct pci_dev *pdev = rpc->rpd->port; + u32 reg32; + int pos; + + /* + * Disable error reporting for the root port device and downstream port + * devices. + */ + set_downstream_devices_error_reporting(pdev, false); + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); + /* Disable Root's interrupt in response to error messages */ + pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, reg32); + + /* Clear Root's error status reg */ + pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32); +} + /** * aer_irq - Root Port's ISR * @irq: IRQ assigned to Root Port * @context: pointer to Root Port data structure * * Invoked when Root Port detects AER messages. - **/ + */ irqreturn_t aer_irq(int irq, void *context) { unsigned int status, id; @@ -96,13 +209,13 @@ irqreturn_t aer_irq(int irq, void *context) /* Read error status */ pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, &status); - if (!(status & ROOT_ERR_STATUS_MASKS)) { + if (!(status & (PCI_ERR_ROOT_UNCOR_RCV|PCI_ERR_ROOT_COR_RCV))) { spin_unlock_irqrestore(&rpc->e_lock, flags); return IRQ_NONE; } /* Read error source and clear error status */ - pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_COR_SRC, &id); + pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_ERR_SRC, &id); pci_write_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, status); /* Store error source for later DPC handler */ @@ -134,7 +247,7 @@ EXPORT_SYMBOL_GPL(aer_irq); * @dev: pointer to the pcie_dev data structure * * Invoked when Root Port's AER service is loaded. - **/ + */ static struct aer_rpc *aer_alloc_rpc(struct pcie_device *dev) { struct aer_rpc *rpc; @@ -143,19 +256,15 @@ static struct aer_rpc *aer_alloc_rpc(struct pcie_device *dev) if (!rpc) return NULL; - /* - * Initialize Root lock access, e_lock, to Root Error Status Reg, - * Root Error ID Reg, and Root error producer/consumer index. - */ + /* Initialize Root lock access, e_lock, to Root Error Status Reg */ spin_lock_init(&rpc->e_lock); rpc->rpd = dev; INIT_WORK(&rpc->dpc_handler, aer_isr); - rpc->prod_idx = rpc->cons_idx = 0; mutex_init(&rpc->rpc_mutex); init_waitqueue_head(&rpc->wait_release); - /* Use PCIE bus function to store rpc into PCIE device */ + /* Use PCIe bus function to store rpc into PCIe device */ set_service_data(dev, rpc); return rpc; @@ -166,7 +275,7 @@ static struct aer_rpc *aer_alloc_rpc(struct pcie_device *dev) * @dev: pointer to the pcie_dev data structure * * Invoked when PCI Express bus unloads or AER probe fails. - **/ + */ static void aer_remove(struct pcie_device *dev) { struct aer_rpc *rpc = get_service_data(dev); @@ -178,7 +287,8 @@ static void aer_remove(struct pcie_device *dev) wait_event(rpc->wait_release, rpc->prod_idx == rpc->cons_idx); - aer_delete_rootport(rpc); + aer_disable_rootport(rpc); + kfree(rpc); set_service_data(dev, NULL); } } @@ -189,8 +299,8 @@ static void aer_remove(struct pcie_device *dev) * @id: pointer to the service id data structure * * Invoked when PCI Express bus loads AER service driver. - **/ -static int __devinit aer_probe(struct pcie_device *dev) + */ +static int aer_probe(struct pcie_device *dev) { int status; struct aer_rpc *rpc; @@ -229,41 +339,30 @@ static int __devinit aer_probe(struct pcie_device *dev) * @dev: pointer to Root Port's pci_dev data structure * * Invoked by Port Bus driver when performing link reset at Root Port. - **/ + */ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) { - u16 p2p_ctrl; - u32 status; + u32 reg32; int pos; pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); /* Disable Root's interrupt in response to error messages */ - pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, 0); - - /* Assert Secondary Bus Reset */ - pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &p2p_ctrl); - p2p_ctrl |= PCI_CB_BRIDGE_CTL_CB_RESET; - pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl); + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32); - /* De-assert Secondary Bus Reset */ - p2p_ctrl &= ~PCI_CB_BRIDGE_CTL_CB_RESET; - pci_write_config_word(dev, PCI_BRIDGE_CONTROL, p2p_ctrl); - - /* - * System software must wait for at least 100ms from the end - * of a reset of one or more device before it is permitted - * to issue Configuration Requests to those devices. - */ - msleep(200); + pci_reset_bridge_secondary_bus(dev); dev_printk(KERN_DEBUG, &dev->dev, "Root Port link has been reset\n"); + /* Clear Root Error Status */ + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, ®32); + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, reg32); + /* Enable Root Port's interrupt in response to error messages */ - pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &status); - pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, status); - pci_write_config_dword(dev, - pos + PCI_ERR_ROOT_COMMAND, - ROOT_PORT_INTR_ON_MESG_MASK); + pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, ®32); + reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; + pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32); return PCI_ERS_RESULT_RECOVERED; } @@ -274,7 +373,7 @@ static pci_ers_result_t aer_root_reset(struct pci_dev *dev) * @error: error severity being notified by port bus * * Invoked by Port Bus driver during error recovery. - **/ + */ static pci_ers_result_t aer_error_detected(struct pci_dev *dev, enum pci_channel_state error) { @@ -287,7 +386,7 @@ static pci_ers_result_t aer_error_detected(struct pci_dev *dev, * @dev: pointer to Root Port's pci_dev data structure * * Invoked by Port Bus driver during nonfatal recovery. - **/ + */ static void aer_error_resume(struct pci_dev *dev) { int pos; @@ -295,9 +394,8 @@ static void aer_error_resume(struct pci_dev *dev) u16 reg16; /* Clean up Root device status */ - pos = pci_pcie_cap(dev); - pci_read_config_word(dev, pos + PCI_EXP_DEVSTA, ®16); - pci_write_config_word(dev, pos + PCI_EXP_DEVSTA, reg16); + pcie_capability_read_word(dev, PCI_EXP_DEVSTA, ®16); + pcie_capability_write_word(dev, PCI_EXP_DEVSTA, reg16); /* Clean AER Root Error Status */ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); @@ -314,12 +412,10 @@ static void aer_error_resume(struct pci_dev *dev) * aer_service_init - register AER root service driver * * Invoked when AER root service driver is loaded. - **/ + */ static int __init aer_service_init(void) { - if (pcie_aer_disable) - return -ENXIO; - if (!pci_msi_enabled()) + if (!pci_aer_available() || aer_acpi_firmware_first()) return -ENXIO; return pcie_port_service_register(&aerdriver); } @@ -328,7 +424,7 @@ static int __init aer_service_init(void) * aer_service_exit - unregister AER root service driver * * Invoked when AER root service driver is unloaded. - **/ + */ static void __exit aer_service_exit(void) { pcie_port_service_unregister(&aerdriver); diff --git a/drivers/pci/pcie/aer/aerdrv.h b/drivers/pci/pcie/aer/aerdrv.h index bd833ea3ba4..84420b7c945 100644 --- a/drivers/pci/pcie/aer/aerdrv.h +++ b/drivers/pci/pcie/aer/aerdrv.h @@ -13,13 +13,6 @@ #include <linux/aer.h> #include <linux/interrupt.h> -#define AER_NONFATAL 0 -#define AER_FATAL 1 -#define AER_CORRECTABLE 2 - -/* Root Error Status Register Bits */ -#define ROOT_ERR_STATUS_MASKS 0x0f - #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ PCI_EXP_RTCTL_SENFEE| \ PCI_EXP_RTCTL_SEFEE) @@ -38,13 +31,6 @@ PCI_ERR_UNC_UNX_COMP| \ PCI_ERR_UNC_MALF_TLP) -struct header_log_regs { - unsigned int dw0; - unsigned int dw1; - unsigned int dw2; - unsigned int dw3; -}; - #define AER_MAX_MULTI_ERR_DEVICES 5 /* Not likely to have more */ struct aer_err_info { struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; @@ -62,7 +48,7 @@ struct aer_err_info { unsigned int status; /* COR/UNCOR Error Status */ unsigned int mask; /* COR/UNCOR Error Mask */ - struct header_log_regs tlp; /* TLP Header */ + struct aer_header_log_regs tlp; /* TLP Header */ }; struct aer_err_source { @@ -97,6 +83,9 @@ struct aer_broadcast_data { static inline pci_ers_result_t merge_result(enum pci_ers_result orig, enum pci_ers_result new) { + if (new == PCI_ERS_RESULT_NO_AER_DRIVER) + return PCI_ERS_RESULT_NO_AER_DRIVER; + if (new == PCI_ERS_RESULT_NONE) return orig; @@ -107,7 +96,7 @@ static inline pci_ers_result_t merge_result(enum pci_ers_result orig, break; case PCI_ERS_RESULT_DISCONNECT: if (new == PCI_ERS_RESULT_NEED_RESET) - orig = new; + orig = PCI_ERS_RESULT_NEED_RESET; break; default: break; @@ -117,21 +106,27 @@ static inline pci_ers_result_t merge_result(enum pci_ers_result orig, } extern struct bus_type pcie_port_bus_type; -extern void aer_enable_rootport(struct aer_rpc *rpc); -extern void aer_delete_rootport(struct aer_rpc *rpc); -extern int aer_init(struct pcie_device *dev); -extern void aer_isr(struct work_struct *work); -extern void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); -extern void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info); -extern irqreturn_t aer_irq(int irq, void *context); - -#ifdef CONFIG_ACPI -extern int aer_osc_setup(struct pcie_device *pciedev); +int aer_init(struct pcie_device *dev); +void aer_isr(struct work_struct *work); +void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); +void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info); +irqreturn_t aer_irq(int irq, void *context); + +#ifdef CONFIG_ACPI_APEI +int pcie_aer_get_firmware_first(struct pci_dev *pci_dev); #else -static inline int aer_osc_setup(struct pcie_device *pciedev) +static inline int pcie_aer_get_firmware_first(struct pci_dev *pci_dev) { + if (pci_dev->__aer_firmware_first_valid) + return pci_dev->__aer_firmware_first; return 0; } #endif +static inline void pcie_aer_force_firmware_first(struct pci_dev *pci_dev, + int enable) +{ + pci_dev->__aer_firmware_first = !!enable; + pci_dev->__aer_firmware_first_valid = 1; +} #endif /* _AERDRV_H_ */ diff --git a/drivers/pci/pcie/aer/aerdrv_acpi.c b/drivers/pci/pcie/aer/aerdrv_acpi.c index 8edb2f300e8..01906576ab9 100644 --- a/drivers/pci/pcie/aer/aerdrv_acpi.c +++ b/drivers/pci/pcie/aer/aerdrv_acpi.c @@ -16,40 +16,126 @@ #include <linux/acpi.h> #include <linux/pci-acpi.h> #include <linux/delay.h> +#include <acpi/apei.h> #include "aerdrv.h" -/** - * aer_osc_setup - run ACPI _OSC method - * @pciedev: pcie_device which AER is being enabled on - * - * @return: Zero on success. Nonzero otherwise. - * - * Invoked when PCIE bus loads AER service driver. To avoid conflict with - * BIOS AER support requires BIOS to yield AER control to OS native driver. - **/ -int aer_osc_setup(struct pcie_device *pciedev) +#ifdef CONFIG_ACPI_APEI +static inline int hest_match_pci(struct acpi_hest_aer_common *p, + struct pci_dev *pci) { - acpi_status status = AE_NOT_FOUND; - struct pci_dev *pdev = pciedev->port; - acpi_handle handle = NULL; - - if (acpi_pci_disabled) - return -1; - - handle = acpi_find_root_bridge_handle(pdev); - if (handle) { - status = acpi_pci_osc_control_set(handle, - OSC_PCI_EXPRESS_AER_CONTROL | - OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL); - } + return ACPI_HEST_SEGMENT(p->bus) == pci_domain_nr(pci->bus) && + ACPI_HEST_BUS(p->bus) == pci->bus->number && + p->device == PCI_SLOT(pci->devfn) && + p->function == PCI_FUNC(pci->devfn); +} + +static inline bool hest_match_type(struct acpi_hest_header *hest_hdr, + struct pci_dev *dev) +{ + u16 hest_type = hest_hdr->type; + u8 pcie_type = pci_pcie_type(dev); + + if ((hest_type == ACPI_HEST_TYPE_AER_ROOT_PORT && + pcie_type == PCI_EXP_TYPE_ROOT_PORT) || + (hest_type == ACPI_HEST_TYPE_AER_ENDPOINT && + pcie_type == PCI_EXP_TYPE_ENDPOINT) || + (hest_type == ACPI_HEST_TYPE_AER_BRIDGE && + (dev->class >> 16) == PCI_BASE_CLASS_BRIDGE)) + return true; + return false; +} + +struct aer_hest_parse_info { + struct pci_dev *pci_dev; + int firmware_first; +}; + +static int hest_source_is_pcie_aer(struct acpi_hest_header *hest_hdr) +{ + if (hest_hdr->type == ACPI_HEST_TYPE_AER_ROOT_PORT || + hest_hdr->type == ACPI_HEST_TYPE_AER_ENDPOINT || + hest_hdr->type == ACPI_HEST_TYPE_AER_BRIDGE) + return 1; + return 0; +} + +static int aer_hest_parse(struct acpi_hest_header *hest_hdr, void *data) +{ + struct aer_hest_parse_info *info = data; + struct acpi_hest_aer_common *p; + int ff; + + if (!hest_source_is_pcie_aer(hest_hdr)) + return 0; + + p = (struct acpi_hest_aer_common *)(hest_hdr + 1); + ff = !!(p->flags & ACPI_HEST_FIRMWARE_FIRST); - if (ACPI_FAILURE(status)) { - dev_printk(KERN_DEBUG, &pciedev->device, "AER service couldn't " - "init device: %s\n", - (status == AE_SUPPORT || status == AE_NOT_FOUND) ? - "no _OSC support" : "_OSC failed"); - return -1; + /* + * If no specific device is supplied, determine whether + * FIRMWARE_FIRST is set for *any* PCIe device. + */ + if (!info->pci_dev) { + info->firmware_first |= ff; + return 0; } + /* Otherwise, check the specific device */ + if (p->flags & ACPI_HEST_GLOBAL) { + if (hest_match_type(hest_hdr, info->pci_dev)) + info->firmware_first = ff; + } else + if (hest_match_pci(p, info->pci_dev)) + info->firmware_first = ff; + return 0; } + +static void aer_set_firmware_first(struct pci_dev *pci_dev) +{ + int rc; + struct aer_hest_parse_info info = { + .pci_dev = pci_dev, + .firmware_first = 0, + }; + + rc = apei_hest_parse(aer_hest_parse, &info); + + if (rc) + pci_dev->__aer_firmware_first = 0; + else + pci_dev->__aer_firmware_first = info.firmware_first; + pci_dev->__aer_firmware_first_valid = 1; +} + +int pcie_aer_get_firmware_first(struct pci_dev *dev) +{ + if (!pci_is_pcie(dev)) + return 0; + + if (!dev->__aer_firmware_first_valid) + aer_set_firmware_first(dev); + return dev->__aer_firmware_first; +} + +static bool aer_firmware_first; + +/** + * aer_acpi_firmware_first - Check if APEI should control AER. + */ +bool aer_acpi_firmware_first(void) +{ + static bool parsed = false; + struct aer_hest_parse_info info = { + .pci_dev = NULL, /* Check all PCIe devices */ + .firmware_first = 0, + }; + + if (!parsed) { + apei_hest_parse(aer_hest_parse, &info); + aer_firmware_first = info.firmware_first; + parsed = true; + } + return aer_firmware_first; +} +#endif diff --git a/drivers/pci/pcie/aer/aerdrv_core.c b/drivers/pci/pcie/aer/aerdrv_core.c index ae672ca8033..5653ea94547 100644 --- a/drivers/pci/pcie/aer/aerdrv_core.c +++ b/drivers/pci/pcie/aer/aerdrv_core.c @@ -23,61 +23,37 @@ #include <linux/pm.h> #include <linux/suspend.h> #include <linux/delay.h> +#include <linux/slab.h> +#include <linux/kfifo.h> #include "aerdrv.h" -static int forceload; -static int nosourceid; +static bool forceload; +static bool nosourceid; module_param(forceload, bool, 0); module_param(nosourceid, bool, 0); +#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ + PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) + int pci_enable_pcie_error_reporting(struct pci_dev *dev) { - u16 reg16 = 0; - int pos; - - if (dev->aer_firmware_first) - return -EIO; - - pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); - if (!pos) + if (pcie_aer_get_firmware_first(dev)) return -EIO; - pos = pci_pcie_cap(dev); - if (!pos) + if (!pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) return -EIO; - pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16); - reg16 = reg16 | - PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE; - pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, reg16); - - return 0; + return pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); } EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting); int pci_disable_pcie_error_reporting(struct pci_dev *dev) { - u16 reg16 = 0; - int pos; - - if (dev->aer_firmware_first) - return -EIO; - - pos = pci_pcie_cap(dev); - if (!pos) + if (pcie_aer_get_firmware_first(dev)) return -EIO; - pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16); - reg16 = reg16 & ~(PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE); - pci_write_config_word(dev, pos+PCI_EXP_DEVCTL, reg16); - - return 0; + return pcie_capability_clear_word(dev, PCI_EXP_DEVCTL, + PCI_EXP_AER_FLAGS); } EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting); @@ -98,99 +74,44 @@ int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status); -static int set_device_error_reporting(struct pci_dev *dev, void *data) -{ - bool enable = *((bool *)data); - - if ((dev->pcie_type == PCI_EXP_TYPE_ROOT_PORT) || - (dev->pcie_type == PCI_EXP_TYPE_UPSTREAM) || - (dev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)) { - if (enable) - pci_enable_pcie_error_reporting(dev); - else - pci_disable_pcie_error_reporting(dev); - } - - if (enable) - pcie_set_ecrc_checking(dev); - - return 0; -} - /** - * set_downstream_devices_error_reporting - enable/disable the error reporting bits on the root port and its downstream ports. - * @dev: pointer to root port's pci_dev data structure - * @enable: true = enable error reporting, false = disable error reporting. + * add_error_device - list device to be handled + * @e_info: pointer to error info + * @dev: pointer to pci_dev to be added */ -static void set_downstream_devices_error_reporting(struct pci_dev *dev, - bool enable) -{ - set_device_error_reporting(dev, &enable); - - if (!dev->subordinate) - return; - pci_walk_bus(dev->subordinate, set_device_error_reporting, &enable); -} - -static inline int compare_device_id(struct pci_dev *dev, - struct aer_err_info *e_info) -{ - if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) { - /* - * Device ID match - */ - return 1; - } - - return 0; -} - static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) { if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) { e_info->dev[e_info->error_dev_num] = dev; e_info->error_dev_num++; - return 1; + return 0; } - - return 0; + return -ENOSPC; } - -#define PCI_BUS(x) (((x) >> 8) & 0xff) - -static int find_device_iter(struct pci_dev *dev, void *data) +/** + * is_error_source - check whether the device is source of reported error + * @dev: pointer to pci_dev to be checked + * @e_info: pointer to reported error info + */ +static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) { int pos; - u32 status; - u32 mask; + u32 status, mask; u16 reg16; - int result; - struct aer_err_info *e_info = (struct aer_err_info *)data; /* * When bus id is equal to 0, it might be a bad id * reported by root port. */ - if (!nosourceid && (PCI_BUS(e_info->id) != 0)) { - result = compare_device_id(dev, e_info); - if (result) - add_error_device(e_info, dev); + if (!nosourceid && (PCI_BUS_NUM(e_info->id) != 0)) { + /* Device ID match? */ + if (e_info->id == ((dev->bus->number << 8) | dev->devfn)) + return true; - /* - * If there is no multiple error, we stop - * or continue based on the id comparing. - */ + /* Continue id comparing if there is no multiple error */ if (!e_info->multi_error_valid) - return result; - - /* - * If there are multiple errors and id does match, - * We need continue to search other devices under - * the root port. Return 0 means that. - */ - if (result) - return 0; + return false; } /* @@ -199,80 +120,98 @@ static int find_device_iter(struct pci_dev *dev, void *data) * 2) bus id is equal to 0. Some ports might lose the bus * id of error source id; * 3) There are multiple errors and prior id comparing fails; - * We check AER status registers to find the initial reporter. + * We check AER status registers to find possible reporter. */ if (atomic_read(&dev->enable_cnt) == 0) - return 0; - pos = pci_pcie_cap(dev); - if (!pos) - return 0; + return false; + /* Check if AER is enabled */ - pci_read_config_word(dev, pos+PCI_EXP_DEVCTL, ®16); - if (!(reg16 & ( - PCI_EXP_DEVCTL_CERE | - PCI_EXP_DEVCTL_NFERE | - PCI_EXP_DEVCTL_FERE | - PCI_EXP_DEVCTL_URRE))) - return 0; + pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); + if (!(reg16 & PCI_EXP_AER_FLAGS)) + return false; + pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (!pos) - return 0; + return false; - status = 0; - mask = 0; + /* Check if error is recorded */ if (e_info->severity == AER_CORRECTABLE) { pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status); pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask); - if (status & ~mask) { - add_error_device(e_info, dev); - goto added; - } } else { pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status); pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask); - if (status & ~mask) { - add_error_device(e_info, dev); - goto added; - } } + if (status & ~mask) + return true; - return 0; + return false; +} -added: - if (e_info->multi_error_valid) - return 0; - else - return 1; +static int find_device_iter(struct pci_dev *dev, void *data) +{ + struct aer_err_info *e_info = (struct aer_err_info *)data; + + if (is_error_source(dev, e_info)) { + /* List this device */ + if (add_error_device(e_info, dev)) { + /* We cannot handle more... Stop iteration */ + /* TODO: Should print error message here? */ + return 1; + } + + /* If there is only a single error, stop iteration */ + if (!e_info->multi_error_valid) + return 1; + } + return 0; } /** * find_source_device - search through device hierarchy for source device * @parent: pointer to Root Port pci_dev data structure - * @err_info: including detailed error information such like id + * @e_info: including detailed error information such like id + * + * Return true if found. * - * Invoked when error is detected at the Root Port. + * Invoked by DPC when error is detected at the Root Port. + * Caller of this function must set id, severity, and multi_error_valid of + * struct aer_err_info pointed by @e_info properly. This function must fill + * e_info->error_dev_num and e_info->dev[], based on the given information. */ -static void find_source_device(struct pci_dev *parent, +static bool find_source_device(struct pci_dev *parent, struct aer_err_info *e_info) { struct pci_dev *dev = parent; int result; + /* Must reset in this function */ + e_info->error_dev_num = 0; + /* Is Root Port an agent that sends error message? */ result = find_device_iter(dev, e_info); if (result) - return; + return true; pci_walk_bus(parent->subordinate, find_device_iter, e_info); + + if (!e_info->error_dev_num) { + dev_printk(KERN_DEBUG, &parent->dev, + "can't find device of ID%04x\n", + e_info->id); + return false; + } + return true; } static int report_error_detected(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); dev->error_state = result_data->state; if (!dev->driver || @@ -291,64 +230,89 @@ static int report_error_detected(struct pci_dev *dev, void *data) dev->driver ? "no AER-aware driver" : "no driver"); } - return 0; + + /* + * If there's any device in the subtree that does not + * have an error_detected callback, returning + * PCI_ERS_RESULT_NO_AER_DRIVER prevents calling of + * the subsequent mmio_enabled/slot_reset/resume + * callbacks of "any" device in the subtree. All the + * devices in the subtree are left in the error state + * without recovery. + */ + + if (!(dev->hdr_type & PCI_HEADER_TYPE_BRIDGE)) + vote = PCI_ERS_RESULT_NO_AER_DRIVER; + else + vote = PCI_ERS_RESULT_NONE; + } else { + err_handler = dev->driver->err_handler; + vote = err_handler->error_detected(dev, result_data->state); } - err_handler = dev->driver->err_handler; - vote = err_handler->error_detected(dev, result_data->state); result_data->result = merge_result(result_data->result, vote); + device_unlock(&dev->dev); return 0; } static int report_mmio_enabled(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); if (!dev->driver || !dev->driver->err_handler || !dev->driver->err_handler->mmio_enabled) - return 0; + goto out; err_handler = dev->driver->err_handler; vote = err_handler->mmio_enabled(dev); result_data->result = merge_result(result_data->result, vote); +out: + device_unlock(&dev->dev); return 0; } static int report_slot_reset(struct pci_dev *dev, void *data) { pci_ers_result_t vote; - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; struct aer_broadcast_data *result_data; result_data = (struct aer_broadcast_data *) data; + device_lock(&dev->dev); if (!dev->driver || !dev->driver->err_handler || !dev->driver->err_handler->slot_reset) - return 0; + goto out; err_handler = dev->driver->err_handler; vote = err_handler->slot_reset(dev); result_data->result = merge_result(result_data->result, vote); +out: + device_unlock(&dev->dev); return 0; } static int report_resume(struct pci_dev *dev, void *data) { - struct pci_error_handlers *err_handler; + const struct pci_error_handlers *err_handler; + device_lock(&dev->dev); dev->error_state = pci_channel_io_normal; if (!dev->driver || !dev->driver->err_handler || !dev->driver->err_handler->resume) - return 0; + goto out; err_handler = dev->driver->err_handler; err_handler->resume(dev); +out: + device_unlock(&dev->dev); return 0; } @@ -402,82 +366,79 @@ static pci_ers_result_t broadcast_error_message(struct pci_dev *dev, return result_data.result; } -struct find_aer_service_data { - struct pcie_port_service_driver *aer_driver; - int is_downstream; -}; +/** + * default_reset_link - default reset function + * @dev: pointer to pci_dev data structure + * + * Invoked when performing link reset on a Downstream Port or a + * Root Port with no aer driver. + */ +static pci_ers_result_t default_reset_link(struct pci_dev *dev) +{ + pci_reset_bridge_secondary_bus(dev); + dev_printk(KERN_DEBUG, &dev->dev, "downstream link has been reset\n"); + return PCI_ERS_RESULT_RECOVERED; +} static int find_aer_service_iter(struct device *device, void *data) { - struct device_driver *driver; - struct pcie_port_service_driver *service_driver; - struct find_aer_service_data *result; - - result = (struct find_aer_service_data *) data; - - if (device->bus == &pcie_port_bus_type) { - struct pcie_device *pcie = to_pcie_device(device); + struct pcie_port_service_driver *service_driver, **drv; - if (pcie->port->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) - result->is_downstream = 1; + drv = (struct pcie_port_service_driver **) data; - driver = device->driver; - if (driver) { - service_driver = to_service_driver(driver); - if (service_driver->service == PCIE_PORT_SERVICE_AER) { - result->aer_driver = service_driver; - return 1; - } + if (device->bus == &pcie_port_bus_type && device->driver) { + service_driver = to_service_driver(device->driver); + if (service_driver->service == PCIE_PORT_SERVICE_AER) { + *drv = service_driver; + return 1; } } return 0; } -static void find_aer_service(struct pci_dev *dev, - struct find_aer_service_data *data) +static struct pcie_port_service_driver *find_aer_service(struct pci_dev *dev) { - int retval; - retval = device_for_each_child(&dev->dev, data, find_aer_service_iter); + struct pcie_port_service_driver *drv = NULL; + + device_for_each_child(&dev->dev, &drv, find_aer_service_iter); + + return drv; } -static pci_ers_result_t reset_link(struct pcie_device *aerdev, - struct pci_dev *dev) +static pci_ers_result_t reset_link(struct pci_dev *dev) { struct pci_dev *udev; pci_ers_result_t status; - struct find_aer_service_data data; + struct pcie_port_service_driver *driver; - if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) + if (dev->hdr_type & PCI_HEADER_TYPE_BRIDGE) { + /* Reset this port for all subordinates */ udev = dev; - else + } else { + /* Reset the upstream component (likely downstream port) */ udev = dev->bus->self; + } - data.is_downstream = 0; - data.aer_driver = NULL; - find_aer_service(udev, &data); + /* Use the aer driver of the component firstly */ + driver = find_aer_service(udev); - /* - * Use the aer driver of the error agent firstly. - * If it hasn't the aer driver, use the root port's - */ - if (!data.aer_driver || !data.aer_driver->reset_link) { - if (data.is_downstream && - aerdev->device.driver && - to_service_driver(aerdev->device.driver)->reset_link) { - data.aer_driver = - to_service_driver(aerdev->device.driver); - } else { - dev_printk(KERN_DEBUG, &dev->dev, "no link-reset " - "support\n"); - return PCI_ERS_RESULT_DISCONNECT; - } + if (driver && driver->reset_link) { + status = driver->reset_link(udev); + } else if (pci_pcie_type(udev) == PCI_EXP_TYPE_DOWNSTREAM || + pci_pcie_type(udev) == PCI_EXP_TYPE_ROOT_PORT) { + status = default_reset_link(udev); + } else { + dev_printk(KERN_DEBUG, &dev->dev, + "no link-reset support at upstream device %s\n", + pci_name(udev)); + return PCI_ERS_RESULT_DISCONNECT; } - status = data.aer_driver->reset_link(udev); if (status != PCI_ERS_RESULT_RECOVERED) { - dev_printk(KERN_DEBUG, &dev->dev, "link reset at upstream " - "device %s failed\n", pci_name(udev)); + dev_printk(KERN_DEBUG, &dev->dev, + "link reset at upstream device %s failed\n", + pci_name(udev)); return PCI_ERS_RESULT_DISCONNECT; } @@ -486,7 +447,6 @@ static pci_ers_result_t reset_link(struct pcie_device *aerdev, /** * do_recovery - handle nonfatal/fatal error recovery process - * @aerdev: pointer to a pcie_device data structure of root port * @dev: pointer to a pci_dev data structure of agent detecting an error * @severity: error severity type * @@ -494,9 +454,7 @@ static pci_ers_result_t reset_link(struct pcie_device *aerdev, * error detected message to all downstream drivers within a hierarchy in * question and return the returned code. */ -static pci_ers_result_t do_recovery(struct pcie_device *aerdev, - struct pci_dev *dev, - int severity) +static void do_recovery(struct pci_dev *dev, int severity) { pci_ers_result_t status, result = PCI_ERS_RESULT_RECOVERED; enum pci_channel_state state; @@ -512,11 +470,9 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev, report_error_detected); if (severity == AER_FATAL) { - result = reset_link(aerdev, dev); - if (result != PCI_ERS_RESULT_RECOVERED) { - /* TODO: Should panic here? */ - return result; - } + result = reset_link(dev); + if (result != PCI_ERS_RESULT_RECOVERED) + goto failed; } if (status == PCI_ERS_RESULT_CAN_RECOVER) @@ -537,13 +493,20 @@ static pci_ers_result_t do_recovery(struct pcie_device *aerdev, report_slot_reset); } - if (status == PCI_ERS_RESULT_RECOVERED) - broadcast_error_message(dev, + if (status != PCI_ERS_RESULT_RECOVERED) + goto failed; + + broadcast_error_message(dev, state, "resume", report_resume); - return status; + dev_info(&dev->dev, "AER: Device recovery successful\n"); + return; + +failed: + /* TODO: Should kernel panic here? */ + dev_info(&dev->dev, "AER: Device recovery failed\n"); } /** @@ -558,127 +521,87 @@ static void handle_error_source(struct pcie_device *aerdev, struct pci_dev *dev, struct aer_err_info *info) { - pci_ers_result_t status = 0; int pos; if (info->severity == AER_CORRECTABLE) { /* - * Correctable error does not need software intevention. + * Correctable error does not need software intervention. * No need to go through error recovery process. */ pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); if (pos) pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, info->status); - } else { - status = do_recovery(aerdev, dev, info->severity); - if (status == PCI_ERS_RESULT_RECOVERED) { - dev_printk(KERN_DEBUG, &dev->dev, "AER driver " - "successfully recovered\n"); - } else { - /* TODO: Should kernel panic here? */ - dev_printk(KERN_DEBUG, &dev->dev, "AER driver didn't " - "recover\n"); - } - } + } else + do_recovery(dev, info->severity); } -/** - * aer_enable_rootport - enable Root Port's interrupts when receiving messages - * @rpc: pointer to a Root Port data structure - * - * Invoked when PCIE bus loads AER service driver. - */ -void aer_enable_rootport(struct aer_rpc *rpc) -{ - struct pci_dev *pdev = rpc->rpd->port; - int pos, aer_pos; - u16 reg16; - u32 reg32; - - pos = pci_pcie_cap(pdev); - /* Clear PCIE Capability's Device Status */ - pci_read_config_word(pdev, pos+PCI_EXP_DEVSTA, ®16); - pci_write_config_word(pdev, pos+PCI_EXP_DEVSTA, reg16); - - /* Disable system error generation in response to error messages */ - pci_read_config_word(pdev, pos + PCI_EXP_RTCTL, ®16); - reg16 &= ~(SYSTEM_ERROR_INTR_ON_MESG_MASK); - pci_write_config_word(pdev, pos + PCI_EXP_RTCTL, reg16); - - aer_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); - /* Clear error status */ - pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32); - pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32); - pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, ®32); - pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32); +#ifdef CONFIG_ACPI_APEI_PCIEAER +static void aer_recover_work_func(struct work_struct *work); - /* - * Enable error reporting for the root port device and downstream port - * devices. - */ - set_downstream_devices_error_reporting(pdev, true); +#define AER_RECOVER_RING_ORDER 4 +#define AER_RECOVER_RING_SIZE (1 << AER_RECOVER_RING_ORDER) - /* Enable Root Port's interrupt in response to error messages */ - pci_write_config_dword(pdev, - aer_pos + PCI_ERR_ROOT_COMMAND, - ROOT_PORT_INTR_ON_MESG_MASK); -} +struct aer_recover_entry { + u8 bus; + u8 devfn; + u16 domain; + int severity; + struct aer_capability_regs *regs; +}; -/** - * disable_root_aer - disable Root Port's interrupts when receiving messages - * @rpc: pointer to a Root Port data structure - * - * Invoked when PCIE bus unloads AER service driver. +static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, + AER_RECOVER_RING_SIZE); +/* + * Mutual exclusion for writers of aer_recover_ring, reader side don't + * need lock, because there is only one reader and lock is not needed + * between reader and writer. */ -static void disable_root_aer(struct aer_rpc *rpc) -{ - struct pci_dev *pdev = rpc->rpd->port; - u32 reg32; - int pos; +static DEFINE_SPINLOCK(aer_recover_ring_lock); +static DECLARE_WORK(aer_recover_work, aer_recover_work_func); - /* - * Disable error reporting for the root port device and downstream port - * devices. - */ - set_downstream_devices_error_reporting(pdev, false); - - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR); - /* Disable Root's interrupt in response to error messages */ - pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, 0); - - /* Clear Root's error status reg */ - pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, ®32); - pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32); -} - -/** - * get_e_source - retrieve an error source - * @rpc: pointer to the root port which holds an error - * - * Invoked by DPC handler to consume an error. - */ -static struct aer_err_source *get_e_source(struct aer_rpc *rpc) +void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, + int severity, struct aer_capability_regs *aer_regs) { - struct aer_err_source *e_source; unsigned long flags; + struct aer_recover_entry entry = { + .bus = bus, + .devfn = devfn, + .domain = domain, + .severity = severity, + .regs = aer_regs, + }; + + spin_lock_irqsave(&aer_recover_ring_lock, flags); + if (kfifo_put(&aer_recover_ring, entry)) + schedule_work(&aer_recover_work); + else + pr_err("AER recover: Buffer overflow when recovering AER for %04x:%02x:%02x:%x\n", + domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); + spin_unlock_irqrestore(&aer_recover_ring_lock, flags); +} +EXPORT_SYMBOL_GPL(aer_recover_queue); - /* Lock access to Root error producer/consumer index */ - spin_lock_irqsave(&rpc->e_lock, flags); - if (rpc->prod_idx == rpc->cons_idx) { - spin_unlock_irqrestore(&rpc->e_lock, flags); - return NULL; +static void aer_recover_work_func(struct work_struct *work) +{ + struct aer_recover_entry entry; + struct pci_dev *pdev; + + while (kfifo_get(&aer_recover_ring, &entry)) { + pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, + entry.devfn); + if (!pdev) { + pr_err("AER recover: Can not find pci_dev for %04x:%02x:%02x:%x\n", + entry.domain, entry.bus, + PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); + continue; + } + cper_print_aer(pdev, entry.severity, entry.regs); + do_recovery(pdev, entry.severity); + pci_dev_put(pdev); } - e_source = &rpc->e_sources[rpc->cons_idx]; - rpc->cons_idx++; - if (rpc->cons_idx == AER_ERROR_SOURCES_MAX) - rpc->cons_idx = 0; - spin_unlock_irqrestore(&rpc->e_lock, flags); - - return e_source; } +#endif /** * get_device_error_info - read error status from dev and store it to info @@ -686,11 +609,14 @@ static struct aer_err_source *get_e_source(struct aer_rpc *rpc) * @info: pointer to structure to store the error record * * Return 1 on success, 0 on error. + * + * Note that @info is reused among all error devices. Clear fields properly. */ static int get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) { int pos, temp; + /* Must reset in this function */ info->status = 0; info->tlp_header_valid = 0; @@ -743,12 +669,6 @@ static inline void aer_process_err_devices(struct pcie_device *p_device, { int i; - if (!e_info->dev[0]) { - dev_printk(KERN_DEBUG, &p_device->port->dev, - "can't find device of ID%04x\n", - e_info->id); - } - /* Report all before handle them, not to lost records by reset etc. */ for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { if (get_device_error_info(e_info->dev[i], e_info)) @@ -769,11 +689,10 @@ static void aer_isr_one_error(struct pcie_device *p_device, struct aer_err_source *e_src) { struct aer_err_info *e_info; - int i; /* struct aer_err_info might be big, so we allocate it with slab */ e_info = kmalloc(sizeof(struct aer_err_info), GFP_KERNEL); - if (e_info == NULL) { + if (!e_info) { dev_printk(KERN_DEBUG, &p_device->port->dev, "Can't allocate mem when processing AER errors\n"); return; @@ -783,37 +702,73 @@ static void aer_isr_one_error(struct pcie_device *p_device, * There is a possibility that both correctable error and * uncorrectable error being logged. Report correctable error first. */ - for (i = 1; i & ROOT_ERR_STATUS_MASKS ; i <<= 2) { - if (i > 4) - break; - if (!(e_src->status & i)) - continue; + if (e_src->status & PCI_ERR_ROOT_COR_RCV) { + e_info->id = ERR_COR_ID(e_src->id); + e_info->severity = AER_CORRECTABLE; - memset(e_info, 0, sizeof(struct aer_err_info)); + if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV) + e_info->multi_error_valid = 1; + else + e_info->multi_error_valid = 0; - /* Init comprehensive error information */ - if (i & PCI_ERR_ROOT_COR_RCV) { - e_info->id = ERR_COR_ID(e_src->id); - e_info->severity = AER_CORRECTABLE; - } else { - e_info->id = ERR_UNCOR_ID(e_src->id); - e_info->severity = ((e_src->status >> 6) & 1); - } - if (e_src->status & - (PCI_ERR_ROOT_MULTI_COR_RCV | - PCI_ERR_ROOT_MULTI_UNCOR_RCV)) + aer_print_port_info(p_device->port, e_info); + + if (find_source_device(p_device->port, e_info)) + aer_process_err_devices(p_device, e_info); + } + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { + e_info->id = ERR_UNCOR_ID(e_src->id); + + if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) + e_info->severity = AER_FATAL; + else + e_info->severity = AER_NONFATAL; + + if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV) e_info->multi_error_valid = 1; + else + e_info->multi_error_valid = 0; aer_print_port_info(p_device->port, e_info); - find_source_device(p_device->port, e_info); - aer_process_err_devices(p_device, e_info); + if (find_source_device(p_device->port, e_info)) + aer_process_err_devices(p_device, e_info); } kfree(e_info); } /** + * get_e_source - retrieve an error source + * @rpc: pointer to the root port which holds an error + * @e_src: pointer to store retrieved error source + * + * Return 1 if an error source is retrieved, otherwise 0. + * + * Invoked by DPC handler to consume an error. + */ +static int get_e_source(struct aer_rpc *rpc, struct aer_err_source *e_src) +{ + unsigned long flags; + + /* Lock access to Root error producer/consumer index */ + spin_lock_irqsave(&rpc->e_lock, flags); + if (rpc->prod_idx == rpc->cons_idx) { + spin_unlock_irqrestore(&rpc->e_lock, flags); + return 0; + } + + *e_src = rpc->e_sources[rpc->cons_idx]; + rpc->cons_idx++; + if (rpc->cons_idx == AER_ERROR_SOURCES_MAX) + rpc->cons_idx = 0; + spin_unlock_irqrestore(&rpc->e_lock, flags); + + return 1; +} + +/** * aer_isr - consume errors detected by root port * @work: definition of this work item * @@ -823,34 +778,17 @@ void aer_isr(struct work_struct *work) { struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler); struct pcie_device *p_device = rpc->rpd; - struct aer_err_source *e_src; + struct aer_err_source uninitialized_var(e_src); mutex_lock(&rpc->rpc_mutex); - e_src = get_e_source(rpc); - while (e_src) { - aer_isr_one_error(p_device, e_src); - e_src = get_e_source(rpc); - } + while (get_e_source(rpc, &e_src)) + aer_isr_one_error(p_device, &e_src); mutex_unlock(&rpc->rpc_mutex); wake_up(&rpc->wait_release); } /** - * aer_delete_rootport - disable root port aer and delete service data - * @rpc: pointer to a root port device being deleted - * - * Invoked when AER service unloaded on a specific Root Port - */ -void aer_delete_rootport(struct aer_rpc *rpc) -{ - /* Disable root port AER itself */ - disable_root_aer(rpc); - - kfree(rpc); -} - -/** * aer_init - provide AER initialization * @dev: pointer to AER pcie device * @@ -858,22 +796,10 @@ void aer_delete_rootport(struct aer_rpc *rpc) */ int aer_init(struct pcie_device *dev) { - if (dev->port->aer_firmware_first) { - dev_printk(KERN_DEBUG, &dev->device, - "PCIe errors handled by platform firmware.\n"); - goto out; - } - - if (aer_osc_setup(dev)) - goto out; - - return 0; -out: if (forceload) { dev_printk(KERN_DEBUG, &dev->device, "aerdrv forceload requested.\n"); - dev->port->aer_firmware_first = 0; - return 0; + pcie_aer_force_firmware_first(dev->port, 0); } - return -ENXIO; + return 0; } diff --git a/drivers/pci/pcie/aer/aerdrv_errprint.c b/drivers/pci/pcie/aer/aerdrv_errprint.c index 44acde72294..36ed31b5219 100644 --- a/drivers/pci/pcie/aer/aerdrv_errprint.c +++ b/drivers/pci/pcie/aer/aerdrv_errprint.c @@ -19,9 +19,13 @@ #include <linux/errno.h> #include <linux/pm.h> #include <linux/suspend.h> +#include <linux/cper.h> #include "aerdrv.h" +#define CREATE_TRACE_POINTS +#include <trace/events/ras.h> + #define AER_AGENT_RECEIVER 0 #define AER_AGENT_REQUESTER 1 #define AER_AGENT_COMPLETER 2 @@ -57,86 +61,44 @@ (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ AER_TRANSACTION_LAYER_ERROR) -#define AER_PR(info, pdev, fmt, args...) \ - printk("%s%s %s: " fmt, (info->severity == AER_CORRECTABLE) ? \ - KERN_WARNING : KERN_ERR, dev_driver_string(&pdev->dev), \ - dev_name(&pdev->dev), ## args) - /* * AER error strings */ -static char *aer_error_severity_string[] = { +static const char *aer_error_severity_string[] = { "Uncorrected (Non-Fatal)", "Uncorrected (Fatal)", "Corrected" }; -static char *aer_error_layer[] = { +static const char *aer_error_layer[] = { "Physical Layer", "Data Link Layer", "Transaction Layer" }; -static char *aer_correctable_error_string[] = { - "Receiver Error ", /* Bit Position 0 */ - NULL, - NULL, - NULL, - NULL, - NULL, - "Bad TLP ", /* Bit Position 6 */ - "Bad DLLP ", /* Bit Position 7 */ - "RELAY_NUM Rollover ", /* Bit Position 8 */ - NULL, - NULL, - NULL, - "Replay Timer Timeout ", /* Bit Position 12 */ - "Advisory Non-Fatal ", /* Bit Position 13 */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, + +static const char *aer_correctable_error_string[] = { + "Receiver Error", /* Bit Position 0 */ NULL, NULL, NULL, NULL, NULL, + "Bad TLP", /* Bit Position 6 */ + "Bad DLLP", /* Bit Position 7 */ + "RELAY_NUM Rollover", /* Bit Position 8 */ NULL, NULL, NULL, + "Replay Timer Timeout", /* Bit Position 12 */ + "Advisory Non-Fatal", /* Bit Position 13 */ }; -static char *aer_uncorrectable_error_string[] = { - NULL, - NULL, - NULL, - NULL, - "Data Link Protocol ", /* Bit Position 4 */ - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - "Poisoned TLP ", /* Bit Position 12 */ - "Flow Control Protocol ", /* Bit Position 13 */ - "Completion Timeout ", /* Bit Position 14 */ - "Completer Abort ", /* Bit Position 15 */ - "Unexpected Completion ", /* Bit Position 16 */ - "Receiver Overflow ", /* Bit Position 17 */ - "Malformed TLP ", /* Bit Position 18 */ - "ECRC ", /* Bit Position 19 */ - "Unsupported Request ", /* Bit Position 20 */ +static const char *aer_uncorrectable_error_string[] = { NULL, NULL, NULL, NULL, + "Data Link Protocol", /* Bit Position 4 */ NULL, NULL, NULL, @@ -144,20 +106,44 @@ static char *aer_uncorrectable_error_string[] = { NULL, NULL, NULL, + "Poisoned TLP", /* Bit Position 12 */ + "Flow Control Protocol", /* Bit Position 13 */ + "Completion Timeout", /* Bit Position 14 */ + "Completer Abort", /* Bit Position 15 */ + "Unexpected Completion", /* Bit Position 16 */ + "Receiver Overflow", /* Bit Position 17 */ + "Malformed TLP", /* Bit Position 18 */ + "ECRC", /* Bit Position 19 */ + "Unsupported Request", /* Bit Position 20 */ }; -static char *aer_agent_string[] = { +static const char *aer_agent_string[] = { "Receiver ID", "Requester ID", "Completer ID", "Transmitter ID" }; -static void __aer_print_error(struct aer_err_info *info, struct pci_dev *dev) +static void __print_tlp_header(struct pci_dev *dev, + struct aer_header_log_regs *t) { - int i, status; - char *errmsg = NULL; + unsigned char *tlp = (unsigned char *)&t; + + dev_err(&dev->dev, " TLP Header:" + " %02x%02x%02x%02x %02x%02x%02x%02x" + " %02x%02x%02x%02x %02x%02x%02x%02x\n", + *(tlp + 3), *(tlp + 2), *(tlp + 1), *tlp, + *(tlp + 7), *(tlp + 6), *(tlp + 5), *(tlp + 4), + *(tlp + 11), *(tlp + 10), *(tlp + 9), + *(tlp + 8), *(tlp + 15), *(tlp + 14), + *(tlp + 13), *(tlp + 12)); +} +static void __aer_print_error(struct pci_dev *dev, + struct aer_err_info *info) +{ + int i, status; + const char *errmsg = NULL; status = (info->status & ~info->mask); for (i = 0; i < 32; i++) { @@ -165,61 +151,54 @@ static void __aer_print_error(struct aer_err_info *info, struct pci_dev *dev) continue; if (info->severity == AER_CORRECTABLE) - errmsg = aer_correctable_error_string[i]; + errmsg = i < ARRAY_SIZE(aer_correctable_error_string) ? + aer_correctable_error_string[i] : NULL; else - errmsg = aer_uncorrectable_error_string[i]; + errmsg = i < ARRAY_SIZE(aer_uncorrectable_error_string) ? + aer_uncorrectable_error_string[i] : NULL; if (errmsg) - AER_PR(info, dev, " [%2d] %s%s\n", i, errmsg, + dev_err(&dev->dev, " [%2d] %-22s%s\n", i, errmsg, info->first_error == i ? " (First)" : ""); else - AER_PR(info, dev, " [%2d] Unknown Error Bit%s\n", i, - info->first_error == i ? " (First)" : ""); + dev_err(&dev->dev, " [%2d] Unknown Error Bit%s\n", + i, info->first_error == i ? " (First)" : ""); } } void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) { + int layer, agent; int id = ((dev->bus->number << 8) | dev->devfn); - if (info->status == 0) { - AER_PR(info, dev, - "PCIE Bus Error: severity=%s, type=Unaccessible, " - "id=%04x(Unregistered Agent ID)\n", + if (!info->status) { + dev_err(&dev->dev, "PCIe Bus Error: severity=%s, type=Unaccessible, id=%04x(Unregistered Agent ID)\n", aer_error_severity_string[info->severity], id); - } else { - int layer, agent; - - layer = AER_GET_LAYER_ERROR(info->severity, info->status); - agent = AER_GET_AGENT(info->severity, info->status); - - AER_PR(info, dev, - "PCIE Bus Error: severity=%s, type=%s, id=%04x(%s)\n", - aer_error_severity_string[info->severity], - aer_error_layer[layer], id, aer_agent_string[agent]); - - AER_PR(info, dev, - " device [%04x:%04x] error status/mask=%08x/%08x\n", - dev->vendor, dev->device, info->status, info->mask); - - __aer_print_error(info, dev); - - if (info->tlp_header_valid) { - unsigned char *tlp = (unsigned char *) &info->tlp; - AER_PR(info, dev, " TLP Header:" - " %02x%02x%02x%02x %02x%02x%02x%02x" - " %02x%02x%02x%02x %02x%02x%02x%02x\n", - *(tlp + 3), *(tlp + 2), *(tlp + 1), *tlp, - *(tlp + 7), *(tlp + 6), *(tlp + 5), *(tlp + 4), - *(tlp + 11), *(tlp + 10), *(tlp + 9), - *(tlp + 8), *(tlp + 15), *(tlp + 14), - *(tlp + 13), *(tlp + 12)); - } + goto out; } + layer = AER_GET_LAYER_ERROR(info->severity, info->status); + agent = AER_GET_AGENT(info->severity, info->status); + + dev_err(&dev->dev, "PCIe Bus Error: severity=%s, type=%s, id=%04x(%s)\n", + aer_error_severity_string[info->severity], + aer_error_layer[layer], id, aer_agent_string[agent]); + + dev_err(&dev->dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", + dev->vendor, dev->device, + info->status, info->mask); + + __aer_print_error(dev, info); + + if (info->tlp_header_valid) + __print_tlp_header(dev, &info->tlp); + +out: if (info->id && info->error_dev_num > 1 && info->id == id) - AER_PR(info, dev, - " Error of this Agent(%04x) is reported first\n", id); + dev_err(&dev->dev, " Error of this Agent(%04x) is reported first\n", id); + + trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask), + info->severity); } void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) @@ -228,3 +207,59 @@ void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) info->multi_error_valid ? "Multiple " : "", aer_error_severity_string[info->severity], info->id); } + +#ifdef CONFIG_ACPI_APEI_PCIEAER +int cper_severity_to_aer(int cper_severity) +{ + switch (cper_severity) { + case CPER_SEV_RECOVERABLE: + return AER_NONFATAL; + case CPER_SEV_FATAL: + return AER_FATAL; + default: + return AER_CORRECTABLE; + } +} +EXPORT_SYMBOL_GPL(cper_severity_to_aer); + +void cper_print_aer(struct pci_dev *dev, int cper_severity, + struct aer_capability_regs *aer) +{ + int aer_severity, layer, agent, status_strs_size, tlp_header_valid = 0; + u32 status, mask; + const char **status_strs; + + aer_severity = cper_severity_to_aer(cper_severity); + + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; + mask = aer->cor_mask; + status_strs = aer_correctable_error_string; + status_strs_size = ARRAY_SIZE(aer_correctable_error_string); + } else { + status = aer->uncor_status; + mask = aer->uncor_mask; + status_strs = aer_uncorrectable_error_string; + status_strs_size = ARRAY_SIZE(aer_uncorrectable_error_string); + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + + dev_err(&dev->dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask); + cper_print_bits("", status, status_strs, status_strs_size); + dev_err(&dev->dev, "aer_layer=%s, aer_agent=%s\n", + aer_error_layer[layer], aer_agent_string[agent]); + + if (aer_severity != AER_CORRECTABLE) + dev_err(&dev->dev, "aer_uncor_severity: 0x%08x\n", + aer->uncor_severity); + + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); + + trace_aer_event(dev_name(&dev->dev), (status & ~mask), + aer_severity); +} +#endif diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index 5a01fc7fbf0..e1e7026b838 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -1,6 +1,6 @@ /* * File: drivers/pci/pcie/aspm.c - * Enabling PCIE link L0s/L1 state and Clock Power Management + * Enabling PCIe link L0s/L1 state and Clock Power Management * * Copyright (C) 2007 Intel * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) @@ -69,13 +69,22 @@ struct pcie_link_state { }; static int aspm_disabled, aspm_force; +static bool aspm_support_enabled = true; static DEFINE_MUTEX(aspm_lock); static LIST_HEAD(link_list); #define POLICY_DEFAULT 0 /* BIOS default setting */ #define POLICY_PERFORMANCE 1 /* high performance */ #define POLICY_POWERSAVE 2 /* high power saving */ + +#ifdef CONFIG_PCIEASPM_PERFORMANCE +static int aspm_policy = POLICY_PERFORMANCE; +#elif defined CONFIG_PCIEASPM_POWERSAVE +static int aspm_policy = POLICY_POWERSAVE; +#else static int aspm_policy; +#endif + static const char *policy_str[] = { [POLICY_DEFAULT] = "default", [POLICY_PERFORMANCE] = "performance", @@ -116,21 +125,16 @@ static int policy_to_clkpm_state(struct pcie_link_state *link) static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) { - int pos; - u16 reg16; struct pci_dev *child; struct pci_bus *linkbus = link->pdev->subordinate; list_for_each_entry(child, &linkbus->devices, bus_list) { - pos = pci_pcie_cap(child); - if (!pos) - return; - pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); if (enable) - reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN; + pcie_capability_set_word(child, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN); else - reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN; - pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16); + pcie_capability_clear_word(child, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_CLKREQ_EN); } link->clkpm_enabled = !!enable; } @@ -139,7 +143,7 @@ static void pcie_set_clkpm(struct pcie_link_state *link, int enable) { /* Don't enable Clock PM if the link is not Clock PM capable */ if (!link->clkpm_capable && enable) - return; + enable = 0; /* Need nothing if the specified equals to current state */ if (link->clkpm_enabled == enable) return; @@ -148,7 +152,7 @@ static void pcie_set_clkpm(struct pcie_link_state *link, int enable) static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) { - int pos, capable = 1, enabled = 1; + int capable = 1, enabled = 1; u32 reg32; u16 reg16; struct pci_dev *child; @@ -156,16 +160,13 @@ static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) /* All functions should have the same cap and state, take the worst */ list_for_each_entry(child, &linkbus->devices, bus_list) { - pos = pci_pcie_cap(child); - if (!pos) - return; - pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32); + pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { capable = 0; enabled = 0; break; } - pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16); + pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) enabled = 0; } @@ -181,7 +182,7 @@ static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) */ static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) { - int ppos, cpos, same_clock = 1; + int same_clock = 1; u16 reg16, parent_reg, child_reg[8]; unsigned long start_jiffies; struct pci_dev *child, *parent = link->pdev; @@ -194,46 +195,43 @@ static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) BUG_ON(!pci_is_pcie(child)); /* Check downstream component if bit Slot Clock Configuration is 1 */ - cpos = pci_pcie_cap(child); - pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16); + pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); if (!(reg16 & PCI_EXP_LNKSTA_SLC)) same_clock = 0; /* Check upstream component if bit Slot Clock Configuration is 1 */ - ppos = pci_pcie_cap(parent); - pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); + pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); if (!(reg16 & PCI_EXP_LNKSTA_SLC)) same_clock = 0; /* Configure downstream component, all functions */ list_for_each_entry(child, &linkbus->devices, bus_list) { - cpos = pci_pcie_cap(child); - pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16); + pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); child_reg[PCI_FUNC(child->devfn)] = reg16; if (same_clock) reg16 |= PCI_EXP_LNKCTL_CCC; else reg16 &= ~PCI_EXP_LNKCTL_CCC; - pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16); + pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); } /* Configure upstream component */ - pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16); + pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); parent_reg = reg16; if (same_clock) reg16 |= PCI_EXP_LNKCTL_CCC; else reg16 &= ~PCI_EXP_LNKCTL_CCC; - pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); /* Retrain link */ reg16 |= PCI_EXP_LNKCTL_RL; - pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); /* Wait for link training end. Break out after waiting for timeout */ start_jiffies = jiffies; for (;;) { - pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16); + pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); if (!(reg16 & PCI_EXP_LNKSTA_LT)) break; if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) @@ -244,14 +242,11 @@ static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) return; /* Training failed. Restore common clock configurations */ - dev_printk(KERN_ERR, &parent->dev, - "ASPM: Could not configure common clock\n"); - list_for_each_entry(child, &linkbus->devices, bus_list) { - cpos = pci_pcie_cap(child); - pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, - child_reg[PCI_FUNC(child->devfn)]); - } - pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg); + dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); + list_for_each_entry(child, &linkbus->devices, bus_list) + pcie_capability_write_word(child, PCI_EXP_LNKCTL, + child_reg[PCI_FUNC(child->devfn)]); + pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); } /* Convert L0s latency encoding to ns */ @@ -296,16 +291,14 @@ struct aspm_register_info { static void pcie_get_aspm_reg(struct pci_dev *pdev, struct aspm_register_info *info) { - int pos; u16 reg16; u32 reg32; - pos = pci_pcie_cap(pdev); - pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32); + pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; - pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); + pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; } @@ -403,7 +396,7 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) * do ASPM for now. */ list_for_each_entry(child, &linkbus->devices, bus_list) { - if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) { + if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { link->aspm_disable = ASPM_STATE_ALL; break; } @@ -411,17 +404,15 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Get and check endpoint acceptable latencies */ list_for_each_entry(child, &linkbus->devices, bus_list) { - int pos; u32 reg32, encoding; struct aspm_latency *acceptable = &link->acceptable[PCI_FUNC(child->devfn)]; - if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT && - child->pcie_type != PCI_EXP_TYPE_LEG_END) + if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && + pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) continue; - pos = pci_pcie_cap(child); - pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); + pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); /* Calculate endpoint L0s acceptable latency */ encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; acceptable->l0s = calc_l0s_acceptable(encoding); @@ -435,13 +426,8 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) { - u16 reg16; - int pos = pci_pcie_cap(pdev); - - pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16); - reg16 &= ~0x3; - reg16 |= val; - pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16); + pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, val); } static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) @@ -456,12 +442,12 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) return; /* Convert ASPM state to upstream/downstream ASPM register state */ if (state & ASPM_STATE_L0S_UP) - dwstream |= PCIE_LINK_STATE_L0S; + dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; if (state & ASPM_STATE_L0S_DW) - upstream |= PCIE_LINK_STATE_L0S; + upstream |= PCI_EXP_LNKCTL_ASPM_L0S; if (state & ASPM_STATE_L1) { - upstream |= PCIE_LINK_STATE_L1; - dwstream |= PCIE_LINK_STATE_L1; + upstream |= PCI_EXP_LNKCTL_ASPM_L1; + dwstream |= PCI_EXP_LNKCTL_ASPM_L1; } /* * Spec 2.0 suggests all functions should be configured the @@ -496,25 +482,32 @@ static void free_link_state(struct pcie_link_state *link) static int pcie_aspm_sanity_check(struct pci_dev *pdev) { struct pci_dev *child; - int pos; u32 reg32; + /* - * Some functions in a slot might not all be PCIE functions, + * Some functions in a slot might not all be PCIe functions, * very strange. Disable ASPM for the whole slot */ list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { - pos = pci_pcie_cap(child); - if (!pos) + if (!pci_is_pcie(child)) return -EINVAL; + + /* + * If ASPM is disabled then we're not going to change + * the BIOS state. It's safe to continue even if it's a + * pre-1.1 device + */ + + if (aspm_disabled) + continue; + /* * Disable ASPM for pre-1.1 PCIe device, we follow MS to use * RBER bit to determine if a function is 1.1 version device */ - pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32); + pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { - dev_printk(KERN_INFO, &child->dev, "disabling ASPM" - " on pre-1.1 PCIe device. You can enable it" - " with 'pcie_aspm=force'\n"); + dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); return -EINVAL; } } @@ -532,7 +525,7 @@ static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) INIT_LIST_HEAD(&link->children); INIT_LIST_HEAD(&link->link); link->pdev = pdev; - if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) { struct pcie_link_state *parent; parent = pdev->bus->parent->self->link_state; if (!parent) { @@ -555,7 +548,7 @@ static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) /* * pcie_aspm_init_link_state: Initiate PCI express link state. - * It is called after the pcie and its children devices are scaned. + * It is called after the pcie and its children devices are scanned. * @pdev: the root port or switch downstream port */ void pcie_aspm_init_link_state(struct pci_dev *pdev) @@ -563,14 +556,17 @@ void pcie_aspm_init_link_state(struct pci_dev *pdev) struct pcie_link_state *link; int blacklist = !!pcie_aspm_sanity_check(pdev); - if (aspm_disabled || !pci_is_pcie(pdev) || pdev->link_state) + if (!aspm_support_enabled) + return; + + if (!pci_is_pcie(pdev) || pdev->link_state) return; - if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && - pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) return; /* VIA has a strange chipset, root port is under a bridge */ - if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT && + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && pdev->bus->self) return; @@ -588,11 +584,23 @@ void pcie_aspm_init_link_state(struct pci_dev *pdev) * update through pcie_aspm_cap_init(). */ pcie_aspm_cap_init(link, blacklist); - pcie_config_aspm_path(link); /* Setup initial Clock PM state */ pcie_clkpm_cap_init(link, blacklist); - pcie_set_clkpm(link, policy_to_clkpm_state(link)); + + /* + * At this stage drivers haven't had an opportunity to change the + * link policy setting. Enabling ASPM on broken hardware can cripple + * it even before the driver has had a chance to disable ASPM, so + * default to a safe level right now. If we're enabling ASPM beyond + * the BIOS's expectation, we'll do so once pci_enable_device() is + * called. + */ + if (aspm_policy != POLICY_POWERSAVE) { + pcie_config_aspm_path(link); + pcie_set_clkpm(link, policy_to_clkpm_state(link)); + } + unlock: mutex_unlock(&aspm_lock); out: @@ -615,8 +623,8 @@ static void pcie_update_aspm_capable(struct pcie_link_state *root) if (link->root != root) continue; list_for_each_entry(child, &linkbus->devices, bus_list) { - if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) && - (child->pcie_type != PCI_EXP_TYPE_LEG_END)) + if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && + (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) continue; pcie_aspm_check_latency(child); } @@ -629,11 +637,7 @@ void pcie_aspm_exit_link_state(struct pci_dev *pdev) struct pci_dev *parent = pdev->bus->self; struct pcie_link_state *link, *root, *parent_link; - if (aspm_disabled || !pci_is_pcie(pdev) || - !parent || !parent->link_state) - return; - if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && - (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) + if (!parent || !parent->link_state) return; down_read(&pci_bus_sem); @@ -673,8 +677,8 @@ void pcie_aspm_pm_state_change(struct pci_dev *pdev) if (aspm_disabled || !pci_is_pcie(pdev) || !link) return; - if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && - (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)) + if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) return; /* * Devices changed PM state, we should recheck if latency @@ -688,24 +692,58 @@ void pcie_aspm_pm_state_change(struct pci_dev *pdev) up_read(&pci_bus_sem); } -/* - * pci_disable_link_state - disable pci device's link state, so the link will - * never enter specific states - */ -void pci_disable_link_state(struct pci_dev *pdev, int state) +void pcie_aspm_powersave_config_link(struct pci_dev *pdev) +{ + struct pcie_link_state *link = pdev->link_state; + + if (aspm_disabled || !pci_is_pcie(pdev) || !link) + return; + + if (aspm_policy != POLICY_POWERSAVE) + return; + + if ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM)) + return; + + down_read(&pci_bus_sem); + mutex_lock(&aspm_lock); + pcie_config_aspm_path(link); + pcie_set_clkpm(link, policy_to_clkpm_state(link)); + mutex_unlock(&aspm_lock); + up_read(&pci_bus_sem); +} + +static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem, + bool force) { struct pci_dev *parent = pdev->bus->self; struct pcie_link_state *link; - if (aspm_disabled || !pci_is_pcie(pdev)) + if (!pci_is_pcie(pdev)) return; - if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT || - pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) + + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || + pci_pcie_type(pdev) == PCI_EXP_TYPE_DOWNSTREAM) parent = pdev; if (!parent || !parent->link_state) return; - down_read(&pci_bus_sem); + /* + * A driver requested that ASPM be disabled on this device, but + * if we don't have permission to manage ASPM (e.g., on ACPI + * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and + * the _OSC method), we can't honor that request. Windows has + * a similar mechanism using "PciASPMOptOut", which is also + * ignored in this situation. + */ + if (aspm_disabled && !force) { + dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); + return; + } + + if (sem) + down_read(&pci_bus_sem); mutex_lock(&aspm_lock); link = parent->link_state; if (state & PCIE_LINK_STATE_L0S) @@ -719,15 +757,56 @@ void pci_disable_link_state(struct pci_dev *pdev, int state) pcie_set_clkpm(link, 0); } mutex_unlock(&aspm_lock); - up_read(&pci_bus_sem); + if (sem) + up_read(&pci_bus_sem); +} + +void pci_disable_link_state_locked(struct pci_dev *pdev, int state) +{ + __pci_disable_link_state(pdev, state, false, false); +} +EXPORT_SYMBOL(pci_disable_link_state_locked); + +/** + * pci_disable_link_state - Disable device's link state, so the link will + * never enter specific states. Note that if the BIOS didn't grant ASPM + * control to the OS, this does nothing because we can't touch the LNKCTL + * register. + * + * @pdev: PCI device + * @state: ASPM link state to disable + */ +void pci_disable_link_state(struct pci_dev *pdev, int state) +{ + __pci_disable_link_state(pdev, state, true, false); } EXPORT_SYMBOL(pci_disable_link_state); +void pcie_clear_aspm(struct pci_bus *bus) +{ + struct pci_dev *child; + + if (aspm_force) + return; + + /* + * Clear any ASPM setup that the firmware has carried out on this bus + */ + list_for_each_entry(child, &bus->devices, bus_list) { + __pci_disable_link_state(child, PCIE_LINK_STATE_L0S | + PCIE_LINK_STATE_L1 | + PCIE_LINK_STATE_CLKPM, + false, true); + } +} + static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) { int i; struct pcie_link_state *link; + if (aspm_disabled) + return -EPERM; for (i = 0; i < ARRAY_SIZE(policy_str); i++) if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) break; @@ -782,6 +861,8 @@ static ssize_t link_state_store(struct device *dev, struct pcie_link_state *link, *root = pdev->link_state->root; u32 val = buf[0] - '0', state = 0; + if (aspm_disabled) + return -EPERM; if (n < 1 || val > 3) return -EINVAL; @@ -843,8 +924,8 @@ void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) struct pcie_link_state *link_state = pdev->link_state; if (!pci_is_pcie(pdev) || - (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && - pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) + (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) return; if (link_state->aspm_support) @@ -860,8 +941,8 @@ void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) struct pcie_link_state *link_state = pdev->link_state; if (!pci_is_pcie(pdev) || - (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT && - pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) + (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT && + pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM) || !link_state) return; if (link_state->aspm_support) @@ -876,11 +957,13 @@ void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) static int __init pcie_aspm_disable(char *str) { if (!strcmp(str, "off")) { + aspm_policy = POLICY_DEFAULT; aspm_disabled = 1; + aspm_support_enabled = false; printk(KERN_INFO "PCIe ASPM is disabled\n"); } else if (!strcmp(str, "force")) { aspm_force = 1; - printk(KERN_INFO "PCIe ASPM is forcedly enabled\n"); + printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); } return 1; } @@ -889,19 +972,20 @@ __setup("pcie_aspm=", pcie_aspm_disable); void pcie_no_aspm(void) { - if (!aspm_force) + /* + * Disabling ASPM is intended to prevent the kernel from modifying + * existing hardware state, not to clear existing state. To that end: + * (a) set policy to POLICY_DEFAULT in order to avoid changing state + * (b) prevent userspace from changing policy + */ + if (!aspm_force) { + aspm_policy = POLICY_DEFAULT; aspm_disabled = 1; + } } -/** - * pcie_aspm_enabled - is PCIe ASPM enabled? - * - * Returns true if ASPM has not been disabled by the command-line option - * pcie_aspm=off. - **/ -int pcie_aspm_enabled(void) +bool pcie_aspm_support_enabled(void) { - return !aspm_disabled; + return aspm_support_enabled; } -EXPORT_SYMBOL(pcie_aspm_enabled); - +EXPORT_SYMBOL(pcie_aspm_support_enabled); diff --git a/drivers/pci/pcie/pme.c b/drivers/pci/pcie/pme.c new file mode 100644 index 00000000000..82e06a86cd7 --- /dev/null +++ b/drivers/pci/pcie/pme.c @@ -0,0 +1,438 @@ +/* + * PCIe Native PME support + * + * Copyright (C) 2007 - 2009 Intel Corp + * Copyright (C) 2007 - 2009 Shaohua Li <shaohua.li@intel.com> + * Copyright (C) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. + * + * This file is subject to the terms and conditions of the GNU General Public + * License V2. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/slab.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/device.h> +#include <linux/pcieport_if.h> +#include <linux/pm_runtime.h> + +#include "../pci.h" +#include "portdrv.h" + +/* + * If this switch is set, MSI will not be used for PCIe PME signaling. This + * causes the PCIe port driver to use INTx interrupts only, but it turns out + * that using MSI for PCIe PME signaling doesn't play well with PCIe PME-based + * wake-up from system sleep states. + */ +bool pcie_pme_msi_disabled; + +static int __init pcie_pme_setup(char *str) +{ + if (!strncmp(str, "nomsi", 5)) + pcie_pme_msi_disabled = true; + + return 1; +} +__setup("pcie_pme=", pcie_pme_setup); + +struct pcie_pme_service_data { + spinlock_t lock; + struct pcie_device *srv; + struct work_struct work; + bool noirq; /* Don't enable the PME interrupt used by this service. */ +}; + +/** + * pcie_pme_interrupt_enable - Enable/disable PCIe PME interrupt generation. + * @dev: PCIe root port or event collector. + * @enable: Enable or disable the interrupt. + */ +void pcie_pme_interrupt_enable(struct pci_dev *dev, bool enable) +{ + if (enable) + pcie_capability_set_word(dev, PCI_EXP_RTCTL, + PCI_EXP_RTCTL_PMEIE); + else + pcie_capability_clear_word(dev, PCI_EXP_RTCTL, + PCI_EXP_RTCTL_PMEIE); +} + +/** + * pcie_pme_walk_bus - Scan a PCI bus for devices asserting PME#. + * @bus: PCI bus to scan. + * + * Scan given PCI bus and all buses under it for devices asserting PME#. + */ +static bool pcie_pme_walk_bus(struct pci_bus *bus) +{ + struct pci_dev *dev; + bool ret = false; + + list_for_each_entry(dev, &bus->devices, bus_list) { + /* Skip PCIe devices in case we started from a root port. */ + if (!pci_is_pcie(dev) && pci_check_pme_status(dev)) { + if (dev->pme_poll) + dev->pme_poll = false; + + pci_wakeup_event(dev); + pm_request_resume(&dev->dev); + ret = true; + } + + if (dev->subordinate && pcie_pme_walk_bus(dev->subordinate)) + ret = true; + } + + return ret; +} + +/** + * pcie_pme_from_pci_bridge - Check if PCIe-PCI bridge generated a PME. + * @bus: Secondary bus of the bridge. + * @devfn: Device/function number to check. + * + * PME from PCI devices under a PCIe-PCI bridge may be converted to an in-band + * PCIe PME message. In such that case the bridge should use the Requester ID + * of device/function number 0 on its secondary bus. + */ +static bool pcie_pme_from_pci_bridge(struct pci_bus *bus, u8 devfn) +{ + struct pci_dev *dev; + bool found = false; + + if (devfn) + return false; + + dev = pci_dev_get(bus->self); + if (!dev) + return false; + + if (pci_is_pcie(dev) && pci_pcie_type(dev) == PCI_EXP_TYPE_PCI_BRIDGE) { + down_read(&pci_bus_sem); + if (pcie_pme_walk_bus(bus)) + found = true; + up_read(&pci_bus_sem); + } + + pci_dev_put(dev); + return found; +} + +/** + * pcie_pme_handle_request - Find device that generated PME and handle it. + * @port: Root port or event collector that generated the PME interrupt. + * @req_id: PCIe Requester ID of the device that generated the PME. + */ +static void pcie_pme_handle_request(struct pci_dev *port, u16 req_id) +{ + u8 busnr = req_id >> 8, devfn = req_id & 0xff; + struct pci_bus *bus; + struct pci_dev *dev; + bool found = false; + + /* First, check if the PME is from the root port itself. */ + if (port->devfn == devfn && port->bus->number == busnr) { + if (port->pme_poll) + port->pme_poll = false; + + if (pci_check_pme_status(port)) { + pm_request_resume(&port->dev); + found = true; + } else { + /* + * Apparently, the root port generated the PME on behalf + * of a non-PCIe device downstream. If this is done by + * a root port, the Requester ID field in its status + * register may contain either the root port's, or the + * source device's information (PCI Express Base + * Specification, Rev. 2.0, Section 6.1.9). + */ + down_read(&pci_bus_sem); + found = pcie_pme_walk_bus(port->subordinate); + up_read(&pci_bus_sem); + } + goto out; + } + + /* Second, find the bus the source device is on. */ + bus = pci_find_bus(pci_domain_nr(port->bus), busnr); + if (!bus) + goto out; + + /* Next, check if the PME is from a PCIe-PCI bridge. */ + found = pcie_pme_from_pci_bridge(bus, devfn); + if (found) + goto out; + + /* Finally, try to find the PME source on the bus. */ + down_read(&pci_bus_sem); + list_for_each_entry(dev, &bus->devices, bus_list) { + pci_dev_get(dev); + if (dev->devfn == devfn) { + found = true; + break; + } + pci_dev_put(dev); + } + up_read(&pci_bus_sem); + + if (found) { + /* The device is there, but we have to check its PME status. */ + found = pci_check_pme_status(dev); + if (found) { + if (dev->pme_poll) + dev->pme_poll = false; + + pci_wakeup_event(dev); + pm_request_resume(&dev->dev); + } + pci_dev_put(dev); + } else if (devfn) { + /* + * The device is not there, but we can still try to recover by + * assuming that the PME was reported by a PCIe-PCI bridge that + * used devfn different from zero. + */ + dev_dbg(&port->dev, "PME interrupt generated for non-existent device %02x:%02x.%d\n", + busnr, PCI_SLOT(devfn), PCI_FUNC(devfn)); + found = pcie_pme_from_pci_bridge(bus, 0); + } + + out: + if (!found) + dev_dbg(&port->dev, "Spurious native PME interrupt!\n"); +} + +/** + * pcie_pme_work_fn - Work handler for PCIe PME interrupt. + * @work: Work structure giving access to service data. + */ +static void pcie_pme_work_fn(struct work_struct *work) +{ + struct pcie_pme_service_data *data = + container_of(work, struct pcie_pme_service_data, work); + struct pci_dev *port = data->srv->port; + u32 rtsta; + + spin_lock_irq(&data->lock); + + for (;;) { + if (data->noirq) + break; + + pcie_capability_read_dword(port, PCI_EXP_RTSTA, &rtsta); + if (rtsta & PCI_EXP_RTSTA_PME) { + /* + * Clear PME status of the port. If there are other + * pending PMEs, the status will be set again. + */ + pcie_clear_root_pme_status(port); + + spin_unlock_irq(&data->lock); + pcie_pme_handle_request(port, rtsta & 0xffff); + spin_lock_irq(&data->lock); + + continue; + } + + /* No need to loop if there are no more PMEs pending. */ + if (!(rtsta & PCI_EXP_RTSTA_PENDING)) + break; + + spin_unlock_irq(&data->lock); + cpu_relax(); + spin_lock_irq(&data->lock); + } + + if (!data->noirq) + pcie_pme_interrupt_enable(port, true); + + spin_unlock_irq(&data->lock); +} + +/** + * pcie_pme_irq - Interrupt handler for PCIe root port PME interrupt. + * @irq: Interrupt vector. + * @context: Interrupt context pointer. + */ +static irqreturn_t pcie_pme_irq(int irq, void *context) +{ + struct pci_dev *port; + struct pcie_pme_service_data *data; + u32 rtsta; + unsigned long flags; + + port = ((struct pcie_device *)context)->port; + data = get_service_data((struct pcie_device *)context); + + spin_lock_irqsave(&data->lock, flags); + pcie_capability_read_dword(port, PCI_EXP_RTSTA, &rtsta); + + if (!(rtsta & PCI_EXP_RTSTA_PME)) { + spin_unlock_irqrestore(&data->lock, flags); + return IRQ_NONE; + } + + pcie_pme_interrupt_enable(port, false); + spin_unlock_irqrestore(&data->lock, flags); + + /* We don't use pm_wq, because it's freezable. */ + schedule_work(&data->work); + + return IRQ_HANDLED; +} + +/** + * pcie_pme_set_native - Set the PME interrupt flag for given device. + * @dev: PCI device to handle. + * @ign: Ignored. + */ +static int pcie_pme_set_native(struct pci_dev *dev, void *ign) +{ + dev_info(&dev->dev, "Signaling PME through PCIe PME interrupt\n"); + + device_set_run_wake(&dev->dev, true); + dev->pme_interrupt = true; + return 0; +} + +/** + * pcie_pme_mark_devices - Set the PME interrupt flag for devices below a port. + * @port: PCIe root port or event collector to handle. + * + * For each device below given root port, including the port itself (or for each + * root complex integrated endpoint if @port is a root complex event collector) + * set the flag indicating that it can signal run-time wake-up events via PCIe + * PME interrupts. + */ +static void pcie_pme_mark_devices(struct pci_dev *port) +{ + pcie_pme_set_native(port, NULL); + if (port->subordinate) { + pci_walk_bus(port->subordinate, pcie_pme_set_native, NULL); + } else { + struct pci_bus *bus = port->bus; + struct pci_dev *dev; + + /* Check if this is a root port event collector. */ + if (pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC || !bus) + return; + + down_read(&pci_bus_sem); + list_for_each_entry(dev, &bus->devices, bus_list) + if (pci_is_pcie(dev) + && pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) + pcie_pme_set_native(dev, NULL); + up_read(&pci_bus_sem); + } +} + +/** + * pcie_pme_probe - Initialize PCIe PME service for given root port. + * @srv: PCIe service to initialize. + */ +static int pcie_pme_probe(struct pcie_device *srv) +{ + struct pci_dev *port; + struct pcie_pme_service_data *data; + int ret; + + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + spin_lock_init(&data->lock); + INIT_WORK(&data->work, pcie_pme_work_fn); + data->srv = srv; + set_service_data(srv, data); + + port = srv->port; + pcie_pme_interrupt_enable(port, false); + pcie_clear_root_pme_status(port); + + ret = request_irq(srv->irq, pcie_pme_irq, IRQF_SHARED, "PCIe PME", srv); + if (ret) { + kfree(data); + } else { + pcie_pme_mark_devices(port); + pcie_pme_interrupt_enable(port, true); + } + + return ret; +} + +/** + * pcie_pme_suspend - Suspend PCIe PME service device. + * @srv: PCIe service device to suspend. + */ +static int pcie_pme_suspend(struct pcie_device *srv) +{ + struct pcie_pme_service_data *data = get_service_data(srv); + struct pci_dev *port = srv->port; + + spin_lock_irq(&data->lock); + pcie_pme_interrupt_enable(port, false); + pcie_clear_root_pme_status(port); + data->noirq = true; + spin_unlock_irq(&data->lock); + + synchronize_irq(srv->irq); + + return 0; +} + +/** + * pcie_pme_resume - Resume PCIe PME service device. + * @srv - PCIe service device to resume. + */ +static int pcie_pme_resume(struct pcie_device *srv) +{ + struct pcie_pme_service_data *data = get_service_data(srv); + struct pci_dev *port = srv->port; + + spin_lock_irq(&data->lock); + data->noirq = false; + pcie_clear_root_pme_status(port); + pcie_pme_interrupt_enable(port, true); + spin_unlock_irq(&data->lock); + + return 0; +} + +/** + * pcie_pme_remove - Prepare PCIe PME service device for removal. + * @srv - PCIe service device to remove. + */ +static void pcie_pme_remove(struct pcie_device *srv) +{ + pcie_pme_suspend(srv); + free_irq(srv->irq, srv); + kfree(get_service_data(srv)); +} + +static struct pcie_port_service_driver pcie_pme_driver = { + .name = "pcie_pme", + .port_type = PCI_EXP_TYPE_ROOT_PORT, + .service = PCIE_PORT_SERVICE_PME, + + .probe = pcie_pme_probe, + .suspend = pcie_pme_suspend, + .resume = pcie_pme_resume, + .remove = pcie_pme_remove, +}; + +/** + * pcie_pme_service_init - Register the PCIe PME service driver. + */ +static int __init pcie_pme_service_init(void) +{ + return pcie_port_service_register(&pcie_pme_driver); +} + +module_init(pcie_pme_service_init); diff --git a/drivers/pci/pcie/portdrv.h b/drivers/pci/pcie/portdrv.h index aaeb9d21cba..d525548404d 100644 --- a/drivers/pci/pcie/portdrv.h +++ b/drivers/pci/pcie/portdrv.h @@ -14,20 +14,70 @@ #define PCIE_PORT_DEVICE_MAXSERVICES 4 /* * According to the PCI Express Base Specification 2.0, the indices of - * the MSI-X table entires used by port services must not exceed 31 + * the MSI-X table entries used by port services must not exceed 31 */ #define PCIE_PORT_MAX_MSIX_ENTRIES 32 #define get_descriptor_id(type, service) (((type - 4) << 4) | service) extern struct bus_type pcie_port_bus_type; -extern int pcie_port_device_register(struct pci_dev *dev); +int pcie_port_device_register(struct pci_dev *dev); #ifdef CONFIG_PM -extern int pcie_port_device_suspend(struct device *dev); -extern int pcie_port_device_resume(struct device *dev); +int pcie_port_device_suspend(struct device *dev); +int pcie_port_device_resume(struct device *dev); #endif -extern void pcie_port_device_remove(struct pci_dev *dev); -extern int __must_check pcie_port_bus_register(void); -extern void pcie_port_bus_unregister(void); +void pcie_port_device_remove(struct pci_dev *dev); +int __must_check pcie_port_bus_register(void); +void pcie_port_bus_unregister(void); + +struct pci_dev; + +void pcie_clear_root_pme_status(struct pci_dev *dev); + +#ifdef CONFIG_HOTPLUG_PCI_PCIE +extern bool pciehp_msi_disabled; + +static inline bool pciehp_no_msi(void) +{ + return pciehp_msi_disabled; +} + +#else /* !CONFIG_HOTPLUG_PCI_PCIE */ +static inline bool pciehp_no_msi(void) { return false; } +#endif /* !CONFIG_HOTPLUG_PCI_PCIE */ + +#ifdef CONFIG_PCIE_PME +extern bool pcie_pme_msi_disabled; + +static inline void pcie_pme_disable_msi(void) +{ + pcie_pme_msi_disabled = true; +} + +static inline bool pcie_pme_no_msi(void) +{ + return pcie_pme_msi_disabled; +} + +void pcie_pme_interrupt_enable(struct pci_dev *dev, bool enable); +#else /* !CONFIG_PCIE_PME */ +static inline void pcie_pme_disable_msi(void) {} +static inline bool pcie_pme_no_msi(void) { return false; } +static inline void pcie_pme_interrupt_enable(struct pci_dev *dev, bool en) {} +#endif /* !CONFIG_PCIE_PME */ + +#ifdef CONFIG_ACPI +int pcie_port_acpi_setup(struct pci_dev *port, int *mask); + +static inline int pcie_port_platform_notify(struct pci_dev *port, int *mask) +{ + return pcie_port_acpi_setup(port, mask); +} +#else /* !CONFIG_ACPI */ +static inline int pcie_port_platform_notify(struct pci_dev *port, int *mask) +{ + return 0; +} +#endif /* !CONFIG_ACPI */ #endif /* _PORTDRV_H_ */ diff --git a/drivers/pci/pcie/portdrv_acpi.c b/drivers/pci/pcie/portdrv_acpi.c new file mode 100644 index 00000000000..b4d2894ee3f --- /dev/null +++ b/drivers/pci/pcie/portdrv_acpi.c @@ -0,0 +1,63 @@ +/* + * PCIe Port Native Services Support, ACPI-Related Part + * + * Copyright (C) 2010 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. + * + * This file is subject to the terms and conditions of the GNU General Public + * License V2. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#include <linux/pci.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/acpi.h> +#include <linux/pci-acpi.h> +#include <linux/pcieport_if.h> + +#include "aer/aerdrv.h" +#include "../pci.h" +#include "portdrv.h" + +/** + * pcie_port_acpi_setup - Request the BIOS to release control of PCIe services. + * @port: PCIe Port service for a root port or event collector. + * @srv_mask: Bit mask of services that can be enabled for @port. + * + * Invoked when @port is identified as a PCIe port device. To avoid conflicts + * with the BIOS PCIe port native services support requires the BIOS to yield + * control of these services to the kernel. The mask of services that the BIOS + * allows to be enabled for @port is written to @srv_mask. + * + * NOTE: It turns out that we cannot do that for individual port services + * separately, because that would make some systems work incorrectly. + */ +int pcie_port_acpi_setup(struct pci_dev *port, int *srv_mask) +{ + struct acpi_pci_root *root; + acpi_handle handle; + u32 flags; + + if (acpi_pci_disabled) + return 0; + + handle = acpi_find_root_bridge_handle(port); + if (!handle) + return -EINVAL; + + root = acpi_pci_find_root(handle); + if (!root) + return -ENODEV; + + flags = root->osc_control_set; + + *srv_mask = PCIE_PORT_SERVICE_VC; + if (flags & OSC_PCI_EXPRESS_NATIVE_HP_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_HP; + if (flags & OSC_PCI_EXPRESS_PME_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_PME; + if (flags & OSC_PCI_EXPRESS_AER_CONTROL) + *srv_mask |= PCIE_PORT_SERVICE_AER; + + return 0; +} diff --git a/drivers/pci/pcie/portdrv_bus.c b/drivers/pci/pcie/portdrv_bus.c index 18bf90f748f..87e79a6ffb5 100644 --- a/drivers/pci/pcie/portdrv_bus.c +++ b/drivers/pci/pcie/portdrv_bus.c @@ -18,8 +18,8 @@ static int pcie_port_bus_match(struct device *dev, struct device_driver *drv); struct bus_type pcie_port_bus_type = { - .name = "pci_express", - .match = pcie_port_bus_match, + .name = "pci_express", + .match = pcie_port_bus_match, }; EXPORT_SYMBOL_GPL(pcie_port_bus_type); @@ -38,7 +38,7 @@ static int pcie_port_bus_match(struct device *dev, struct device_driver *drv) return 0; if ((driver->port_type != PCIE_ANY_PORT) && - (driver->port_type != pciedev->port->pcie_type)) + (driver->port_type != pci_pcie_type(pciedev->port))) return 0; return 1; diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c index 413262eb95b..2f0ce668a77 100644 --- a/drivers/pci/pcie/portdrv_core.c +++ b/drivers/pci/pcie/portdrv_core.c @@ -14,10 +14,22 @@ #include <linux/string.h> #include <linux/slab.h> #include <linux/pcieport_if.h> +#include <linux/aer.h> #include "../pci.h" #include "portdrv.h" +bool pciehp_msi_disabled; + +static int __init pciehp_setup(char *str) +{ + if (!strncmp(str, "nomsi", 5)) + pciehp_msi_disabled = true; + + return 1; +} +__setup("pcie_hp=", pciehp_setup); + /** * release_pcie_device - free PCI Express port service device structure * @dev: Port service device to release @@ -27,14 +39,14 @@ */ static void release_pcie_device(struct device *dev) { - kfree(to_pcie_device(dev)); + kfree(to_pcie_device(dev)); } /** * pcie_port_msix_add_entry - add entry to given array of MSI-X entries * @entries: Array of MSI-X entries * @new_entry: Index of the entry to add to the array - * @nr_entries: Number of entries aleady in the array + * @nr_entries: Number of entries already in the array * * Return value: Position of the added entry in the array */ @@ -67,9 +79,10 @@ static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) u16 reg16; u32 reg32; - nr_entries = pci_msix_table_size(dev); - if (!nr_entries) - return -EINVAL; + nr_entries = pci_msix_vec_count(dev); + if (nr_entries < 0) + return nr_entries; + BUG_ON(!nr_entries); if (nr_entries > PCIE_PORT_MAX_MSIX_ENTRIES) nr_entries = PCIE_PORT_MAX_MSIX_ENTRIES; @@ -86,7 +99,7 @@ static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) for (i = 0; i < nr_entries; i++) msix_entries[i].entry = i; - status = pci_enable_msix(dev, msix_entries, nr_entries); + status = pci_enable_msix_exact(dev, msix_entries, nr_entries); if (status) goto Exit; @@ -108,8 +121,7 @@ static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) * the value in this field indicates which MSI-X Table entry is * used to generate the interrupt message." */ - pos = pci_pcie_cap(dev); - pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16); + pcie_capability_read_word(dev, PCI_EXP_FLAGS, ®16); entry = (reg16 & PCI_EXP_FLAGS_IRQ) >> 9; if (entry >= nr_entries) goto Error; @@ -159,7 +171,7 @@ static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) pci_disable_msix(dev); /* Now allocate the MSI-X vectors for real */ - status = pci_enable_msix(dev, msix_entries, nvec); + status = pci_enable_msix_exact(dev, msix_entries, nvec); if (status) goto Exit; } @@ -186,16 +198,32 @@ static int pcie_port_enable_msix(struct pci_dev *dev, int *vectors, int mask) */ static int init_service_irqs(struct pci_dev *dev, int *irqs, int mask) { - int i, irq; + int i, irq = -1; + + /* + * If MSI cannot be used for PCIe PME or hotplug, we have to use + * INTx or other interrupts, e.g. system shared interrupt. + */ + if (((mask & PCIE_PORT_SERVICE_PME) && pcie_pme_no_msi()) || + ((mask & PCIE_PORT_SERVICE_HP) && pciehp_no_msi())) { + if (dev->irq) + irq = dev->irq; + goto no_msi; + } /* Try to use MSI-X if supported */ if (!pcie_port_enable_msix(dev, irqs, mask)) return 0; - /* We're not going to use MSI-X, so try MSI and fall back to INTx */ - irq = -1; - if (!pci_enable_msi(dev) || dev->pin) + + /* + * We're not going to use MSI-X, so try MSI and fall back to INTx. + * If neither MSI/MSI-X nor INTx available, try other interrupt. On + * some platforms, root port doesn't support MSI/MSI-X/INTx in RC mode. + */ + if (!pci_enable_msi(dev) || dev->irq) irq = dev->irq; + no_msi: for (i = 0; i < PCIE_PORT_DEVICE_MAXSERVICES; i++) irqs[i] = irq; irqs[PCIE_PORT_SERVICE_VC_SHIFT] = -1; @@ -225,27 +253,64 @@ static void cleanup_service_irqs(struct pci_dev *dev) */ static int get_port_device_capability(struct pci_dev *dev) { - int services = 0, pos; - u16 reg16; + int services = 0; u32 reg32; + int cap_mask = 0; + int err; + + if (pcie_ports_disabled) + return 0; + + cap_mask = PCIE_PORT_SERVICE_PME | PCIE_PORT_SERVICE_HP + | PCIE_PORT_SERVICE_VC; + if (pci_aer_available()) + cap_mask |= PCIE_PORT_SERVICE_AER; + + if (pcie_ports_auto) { + err = pcie_port_platform_notify(dev, &cap_mask); + if (err) + return 0; + } - pos = pci_pcie_cap(dev); - pci_read_config_word(dev, pos + PCI_EXP_FLAGS, ®16); /* Hot-Plug Capable */ - if (reg16 & PCI_EXP_FLAGS_SLOT) { - pci_read_config_dword(dev, pos + PCI_EXP_SLTCAP, ®32); - if (reg32 & PCI_EXP_SLTCAP_HPC) + if ((cap_mask & PCIE_PORT_SERVICE_HP) && + pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT) { + pcie_capability_read_dword(dev, PCI_EXP_SLTCAP, ®32); + if (reg32 & PCI_EXP_SLTCAP_HPC) { services |= PCIE_PORT_SERVICE_HP; + /* + * Disable hot-plug interrupts in case they have been + * enabled by the BIOS and the hot-plug service driver + * is not loaded. + */ + pcie_capability_clear_word(dev, PCI_EXP_SLTCTL, + PCI_EXP_SLTCTL_CCIE | PCI_EXP_SLTCTL_HPIE); + } } /* AER capable */ - if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) + if ((cap_mask & PCIE_PORT_SERVICE_AER) + && pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR)) { services |= PCIE_PORT_SERVICE_AER; + /* + * Disable AER on this port in case it's been enabled by the + * BIOS (the AER service driver will enable it when necessary). + */ + pci_disable_pcie_error_reporting(dev); + } /* VC support */ if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_VC)) services |= PCIE_PORT_SERVICE_VC; /* Root ports are capable of generating PME too */ - if (dev->pcie_type == PCI_EXP_TYPE_ROOT_PORT) + if ((cap_mask & PCIE_PORT_SERVICE_PME) + && pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) { services |= PCIE_PORT_SERVICE_PME; + /* + * Disable PME interrupt on this port in case it's been enabled + * by the BIOS (the PME service driver will enable it when + * necessary). + */ + pcie_pme_interrupt_enable(dev, false); + } return services; } @@ -275,15 +340,17 @@ static int pcie_device_init(struct pci_dev *pdev, int service, int irq) device->release = release_pcie_device; /* callback to free pcie dev */ dev_set_name(device, "%s:pcie%02x", pci_name(pdev), - get_descriptor_id(pdev->pcie_type, service)); + get_descriptor_id(pci_pcie_type(pdev), service)); device->parent = &pdev->dev; + device_enable_async_suspend(device); retval = device_register(device); - if (retval) - kfree(pcie); - else - get_device(device); - return retval; + if (retval) { + put_device(device); + return retval; + } + + return 0; } /** @@ -298,23 +365,27 @@ int pcie_port_device_register(struct pci_dev *dev) int status, capabilities, i, nr_service; int irqs[PCIE_PORT_DEVICE_MAXSERVICES]; - /* Get and check PCI Express port services */ - capabilities = get_port_device_capability(dev); - if (!capabilities) - return -ENODEV; - /* Enable PCI Express port device */ status = pci_enable_device(dev); if (status) return status; + + /* Get and check PCI Express port services */ + capabilities = get_port_device_capability(dev); + if (!capabilities) + return 0; + pci_set_master(dev); /* * Initialize service irqs. Don't use service devices that * require interrupts if there is no way to generate them. + * However, some drivers may have a polling mode (e.g. pciehp_poll_mode) + * that can be used in the absence of irqs. Allow them to determine + * if that is to be used. */ status = init_service_irqs(dev, irqs, capabilities); if (status) { - capabilities &= PCIE_PORT_SERVICE_VC; + capabilities &= PCIE_PORT_SERVICE_VC | PCIE_PORT_SERVICE_HP; if (!capabilities) goto error_disable; } @@ -346,12 +417,11 @@ static int suspend_iter(struct device *dev, void *data) { struct pcie_port_service_driver *service_driver; - if ((dev->bus == &pcie_port_bus_type) && - (dev->driver)) { - service_driver = to_service_driver(dev->driver); - if (service_driver->suspend) - service_driver->suspend(to_pcie_device(dev)); - } + if ((dev->bus == &pcie_port_bus_type) && dev->driver) { + service_driver = to_service_driver(dev->driver); + if (service_driver->suspend) + service_driver->suspend(to_pcie_device(dev)); + } return 0; } @@ -389,10 +459,8 @@ int pcie_port_device_resume(struct device *dev) static int remove_iter(struct device *dev, void *data) { - if (dev->bus == &pcie_port_bus_type) { - put_device(dev); + if (dev->bus == &pcie_port_bus_type) device_unregister(dev); - } return 0; } @@ -433,12 +501,12 @@ static int pcie_port_probe_service(struct device *dev) pciedev = to_pcie_device(dev); status = driver->probe(pciedev); - if (!status) { - dev_printk(KERN_DEBUG, dev, "service driver %s loaded\n", - driver->name); - get_device(dev); - } - return status; + if (status) + return status; + + dev_printk(KERN_DEBUG, dev, "service driver %s loaded\n", driver->name); + get_device(dev); + return 0; } /** @@ -486,7 +554,10 @@ static void pcie_port_shutdown_service(struct device *dev) {} */ int pcie_port_service_register(struct pcie_port_service_driver *new) { - new->driver.name = (char *)new->name; + if (pcie_ports_disabled) + return -ENODEV; + + new->driver.name = new->name; new->driver.bus = &pcie_port_bus_type; new->driver.probe = pcie_port_probe_service; new->driver.remove = pcie_port_remove_service; @@ -494,6 +565,7 @@ int pcie_port_service_register(struct pcie_port_service_driver *new) return driver_register(&new->driver); } +EXPORT_SYMBOL(pcie_port_service_register); /** * pcie_port_service_unregister - unregister PCI Express port service driver @@ -503,6 +575,4 @@ void pcie_port_service_unregister(struct pcie_port_service_driver *drv) { driver_unregister(&drv->driver); } - -EXPORT_SYMBOL(pcie_port_service_register); EXPORT_SYMBOL(pcie_port_service_unregister); diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index ce52ea34fee..80887eaa066 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -11,10 +11,12 @@ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/pm.h> +#include <linux/pm_runtime.h> #include <linux/init.h> -#include <linux/slab.h> #include <linux/pcieport_if.h> #include <linux/aer.h> +#include <linux/dmi.h> +#include <linux/pci-aspm.h> #include "portdrv.h" #include "aer/aerdrv.h" @@ -24,13 +26,47 @@ */ #define DRIVER_VERSION "v1.0" #define DRIVER_AUTHOR "tom.l.nguyen@intel.com" -#define DRIVER_DESC "PCIE Port Bus Driver" +#define DRIVER_DESC "PCIe Port Bus Driver" MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_LICENSE("GPL"); +/* If this switch is set, PCIe port native services should not be enabled. */ +bool pcie_ports_disabled; + +/* + * If this switch is set, ACPI _OSC will be used to determine whether or not to + * enable PCIe port native services. + */ +bool pcie_ports_auto = true; + +static int __init pcie_port_setup(char *str) +{ + if (!strncmp(str, "compat", 6)) { + pcie_ports_disabled = true; + } else if (!strncmp(str, "native", 6)) { + pcie_ports_disabled = false; + pcie_ports_auto = false; + } else if (!strncmp(str, "auto", 4)) { + pcie_ports_disabled = false; + pcie_ports_auto = true; + } + + return 1; +} +__setup("pcie_ports=", pcie_port_setup); + /* global data */ +/** + * pcie_clear_root_pme_status - Clear root port PME interrupt status. + * @dev: PCIe root port or event collector. + */ +void pcie_clear_root_pme_status(struct pci_dev *dev) +{ + pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); +} + static int pcie_portdrv_restore_config(struct pci_dev *dev) { int retval; @@ -43,13 +79,102 @@ static int pcie_portdrv_restore_config(struct pci_dev *dev) } #ifdef CONFIG_PM -static struct dev_pm_ops pcie_portdrv_pm_ops = { +static int pcie_port_resume_noirq(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + + /* + * Some BIOSes forget to clear Root PME Status bits after system wakeup + * which breaks ACPI-based runtime wakeup on PCI Express, so clear those + * bits now just in case (shouldn't hurt). + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + pcie_clear_root_pme_status(pdev); + return 0; +} + +#ifdef CONFIG_PM_RUNTIME +struct d3cold_info { + bool no_d3cold; + unsigned int d3cold_delay; +}; + +static int pci_dev_d3cold_info(struct pci_dev *pdev, void *data) +{ + struct d3cold_info *info = data; + + info->d3cold_delay = max_t(unsigned int, pdev->d3cold_delay, + info->d3cold_delay); + if (pdev->no_d3cold) + info->no_d3cold = true; + return 0; +} + +static int pcie_port_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct d3cold_info d3cold_info = { + .no_d3cold = false, + .d3cold_delay = PCI_PM_D3_WAIT, + }; + + /* + * If any subordinate device disable D3cold, we should not put + * the port into D3cold. The D3cold delay of port should be + * the max of that of all subordinate devices. + */ + pci_walk_bus(pdev->subordinate, pci_dev_d3cold_info, &d3cold_info); + pdev->no_d3cold = d3cold_info.no_d3cold; + pdev->d3cold_delay = d3cold_info.d3cold_delay; + return 0; +} + +static int pcie_port_runtime_resume(struct device *dev) +{ + return 0; +} + +static int pci_dev_pme_poll(struct pci_dev *pdev, void *data) +{ + bool *pme_poll = data; + + if (pdev->pme_poll) + *pme_poll = true; + return 0; +} + +static int pcie_port_runtime_idle(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + bool pme_poll = false; + + /* + * If any subordinate device needs pme poll, we should keep + * the port in D0, because we need port in D0 to poll it. + */ + pci_walk_bus(pdev->subordinate, pci_dev_pme_poll, &pme_poll); + /* Delay for a short while to prevent too frequent suspend/resume */ + if (!pme_poll) + pm_schedule_suspend(dev, 10); + return -EBUSY; +} +#else +#define pcie_port_runtime_suspend NULL +#define pcie_port_runtime_resume NULL +#define pcie_port_runtime_idle NULL +#endif + +static const struct dev_pm_ops pcie_portdrv_pm_ops = { .suspend = pcie_port_device_suspend, .resume = pcie_port_device_resume, .freeze = pcie_port_device_suspend, .thaw = pcie_port_device_resume, .poweroff = pcie_port_device_suspend, .restore = pcie_port_device_resume, + .resume_noirq = pcie_port_resume_noirq, + .runtime_suspend = pcie_port_runtime_suspend, + .runtime_resume = pcie_port_runtime_resume, + .runtime_idle = pcie_port_runtime_idle, }; #define PCIE_PORTDRV_PM_OPS (&pcie_portdrv_pm_ops) @@ -63,38 +188,41 @@ static struct dev_pm_ops pcie_portdrv_pm_ops = { * pcie_portdrv_probe - Probe PCI-Express port devices * @dev: PCI-Express port device being probed * - * If detected invokes the pcie_port_device_register() method for + * If detected invokes the pcie_port_device_register() method for * this port device. * */ -static int __devinit pcie_portdrv_probe(struct pci_dev *dev, +static int pcie_portdrv_probe(struct pci_dev *dev, const struct pci_device_id *id) { int status; if (!pci_is_pcie(dev) || - ((dev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) && - (dev->pcie_type != PCI_EXP_TYPE_UPSTREAM) && - (dev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))) + ((pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT) && + (pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM) && + (pci_pcie_type(dev) != PCI_EXP_TYPE_DOWNSTREAM))) return -ENODEV; - if (!dev->irq && dev->pin) { - dev_warn(&dev->dev, "device [%04x:%04x] has invalid IRQ; " - "check vendor BIOS\n", dev->vendor, dev->device); + if (!dev->irq && dev->pin) { + dev_warn(&dev->dev, "device [%04x:%04x] has invalid IRQ; check vendor BIOS\n", + dev->vendor, dev->device); } status = pcie_port_device_register(dev); if (status) return status; pci_save_state(dev); - + /* + * D3cold may not work properly on some PCIe port, so disable + * it by default. + */ + dev->d3cold_allowed = false; return 0; } -static void pcie_portdrv_remove (struct pci_dev *dev) +static void pcie_portdrv_remove(struct pci_dev *dev) { pcie_port_device_remove(dev); - pci_disable_device(dev); } static int error_detected_iter(struct device *device, void *data) @@ -129,14 +257,11 @@ static int error_detected_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, enum pci_channel_state error) { - struct aer_broadcast_data result_data = - {error, PCI_ERS_RESULT_CAN_RECOVER}; - int retval; - - /* can not fail */ - retval = device_for_each_child(&dev->dev, &result_data, error_detected_iter); + struct aer_broadcast_data data = {error, PCI_ERS_RESULT_CAN_RECOVER}; - return result_data.result; + /* get true return value from &data */ + device_for_each_child(&dev->dev, &data, error_detected_iter); + return data.result; } static int mmio_enabled_iter(struct device *device, void *data) @@ -167,10 +292,9 @@ static int mmio_enabled_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_mmio_enabled(struct pci_dev *dev) { pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; - int retval; /* get true return value from &status */ - retval = device_for_each_child(&dev->dev, &status, mmio_enabled_iter); + device_for_each_child(&dev->dev, &status, mmio_enabled_iter); return status; } @@ -202,7 +326,6 @@ static int slot_reset_iter(struct device *device, void *data) static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev) { pci_ers_result_t status = PCI_ERS_RESULT_RECOVERED; - int retval; /* If fatal, restore cfg space for possible link reset at upstream */ if (dev->error_state == pci_channel_io_frozen) { @@ -213,8 +336,7 @@ static pci_ers_result_t pcie_portdrv_slot_reset(struct pci_dev *dev) } /* get true return value from &status */ - retval = device_for_each_child(&dev->dev, &status, slot_reset_iter); - + device_for_each_child(&dev->dev, &status, slot_reset_iter); return status; } @@ -240,9 +362,7 @@ static int resume_iter(struct device *device, void *data) static void pcie_portdrv_err_resume(struct pci_dev *dev) { - int retval; - /* nothing to do with error value, if it ever happens */ - retval = device_for_each_child(&dev->dev, NULL, resume_iter); + device_for_each_child(&dev->dev, NULL, resume_iter); } /* @@ -255,11 +375,11 @@ static const struct pci_device_id port_pci_ids[] = { { }; MODULE_DEVICE_TABLE(pci, port_pci_ids); -static struct pci_error_handlers pcie_portdrv_err_handler = { - .error_detected = pcie_portdrv_error_detected, - .mmio_enabled = pcie_portdrv_mmio_enabled, - .slot_reset = pcie_portdrv_slot_reset, - .resume = pcie_portdrv_err_resume, +static const struct pci_error_handlers pcie_portdrv_err_handler = { + .error_detected = pcie_portdrv_error_detected, + .mmio_enabled = pcie_portdrv_mmio_enabled, + .slot_reset = pcie_portdrv_slot_reset, + .resume = pcie_portdrv_err_resume, }; static struct pci_driver pcie_portdriver = { @@ -269,15 +389,44 @@ static struct pci_driver pcie_portdriver = { .probe = pcie_portdrv_probe, .remove = pcie_portdrv_remove, - .err_handler = &pcie_portdrv_err_handler, + .err_handler = &pcie_portdrv_err_handler, + + .driver.pm = PCIE_PORTDRV_PM_OPS, +}; + +static int __init dmi_pcie_pme_disable_msi(const struct dmi_system_id *d) +{ + pr_notice("%s detected: will not use MSI for PCIe PME signaling\n", + d->ident); + pcie_pme_disable_msi(); + return 0; +} - .driver.pm = PCIE_PORTDRV_PM_OPS, +static struct dmi_system_id __initdata pcie_portdrv_dmi_table[] = { + /* + * Boxes that should not use MSI for PCIe PME signaling. + */ + { + .callback = dmi_pcie_pme_disable_msi, + .ident = "MSI Wind U-100", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, + "MICRO-STAR INTERNATIONAL CO., LTD"), + DMI_MATCH(DMI_PRODUCT_NAME, "U-100"), + }, + }, + {} }; static int __init pcie_portdrv_init(void) { int retval; + if (pcie_ports_disabled) + return pci_register_driver(&pcie_portdriver); + + dmi_check_system(pcie_portdrv_dmi_table); + retval = pcie_port_bus_register(); if (retval) { printk(KERN_WARNING "PCIE: bus_register error: %d\n", retval); @@ -290,11 +439,4 @@ static int __init pcie_portdrv_init(void) return retval; } -static void __exit pcie_portdrv_exit(void) -{ - pci_unregister_driver(&pcie_portdriver); - pcie_port_bus_unregister(); -} - module_init(pcie_portdrv_init); -module_exit(pcie_portdrv_exit); |