diff options
Diffstat (limited to 'drivers/edac/amd64_edac.c')
| -rw-r--r-- | drivers/edac/amd64_edac.c | 4132 |
1 files changed, 1876 insertions, 2256 deletions
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index c36bf40568c..f8bf00010d4 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1,7 +1,7 @@ #include "amd64_edac.h" -#include <asm/k8.h> +#include <asm/amd_nb.h> -static struct edac_pci_ctl_info *amd64_ctl_pci; +static struct edac_pci_ctl_info *pci_ctl; static int report_gart_errors; module_param(report_gart_errors, int, 0644); @@ -13,10 +13,136 @@ module_param(report_gart_errors, int, 0644); static int ecc_enable_override; module_param(ecc_enable_override, int, 0644); -/* Lookup table for all possible MC control instances */ -struct amd64_pvt; -static struct mem_ctl_info *mci_lookup[MAX_NUMNODES]; -static struct amd64_pvt *pvt_lookup[MAX_NUMNODES]; +static struct msr __percpu *msrs; + +/* + * count successfully initialized driver instances for setup_pci_device() + */ +static atomic_t drv_instances = ATOMIC_INIT(0); + +/* Per-node driver instances */ +static struct mem_ctl_info **mcis; +static struct ecc_settings **ecc_stngs; + +/* + * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing + * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching- + * or higher value'. + * + *FIXME: Produce a better mapping/linearisation. + */ +static const struct scrubrate { + u32 scrubval; /* bit pattern for scrub rate */ + u32 bandwidth; /* bandwidth consumed (bytes/sec) */ +} scrubrates[] = { + { 0x01, 1600000000UL}, + { 0x02, 800000000UL}, + { 0x03, 400000000UL}, + { 0x04, 200000000UL}, + { 0x05, 100000000UL}, + { 0x06, 50000000UL}, + { 0x07, 25000000UL}, + { 0x08, 12284069UL}, + { 0x09, 6274509UL}, + { 0x0A, 3121951UL}, + { 0x0B, 1560975UL}, + { 0x0C, 781440UL}, + { 0x0D, 390720UL}, + { 0x0E, 195300UL}, + { 0x0F, 97650UL}, + { 0x10, 48854UL}, + { 0x11, 24427UL}, + { 0x12, 12213UL}, + { 0x13, 6101UL}, + { 0x14, 3051UL}, + { 0x15, 1523UL}, + { 0x16, 761UL}, + { 0x00, 0UL}, /* scrubbing off */ +}; + +int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, + u32 *val, const char *func) +{ + int err = 0; + + err = pci_read_config_dword(pdev, offset, val); + if (err) + amd64_warn("%s: error reading F%dx%03x.\n", + func, PCI_FUNC(pdev->devfn), offset); + + return err; +} + +int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, + u32 val, const char *func) +{ + int err = 0; + + err = pci_write_config_dword(pdev, offset, val); + if (err) + amd64_warn("%s: error writing to F%dx%03x.\n", + func, PCI_FUNC(pdev->devfn), offset); + + return err; +} + +/* + * + * Depending on the family, F2 DCT reads need special handling: + * + * K8: has a single DCT only + * + * F10h: each DCT has its own set of regs + * DCT0 -> F2x040.. + * DCT1 -> F2x140.. + * + * F15h: we select which DCT we access using F1x10C[DctCfgSel] + * + * F16h: has only 1 DCT + */ +static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + if (addr >= 0x100) + return -EINVAL; + + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} + +static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} + +/* + * Select DCT to which PCI cfg accesses are routed + */ +static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) +{ + u32 reg = 0; + + amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); + reg &= (pvt->model >= 0x30) ? ~3 : ~1; + reg |= dct; + amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); +} + +static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + u8 dct = 0; + + /* For F15 M30h, the second dct is DCT 3, refer to BKDG Section 2.10 */ + if (addr >= 0x140 && addr <= 0x1a0) { + dct = (pvt->model >= 0x30) ? 3 : 1; + addr -= 0x100; + } + + f15h_select_dct(pvt, dct); + + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} /* * Memory scrubber control interface. For K8, memory scrubbing is handled by @@ -36,8 +162,7 @@ static struct amd64_pvt *pvt_lookup[MAX_NUMNODES]; * scan the scrub rate mapping table for a close or matching bandwidth value to * issue. If requested is too big, then use last maximum value found. */ -static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, - u32 min_scrubrate) +static int __set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) { u32 scrubval; int i; @@ -47,142 +172,77 @@ static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, * memory controller and apply to register. Search for the first * bandwidth entry that is greater or equal than the setting requested * and program that. If at last entry, turn off DRAM scrubbing. + * + * If no suitable bandwidth is found, turn off DRAM scrubbing entirely + * by falling back to the last element in scrubrates[]. */ - for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { + for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { /* * skip scrub rates which aren't recommended * (see F10 BKDG, F3x58) */ - if (scrubrates[i].scrubval < min_scrubrate) + if (scrubrates[i].scrubval < min_rate) continue; if (scrubrates[i].bandwidth <= new_bw) break; - - /* - * if no suitable bandwidth found, turn off DRAM scrubbing - * entirely by falling back to the last element in the - * scrubrates array. - */ } scrubval = scrubrates[i].scrubval; - if (scrubval) - edac_printk(KERN_DEBUG, EDAC_MC, - "Setting scrub rate bandwidth: %u\n", - scrubrates[i].bandwidth); - else - edac_printk(KERN_DEBUG, EDAC_MC, "Turning scrubbing off.\n"); - pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F); + pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); + + if (scrubval) + return scrubrates[i].bandwidth; return 0; } -static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 *bandwidth) +static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) { struct amd64_pvt *pvt = mci->pvt_info; - u32 min_scrubrate = 0x0; + u32 min_scrubrate = 0x5; - switch (boot_cpu_data.x86) { - case 0xf: - min_scrubrate = K8_MIN_SCRUB_RATE_BITS; - break; - case 0x10: - min_scrubrate = F10_MIN_SCRUB_RATE_BITS; - break; - case 0x11: - min_scrubrate = F11_MIN_SCRUB_RATE_BITS; - break; + if (pvt->fam == 0xf) + min_scrubrate = 0x0; - default: - amd64_printk(KERN_ERR, "Unsupported family!\n"); - break; - } - return amd64_search_set_scrub_rate(pvt->misc_f3_ctl, *bandwidth, - min_scrubrate); + /* Erratum #505 */ + if (pvt->fam == 0x15 && pvt->model < 0x10) + f15h_select_dct(pvt, 0); + + return __set_scrub_rate(pvt->F3, bw, min_scrubrate); } -static int amd64_get_scrub_rate(struct mem_ctl_info *mci, u32 *bw) +static int get_scrub_rate(struct mem_ctl_info *mci) { struct amd64_pvt *pvt = mci->pvt_info; u32 scrubval = 0; - int status = -1, i, ret = 0; + int i, retval = -EINVAL; - ret = pci_read_config_dword(pvt->misc_f3_ctl, K8_SCRCTRL, &scrubval); - if (ret) - debugf0("Reading K8_SCRCTRL failed\n"); + /* Erratum #505 */ + if (pvt->fam == 0x15 && pvt->model < 0x10) + f15h_select_dct(pvt, 0); - scrubval = scrubval & 0x001F; + amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); - edac_printk(KERN_DEBUG, EDAC_MC, - "pci-read, sdram scrub control value: %d \n", scrubval); + scrubval = scrubval & 0x001F; - for (i = 0; ARRAY_SIZE(scrubrates); i++) { + for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { if (scrubrates[i].scrubval == scrubval) { - *bw = scrubrates[i].bandwidth; - status = 0; + retval = scrubrates[i].bandwidth; break; } } - - return status; -} - -/* Map from a CSROW entry to the mask entry that operates on it */ -static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow) -{ - return csrow >> (pvt->num_dcsm >> 3); -} - -/* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */ -static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow) -{ - if (dct == 0) - return pvt->dcsb0[csrow]; - else - return pvt->dcsb1[csrow]; + return retval; } /* - * Return the 'mask' address the i'th CS entry. This function is needed because - * there number of DCSM registers on Rev E and prior vs Rev F and later is - * different. + * returns true if the SysAddr given by sys_addr matches the + * DRAM base/limit associated with node_id */ -static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow) +static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) { - if (dct == 0) - return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)]; - else - return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)]; -} - - -/* - * In *base and *limit, pass back the full 40-bit base and limit physical - * addresses for the node given by node_id. This information is obtained from - * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The - * base and limit addresses are of type SysAddr, as defined at the start of - * section 3.4.4 (p. 70). They are the lowest and highest physical addresses - * in the address range they represent. - */ -static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id, - u64 *base, u64 *limit) -{ - *base = pvt->dram_base[node_id]; - *limit = pvt->dram_limit[node_id]; -} - -/* - * Return 1 if the SysAddr given by sys_addr matches the base/limit associated - * with node_id - */ -static int amd64_base_limit_match(struct amd64_pvt *pvt, - u64 sys_addr, int node_id) -{ - u64 base, limit, addr; - - amd64_get_base_and_limit(pvt, node_id, &base, &limit); + u64 addr; /* The K8 treats this as a 40-bit value. However, bits 63-40 will be * all ones if the most significant implemented address bit is 1. @@ -192,7 +252,8 @@ static int amd64_base_limit_match(struct amd64_pvt *pvt, */ addr = sys_addr & 0x000000ffffffffffull; - return (addr >= base) && (addr <= limit); + return ((addr >= get_dram_base(pvt, nid)) && + (addr <= get_dram_limit(pvt, nid))); } /* @@ -205,7 +266,7 @@ static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, u64 sys_addr) { struct amd64_pvt *pvt; - int node_id; + u8 node_id; u32 intlv_en, bits; /* @@ -219,90 +280,120 @@ static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, * registers. Therefore we arbitrarily choose to read it from the * register for node 0. */ - intlv_en = pvt->dram_IntlvEn[0]; + intlv_en = dram_intlv_en(pvt, 0); if (intlv_en == 0) { - for (node_id = 0; ; ) { - if (amd64_base_limit_match(pvt, sys_addr, node_id)) - break; - - if (++node_id >= DRAM_REG_COUNT) - goto err_no_match; + for (node_id = 0; node_id < DRAM_RANGES; node_id++) { + if (base_limit_match(pvt, sys_addr, node_id)) + goto found; } - goto found; + goto err_no_match; } - if (unlikely((intlv_en != (0x01 << 8)) && - (intlv_en != (0x03 << 8)) && - (intlv_en != (0x07 << 8)))) { - amd64_printk(KERN_WARNING, "junk value of 0x%x extracted from " - "IntlvEn field of DRAM Base Register for node 0: " - "This probably indicates a BIOS bug.\n", intlv_en); + if (unlikely((intlv_en != 0x01) && + (intlv_en != 0x03) && + (intlv_en != 0x07))) { + amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); return NULL; } bits = (((u32) sys_addr) >> 12) & intlv_en; for (node_id = 0; ; ) { - if ((pvt->dram_limit[node_id] & intlv_en) == bits) + if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) break; /* intlv_sel field matches */ - if (++node_id >= DRAM_REG_COUNT) + if (++node_id >= DRAM_RANGES) goto err_no_match; } /* sanity test for sys_addr */ - if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { - amd64_printk(KERN_WARNING, - "%s(): sys_addr 0x%lx falls outside base/limit " - "address range for node %d with node interleaving " - "enabled.\n", __func__, (unsigned long)sys_addr, - node_id); + if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { + amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" + "range for node %d with node interleaving enabled.\n", + __func__, sys_addr, node_id); return NULL; } found: - return edac_mc_find(node_id); + return edac_mc_find((int)node_id); err_no_match: - debugf2("sys_addr 0x%lx doesn't match any node\n", - (unsigned long)sys_addr); + edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", + (unsigned long)sys_addr); return NULL; } /* - * Extract the DRAM CS base address from selected csrow register. + * compute the CS base address of the @csrow on the DRAM controller @dct. + * For details see F2x[5C:40] in the processor's BKDG */ -static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow) +static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, + u64 *base, u64 *mask) { - return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) << - pvt->dcs_shift; -} + u64 csbase, csmask, base_bits, mask_bits; + u8 addr_shift; -/* - * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way. - */ -static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow) -{ - u64 dcsm_bits, other_bits; - u64 mask; - - /* Extract bits from DRAM CS Mask. */ - dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask; - - other_bits = pvt->dcsm_mask; - other_bits = ~(other_bits << pvt->dcs_shift); + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow]; + base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); + mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); + addr_shift = 4; /* - * The extracted bits from DCSM belong in the spaces represented by - * the cleared bits in other_bits. + * F16h and F15h, models 30h and later need two addr_shift values: + * 8 for high and 6 for low (cf. F16h BKDG). */ - mask = (dcsm_bits << pvt->dcs_shift) | other_bits; + } else if (pvt->fam == 0x16 || + (pvt->fam == 0x15 && pvt->model >= 0x30)) { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow >> 1]; + + *base = (csbase & GENMASK_ULL(15, 5)) << 6; + *base |= (csbase & GENMASK_ULL(30, 19)) << 8; + + *mask = ~0ULL; + /* poke holes for the csmask */ + *mask &= ~((GENMASK_ULL(15, 5) << 6) | + (GENMASK_ULL(30, 19) << 8)); + + *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; + *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; + + return; + } else { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow >> 1]; + addr_shift = 8; + + if (pvt->fam == 0x15) + base_bits = mask_bits = + GENMASK_ULL(30,19) | GENMASK_ULL(13,5); + else + base_bits = mask_bits = + GENMASK_ULL(28,19) | GENMASK_ULL(13,5); + } + + *base = (csbase & base_bits) << addr_shift; - return mask; + *mask = ~0ULL; + /* poke holes for the csmask */ + *mask &= ~(mask_bits << addr_shift); + /* OR them in */ + *mask |= (csmask & mask_bits) << addr_shift; } +#define for_each_chip_select(i, dct, pvt) \ + for (i = 0; i < pvt->csels[dct].b_cnt; i++) + +#define chip_select_base(i, dct, pvt) \ + pvt->csels[dct].csbases[i] + +#define for_each_chip_select_mask(i, dct, pvt) \ + for (i = 0; i < pvt->csels[dct].m_cnt; i++) + /* * @input_addr is an InputAddr associated with the node given by mci. Return the * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). @@ -315,49 +406,29 @@ static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) pvt = mci->pvt_info; - /* - * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS - * base/mask register pair, test the condition shown near the start of - * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E). - */ - for (csrow = 0; csrow < CHIPSELECT_COUNT; csrow++) { - - /* This DRAM chip select is disabled on this node */ - if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) + for_each_chip_select(csrow, 0, pvt) { + if (!csrow_enabled(csrow, 0, pvt)) continue; - base = base_from_dct_base(pvt, csrow); - mask = ~mask_from_dct_mask(pvt, csrow); + get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); + + mask = ~mask; if ((input_addr & mask) == (base & mask)) { - debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", - (unsigned long)input_addr, csrow, - pvt->mc_node_id); + edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", + (unsigned long)input_addr, csrow, + pvt->mc_node_id); return csrow; } } - - debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", - (unsigned long)input_addr, pvt->mc_node_id); + edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", + (unsigned long)input_addr, pvt->mc_node_id); return -1; } /* - * Return the base value defined by the DRAM Base register for the node - * represented by mci. This function returns the full 40-bit value despite the - * fact that the register only stores bits 39-24 of the value. See section - * 3.4.4.1 (BKDG #26094, K8, revA-E) - */ -static inline u64 get_dram_base(struct mem_ctl_info *mci) -{ - struct amd64_pvt *pvt = mci->pvt_info; - - return pvt->dram_base[pvt->mc_node_id]; -} - -/* * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) * for the node represented by mci. Info is passed back in *hole_base, * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if @@ -377,25 +448,23 @@ int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, u64 *hole_offset, u64 *hole_size) { struct amd64_pvt *pvt = mci->pvt_info; - u64 base; /* only revE and later have the DRAM Hole Address Register */ - if (boot_cpu_data.x86 == 0xf && pvt->ext_model < OPTERON_CPU_REV_E) { - debugf1(" revision %d for node %d does not support DHAR\n", - pvt->ext_model, pvt->mc_node_id); + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { + edac_dbg(1, " revision %d for node %d does not support DHAR\n", + pvt->ext_model, pvt->mc_node_id); return 1; } - /* only valid for Fam10h */ - if (boot_cpu_data.x86 == 0x10 && - (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) { - debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); + /* valid for Fam10h and above */ + if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { + edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); return 1; } - if ((pvt->dhar & DHAR_VALID) == 0) { - debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", - pvt->mc_node_id); + if (!dhar_valid(pvt)) { + edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", + pvt->mc_node_id); return 1; } @@ -417,19 +486,15 @@ int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, * addresses in the hole so that they start at 0x100000000. */ - base = dhar_base(pvt->dhar); + *hole_base = dhar_base(pvt); + *hole_size = (1ULL << 32) - *hole_base; - *hole_base = base; - *hole_size = (0x1ull << 32) - base; - - if (boot_cpu_data.x86 > 0xf) - *hole_offset = f10_dhar_offset(pvt->dhar); - else - *hole_offset = k8_dhar_offset(pvt->dhar); + *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) + : k8_dhar_offset(pvt); - debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", - pvt->mc_node_id, (unsigned long)*hole_base, - (unsigned long)*hole_offset, (unsigned long)*hole_size); + edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", + pvt->mc_node_id, (unsigned long)*hole_base, + (unsigned long)*hole_offset, (unsigned long)*hole_size); return 0; } @@ -466,23 +531,23 @@ EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); */ static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) { + struct amd64_pvt *pvt = mci->pvt_info; u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; - int ret = 0; + int ret; - dram_base = get_dram_base(mci); + dram_base = get_dram_base(pvt, pvt->mc_node_id); ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); if (!ret) { - if ((sys_addr >= (1ull << 32)) && - (sys_addr < ((1ull << 32) + hole_size))) { + if ((sys_addr >= (1ULL << 32)) && + (sys_addr < ((1ULL << 32) + hole_size))) { /* use DHAR to translate SysAddr to DramAddr */ dram_addr = sys_addr - hole_offset; - debugf2("using DHAR to translate SysAddr 0x%lx to " - "DramAddr 0x%lx\n", - (unsigned long)sys_addr, - (unsigned long)dram_addr); + edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", + (unsigned long)sys_addr, + (unsigned long)dram_addr); return dram_addr; } @@ -497,11 +562,10 @@ static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture * Programmer's Manual Volume 1 Application Programming. */ - dram_addr = (sys_addr & 0xffffffffffull) - dram_base; + dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; - debugf2("using DRAM Base register to translate SysAddr 0x%lx to " - "DramAddr 0x%lx\n", (unsigned long)sys_addr, - (unsigned long)dram_addr); + edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", + (unsigned long)sys_addr, (unsigned long)dram_addr); return dram_addr; } @@ -533,13 +597,13 @@ static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) * concerning translating a DramAddr to an InputAddr. */ - intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); - input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) + - (dram_addr & 0xfff); + intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); + input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + + (dram_addr & 0xfff); - debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", - intlv_shift, (unsigned long)dram_addr, - (unsigned long)input_addr); + edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", + intlv_shift, (unsigned long)dram_addr, + (unsigned long)input_addr); return input_addr; } @@ -555,158 +619,18 @@ static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) input_addr = dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); - debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", - (unsigned long)sys_addr, (unsigned long)input_addr); + edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n", + (unsigned long)sys_addr, (unsigned long)input_addr); return input_addr; } - -/* - * @input_addr is an InputAddr associated with the node represented by mci. - * Translate @input_addr to a DramAddr and return the result. - */ -static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) -{ - struct amd64_pvt *pvt; - int node_id, intlv_shift; - u64 bits, dram_addr; - u32 intlv_sel; - - /* - * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) - * shows how to translate a DramAddr to an InputAddr. Here we reverse - * this procedure. When translating from a DramAddr to an InputAddr, the - * bits used for node interleaving are discarded. Here we recover these - * bits from the IntlvSel field of the DRAM Limit register (section - * 3.4.4.2) for the node that input_addr is associated with. - */ - pvt = mci->pvt_info; - node_id = pvt->mc_node_id; - BUG_ON((node_id < 0) || (node_id > 7)); - - intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); - - if (intlv_shift == 0) { - debugf1(" InputAddr 0x%lx translates to DramAddr of " - "same value\n", (unsigned long)input_addr); - - return input_addr; - } - - bits = ((input_addr & 0xffffff000ull) << intlv_shift) + - (input_addr & 0xfff); - - intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1); - dram_addr = bits + (intlv_sel << 12); - - debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " - "(%d node interleave bits)\n", (unsigned long)input_addr, - (unsigned long)dram_addr, intlv_shift); - - return dram_addr; -} - -/* - * @dram_addr is a DramAddr that maps to the node represented by mci. Convert - * @dram_addr to a SysAddr. - */ -static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) -{ - struct amd64_pvt *pvt = mci->pvt_info; - u64 hole_base, hole_offset, hole_size, base, limit, sys_addr; - int ret = 0; - - ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, - &hole_size); - if (!ret) { - if ((dram_addr >= hole_base) && - (dram_addr < (hole_base + hole_size))) { - sys_addr = dram_addr + hole_offset; - - debugf1("using DHAR to translate DramAddr 0x%lx to " - "SysAddr 0x%lx\n", (unsigned long)dram_addr, - (unsigned long)sys_addr); - - return sys_addr; - } - } - - amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit); - sys_addr = dram_addr + base; - - /* - * The sys_addr we have computed up to this point is a 40-bit value - * because the k8 deals with 40-bit values. However, the value we are - * supposed to return is a full 64-bit physical address. The AMD - * x86-64 architecture specifies that the most significant implemented - * address bit through bit 63 of a physical address must be either all - * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a - * 64-bit value below. See section 3.4.2 of AMD publication 24592: - * AMD x86-64 Architecture Programmer's Manual Volume 1 Application - * Programming. - */ - sys_addr |= ~((sys_addr & (1ull << 39)) - 1); - - debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", - pvt->mc_node_id, (unsigned long)dram_addr, - (unsigned long)sys_addr); - - return sys_addr; -} - -/* - * @input_addr is an InputAddr associated with the node given by mci. Translate - * @input_addr to a SysAddr. - */ -static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, - u64 input_addr) -{ - return dram_addr_to_sys_addr(mci, - input_addr_to_dram_addr(mci, input_addr)); -} - -/* - * Find the minimum and maximum InputAddr values that map to the given @csrow. - * Pass back these values in *input_addr_min and *input_addr_max. - */ -static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, - u64 *input_addr_min, u64 *input_addr_max) -{ - struct amd64_pvt *pvt; - u64 base, mask; - - pvt = mci->pvt_info; - BUG_ON((csrow < 0) || (csrow >= CHIPSELECT_COUNT)); - - base = base_from_dct_base(pvt, csrow); - mask = mask_from_dct_mask(pvt, csrow); - - *input_addr_min = base & ~mask; - *input_addr_max = base | mask | pvt->dcs_mask_notused; -} - -/* - * Extract error address from MCA NB Address Low (section 3.6.4.5) and MCA NB - * Address High (section 3.6.4.6) register values and return the result. Address - * is located in the info structure (nbeah and nbeal), the encoding is device - * specific. - */ -static u64 extract_error_address(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) -{ - struct amd64_pvt *pvt = mci->pvt_info; - - return pvt->ops->get_error_address(mci, info); -} - - /* Map the Error address to a PAGE and PAGE OFFSET. */ static inline void error_address_to_page_and_offset(u64 error_address, - u32 *page, u32 *offset) + struct err_info *err) { - *page = (u32) (error_address >> PAGE_SHIFT); - *offset = ((u32) error_address) & ~PAGE_MASK; + err->page = (u32) (error_address >> PAGE_SHIFT); + err->offset = ((u32) error_address) & ~PAGE_MASK; } /* @@ -724,320 +648,189 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); if (csrow == -1) - amd64_mc_printk(mci, KERN_ERR, - "Failed to translate InputAddr to csrow for " - "address 0x%lx\n", (unsigned long)sys_addr); + amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " + "address 0x%lx\n", (unsigned long)sys_addr); return csrow; } -static int get_channel_from_ecc_syndrome(unsigned short syndrome); - -static void amd64_cpu_display_info(struct amd64_pvt *pvt) -{ - if (boot_cpu_data.x86 == 0x11) - edac_printk(KERN_DEBUG, EDAC_MC, "F11h CPU detected\n"); - else if (boot_cpu_data.x86 == 0x10) - edac_printk(KERN_DEBUG, EDAC_MC, "F10h CPU detected\n"); - else if (boot_cpu_data.x86 == 0xf) - edac_printk(KERN_DEBUG, EDAC_MC, "%s detected\n", - (pvt->ext_model >= OPTERON_CPU_REV_F) ? - "Rev F or later" : "Rev E or earlier"); - else - /* we'll hardly ever ever get here */ - edac_printk(KERN_ERR, EDAC_MC, "Unknown cpu!\n"); -} +static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); /* * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs * are ECC capable. */ -static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) +static unsigned long determine_edac_cap(struct amd64_pvt *pvt) { - int bit; - enum dev_type edac_cap = EDAC_NONE; + u8 bit; + unsigned long edac_cap = EDAC_FLAG_NONE; - bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= OPTERON_CPU_REV_F) + bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) ? 19 : 17; - if (pvt->dclr0 >> BIT(bit)) + if (pvt->dclr0 & BIT(bit)) edac_cap = EDAC_FLAG_SECDED; return edac_cap; } +static void debug_display_dimm_sizes(struct amd64_pvt *, u8); -static void f10_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt, - int ganged); - -/* Display and decode various NB registers for debug purposes. */ -static void amd64_dump_misc_regs(struct amd64_pvt *pvt) -{ - int ganged; - - debugf1(" nbcap:0x%8.08x DctDualCap=%s DualNode=%s 8-Node=%s\n", - pvt->nbcap, - (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "True" : "False", - (pvt->nbcap & K8_NBCAP_DUAL_NODE) ? "True" : "False", - (pvt->nbcap & K8_NBCAP_8_NODE) ? "True" : "False"); - debugf1(" ECC Capable=%s ChipKill Capable=%s\n", - (pvt->nbcap & K8_NBCAP_SECDED) ? "True" : "False", - (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "True" : "False"); - debugf1(" DramCfg0-low=0x%08x DIMM-ECC=%s Parity=%s Width=%s\n", - pvt->dclr0, - (pvt->dclr0 & BIT(19)) ? "Enabled" : "Disabled", - (pvt->dclr0 & BIT(8)) ? "Enabled" : "Disabled", - (pvt->dclr0 & BIT(11)) ? "128b" : "64b"); - debugf1(" DIMM x4 Present: L0=%s L1=%s L2=%s L3=%s DIMM Type=%s\n", - (pvt->dclr0 & BIT(12)) ? "Y" : "N", - (pvt->dclr0 & BIT(13)) ? "Y" : "N", - (pvt->dclr0 & BIT(14)) ? "Y" : "N", - (pvt->dclr0 & BIT(15)) ? "Y" : "N", - (pvt->dclr0 & BIT(16)) ? "UN-Buffered" : "Buffered"); - - - debugf1(" online-spare: 0x%8.08x\n", pvt->online_spare); - - if (boot_cpu_data.x86 == 0xf) { - debugf1(" dhar: 0x%8.08x Base=0x%08x Offset=0x%08x\n", - pvt->dhar, dhar_base(pvt->dhar), - k8_dhar_offset(pvt->dhar)); - debugf1(" DramHoleValid=%s\n", - (pvt->dhar & DHAR_VALID) ? "True" : "False"); - - debugf1(" dbam-dkt: 0x%8.08x\n", pvt->dbam0); - - /* everything below this point is Fam10h and above */ - return; - - } else { - debugf1(" dhar: 0x%8.08x Base=0x%08x Offset=0x%08x\n", - pvt->dhar, dhar_base(pvt->dhar), - f10_dhar_offset(pvt->dhar)); - debugf1(" DramMemHoistValid=%s DramHoleValid=%s\n", - (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) ? - "True" : "False", - (pvt->dhar & DHAR_VALID) ? - "True" : "False"); - } +static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) +{ + edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); - /* Only if NOT ganged does dcl1 have valid info */ - if (!dct_ganging_enabled(pvt)) { - debugf1(" DramCfg1-low=0x%08x DIMM-ECC=%s Parity=%s " - "Width=%s\n", pvt->dclr1, - (pvt->dclr1 & BIT(19)) ? "Enabled" : "Disabled", - (pvt->dclr1 & BIT(8)) ? "Enabled" : "Disabled", - (pvt->dclr1 & BIT(11)) ? "128b" : "64b"); - debugf1(" DIMM x4 Present: L0=%s L1=%s L2=%s L3=%s " - "DIMM Type=%s\n", - (pvt->dclr1 & BIT(12)) ? "Y" : "N", - (pvt->dclr1 & BIT(13)) ? "Y" : "N", - (pvt->dclr1 & BIT(14)) ? "Y" : "N", - (pvt->dclr1 & BIT(15)) ? "Y" : "N", - (pvt->dclr1 & BIT(16)) ? "UN-Buffered" : "Buffered"); - } + edac_dbg(1, " DIMM type: %sbuffered; all DIMMs support ECC: %s\n", + (dclr & BIT(16)) ? "un" : "", + (dclr & BIT(19)) ? "yes" : "no"); - /* - * Determine if ganged and then dump memory sizes for first controller, - * and if NOT ganged dump info for 2nd controller. - */ - ganged = dct_ganging_enabled(pvt); + edac_dbg(1, " PAR/ERR parity: %s\n", + (dclr & BIT(8)) ? "enabled" : "disabled"); - f10_debug_display_dimm_sizes(0, pvt, ganged); + if (pvt->fam == 0x10) + edac_dbg(1, " DCT 128bit mode width: %s\n", + (dclr & BIT(11)) ? "128b" : "64b"); - if (!ganged) - f10_debug_display_dimm_sizes(1, pvt, ganged); + edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", + (dclr & BIT(12)) ? "yes" : "no", + (dclr & BIT(13)) ? "yes" : "no", + (dclr & BIT(14)) ? "yes" : "no", + (dclr & BIT(15)) ? "yes" : "no"); } -/* Read in both of DBAM registers */ -static void amd64_read_dbam_reg(struct amd64_pvt *pvt) +/* Display and decode various NB registers for debug purposes. */ +static void dump_misc_regs(struct amd64_pvt *pvt) { - int err = 0; - unsigned int reg; + edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); - reg = DBAM0; - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, &pvt->dbam0); - if (err) - goto err_reg; + edac_dbg(1, " NB two channel DRAM capable: %s\n", + (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); - if (boot_cpu_data.x86 >= 0x10) { - reg = DBAM1; - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, &pvt->dbam1); + edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", + (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", + (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); - if (err) - goto err_reg; - } + debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); -err_reg: - debugf0("Error reading F2x%03x.\n", reg); -} + edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); -/* - * NOTE: CPU Revision Dependent code: Rev E and Rev F - * - * Set the DCSB and DCSM mask values depending on the CPU revision value. Also - * set the shift factor for the DCSB and DCSM values. - * - * ->dcs_mask_notused, RevE: - * - * To find the max InputAddr for the csrow, start with the base address and set - * all bits that are "don't care" bits in the test at the start of section - * 3.5.4 (p. 84). - * - * The "don't care" bits are all set bits in the mask and all bits in the gaps - * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS - * represents bits [24:20] and [12:0], which are all bits in the above-mentioned - * gaps. - * - * ->dcs_mask_notused, RevF and later: - * - * To find the max InputAddr for the csrow, start with the base address and set - * all bits that are "don't care" bits in the test at the start of NPT section - * 4.5.4 (p. 87). - * - * The "don't care" bits are all set bits in the mask and all bits in the gaps - * between bit ranges [36:27] and [21:13]. - * - * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0], - * which are all bits in the above-mentioned gaps. - */ -static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt) -{ - if (pvt->ext_model >= OPTERON_CPU_REV_F) { - pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS; - pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS; - pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS; - pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT; + edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", + pvt->dhar, dhar_base(pvt), + (pvt->fam == 0xf) ? k8_dhar_offset(pvt) + : f10_dhar_offset(pvt)); - switch (boot_cpu_data.x86) { - case 0xf: - pvt->num_dcsm = REV_F_DCSM_COUNT; - break; + edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); - case 0x10: - pvt->num_dcsm = F10_DCSM_COUNT; - break; + debug_display_dimm_sizes(pvt, 0); - case 0x11: - pvt->num_dcsm = F11_DCSM_COUNT; - break; + /* everything below this point is Fam10h and above */ + if (pvt->fam == 0xf) + return; - default: - amd64_printk(KERN_ERR, "Unsupported family!\n"); - break; - } + debug_display_dimm_sizes(pvt, 1); + + amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); + + /* Only if NOT ganged does dclr1 have valid info */ + if (!dct_ganging_enabled(pvt)) + debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); +} + +/* + * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] + */ +static void prep_chip_selects(struct amd64_pvt *pvt) +{ + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; + } else if (pvt->fam == 0x15 && pvt->model >= 0x30) { + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; } else { - pvt->dcsb_base = REV_E_DCSB_BASE_BITS; - pvt->dcsm_mask = REV_E_DCSM_MASK_BITS; - pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS; - pvt->dcs_shift = REV_E_DCS_SHIFT; - pvt->num_dcsm = REV_E_DCSM_COUNT; + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; } } /* - * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers + * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers */ -static void amd64_read_dct_base_mask(struct amd64_pvt *pvt) +static void read_dct_base_mask(struct amd64_pvt *pvt) { - int cs, reg, err = 0; + int cs; - amd64_set_dct_base_and_mask(pvt); + prep_chip_selects(pvt); - for (cs = 0; cs < CHIPSELECT_COUNT; cs++) { - reg = K8_DCSB0 + (cs * 4); - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, - &pvt->dcsb0[cs]); - if (unlikely(err)) - debugf0("Reading K8_DCSB0[%d] failed\n", cs); - else - debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsb0[cs], reg); - - /* If DCT are NOT ganged, then read in DCT1's base */ - if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { - reg = F10_DCSB1 + (cs * 4); - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, - &pvt->dcsb1[cs]); - if (unlikely(err)) - debugf0("Reading F10_DCSB1[%d] failed\n", cs); - else - debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsb1[cs], reg); - } else { - pvt->dcsb1[cs] = 0; - } + for_each_chip_select(cs, 0, pvt) { + int reg0 = DCSB0 + (cs * 4); + int reg1 = DCSB1 + (cs * 4); + u32 *base0 = &pvt->csels[0].csbases[cs]; + u32 *base1 = &pvt->csels[1].csbases[cs]; + + if (!amd64_read_dct_pci_cfg(pvt, reg0, base0)) + edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", + cs, *base0, reg0); + + if (pvt->fam == 0xf || dct_ganging_enabled(pvt)) + continue; + + if (!amd64_read_dct_pci_cfg(pvt, reg1, base1)) + edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", + cs, *base1, reg1); } - for (cs = 0; cs < pvt->num_dcsm; cs++) { - reg = K8_DCSB0 + (cs * 4); - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, - &pvt->dcsm0[cs]); - if (unlikely(err)) - debugf0("Reading K8_DCSM0 failed\n"); - else - debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsm0[cs], reg); - - /* If DCT are NOT ganged, then read in DCT1's mask */ - if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { - reg = F10_DCSM1 + (cs * 4); - err = pci_read_config_dword(pvt->dram_f2_ctl, reg, - &pvt->dcsm1[cs]); - if (unlikely(err)) - debugf0("Reading F10_DCSM1[%d] failed\n", cs); - else - debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsm1[cs], reg); - } else - pvt->dcsm1[cs] = 0; + for_each_chip_select_mask(cs, 0, pvt) { + int reg0 = DCSM0 + (cs * 4); + int reg1 = DCSM1 + (cs * 4); + u32 *mask0 = &pvt->csels[0].csmasks[cs]; + u32 *mask1 = &pvt->csels[1].csmasks[cs]; + + if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0)) + edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", + cs, *mask0, reg0); + + if (pvt->fam == 0xf || dct_ganging_enabled(pvt)) + continue; + + if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1)) + edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", + cs, *mask1, reg1); } } -static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt) +static enum mem_type determine_memory_type(struct amd64_pvt *pvt, int cs) { enum mem_type type; - if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= OPTERON_CPU_REV_F) { - /* Rev F and later */ - type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; + /* F15h supports only DDR3 */ + if (pvt->fam >= 0x15) + type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; + else if (pvt->fam == 0x10 || pvt->ext_model >= K8_REV_F) { + if (pvt->dchr0 & DDR3_MODE) + type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; + else + type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2; } else { - /* Rev E and earlier */ type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; } - debugf1(" Memory type is: %s\n", - (type == MEM_DDR2) ? "MEM_DDR2" : - (type == MEM_RDDR2) ? "MEM_RDDR2" : - (type == MEM_DDR) ? "MEM_DDR" : "MEM_RDDR"); + amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); return type; } -/* - * Read the DRAM Configuration Low register. It differs between CG, D & E revs - * and the later RevF memory controllers (DDR vs DDR2) - * - * Return: - * number of memory channels in operation - * Pass back: - * contents of the DCL0_LOW register - */ +/* Get the number of DCT channels the memory controller is using. */ static int k8_early_channel_count(struct amd64_pvt *pvt) { - int flag, err = 0; - - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); - if (err) - return err; + int flag; - if ((boot_cpu_data.x86_model >> 4) >= OPTERON_CPU_REV_F) { + if (pvt->ext_model >= K8_REV_F) /* RevF (NPT) and later */ - flag = pvt->dclr0 & F10_WIDTH_128; - } else { + flag = pvt->dclr0 & WIDTH_128; + else /* RevE and earlier */ flag = pvt->dclr0 & REVE_WIDTH_128; - } /* not used */ pvt->dclr1 = 0; @@ -1045,76 +838,173 @@ static int k8_early_channel_count(struct amd64_pvt *pvt) return (flag) ? 2 : 1; } -/* extract the ERROR ADDRESS for the K8 CPUs */ -static u64 k8_get_error_address(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +/* On F10h and later ErrAddr is MC4_ADDR[47:1] */ +static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) { - return (((u64) (info->nbeah & 0xff)) << 32) + - (info->nbeal & ~0x03); + u64 addr; + u8 start_bit = 1; + u8 end_bit = 47; + + if (pvt->fam == 0xf) { + start_bit = 3; + end_bit = 39; + } + + addr = m->addr & GENMASK_ULL(end_bit, start_bit); + + /* + * Erratum 637 workaround + */ + if (pvt->fam == 0x15) { + struct amd64_pvt *pvt; + u64 cc6_base, tmp_addr; + u32 tmp; + u16 mce_nid; + u8 intlv_en; + + if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) + return addr; + + mce_nid = amd_get_nb_id(m->extcpu); + pvt = mcis[mce_nid]->pvt_info; + + amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); + intlv_en = tmp >> 21 & 0x7; + + /* add [47:27] + 3 trailing bits */ + cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; + + /* reverse and add DramIntlvEn */ + cc6_base |= intlv_en ^ 0x7; + + /* pin at [47:24] */ + cc6_base <<= 24; + + if (!intlv_en) + return cc6_base | (addr & GENMASK_ULL(23, 0)); + + amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); + + /* faster log2 */ + tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); + + /* OR DramIntlvSel into bits [14:12] */ + tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; + + /* add remaining [11:0] bits from original MC4_ADDR */ + tmp_addr |= addr & GENMASK_ULL(11, 0); + + return cc6_base | tmp_addr; + } + + return addr; } -/* - * Read the Base and Limit registers for K8 based Memory controllers; extract - * fields from the 'raw' reg into separate data fields - * - * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN - */ -static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram) +static struct pci_dev *pci_get_related_function(unsigned int vendor, + unsigned int device, + struct pci_dev *related) { - u32 low; - u32 off = dram << 3; /* 8 bytes between DRAM entries */ - int err; + struct pci_dev *dev = NULL; - err = pci_read_config_dword(pvt->addr_f1_ctl, - K8_DRAM_BASE_LOW + off, &low); - if (err) - debugf0("Reading K8_DRAM_BASE_LOW failed\n"); + while ((dev = pci_get_device(vendor, device, dev))) { + if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && + (dev->bus->number == related->bus->number) && + (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) + break; + } - /* Extract parts into separate data entries */ - pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8; - pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7; - pvt->dram_rw_en[dram] = (low & 0x3); + return dev; +} - err = pci_read_config_dword(pvt->addr_f1_ctl, - K8_DRAM_LIMIT_LOW + off, &low); - if (err) - debugf0("Reading K8_DRAM_LIMIT_LOW failed\n"); +static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) +{ + struct amd_northbridge *nb; + struct pci_dev *f1 = NULL; + unsigned int pci_func; + int off = range << 3; + u32 llim; - /* - * Extract parts into separate data entries. Limit is the HIGHEST memory - * location of the region, so lower 24 bits need to be all ones - */ - pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF; - pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7; - pvt->dram_DstNode[dram] = (low & 0x7); + amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); + amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); + + if (pvt->fam == 0xf) + return; + + if (!dram_rw(pvt, range)) + return; + + amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); + amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); + + /* F15h: factor in CC6 save area by reading dst node's limit reg */ + if (pvt->fam != 0x15) + return; + + nb = node_to_amd_nb(dram_dst_node(pvt, range)); + if (WARN_ON(!nb)) + return; + + pci_func = (pvt->model == 0x30) ? PCI_DEVICE_ID_AMD_15H_M30H_NB_F1 + : PCI_DEVICE_ID_AMD_15H_NB_F1; + + f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); + if (WARN_ON(!f1)) + return; + + amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); + + pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); + + /* {[39:27],111b} */ + pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; + + pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); + + /* [47:40] */ + pvt->ranges[range].lim.hi |= llim >> 13; + + pci_dev_put(f1); } -static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info, - u64 SystemAddress) +static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, + struct err_info *err) { - struct mem_ctl_info *src_mci; - unsigned short syndrome; - int channel, csrow; - u32 page, offset; + struct amd64_pvt *pvt = mci->pvt_info; - /* Extract the syndrome parts and form a 16-bit syndrome */ - syndrome = EXTRACT_HIGH_SYNDROME(info->nbsl) << 8; - syndrome |= EXTRACT_LOW_SYNDROME(info->nbsh); + error_address_to_page_and_offset(sys_addr, err); + + /* + * Find out which node the error address belongs to. This may be + * different from the node that detected the error. + */ + err->src_mci = find_mc_by_sys_addr(mci, sys_addr); + if (!err->src_mci) { + amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", + (unsigned long)sys_addr); + err->err_code = ERR_NODE; + return; + } + + /* Now map the sys_addr to a CSROW */ + err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); + if (err->csrow < 0) { + err->err_code = ERR_CSROW; + return; + } /* CHIPKILL enabled */ - if (info->nbcfg & K8_NBCFG_CHIPKILL) { - channel = get_channel_from_ecc_syndrome(syndrome); - if (channel < 0) { + if (pvt->nbcfg & NBCFG_CHIPKILL) { + err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); + if (err->channel < 0) { /* * Syndrome didn't map, so we don't know which of the * 2 DIMMs is in error. So we need to ID 'both' of them * as suspect. */ - amd64_mc_printk(mci, KERN_WARNING, - "unknown syndrome 0x%x - possible error " - "reporting race\n", syndrome); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); + amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " + "possible error reporting race\n", + err->syndrome); + err->err_code = ERR_CHANNEL; return; } } else { @@ -1126,64 +1016,69 @@ static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, * was obtained from email communication with someone at AMD. * (Wish the email was placed in this comment - norsk) */ - channel = ((SystemAddress & BIT(3)) != 0); + err->channel = ((sys_addr & BIT(3)) != 0); } +} - /* - * Find out which node the error address belongs to. This may be - * different from the node that detected the error. - */ - src_mci = find_mc_by_sys_addr(mci, SystemAddress); - if (src_mci) { - amd64_mc_printk(mci, KERN_ERR, - "failed to map error address 0x%lx to a node\n", - (unsigned long)SystemAddress); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); - return; - } +static int ddr2_cs_size(unsigned i, bool dct_width) +{ + unsigned shift = 0; - /* Now map the SystemAddress to a CSROW */ - csrow = sys_addr_to_csrow(src_mci, SystemAddress); - if (csrow < 0) { - edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); - } else { - error_address_to_page_and_offset(SystemAddress, &page, &offset); + if (i <= 2) + shift = i; + else if (!(i & 0x1)) + shift = i >> 1; + else + shift = (i + 1) >> 1; - edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, - channel, EDAC_MOD_STR); - } + return 128 << (shift + !!dct_width); } -/* - * determrine the number of PAGES in for this DIMM's size based on its DRAM - * Address Mapping. - * - * First step is to calc the number of bits to shift a value of 1 left to - * indicate show many pages. Start with the DBAM value as the starting bits, - * then proceed to adjust those shift bits, based on CPU rev and the table. - * See BKDG on the DBAM - */ -static int k8_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map) +static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - int nr_pages; + u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; + + if (pvt->ext_model >= K8_REV_F) { + WARN_ON(cs_mode > 11); + return ddr2_cs_size(cs_mode, dclr & WIDTH_128); + } + else if (pvt->ext_model >= K8_REV_D) { + unsigned diff; + WARN_ON(cs_mode > 10); - if (pvt->ext_model >= OPTERON_CPU_REV_F) { - nr_pages = 1 << (revf_quad_ddr2_shift[dram_map] - PAGE_SHIFT); - } else { /* - * RevE and less section; this line is tricky. It collapses the - * table used by RevD and later to one that matches revisions CG - * and earlier. + * the below calculation, besides trying to win an obfuscated C + * contest, maps cs_mode values to DIMM chip select sizes. The + * mappings are: + * + * cs_mode CS size (mb) + * ======= ============ + * 0 32 + * 1 64 + * 2 128 + * 3 128 + * 4 256 + * 5 512 + * 6 256 + * 7 512 + * 8 1024 + * 9 1024 + * 10 2048 + * + * Basically, it calculates a value with which to shift the + * smallest CS size of 32MB. + * + * ddr[23]_cs_size have a similar purpose. */ - dram_map -= (pvt->ext_model >= OPTERON_CPU_REV_D) ? - (dram_map > 8 ? 4 : (dram_map > 5 ? - 3 : (dram_map > 2 ? 1 : 0))) : 0; + diff = cs_mode/3 + (unsigned)(cs_mode > 5); - /* 25 shift is 32MiB minimum DIMM size in RevE and prior */ - nr_pages = 1 << (dram_map + 25 - PAGE_SHIFT); + return 32 << (cs_mode - diff); + } + else { + WARN_ON(cs_mode > 6); + return 32 << cs_mode; } - - return nr_pages; } /* @@ -1194,316 +1089,276 @@ static int k8_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map) * Pass back: * contents of the DCL0_LOW register */ -static int f10_early_channel_count(struct amd64_pvt *pvt) +static int f1x_early_channel_count(struct amd64_pvt *pvt) { - int err = 0, channels = 0; - u32 dbam; - - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); - if (err) - goto err_reg; + int i, j, channels = 0; - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCLR_1, &pvt->dclr1); - if (err) - goto err_reg; - - /* If we are in 128 bit mode, then we are using 2 channels */ - if (pvt->dclr0 & F10_WIDTH_128) { - debugf0("Data WIDTH is 128 bits - 2 channels\n"); - channels = 2; - return channels; - } + /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ + if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128)) + return 2; /* - * Need to check if in UN-ganged mode: In such, there are 2 channels, - * but they are NOT in 128 bit mode and thus the above 'dcl0' status bit - * will be OFF. + * Need to check if in unganged mode: In such, there are 2 channels, + * but they are not in 128 bit mode and thus the above 'dclr0' status + * bit will be OFF. * * Need to check DCT0[0] and DCT1[0] to see if only one of them has * their CSEnable bit on. If so, then SINGLE DIMM case. */ - debugf0("Data WIDTH is NOT 128 bits - need more decoding\n"); + edac_dbg(0, "Data width is not 128 bits - need more decoding\n"); /* * Check DRAM Bank Address Mapping values for each DIMM to see if there * is more than just one DIMM present in unganged mode. Need to check * both controllers since DIMMs can be placed in either one. */ - channels = 0; - err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM0, &dbam); - if (err) - goto err_reg; - - if (DBAM_DIMM(0, dbam) > 0) - channels++; - if (DBAM_DIMM(1, dbam) > 0) - channels++; - if (DBAM_DIMM(2, dbam) > 0) - channels++; - if (DBAM_DIMM(3, dbam) > 0) - channels++; - - /* If more than 2 DIMMs are present, then we have 2 channels */ - if (channels > 2) - channels = 2; - else if (channels == 0) { - /* No DIMMs on DCT0, so look at DCT1 */ - err = pci_read_config_dword(pvt->dram_f2_ctl, DBAM1, &dbam); - if (err) - goto err_reg; + for (i = 0; i < 2; i++) { + u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); - if (DBAM_DIMM(0, dbam) > 0) - channels++; - if (DBAM_DIMM(1, dbam) > 0) - channels++; - if (DBAM_DIMM(2, dbam) > 0) - channels++; - if (DBAM_DIMM(3, dbam) > 0) - channels++; - - if (channels > 2) - channels = 2; + for (j = 0; j < 4; j++) { + if (DBAM_DIMM(j, dbam) > 0) { + channels++; + break; + } + } } - /* If we found ALL 0 values, then assume just ONE DIMM-ONE Channel */ - if (channels == 0) - channels = 1; + if (channels > 2) + channels = 2; - debugf0("DIMM count= %d\n", channels); + amd64_info("MCT channel count: %d\n", channels); return channels; - -err_reg: - return -1; - } -static int f10_dbam_map_to_pages(struct amd64_pvt *pvt, int dram_map) +static int ddr3_cs_size(unsigned i, bool dct_width) { - return 1 << (revf_quad_ddr2_shift[dram_map] - PAGE_SHIFT); -} + unsigned shift = 0; + int cs_size = 0; -/* Enable extended configuration access via 0xCF8 feature */ -static void amd64_setup(struct amd64_pvt *pvt) -{ - u32 reg; + if (i == 0 || i == 3 || i == 4) + cs_size = -1; + else if (i <= 2) + shift = i; + else if (i == 12) + shift = 7; + else if (!(i & 0x1)) + shift = i >> 1; + else + shift = (i + 1) >> 1; - pci_read_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); + if (cs_size != -1) + cs_size = (128 * (1 << !!dct_width)) << shift; - pvt->flags.cf8_extcfg = !!(reg & F10_NB_CFG_LOW_ENABLE_EXT_CFG); - reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; - pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); + return cs_size; } -/* Restore the extended configuration access via 0xCF8 feature */ -static void amd64_teardown(struct amd64_pvt *pvt) +static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - u32 reg; + u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; - pci_read_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); + WARN_ON(cs_mode > 11); - reg &= ~F10_NB_CFG_LOW_ENABLE_EXT_CFG; - if (pvt->flags.cf8_extcfg) - reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; - pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); + if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) + return ddr3_cs_size(cs_mode, dclr & WIDTH_128); + else + return ddr2_cs_size(cs_mode, dclr & WIDTH_128); } -static u64 f10_get_error_address(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +/* + * F15h supports only 64bit DCT interfaces + */ +static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - return (((u64) (info->nbeah & 0xffff)) << 32) + - (info->nbeal & ~0x01); + WARN_ON(cs_mode > 12); + + return ddr3_cs_size(cs_mode, false); } /* - * Read the Base and Limit registers for F10 based Memory controllers. Extract - * fields from the 'raw' reg into separate data fields. - * - * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN. + * F16h and F15h model 30h have only limited cs_modes. */ -static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram) +static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit; - - low_offset = K8_DRAM_BASE_LOW + (dram << 3); - high_offset = F10_DRAM_BASE_HIGH + (dram << 3); + WARN_ON(cs_mode > 12); - /* read the 'raw' DRAM BASE Address register */ - pci_read_config_dword(pvt->addr_f1_ctl, low_offset, &low_base); - - /* Read from the ECS data register */ - pci_read_config_dword(pvt->addr_f1_ctl, high_offset, &high_base); + if (cs_mode == 6 || cs_mode == 8 || + cs_mode == 9 || cs_mode == 12) + return -1; + else + return ddr3_cs_size(cs_mode, false); +} - /* Extract parts into separate data entries */ - pvt->dram_rw_en[dram] = (low_base & 0x3); +static void read_dram_ctl_register(struct amd64_pvt *pvt) +{ - if (pvt->dram_rw_en[dram] == 0) + if (pvt->fam == 0xf) return; - pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7; - - pvt->dram_base[dram] = (((((u64) high_base & 0x000000FF) << 32) | - ((u64) low_base & 0xFFFF0000))) << 8; - - low_offset = K8_DRAM_LIMIT_LOW + (dram << 3); - high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3); + if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) { + edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", + pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); - /* read the 'raw' LIMIT registers */ - pci_read_config_dword(pvt->addr_f1_ctl, low_offset, &low_limit); + edac_dbg(0, " DCTs operate in %s mode\n", + (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); - /* Read from the ECS data register for the HIGH portion */ - pci_read_config_dword(pvt->addr_f1_ctl, high_offset, &high_limit); + if (!dct_ganging_enabled(pvt)) + edac_dbg(0, " Address range split per DCT: %s\n", + (dct_high_range_enabled(pvt) ? "yes" : "no")); - debugf0(" HW Regs: BASE=0x%08x-%08x LIMIT= 0x%08x-%08x\n", - high_base, low_base, high_limit, low_limit); + edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", + (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), + (dct_memory_cleared(pvt) ? "yes" : "no")); - pvt->dram_DstNode[dram] = (low_limit & 0x7); - pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7; + edac_dbg(0, " channel interleave: %s, " + "interleave bits selector: 0x%x\n", + (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), + dct_sel_interleave_addr(pvt)); + } - /* - * Extract address values and form a LIMIT address. Limit is the HIGHEST - * memory location of the region, so low 24 bits need to be all ones. - */ - low_limit |= 0x0000FFFF; - pvt->dram_limit[dram] = - ((((u64) high_limit << 32) + (u64) low_limit) << 8) | (0xFF); + amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi); } -static void f10_read_dram_ctl_register(struct amd64_pvt *pvt) -{ - int err = 0; - - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCTL_SEL_LOW, - &pvt->dram_ctl_select_low); - if (err) { - debugf0("Reading F10_DCTL_SEL_LOW failed\n"); - } else { - debugf0("DRAM_DCTL_SEL_LOW=0x%x DctSelBaseAddr=0x%x\n", - pvt->dram_ctl_select_low, dct_sel_baseaddr(pvt)); - - debugf0(" DRAM DCTs are=%s DRAM Is=%s DRAM-Ctl-" - "sel-hi-range=%s\n", - (dct_ganging_enabled(pvt) ? "GANGED" : "NOT GANGED"), - (dct_dram_enabled(pvt) ? "Enabled" : "Disabled"), - (dct_high_range_enabled(pvt) ? "Enabled" : "Disabled")); - - debugf0(" DctDatIntLv=%s MemCleared=%s DctSelIntLvAddr=0x%x\n", - (dct_data_intlv_enabled(pvt) ? "Enabled" : "Disabled"), - (dct_memory_cleared(pvt) ? "True " : "False "), - dct_sel_interleave_addr(pvt)); +/* + * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, + * 2.10.12 Memory Interleaving Modes). + */ +static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, + u8 intlv_en, int num_dcts_intlv, + u32 dct_sel) +{ + u8 channel = 0; + u8 select; + + if (!(intlv_en)) + return (u8)(dct_sel); + + if (num_dcts_intlv == 2) { + select = (sys_addr >> 8) & 0x3; + channel = select ? 0x3 : 0; + } else if (num_dcts_intlv == 4) { + u8 intlv_addr = dct_sel_interleave_addr(pvt); + switch (intlv_addr) { + case 0x4: + channel = (sys_addr >> 8) & 0x3; + break; + case 0x5: + channel = (sys_addr >> 9) & 0x3; + break; + } } - - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCTL_SEL_HIGH, - &pvt->dram_ctl_select_high); - if (err) - debugf0("Reading F10_DCTL_SEL_HIGH failed\n"); + return channel; } /* - * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory + * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory * Interleaving Modes. */ -static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, - int hi_range_sel, u32 intlv_en) +static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, + bool hi_range_sel, u8 intlv_en) { - u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1; + u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; if (dct_ganging_enabled(pvt)) - cs = 0; - else if (hi_range_sel) - cs = dct_sel_high; - else if (dct_interleave_enabled(pvt)) { - /* - * see F2x110[DctSelIntLvAddr] - channel interleave mode - */ - if (dct_sel_interleave_addr(pvt) == 0) - cs = sys_addr >> 6 & 1; - else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) { - temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; + return 0; - if (dct_sel_interleave_addr(pvt) & 1) - cs = (sys_addr >> 9 & 1) ^ temp; - else - cs = (sys_addr >> 6 & 1) ^ temp; - } else if (intlv_en & 4) - cs = sys_addr >> 15 & 1; - else if (intlv_en & 2) - cs = sys_addr >> 14 & 1; - else if (intlv_en & 1) - cs = sys_addr >> 13 & 1; - else - cs = sys_addr >> 12 & 1; - } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) - cs = ~dct_sel_high & 1; - else - cs = 0; + if (hi_range_sel) + return dct_sel_high; - return cs; -} + /* + * see F2x110[DctSelIntLvAddr] - channel interleave mode + */ + if (dct_interleave_enabled(pvt)) { + u8 intlv_addr = dct_sel_interleave_addr(pvt); -static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en) -{ - if (intlv_en == 1) - return 1; - else if (intlv_en == 3) - return 2; - else if (intlv_en == 7) - return 3; + /* return DCT select function: 0=DCT0, 1=DCT1 */ + if (!intlv_addr) + return sys_addr >> 6 & 1; + + if (intlv_addr & 0x2) { + u8 shift = intlv_addr & 0x1 ? 9 : 6; + u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; + + return ((sys_addr >> shift) & 1) ^ temp; + } + + return (sys_addr >> (12 + hweight8(intlv_en))) & 1; + } + + if (dct_high_range_enabled(pvt)) + return ~dct_sel_high & 1; return 0; } -/* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */ -static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel, - u32 dct_sel_base_addr, - u64 dct_sel_base_off, - u32 hole_valid, u32 hole_off, - u64 dram_base) +/* Convert the sys_addr to the normalized DCT address */ +static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, + u64 sys_addr, bool hi_rng, + u32 dct_sel_base_addr) { u64 chan_off; + u64 dram_base = get_dram_base(pvt, range); + u64 hole_off = f10_dhar_offset(pvt); + u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; - if (hi_range_sel) { - if (!(dct_sel_base_addr & 0xFFFFF800) && - hole_valid && (sys_addr >= 0x100000000ULL)) - chan_off = hole_off << 16; + if (hi_rng) { + /* + * if + * base address of high range is below 4Gb + * (bits [47:27] at [31:11]) + * DRAM address space on this DCT is hoisted above 4Gb && + * sys_addr > 4Gb + * + * remove hole offset from sys_addr + * else + * remove high range offset from sys_addr + */ + if ((!(dct_sel_base_addr >> 16) || + dct_sel_base_addr < dhar_base(pvt)) && + dhar_valid(pvt) && + (sys_addr >= BIT_64(32))) + chan_off = hole_off; else chan_off = dct_sel_base_off; } else { - if (hole_valid && (sys_addr >= 0x100000000ULL)) - chan_off = hole_off << 16; + /* + * if + * we have a valid hole && + * sys_addr > 4Gb + * + * remove hole + * else + * remove dram base to normalize to DCT address + */ + if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) + chan_off = hole_off; else - chan_off = dram_base & 0xFFFFF8000000ULL; + chan_off = dram_base; } - return (sys_addr & 0x0000FFFFFFFFFFC0ULL) - - (chan_off & 0x0000FFFFFF800000ULL); + return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); } -/* Hack for the time being - Can we get this from BIOS?? */ -#define CH0SPARE_RANK 0 -#define CH1SPARE_RANK 1 - /* * checks if the csrow passed in is marked as SPARED, if so returns the new * spare row */ -static inline int f10_process_possible_spare(int csrow, - u32 cs, struct amd64_pvt *pvt) -{ - u32 swap_done; - u32 bad_dram_cs; - - /* Depending on channel, isolate respective SPARING info */ - if (cs) { - swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare); - bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare); - if (swap_done && (csrow == bad_dram_cs)) - csrow = CH1SPARE_RANK; - } else { - swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare); - bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare); - if (swap_done && (csrow == bad_dram_cs)) - csrow = CH0SPARE_RANK; +static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) +{ + int tmp_cs; + + if (online_spare_swap_done(pvt, dct) && + csrow == online_spare_bad_dramcs(pvt, dct)) { + + for_each_chip_select(tmp_cs, dct, pvt) { + if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { + csrow = tmp_cs; + break; + } + } } return csrow; } @@ -1516,93 +1371,113 @@ static inline int f10_process_possible_spare(int csrow, * -EINVAL: NOT FOUND * 0..csrow = Chip-Select Row */ -static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs) +static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; - u32 cs_base, cs_mask; + u64 cs_base, cs_mask; int cs_found = -EINVAL; int csrow; - mci = mci_lookup[nid]; + mci = mcis[nid]; if (!mci) return cs_found; pvt = mci->pvt_info; - debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs); + edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); - for (csrow = 0; csrow < CHIPSELECT_COUNT; csrow++) { - - cs_base = amd64_get_dct_base(pvt, cs, csrow); - if (!(cs_base & K8_DCSB_CS_ENABLE)) + for_each_chip_select(csrow, dct, pvt) { + if (!csrow_enabled(csrow, dct, pvt)) continue; - /* - * We have an ENABLED CSROW, Isolate just the MASK bits of the - * target: [28:19] and [13:5], which map to [36:27] and [21:13] - * of the actual address. - */ - cs_base &= REV_F_F1Xh_DCSB_BASE_BITS; - - /* - * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and - * [4:0] to become ON. Then mask off bits [28:0] ([36:8]) - */ - cs_mask = amd64_get_dct_mask(pvt, cs, csrow); + get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); - debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n", - csrow, cs_base, cs_mask); + edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", + csrow, cs_base, cs_mask); - cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF; + cs_mask = ~cs_mask; - debugf1(" Final CSMask=0x%x\n", cs_mask); - debugf1(" (InputAddr & ~CSMask)=0x%x " - "(CSBase & ~CSMask)=0x%x\n", - (in_addr & ~cs_mask), (cs_base & ~cs_mask)); + edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", + (in_addr & cs_mask), (cs_base & cs_mask)); - if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) { - cs_found = f10_process_possible_spare(csrow, cs, pvt); + if ((in_addr & cs_mask) == (cs_base & cs_mask)) { + if (pvt->fam == 0x15 && pvt->model >= 0x30) { + cs_found = csrow; + break; + } + cs_found = f10_process_possible_spare(pvt, dct, csrow); - debugf1(" MATCH csrow=%d\n", cs_found); + edac_dbg(1, " MATCH csrow=%d\n", cs_found); break; } } return cs_found; } -/* For a given @dram_range, check if @sys_addr falls within it. */ -static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, - u64 sys_addr, int *nid, int *chan_sel) +/* + * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is + * swapped with a region located at the bottom of memory so that the GPU can use + * the interleaved region and thus two channels. + */ +static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) { - int node_id, cs_found = -EINVAL, high_range = 0; - u32 intlv_en, intlv_sel, intlv_shift, hole_off; - u32 hole_valid, tmp, dct_sel_base, channel; - u64 dram_base, chan_addr, dct_sel_base_off; + u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; - dram_base = pvt->dram_base[dram_range]; - intlv_en = pvt->dram_IntlvEn[dram_range]; + if (pvt->fam == 0x10) { + /* only revC3 and revE have that feature */ + if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) + return sys_addr; + } - node_id = pvt->dram_DstNode[dram_range]; - intlv_sel = pvt->dram_IntlvSel[dram_range]; + amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg); - debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n", - dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]); + if (!(swap_reg & 0x1)) + return sys_addr; - /* - * This assumes that one node's DHAR is the same as all the other - * nodes' DHAR. - */ - hole_off = (pvt->dhar & 0x0000FF80); - hole_valid = (pvt->dhar & 0x1); - dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16; + swap_base = (swap_reg >> 3) & 0x7f; + swap_limit = (swap_reg >> 11) & 0x7f; + rgn_size = (swap_reg >> 20) & 0x7f; + tmp_addr = sys_addr >> 27; - debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n", - hole_off, hole_valid, intlv_sel); + if (!(sys_addr >> 34) && + (((tmp_addr >= swap_base) && + (tmp_addr <= swap_limit)) || + (tmp_addr < rgn_size))) + return sys_addr ^ (u64)swap_base << 27; - if (intlv_en || - (intlv_sel != ((sys_addr >> 12) & intlv_en))) + return sys_addr; +} + +/* For a given @dram_range, check if @sys_addr falls within it. */ +static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, + u64 sys_addr, int *chan_sel) +{ + int cs_found = -EINVAL; + u64 chan_addr; + u32 dct_sel_base; + u8 channel; + bool high_range = false; + + u8 node_id = dram_dst_node(pvt, range); + u8 intlv_en = dram_intlv_en(pvt, range); + u32 intlv_sel = dram_intlv_sel(pvt, range); + + edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", + range, sys_addr, get_dram_limit(pvt, range)); + + if (dhar_valid(pvt) && + dhar_base(pvt) <= sys_addr && + sys_addr < BIT_64(32)) { + amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", + sys_addr); return -EINVAL; + } + + if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) + return -EINVAL; + + sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); dct_sel_base = dct_sel_baseaddr(pvt); @@ -1613,1037 +1488,698 @@ static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt) && ((sys_addr >> 27) >= (dct_sel_base >> 11))) - high_range = 1; - - channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en); - - chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base, - dct_sel_base_off, hole_valid, - hole_off, dram_base); + high_range = true; - intlv_shift = f10_map_intlv_en_to_shift(intlv_en); + channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); - /* remove Node ID (in case of memory interleaving) */ - tmp = chan_addr & 0xFC0; + chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, + high_range, dct_sel_base); - chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp; + /* Remove node interleaving, see F1x120 */ + if (intlv_en) + chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | + (chan_addr & 0xfff); - /* remove channel interleave and hash */ + /* remove channel interleave */ if (dct_interleave_enabled(pvt) && !dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) { - if (dct_sel_interleave_addr(pvt) != 1) - chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL; - else { - tmp = chan_addr & 0xFC0; - chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1) - | tmp; - } + + if (dct_sel_interleave_addr(pvt) != 1) { + if (dct_sel_interleave_addr(pvt) == 0x3) + /* hash 9 */ + chan_addr = ((chan_addr >> 10) << 9) | + (chan_addr & 0x1ff); + else + /* A[6] or hash 6 */ + chan_addr = ((chan_addr >> 7) << 6) | + (chan_addr & 0x3f); + } else + /* A[12] */ + chan_addr = ((chan_addr >> 13) << 12) | + (chan_addr & 0xfff); } - debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n", - chan_addr, (u32)(chan_addr >> 8)); + edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); - cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel); + cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); - if (cs_found >= 0) { - *nid = node_id; + if (cs_found >= 0) *chan_sel = channel; - } + return cs_found; } -static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, - int *node, int *chan_sel) +static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, + u64 sys_addr, int *chan_sel) { - int dram_range, cs_found = -EINVAL; - u64 dram_base, dram_limit; + int cs_found = -EINVAL; + int num_dcts_intlv = 0; + u64 chan_addr, chan_offset; + u64 dct_base, dct_limit; + u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; + u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; - for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) { + u64 dhar_offset = f10_dhar_offset(pvt); + u8 intlv_addr = dct_sel_interleave_addr(pvt); + u8 node_id = dram_dst_node(pvt, range); + u8 intlv_en = dram_intlv_en(pvt, range); - if (!pvt->dram_rw_en[dram_range]) - continue; + amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); + amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); - dram_base = pvt->dram_base[dram_range]; - dram_limit = pvt->dram_limit[dram_range]; + dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); + dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); - if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) { + edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", + range, sys_addr, get_dram_limit(pvt, range)); - cs_found = f10_match_to_this_node(pvt, dram_range, - sys_addr, node, - chan_sel); - if (cs_found >= 0) - break; - } + if (!(get_dram_base(pvt, range) <= sys_addr) && + !(get_dram_limit(pvt, range) >= sys_addr)) + return -EINVAL; + + if (dhar_valid(pvt) && + dhar_base(pvt) <= sys_addr && + sys_addr < BIT_64(32)) { + amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", + sys_addr); + return -EINVAL; + } + + /* Verify sys_addr is within DCT Range. */ + dct_base = (u64) dct_sel_baseaddr(pvt); + dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; + + if (!(dct_cont_base_reg & BIT(0)) && + !(dct_base <= (sys_addr >> 27) && + dct_limit >= (sys_addr >> 27))) + return -EINVAL; + + /* Verify number of dct's that participate in channel interleaving. */ + num_dcts_intlv = (int) hweight8(intlv_en); + + if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) + return -EINVAL; + + channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, + num_dcts_intlv, dct_sel); + + /* Verify we stay within the MAX number of channels allowed */ + if (channel > 3) + return -EINVAL; + + leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); + + /* Get normalized DCT addr */ + if (leg_mmio_hole && (sys_addr >= BIT_64(32))) + chan_offset = dhar_offset; + else + chan_offset = dct_base << 27; + + chan_addr = sys_addr - chan_offset; + + /* remove channel interleave */ + if (num_dcts_intlv == 2) { + if (intlv_addr == 0x4) + chan_addr = ((chan_addr >> 9) << 8) | + (chan_addr & 0xff); + else if (intlv_addr == 0x5) + chan_addr = ((chan_addr >> 10) << 9) | + (chan_addr & 0x1ff); + else + return -EINVAL; + + } else if (num_dcts_intlv == 4) { + if (intlv_addr == 0x4) + chan_addr = ((chan_addr >> 10) << 8) | + (chan_addr & 0xff); + else if (intlv_addr == 0x5) + chan_addr = ((chan_addr >> 11) << 9) | + (chan_addr & 0x1ff); + else + return -EINVAL; + } + + if (dct_offset_en) { + amd64_read_pci_cfg(pvt->F1, + DRAM_CONT_HIGH_OFF + (int) channel * 4, + &tmp); + chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; } + + f15h_select_dct(pvt, channel); + + edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); + + /* + * Find Chip select: + * if channel = 3, then alias it to 1. This is because, in F15 M30h, + * there is support for 4 DCT's, but only 2 are currently functional. + * They are DCT0 and DCT3. But we have read all registers of DCT3 into + * pvt->csels[1]. So we need to use '1' here to get correct info. + * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. + */ + alias_channel = (channel == 3) ? 1 : channel; + + cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); + + if (cs_found >= 0) + *chan_sel = alias_channel; + return cs_found; } -/* - * This the F10h reference code from AMD to map a @sys_addr to NodeID, - * CSROW, Channel. - * - * The @sys_addr is usually an error address received from the hardware. - */ -static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info, - u64 sys_addr) +static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, + u64 sys_addr, + int *chan_sel) { - struct amd64_pvt *pvt = mci->pvt_info; - u32 page, offset; - unsigned short syndrome; - int nid, csrow, chan = 0; - - csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); - - if (csrow >= 0) { - error_address_to_page_and_offset(sys_addr, &page, &offset); + int cs_found = -EINVAL; + unsigned range; - syndrome = EXTRACT_HIGH_SYNDROME(info->nbsl) << 8; - syndrome |= EXTRACT_LOW_SYNDROME(info->nbsh); + for (range = 0; range < DRAM_RANGES; range++) { + if (!dram_rw(pvt, range)) + continue; - /* - * Is CHIPKILL on? If so, then we can attempt to use the - * syndrome to isolate which channel the error was on. - */ - if (pvt->nbcfg & K8_NBCFG_CHIPKILL) - chan = get_channel_from_ecc_syndrome(syndrome); + if (pvt->fam == 0x15 && pvt->model >= 0x30) + cs_found = f15_m30h_match_to_this_node(pvt, range, + sys_addr, + chan_sel); - if (chan >= 0) { - edac_mc_handle_ce(mci, page, offset, syndrome, - csrow, chan, EDAC_MOD_STR); - } else { - /* - * Channel unknown, report all channels on this - * CSROW as failed. - */ - for (chan = 0; chan < mci->csrows[csrow].nr_channels; - chan++) { - edac_mc_handle_ce(mci, page, offset, - syndrome, - csrow, chan, - EDAC_MOD_STR); - } + else if ((get_dram_base(pvt, range) <= sys_addr) && + (get_dram_limit(pvt, range) >= sys_addr)) { + cs_found = f1x_match_to_this_node(pvt, range, + sys_addr, chan_sel); + if (cs_found >= 0) + break; } - - } else { - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); } + return cs_found; } /* - * Input (@index) is the DBAM DIMM value (1 of 4) used as an index into a shift - * table (revf_quad_ddr2_shift) which starts at 128MB DIMM size. Index of 0 - * indicates an empty DIMM slot, as reported by Hardware on empty slots. + * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps + * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW). * - * Normalize to 128MB by subracting 27 bit shift. + * The @sys_addr is usually an error address received from the hardware + * (MCX_ADDR). */ -static int map_dbam_to_csrow_size(int index) +static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, + struct err_info *err) { - int mega_bytes = 0; + struct amd64_pvt *pvt = mci->pvt_info; - if (index > 0 && index <= DBAM_MAX_VALUE) - mega_bytes = ((128 << (revf_quad_ddr2_shift[index]-27))); + error_address_to_page_and_offset(sys_addr, err); - return mega_bytes; + err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); + if (err->csrow < 0) { + err->err_code = ERR_CSROW; + return; + } + + /* + * We need the syndromes for channel detection only when we're + * ganged. Otherwise @chan should already contain the channel at + * this point. + */ + if (dct_ganging_enabled(pvt)) + err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); } /* - * debug routine to display the memory sizes of a DIMM (ganged or not) and it - * CSROWs as well + * debug routine to display the memory sizes of all logical DIMMs and its + * CSROWs */ -static void f10_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt, - int ganged) +static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) { int dimm, size0, size1; - u32 dbam; - u32 *dcsb; + u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; + u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; + + if (pvt->fam == 0xf) { + /* K8 families < revF not supported yet */ + if (pvt->ext_model < K8_REV_F) + return; + else + WARN_ON(ctrl != 0); + } - debugf1(" dbam%d: 0x%8.08x CSROW is %s\n", ctrl, - ctrl ? pvt->dbam1 : pvt->dbam0, - ganged ? "GANGED - dbam1 not used" : "NON-GANGED"); + dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; + dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases + : pvt->csels[0].csbases; - dbam = ctrl ? pvt->dbam1 : pvt->dbam0; - dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0; + edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", + ctrl, dbam); + + edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); /* Dump memory sizes for DIMM and its CSROWs */ for (dimm = 0; dimm < 4; dimm++) { size0 = 0; - if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE) - size0 = map_dbam_to_csrow_size(DBAM_DIMM(dimm, dbam)); + if (dcsb[dimm*2] & DCSB_CS_ENABLE) + size0 = pvt->ops->dbam_to_cs(pvt, ctrl, + DBAM_DIMM(dimm, dbam)); size1 = 0; - if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE) - size1 = map_dbam_to_csrow_size(DBAM_DIMM(dimm, dbam)); - - debugf1(" CTRL-%d DIMM-%d=%5dMB CSROW-%d=%5dMB " - "CSROW-%d=%5dMB\n", - ctrl, - dimm, - size0 + size1, - dimm * 2, - size0, - dimm * 2 + 1, - size1); - } -} - -/* - * Very early hardware probe on pci_probe thread to determine if this module - * supports the hardware. - * - * Return: - * 0 for OK - * 1 for error - */ -static int f10_probe_valid_hardware(struct amd64_pvt *pvt) -{ - int ret = 0; - - /* - * If we are on a DDR3 machine, we don't know yet if - * we support that properly at this time - */ - if ((pvt->dchr0 & F10_DCHR_Ddr3Mode) || - (pvt->dchr1 & F10_DCHR_Ddr3Mode)) { - - amd64_printk(KERN_WARNING, - "%s() This machine is running with DDR3 memory. " - "This is not currently supported. " - "DCHR0=0x%x DCHR1=0x%x\n", - __func__, pvt->dchr0, pvt->dchr1); - - amd64_printk(KERN_WARNING, - " Contact '%s' module MAINTAINER to help add" - " support.\n", - EDAC_MOD_STR); - - ret = 1; + if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) + size1 = pvt->ops->dbam_to_cs(pvt, ctrl, + DBAM_DIMM(dimm, dbam)); + amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", + dimm * 2, size0, + dimm * 2 + 1, size1); } - return ret; } -/* - * There currently are 3 types type of MC devices for AMD Athlon/Opterons - * (as per PCI DEVICE_IDs): - * - * Family K8: That is the Athlon64 and Opteron CPUs. They all have the same PCI - * DEVICE ID, even though there is differences between the different Revisions - * (CG,D,E,F). - * - * Family F10h and F11h. - * - */ -static struct amd64_family_type amd64_family_types[] = { +static struct amd64_family_type family_types[] = { [K8_CPUS] = { - .ctl_name = "RevF", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_K8_NB_MISC, + .ctl_name = "K8", + .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, + .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, .ops = { - .early_channel_count = k8_early_channel_count, - .get_error_address = k8_get_error_address, - .read_dram_base_limit = k8_read_dram_base_limit, - .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, - .dbam_map_to_pages = k8_dbam_map_to_pages, + .early_channel_count = k8_early_channel_count, + .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, + .dbam_to_cs = k8_dbam_to_chip_select, + .read_dct_pci_cfg = k8_read_dct_pci_cfg, } }, [F10_CPUS] = { - .ctl_name = "Family 10h", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_10H_NB_MAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_10H_NB_MISC, + .ctl_name = "F10h", + .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, + .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, .ops = { - .probe_valid_hardware = f10_probe_valid_hardware, - .early_channel_count = f10_early_channel_count, - .get_error_address = f10_get_error_address, - .read_dram_base_limit = f10_read_dram_base_limit, - .read_dram_ctl_register = f10_read_dram_ctl_register, - .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, - .dbam_map_to_pages = f10_dbam_map_to_pages, + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f10_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, } }, - [F11_CPUS] = { - .ctl_name = "Family 11h", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_11H_NB_MAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_11H_NB_MISC, + [F15_CPUS] = { + .ctl_name = "F15h", + .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, .ops = { - .probe_valid_hardware = f10_probe_valid_hardware, - .early_channel_count = f10_early_channel_count, - .get_error_address = f10_get_error_address, - .read_dram_base_limit = f10_read_dram_base_limit, - .read_dram_ctl_register = f10_read_dram_ctl_register, - .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, - .dbam_map_to_pages = f10_dbam_map_to_pages, + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f15_dbam_to_chip_select, + .read_dct_pci_cfg = f15_read_dct_pci_cfg, + } + }, + [F15_M30H_CPUS] = { + .ctl_name = "F15h_M30h", + .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f15_read_dct_pci_cfg, + } + }, + [F16_CPUS] = { + .ctl_name = "F16h", + .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, + } + }, + [F16_M30H_CPUS] = { + .ctl_name = "F16h_M30h", + .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, } }, }; -static struct pci_dev *pci_get_related_function(unsigned int vendor, - unsigned int device, - struct pci_dev *related) -{ - struct pci_dev *dev = NULL; - - dev = pci_get_device(vendor, device, dev); - while (dev) { - if ((dev->bus->number == related->bus->number) && - (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) - break; - dev = pci_get_device(vendor, device, dev); - } - - return dev; -} - /* - * syndrome mapping table for ECC ChipKill devices - * - * The comment in each row is the token (nibble) number that is in error. - * The least significant nibble of the syndrome is the mask for the bits - * that are in error (need to be toggled) for the particular nibble. - * - * Each row contains 16 entries. - * The first entry (0th) is the channel number for that row of syndromes. - * The remaining 15 entries are the syndromes for the respective Error - * bit mask index. - * - * 1st index entry is 0x0001 mask, indicating that the rightmost bit is the - * bit in error. - * The 2nd index entry is 0x0010 that the second bit is damaged. - * The 3rd index entry is 0x0011 indicating that the rightmost 2 bits - * are damaged. - * Thus so on until index 15, 0x1111, whose entry has the syndrome - * indicating that all 4 bits are damaged. - * - * A search is performed on this table looking for a given syndrome. - * - * See the AMD documentation for ECC syndromes. This ECC table is valid - * across all the versions of the AMD64 processors. + * These are tables of eigenvectors (one per line) which can be used for the + * construction of the syndrome tables. The modified syndrome search algorithm + * uses those to find the symbol in error and thus the DIMM. * - * A fast lookup is to use the LAST four bits of the 16-bit syndrome as a - * COLUMN index, then search all ROWS of that column, looking for a match - * with the input syndrome. The ROW value will be the token number. - * - * The 0'th entry on that row, can be returned as the CHANNEL (0 or 1) of this - * error. + * Algorithm courtesy of Ross LaFetra from AMD. */ -#define NUMBER_ECC_ROWS 36 -static const unsigned short ecc_chipkill_syndromes[NUMBER_ECC_ROWS][16] = { - /* Channel 0 syndromes */ - {/*0*/ 0, 0xe821, 0x7c32, 0x9413, 0xbb44, 0x5365, 0xc776, 0x2f57, - 0xdd88, 0x35a9, 0xa1ba, 0x499b, 0x66cc, 0x8eed, 0x1afe, 0xf2df }, - {/*1*/ 0, 0x5d31, 0xa612, 0xfb23, 0x9584, 0xc8b5, 0x3396, 0x6ea7, - 0xeac8, 0xb7f9, 0x4cda, 0x11eb, 0x7f4c, 0x227d, 0xd95e, 0x846f }, - {/*2*/ 0, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, - 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f }, - {/*3*/ 0, 0x2021, 0x3032, 0x1013, 0x4044, 0x6065, 0x7076, 0x5057, - 0x8088, 0xa0a9, 0xb0ba, 0x909b, 0xc0cc, 0xe0ed, 0xf0fe, 0xd0df }, - {/*4*/ 0, 0x5041, 0xa082, 0xf0c3, 0x9054, 0xc015, 0x30d6, 0x6097, - 0xe0a8, 0xb0e9, 0x402a, 0x106b, 0x70fc, 0x20bd, 0xd07e, 0x803f }, - {/*5*/ 0, 0xbe21, 0xd732, 0x6913, 0x2144, 0x9f65, 0xf676, 0x4857, - 0x3288, 0x8ca9, 0xe5ba, 0x5b9b, 0x13cc, 0xaded, 0xc4fe, 0x7adf }, - {/*6*/ 0, 0x4951, 0x8ea2, 0xc7f3, 0x5394, 0x1ac5, 0xdd36, 0x9467, - 0xa1e8, 0xe8b9, 0x2f4a, 0x661b, 0xf27c, 0xbb2d, 0x7cde, 0x358f }, - {/*7*/ 0, 0x74e1, 0x9872, 0xec93, 0xd6b4, 0xa255, 0x4ec6, 0x3a27, - 0x6bd8, 0x1f39, 0xf3aa, 0x874b, 0xbd6c, 0xc98d, 0x251e, 0x51ff }, - {/*8*/ 0, 0x15c1, 0x2a42, 0x3f83, 0xcef4, 0xdb35, 0xe4b6, 0xf177, - 0x4758, 0x5299, 0x6d1a, 0x78db, 0x89ac, 0x9c6d, 0xa3ee, 0xb62f }, - {/*9*/ 0, 0x3d01, 0x1602, 0x2b03, 0x8504, 0xb805, 0x9306, 0xae07, - 0xca08, 0xf709, 0xdc0a, 0xe10b, 0x4f0c, 0x720d, 0x590e, 0x640f }, - {/*a*/ 0, 0x9801, 0xec02, 0x7403, 0x6b04, 0xf305, 0x8706, 0x1f07, - 0xbd08, 0x2509, 0x510a, 0xc90b, 0xd60c, 0x4e0d, 0x3a0e, 0xa20f }, - {/*b*/ 0, 0xd131, 0x6212, 0xb323, 0x3884, 0xe9b5, 0x5a96, 0x8ba7, - 0x1cc8, 0xcdf9, 0x7eda, 0xafeb, 0x244c, 0xf57d, 0x465e, 0x976f }, - {/*c*/ 0, 0xe1d1, 0x7262, 0x93b3, 0xb834, 0x59e5, 0xca56, 0x2b87, - 0xdc18, 0x3dc9, 0xae7a, 0x4fab, 0x542c, 0x85fd, 0x164e, 0xf79f }, - {/*d*/ 0, 0x6051, 0xb0a2, 0xd0f3, 0x1094, 0x70c5, 0xa036, 0xc067, - 0x20e8, 0x40b9, 0x904a, 0x601b, 0x307c, 0x502d, 0x80de, 0xe08f }, - {/*e*/ 0, 0xa4c1, 0xf842, 0x5c83, 0xe6f4, 0x4235, 0x1eb6, 0xba77, - 0x7b58, 0xdf99, 0x831a, 0x27db, 0x9dac, 0x396d, 0x65ee, 0xc12f }, - {/*f*/ 0, 0x11c1, 0x2242, 0x3383, 0xc8f4, 0xd935, 0xeab6, 0xfb77, - 0x4c58, 0x5d99, 0x6e1a, 0x7fdb, 0x84ac, 0x956d, 0xa6ee, 0xb72f }, - - /* Channel 1 syndromes */ - {/*10*/ 1, 0x45d1, 0x8a62, 0xcfb3, 0x5e34, 0x1be5, 0xd456, 0x9187, - 0xa718, 0xe2c9, 0x2d7a, 0x68ab, 0xf92c, 0xbcfd, 0x734e, 0x369f }, - {/*11*/ 1, 0x63e1, 0xb172, 0xd293, 0x14b4, 0x7755, 0xa5c6, 0xc627, - 0x28d8, 0x4b39, 0x99aa, 0xfa4b, 0x3c6c, 0x5f8d, 0x8d1e, 0xeeff }, - {/*12*/ 1, 0xb741, 0xd982, 0x6ec3, 0x2254, 0x9515, 0xfbd6, 0x4c97, - 0x33a8, 0x84e9, 0xea2a, 0x5d6b, 0x11fc, 0xa6bd, 0xc87e, 0x7f3f }, - {/*13*/ 1, 0xdd41, 0x6682, 0xbbc3, 0x3554, 0xe815, 0x53d6, 0xce97, - 0x1aa8, 0xc7e9, 0x7c2a, 0xa1fb, 0x2ffc, 0xf2bd, 0x497e, 0x943f }, - {/*14*/ 1, 0x2bd1, 0x3d62, 0x16b3, 0x4f34, 0x64e5, 0x7256, 0x5987, - 0x8518, 0xaec9, 0xb87a, 0x93ab, 0xca2c, 0xe1fd, 0xf74e, 0xdc9f }, - {/*15*/ 1, 0x83c1, 0xc142, 0x4283, 0xa4f4, 0x2735, 0x65b6, 0xe677, - 0xf858, 0x7b99, 0x391a, 0xbadb, 0x5cac, 0xdf6d, 0x9dee, 0x1e2f }, - {/*16*/ 1, 0x8fd1, 0xc562, 0x4ab3, 0xa934, 0x26e5, 0x6c56, 0xe387, - 0xfe18, 0x71c9, 0x3b7a, 0xb4ab, 0x572c, 0xd8fd, 0x924e, 0x1d9f }, - {/*17*/ 1, 0x4791, 0x89e2, 0xce73, 0x5264, 0x15f5, 0xdb86, 0x9c17, - 0xa3b8, 0xe429, 0x2a5a, 0x6dcb, 0xf1dc, 0xb64d, 0x783e, 0x3faf }, - {/*18*/ 1, 0x5781, 0xa9c2, 0xfe43, 0x92a4, 0xc525, 0x3b66, 0x6ce7, - 0xe3f8, 0xb479, 0x4a3a, 0x1dbb, 0x715c, 0x26dd, 0xd89e, 0x8f1f }, - {/*19*/ 1, 0xbf41, 0xd582, 0x6ac3, 0x2954, 0x9615, 0xfcd6, 0x4397, - 0x3ea8, 0x81e9, 0xeb2a, 0x546b, 0x17fc, 0xa8bd, 0xc27e, 0x7d3f }, - {/*1a*/ 1, 0x9891, 0xe1e2, 0x7273, 0x6464, 0xf7f5, 0x8586, 0x1617, - 0xb8b8, 0x2b29, 0x595a, 0xcacb, 0xdcdc, 0x4f4d, 0x3d3e, 0xaeaf }, - {/*1b*/ 1, 0xcce1, 0x4472, 0x8893, 0xfdb4, 0x3f55, 0xb9c6, 0x7527, - 0x56d8, 0x9a39, 0x12aa, 0xde4b, 0xab6c, 0x678d, 0xef1e, 0x23ff }, - {/*1c*/ 1, 0xa761, 0xf9b2, 0x5ed3, 0xe214, 0x4575, 0x1ba6, 0xbcc7, - 0x7328, 0xd449, 0x8a9a, 0x2dfb, 0x913c, 0x365d, 0x688e, 0xcfef }, - {/*1d*/ 1, 0xff61, 0x55b2, 0xaad3, 0x7914, 0x8675, 0x2ca6, 0xd3c7, - 0x9e28, 0x6149, 0xcb9a, 0x34fb, 0xe73c, 0x185d, 0xb28e, 0x4def }, - {/*1e*/ 1, 0x5451, 0xa8a2, 0xfcf3, 0x9694, 0xc2c5, 0x3e36, 0x6a67, - 0xebe8, 0xbfb9, 0x434a, 0x171b, 0x7d7c, 0x292d, 0xd5de, 0x818f }, - {/*1f*/ 1, 0x6fc1, 0xb542, 0xda83, 0x19f4, 0x7635, 0xacb6, 0xc377, - 0x2e58, 0x4199, 0x9b1a, 0xf4db, 0x37ac, 0x586d, 0x82ee, 0xed2f }, - - /* ECC bits are also in the set of tokens and they too can go bad - * first 2 cover channel 0, while the second 2 cover channel 1 - */ - {/*20*/ 0, 0xbe01, 0xd702, 0x6903, 0x2104, 0x9f05, 0xf606, 0x4807, - 0x3208, 0x8c09, 0xe50a, 0x5b0b, 0x130c, 0xad0d, 0xc40e, 0x7a0f }, - {/*21*/ 0, 0x4101, 0x8202, 0xc303, 0x5804, 0x1905, 0xda06, 0x9b07, - 0xac08, 0xed09, 0x2e0a, 0x6f0b, 0x640c, 0xb50d, 0x760e, 0x370f }, - {/*22*/ 1, 0xc441, 0x4882, 0x8cc3, 0xf654, 0x3215, 0xbed6, 0x7a97, - 0x5ba8, 0x9fe9, 0x132a, 0xd76b, 0xadfc, 0x69bd, 0xe57e, 0x213f }, - {/*23*/ 1, 0x7621, 0x9b32, 0xed13, 0xda44, 0xac65, 0x4176, 0x3757, - 0x6f88, 0x19a9, 0xf4ba, 0x829b, 0xb5cc, 0xc3ed, 0x2efe, 0x58df } +static const u16 x4_vectors[] = { + 0x2f57, 0x1afe, 0x66cc, 0xdd88, + 0x11eb, 0x3396, 0x7f4c, 0xeac8, + 0x0001, 0x0002, 0x0004, 0x0008, + 0x1013, 0x3032, 0x4044, 0x8088, + 0x106b, 0x30d6, 0x70fc, 0xe0a8, + 0x4857, 0xc4fe, 0x13cc, 0x3288, + 0x1ac5, 0x2f4a, 0x5394, 0xa1e8, + 0x1f39, 0x251e, 0xbd6c, 0x6bd8, + 0x15c1, 0x2a42, 0x89ac, 0x4758, + 0x2b03, 0x1602, 0x4f0c, 0xca08, + 0x1f07, 0x3a0e, 0x6b04, 0xbd08, + 0x8ba7, 0x465e, 0x244c, 0x1cc8, + 0x2b87, 0x164e, 0x642c, 0xdc18, + 0x40b9, 0x80de, 0x1094, 0x20e8, + 0x27db, 0x1eb6, 0x9dac, 0x7b58, + 0x11c1, 0x2242, 0x84ac, 0x4c58, + 0x1be5, 0x2d7a, 0x5e34, 0xa718, + 0x4b39, 0x8d1e, 0x14b4, 0x28d8, + 0x4c97, 0xc87e, 0x11fc, 0x33a8, + 0x8e97, 0x497e, 0x2ffc, 0x1aa8, + 0x16b3, 0x3d62, 0x4f34, 0x8518, + 0x1e2f, 0x391a, 0x5cac, 0xf858, + 0x1d9f, 0x3b7a, 0x572c, 0xfe18, + 0x15f5, 0x2a5a, 0x5264, 0xa3b8, + 0x1dbb, 0x3b66, 0x715c, 0xe3f8, + 0x4397, 0xc27e, 0x17fc, 0x3ea8, + 0x1617, 0x3d3e, 0x6464, 0xb8b8, + 0x23ff, 0x12aa, 0xab6c, 0x56d8, + 0x2dfb, 0x1ba6, 0x913c, 0x7328, + 0x185d, 0x2ca6, 0x7914, 0x9e28, + 0x171b, 0x3e36, 0x7d7c, 0xebe8, + 0x4199, 0x82ee, 0x19f4, 0x2e58, + 0x4807, 0xc40e, 0x130c, 0x3208, + 0x1905, 0x2e0a, 0x5804, 0xac08, + 0x213f, 0x132a, 0xadfc, 0x5ba8, + 0x19a9, 0x2efe, 0xb5cc, 0x6f88, }; -/* - * Given the syndrome argument, scan each of the channel tables for a syndrome - * match. Depending on which table it is found, return the channel number. - */ -static int get_channel_from_ecc_syndrome(unsigned short syndrome) -{ - int row; - int column; - - /* Determine column to scan */ - column = syndrome & 0xF; - - /* Scan all rows, looking for syndrome, or end of table */ - for (row = 0; row < NUMBER_ECC_ROWS; row++) { - if (ecc_chipkill_syndromes[row][column] == syndrome) - return ecc_chipkill_syndromes[row][0]; - } - - debugf0("syndrome(%x) not found\n", syndrome); - return -1; -} - -/* - * Check for valid error in the NB Status High register. If so, proceed to read - * NB Status Low, NB Address Low and NB Address High registers and store data - * into error structure. - * - * Returns: - * - 1: if hardware regs contains valid error info - * - 0: if no valid error is indicated - */ -static int amd64_get_error_info_regs(struct mem_ctl_info *mci, - struct amd64_error_info_regs *regs) -{ - struct amd64_pvt *pvt; - struct pci_dev *misc_f3_ctl; - int err = 0; - - pvt = mci->pvt_info; - misc_f3_ctl = pvt->misc_f3_ctl; - - err = pci_read_config_dword(misc_f3_ctl, K8_NBSH, ®s->nbsh); - if (err) - goto err_reg; - - if (!(regs->nbsh & K8_NBSH_VALID_BIT)) - return 0; - - /* valid error, read remaining error information registers */ - err = pci_read_config_dword(misc_f3_ctl, K8_NBSL, ®s->nbsl); - if (err) - goto err_reg; - - err = pci_read_config_dword(misc_f3_ctl, K8_NBEAL, ®s->nbeal); - if (err) - goto err_reg; - - err = pci_read_config_dword(misc_f3_ctl, K8_NBEAH, ®s->nbeah); - if (err) - goto err_reg; - - err = pci_read_config_dword(misc_f3_ctl, K8_NBCFG, ®s->nbcfg); - if (err) - goto err_reg; - - return 1; - -err_reg: - debugf0("Reading error info register failed\n"); - return 0; -} +static const u16 x8_vectors[] = { + 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, + 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, + 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, + 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80, + 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780, + 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080, + 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080, + 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080, + 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80, + 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580, + 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880, + 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280, + 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180, + 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580, + 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280, + 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180, + 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080, + 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, + 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, +}; -/* - * This function is called to retrieve the error data from hardware and store it - * in the info structure. - * - * Returns: - * - 1: if a valid error is found - * - 0: if no error is found - */ -static int amd64_get_error_info(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, + unsigned v_dim) { - struct amd64_pvt *pvt; - struct amd64_error_info_regs regs; + unsigned int i, err_sym; - pvt = mci->pvt_info; + for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { + u16 s = syndrome; + unsigned v_idx = err_sym * v_dim; + unsigned v_end = (err_sym + 1) * v_dim; - if (!amd64_get_error_info_regs(mci, info)) - return 0; + /* walk over all 16 bits of the syndrome */ + for (i = 1; i < (1U << 16); i <<= 1) { - /* - * Here's the problem with the K8's EDAC reporting: There are four - * registers which report pieces of error information. They are shared - * between CEs and UEs. Furthermore, contrary to what is stated in the - * BKDG, the overflow bit is never used! Every error always updates the - * reporting registers. - * - * Can you see the race condition? All four error reporting registers - * must be read before a new error updates them! There is no way to read - * all four registers atomically. The best than can be done is to detect - * that a race has occured and then report the error without any kind of - * precision. - * - * What is still positive is that errors are still reported and thus - * problems can still be detected - just not localized because the - * syndrome and address are spread out across registers. - * - * Grrrrr!!!!! Here's hoping that AMD fixes this in some future K8 rev. - * UEs and CEs should have separate register sets with proper overflow - * bits that are used! At very least the problem can be fixed by - * honoring the ErrValid bit in 'nbsh' and not updating registers - just - * set the overflow bit - unless the current error is CE and the new - * error is UE which would be the only situation for overwriting the - * current values. - */ + /* if bit is set in that eigenvector... */ + if (v_idx < v_end && vectors[v_idx] & i) { + u16 ev_comp = vectors[v_idx++]; - regs = *info; + /* ... and bit set in the modified syndrome, */ + if (s & i) { + /* remove it. */ + s ^= ev_comp; - /* Use info from the second read - most current */ - if (unlikely(!amd64_get_error_info_regs(mci, info))) - return 0; + if (!s) + return err_sym; + } - /* clear the error bits in hardware */ - pci_write_bits32(pvt->misc_f3_ctl, K8_NBSH, 0, K8_NBSH_VALID_BIT); - - /* Check for the possible race condition */ - if ((regs.nbsh != info->nbsh) || - (regs.nbsl != info->nbsl) || - (regs.nbeah != info->nbeah) || - (regs.nbeal != info->nbeal)) { - amd64_mc_printk(mci, KERN_WARNING, - "hardware STATUS read access race condition " - "detected!\n"); - return 0; + } else if (s & i) + /* can't get to zero, move to next symbol */ + break; + } } - return 1; -} - -static inline void amd64_decode_gart_tlb_error(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) -{ - u32 err_code; - u32 ec_tt; /* error code transaction type (2b) */ - u32 ec_ll; /* error code cache level (2b) */ - - err_code = EXTRACT_ERROR_CODE(info->nbsl); - ec_ll = EXTRACT_LL_CODE(err_code); - ec_tt = EXTRACT_TT_CODE(err_code); - amd64_mc_printk(mci, KERN_ERR, - "GART TLB event: transaction type(%s), " - "cache level(%s)\n", tt_msgs[ec_tt], ll_msgs[ec_ll]); + edac_dbg(0, "syndrome(%x) not found\n", syndrome); + return -1; } -static inline void amd64_decode_mem_cache_error(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +static int map_err_sym_to_channel(int err_sym, int sym_size) { - u32 err_code; - u32 ec_rrrr; /* error code memory transaction (4b) */ - u32 ec_tt; /* error code transaction type (2b) */ - u32 ec_ll; /* error code cache level (2b) */ - - err_code = EXTRACT_ERROR_CODE(info->nbsl); - ec_ll = EXTRACT_LL_CODE(err_code); - ec_tt = EXTRACT_TT_CODE(err_code); - ec_rrrr = EXTRACT_RRRR_CODE(err_code); + if (sym_size == 4) + switch (err_sym) { + case 0x20: + case 0x21: + return 0; + break; + case 0x22: + case 0x23: + return 1; + break; + default: + return err_sym >> 4; + break; + } + /* x8 symbols */ + else + switch (err_sym) { + /* imaginary bits not in a DIMM */ + case 0x10: + WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n", + err_sym); + return -1; + break; - amd64_mc_printk(mci, KERN_ERR, - "cache hierarchy error: memory transaction type(%s), " - "transaction type(%s), cache level(%s)\n", - rrrr_msgs[ec_rrrr], tt_msgs[ec_tt], ll_msgs[ec_ll]); + case 0x11: + return 0; + break; + case 0x12: + return 1; + break; + default: + return err_sym >> 3; + break; + } + return -1; } - -/* - * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR - * ADDRESS and process. - */ -static void amd64_handle_ce(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) { struct amd64_pvt *pvt = mci->pvt_info; - u64 SystemAddress; - - /* Ensure that the Error Address is VALID */ - if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { - amd64_mc_printk(mci, KERN_ERR, - "HW has no ERROR_ADDRESS available\n"); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); - return; + int err_sym = -1; + + if (pvt->ecc_sym_sz == 8) + err_sym = decode_syndrome(syndrome, x8_vectors, + ARRAY_SIZE(x8_vectors), + pvt->ecc_sym_sz); + else if (pvt->ecc_sym_sz == 4) + err_sym = decode_syndrome(syndrome, x4_vectors, + ARRAY_SIZE(x4_vectors), + pvt->ecc_sym_sz); + else { + amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); + return err_sym; } - SystemAddress = extract_error_address(mci, info); - - amd64_mc_printk(mci, KERN_ERR, - "CE ERROR_ADDRESS= 0x%llx\n", SystemAddress); - - pvt->ops->map_sysaddr_to_csrow(mci, info, SystemAddress); + return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); } -/* Handle any Un-correctable Errors (UEs) */ -static void amd64_handle_ue(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) +static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err, + u8 ecc_type) { - int csrow; - u64 SystemAddress; - u32 page, offset; - struct mem_ctl_info *log_mci, *src_mci = NULL; - - log_mci = mci; - - if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { - amd64_mc_printk(mci, KERN_CRIT, - "HW has no ERROR_ADDRESS available\n"); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); - return; - } - - SystemAddress = extract_error_address(mci, info); + enum hw_event_mc_err_type err_type; + const char *string; - /* - * Find out which node the error address belongs to. This may be - * different from the node that detected the error. - */ - src_mci = find_mc_by_sys_addr(mci, SystemAddress); - if (!src_mci) { - amd64_mc_printk(mci, KERN_CRIT, - "ERROR ADDRESS (0x%lx) value NOT mapped to a MC\n", - (unsigned long)SystemAddress); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); + if (ecc_type == 2) + err_type = HW_EVENT_ERR_CORRECTED; + else if (ecc_type == 1) + err_type = HW_EVENT_ERR_UNCORRECTED; + else { + WARN(1, "Something is rotten in the state of Denmark.\n"); return; } - log_mci = src_mci; - - csrow = sys_addr_to_csrow(log_mci, SystemAddress); - if (csrow < 0) { - amd64_mc_printk(mci, KERN_CRIT, - "ERROR_ADDRESS (0x%lx) value NOT mapped to 'csrow'\n", - (unsigned long)SystemAddress); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); - } else { - error_address_to_page_and_offset(SystemAddress, &page, &offset); - edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); - } -} - -static void amd64_decode_bus_error(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info) -{ - u32 err_code, ext_ec; - u32 ec_pp; /* error code participating processor (2p) */ - u32 ec_to; /* error code timed out (1b) */ - u32 ec_rrrr; /* error code memory transaction (4b) */ - u32 ec_ii; /* error code memory or I/O (2b) */ - u32 ec_ll; /* error code cache level (2b) */ - - ext_ec = EXTRACT_EXT_ERROR_CODE(info->nbsl); - err_code = EXTRACT_ERROR_CODE(info->nbsl); - - ec_ll = EXTRACT_LL_CODE(err_code); - ec_ii = EXTRACT_II_CODE(err_code); - ec_rrrr = EXTRACT_RRRR_CODE(err_code); - ec_to = EXTRACT_TO_CODE(err_code); - ec_pp = EXTRACT_PP_CODE(err_code); - - amd64_mc_printk(mci, KERN_ERR, - "BUS ERROR:\n" - " time-out(%s) mem or i/o(%s)\n" - " participating processor(%s)\n" - " memory transaction type(%s)\n" - " cache level(%s) Error Found by: %s\n", - to_msgs[ec_to], - ii_msgs[ec_ii], - pp_msgs[ec_pp], - rrrr_msgs[ec_rrrr], - ll_msgs[ec_ll], - (info->nbsh & K8_NBSH_ERR_SCRUBER) ? - "Scrubber" : "Normal Operation"); - - /* If this was an 'observed' error, early out */ - if (ec_pp == K8_NBSL_PP_OBS) - return; /* We aren't the node involved */ - - /* Parse out the extended error code for ECC events */ - switch (ext_ec) { - /* F10 changed to one Extended ECC error code */ - case F10_NBSL_EXT_ERR_RES: /* Reserved field */ - case F10_NBSL_EXT_ERR_ECC: /* F10 ECC ext err code */ + switch (err->err_code) { + case DECODE_OK: + string = ""; + break; + case ERR_NODE: + string = "Failed to map error addr to a node"; + break; + case ERR_CSROW: + string = "Failed to map error addr to a csrow"; + break; + case ERR_CHANNEL: + string = "unknown syndrome - possible error reporting race"; break; - default: - amd64_mc_printk(mci, KERN_ERR, "NOT ECC: no special error " - "handling for this error\n"); - return; + string = "WTF error"; + break; } - if (info->nbsh & K8_NBSH_CECC) - amd64_handle_ce(mci, info); - else if (info->nbsh & K8_NBSH_UECC) - amd64_handle_ue(mci, info); - - /* - * If main error is CE then overflow must be CE. If main error is UE - * then overflow is unknown. We'll call the overflow a CE - if - * panic_on_ue is set then we're already panic'ed and won't arrive - * here. Else, then apparently someone doesn't think that UE's are - * catastrophic. - */ - if (info->nbsh & K8_NBSH_OVERFLOW) - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR - "Error Overflow set"); + edac_mc_handle_error(err_type, mci, 1, + err->page, err->offset, err->syndrome, + err->csrow, err->channel, -1, + string, ""); } -int amd64_process_error_info(struct mem_ctl_info *mci, - struct amd64_error_info_regs *info, - int handle_errors) +static inline void decode_bus_error(int node_id, struct mce *m) { - struct amd64_pvt *pvt; - struct amd64_error_info_regs *regs; - u32 err_code, ext_ec; - int gart_tlb_error = 0; - - pvt = mci->pvt_info; - - /* If caller doesn't want us to process the error, return */ - if (!handle_errors) - return 1; - - regs = info; - - debugf1("NorthBridge ERROR: mci(0x%p)\n", mci); - debugf1(" MC node(%d) Error-Address(0x%.8x-%.8x)\n", - pvt->mc_node_id, regs->nbeah, regs->nbeal); - debugf1(" nbsh(0x%.8x) nbsl(0x%.8x)\n", - regs->nbsh, regs->nbsl); - debugf1(" Valid Error=%s Overflow=%s\n", - (regs->nbsh & K8_NBSH_VALID_BIT) ? "True" : "False", - (regs->nbsh & K8_NBSH_OVERFLOW) ? "True" : "False"); - debugf1(" Err Uncorrected=%s MCA Error Reporting=%s\n", - (regs->nbsh & K8_NBSH_UNCORRECTED_ERR) ? - "True" : "False", - (regs->nbsh & K8_NBSH_ERR_ENABLE) ? - "True" : "False"); - debugf1(" MiscErr Valid=%s ErrAddr Valid=%s PCC=%s\n", - (regs->nbsh & K8_NBSH_MISC_ERR_VALID) ? - "True" : "False", - (regs->nbsh & K8_NBSH_VALID_ERROR_ADDR) ? - "True" : "False", - (regs->nbsh & K8_NBSH_PCC) ? - "True" : "False"); - debugf1(" CECC=%s UECC=%s Found by Scruber=%s\n", - (regs->nbsh & K8_NBSH_CECC) ? - "True" : "False", - (regs->nbsh & K8_NBSH_UECC) ? - "True" : "False", - (regs->nbsh & K8_NBSH_ERR_SCRUBER) ? - "True" : "False"); - debugf1(" CORE0=%s CORE1=%s CORE2=%s CORE3=%s\n", - (regs->nbsh & K8_NBSH_CORE0) ? "True" : "False", - (regs->nbsh & K8_NBSH_CORE1) ? "True" : "False", - (regs->nbsh & K8_NBSH_CORE2) ? "True" : "False", - (regs->nbsh & K8_NBSH_CORE3) ? "True" : "False"); - - - err_code = EXTRACT_ERROR_CODE(regs->nbsl); - - /* Determine which error type: - * 1) GART errors - non-fatal, developmental events - * 2) MEMORY errors - * 3) BUS errors - * 4) Unknown error - */ - if (TEST_TLB_ERROR(err_code)) { - /* - * GART errors are intended to help graphics driver developers - * to detect bad GART PTEs. It is recommended by AMD to disable - * GART table walk error reporting by default[1] (currently - * being disabled in mce_cpu_quirks()) and according to the - * comment in mce_cpu_quirks(), such GART errors can be - * incorrectly triggered. We may see these errors anyway and - * unless requested by the user, they won't be reported. - * - * [1] section 13.10.1 on BIOS and Kernel Developers Guide for - * AMD NPT family 0Fh processors - */ - if (report_gart_errors == 0) - return 1; - - /* - * Only if GART error reporting is requested should we generate - * any logs. - */ - gart_tlb_error = 1; - - debugf1("GART TLB error\n"); - amd64_decode_gart_tlb_error(mci, info); - } else if (TEST_MEM_ERROR(err_code)) { - debugf1("Memory/Cache error\n"); - amd64_decode_mem_cache_error(mci, info); - } else if (TEST_BUS_ERROR(err_code)) { - debugf1("Bus (Link/DRAM) error\n"); - amd64_decode_bus_error(mci, info); - } else { - /* shouldn't reach here! */ - amd64_mc_printk(mci, KERN_WARNING, - "%s(): unknown MCE error 0x%x\n", __func__, - err_code); - } - - ext_ec = EXTRACT_EXT_ERROR_CODE(regs->nbsl); - amd64_mc_printk(mci, KERN_ERR, - "ExtErr=(0x%x) %s\n", ext_ec, ext_msgs[ext_ec]); - - if (((ext_ec >= F10_NBSL_EXT_ERR_CRC && - ext_ec <= F10_NBSL_EXT_ERR_TGT) || - (ext_ec == F10_NBSL_EXT_ERR_RMW)) && - EXTRACT_LDT_LINK(info->nbsh)) { + struct mem_ctl_info *mci = mcis[node_id]; + struct amd64_pvt *pvt = mci->pvt_info; + u8 ecc_type = (m->status >> 45) & 0x3; + u8 xec = XEC(m->status, 0x1f); + u16 ec = EC(m->status); + u64 sys_addr; + struct err_info err; + + /* Bail out early if this was an 'observed' error */ + if (PP(ec) == NBSL_PP_OBS) + return; - amd64_mc_printk(mci, KERN_ERR, - "Error on hypertransport link: %s\n", - htlink_msgs[ - EXTRACT_LDT_LINK(info->nbsh)]); - } + /* Do only ECC errors */ + if (xec && xec != F10_NBSL_EXT_ERR_ECC) + return; - /* - * Check the UE bit of the NB status high register, if set generate some - * logs. If NOT a GART error, then process the event as a NO-INFO event. - * If it was a GART error, skip that process. - */ - if (regs->nbsh & K8_NBSH_UNCORRECTED_ERR) { - amd64_mc_printk(mci, KERN_CRIT, "uncorrected error\n"); - if (!gart_tlb_error) - edac_mc_handle_ue_no_info(mci, "UE bit is set\n"); - } + memset(&err, 0, sizeof(err)); - if (regs->nbsh & K8_NBSH_PCC) - amd64_mc_printk(mci, KERN_CRIT, - "PCC (processor context corrupt) set\n"); + sys_addr = get_error_address(pvt, m); - return 1; -} -EXPORT_SYMBOL_GPL(amd64_process_error_info); + if (ecc_type == 2) + err.syndrome = extract_syndrome(m->status); -/* - * The main polling 'check' function, called FROM the edac core to perform the - * error checking and if an error is encountered, error processing. - */ -static void amd64_check(struct mem_ctl_info *mci) -{ - struct amd64_error_info_regs info; + pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); - if (amd64_get_error_info(mci, &info)) - amd64_process_error_info(mci, &info, 1); + __log_bus_error(mci, &err, ecc_type); } /* - * Input: - * 1) struct amd64_pvt which contains pvt->dram_f2_ctl pointer - * 2) AMD Family index value - * - * Ouput: - * Upon return of 0, the following filled in: - * - * struct pvt->addr_f1_ctl - * struct pvt->misc_f3_ctl - * - * Filled in with related device funcitions of 'dram_f2_ctl' - * These devices are "reserved" via the pci_get_device() - * - * Upon return of 1 (error status): - * - * Nothing reserved + * Use pvt->F2 which contains the F2 CPU PCI device to get the related + * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. */ -static int amd64_reserve_mc_sibling_devices(struct amd64_pvt *pvt, int mc_idx) +static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) { - const struct amd64_family_type *amd64_dev = &amd64_family_types[mc_idx]; - /* Reserve the ADDRESS MAP Device */ - pvt->addr_f1_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, - amd64_dev->addr_f1_ctl, - pvt->dram_f2_ctl); - - if (!pvt->addr_f1_ctl) { - amd64_printk(KERN_ERR, "error address map device not found: " - "vendor %x device 0x%x (broken BIOS?)\n", - PCI_VENDOR_ID_AMD, amd64_dev->addr_f1_ctl); - return 1; + pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); + if (!pvt->F1) { + amd64_err("error address map device not found: " + "vendor %x device 0x%x (broken BIOS?)\n", + PCI_VENDOR_ID_AMD, f1_id); + return -ENODEV; } /* Reserve the MISC Device */ - pvt->misc_f3_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, - amd64_dev->misc_f3_ctl, - pvt->dram_f2_ctl); + pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); + if (!pvt->F3) { + pci_dev_put(pvt->F1); + pvt->F1 = NULL; - if (!pvt->misc_f3_ctl) { - pci_dev_put(pvt->addr_f1_ctl); - pvt->addr_f1_ctl = NULL; + amd64_err("error F3 device not found: " + "vendor %x device 0x%x (broken BIOS?)\n", + PCI_VENDOR_ID_AMD, f3_id); - amd64_printk(KERN_ERR, "error miscellaneous device not found: " - "vendor %x device 0x%x (broken BIOS?)\n", - PCI_VENDOR_ID_AMD, amd64_dev->misc_f3_ctl); - return 1; + return -ENODEV; } - - debugf1(" Addr Map device PCI Bus ID:\t%s\n", - pci_name(pvt->addr_f1_ctl)); - debugf1(" DRAM MEM-CTL PCI Bus ID:\t%s\n", - pci_name(pvt->dram_f2_ctl)); - debugf1(" Misc device PCI Bus ID:\t%s\n", - pci_name(pvt->misc_f3_ctl)); + edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); + edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); + edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); return 0; } -static void amd64_free_mc_sibling_devices(struct amd64_pvt *pvt) +static void free_mc_sibling_devs(struct amd64_pvt *pvt) { - pci_dev_put(pvt->addr_f1_ctl); - pci_dev_put(pvt->misc_f3_ctl); + pci_dev_put(pvt->F1); + pci_dev_put(pvt->F3); } /* * Retrieve the hardware registers of the memory controller (this includes the * 'Address Map' and 'Misc' device regs) */ -static void amd64_read_mc_registers(struct amd64_pvt *pvt) +static void read_mc_regs(struct amd64_pvt *pvt) { + unsigned range; u64 msr_val; - int dram, err = 0; + u32 tmp; /* * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since * those are Read-As-Zero */ - rdmsrl(MSR_K8_TOP_MEM1, msr_val); - pvt->top_mem = msr_val >> 23; - debugf0(" TOP_MEM=0x%08llx\n", pvt->top_mem); + rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); + edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); /* check first whether TOP_MEM2 is enabled */ rdmsrl(MSR_K8_SYSCFG, msr_val); if (msr_val & (1U << 21)) { - rdmsrl(MSR_K8_TOP_MEM2, msr_val); - pvt->top_mem2 = msr_val >> 23; - debugf0(" TOP_MEM2=0x%08llx\n", pvt->top_mem2); + rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); + edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); } else - debugf0(" TOP_MEM2 disabled.\n"); - - amd64_cpu_display_info(pvt); + edac_dbg(0, " TOP_MEM2 disabled\n"); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCAP, &pvt->nbcap); - if (err) - goto err_reg; + amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); - if (pvt->ops->read_dram_ctl_register) - pvt->ops->read_dram_ctl_register(pvt); + read_dram_ctl_register(pvt); - for (dram = 0; dram < DRAM_REG_COUNT; dram++) { - /* - * Call CPU specific READ function to get the DRAM Base and - * Limit values from the DCT. - */ - pvt->ops->read_dram_base_limit(pvt, dram); + for (range = 0; range < DRAM_RANGES; range++) { + u8 rw; - /* - * Only print out debug info on rows with both R and W Enabled. - * Normal processing, compiler should optimize this whole 'if' - * debug output block away. - */ - if (pvt->dram_rw_en[dram] != 0) { - debugf1(" DRAM_BASE[%d]: 0x%8.08x-%8.08x " - "DRAM_LIMIT: 0x%8.08x-%8.08x\n", - dram, - (u32)(pvt->dram_base[dram] >> 32), - (u32)(pvt->dram_base[dram] & 0xFFFFFFFF), - (u32)(pvt->dram_limit[dram] >> 32), - (u32)(pvt->dram_limit[dram] & 0xFFFFFFFF)); - debugf1(" IntlvEn=%s %s %s " - "IntlvSel=%d DstNode=%d\n", - pvt->dram_IntlvEn[dram] ? - "Enabled" : "Disabled", - (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W", - (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R", - pvt->dram_IntlvSel[dram], - pvt->dram_DstNode[dram]); - } - } + /* read settings for this DRAM range */ + read_dram_base_limit_regs(pvt, range); - amd64_read_dct_base_mask(pvt); + rw = dram_rw(pvt, range); + if (!rw) + continue; - err = pci_read_config_dword(pvt->addr_f1_ctl, K8_DHAR, &pvt->dhar); - if (err) - goto err_reg; + edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", + range, + get_dram_base(pvt, range), + get_dram_limit(pvt, range)); + + edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", + dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", + (rw & 0x1) ? "R" : "-", + (rw & 0x2) ? "W" : "-", + dram_intlv_sel(pvt, range), + dram_dst_node(pvt, range)); + } - amd64_read_dbam_reg(pvt); + read_dct_base_mask(pvt); - err = pci_read_config_dword(pvt->misc_f3_ctl, - F10_ONLINE_SPARE, &pvt->online_spare); - if (err) - goto err_reg; + amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); + amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0); - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); - if (err) - goto err_reg; + amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCHR_0, &pvt->dchr0); - if (err) - goto err_reg; + amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0); + amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0); if (!dct_ganging_enabled(pvt)) { - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCLR_1, - &pvt->dclr1); - if (err) - goto err_reg; - - err = pci_read_config_dword(pvt->dram_f2_ctl, F10_DCHR_1, - &pvt->dchr1); - if (err) - goto err_reg; + amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1); + amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1); } - amd64_dump_misc_regs(pvt); + pvt->ecc_sym_sz = 4; -err_reg: - debugf0("Reading an MC register failed\n"); + if (pvt->fam >= 0x10) { + amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); + if (pvt->fam != 0x16) + /* F16h has only DCT0 */ + amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1); + /* F10h, revD and later can do x8 ECC too */ + if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) + pvt->ecc_sym_sz = 8; + } + dump_misc_regs(pvt); } /* * NOTE: CPU Revision Dependent code * * Input: - * @csrow_nr ChipSelect Row Number (0..CHIPSELECT_COUNT-1) + * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) * k8 private pointer to --> * DRAM Bank Address mapping register * node_id @@ -2673,9 +2209,11 @@ err_reg: * encompasses * */ -static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) +static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) { - u32 dram_map, nr_pages; + u32 cs_mode, nr_pages; + u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; + /* * The math on this doesn't look right on the surface because x/2*4 can @@ -2684,19 +2222,13 @@ static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) * number of bits to shift the DBAM register to extract the proper CSROW * field. */ - dram_map = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; + cs_mode = DBAM_DIMM(csrow_nr / 2, dbam); - nr_pages = pvt->ops->dbam_map_to_pages(pvt, dram_map); - - /* - * If dual channel then double the memory size of single channel. - * Channel count is 1 or 2 - */ - nr_pages <<= (pvt->channel_count - 1); + nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT); - debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, dram_map); - debugf0(" nr_pages= %u channel-count = %d\n", - nr_pages, pvt->channel_count); + edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", + csrow_nr, dct, cs_mode); + edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); return nr_pages; } @@ -2705,501 +2237,551 @@ static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) * Initialize the array of csrow attribute instances, based on the values * from pci config hardware registers. */ -static int amd64_init_csrows(struct mem_ctl_info *mci) +static int init_csrows(struct mem_ctl_info *mci) { + struct amd64_pvt *pvt = mci->pvt_info; struct csrow_info *csrow; - struct amd64_pvt *pvt; - u64 input_addr_min, input_addr_max, sys_addr; - int i, err = 0, empty = 1; + struct dimm_info *dimm; + enum edac_type edac_mode; + enum mem_type mtype; + int i, j, empty = 1; + int nr_pages = 0; + u32 val; - pvt = mci->pvt_info; + amd64_read_pci_cfg(pvt->F3, NBCFG, &val); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &pvt->nbcfg); - if (err) - debugf0("Reading K8_NBCFG failed\n"); + pvt->nbcfg = val; + + edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", + pvt->mc_node_id, val, + !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); - debugf0("NBCFG= 0x%x CHIPKILL= %s DRAM ECC= %s\n", pvt->nbcfg, - (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled" - ); + /* + * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. + */ + for_each_chip_select(i, 0, pvt) { + bool row_dct0 = !!csrow_enabled(i, 0, pvt); + bool row_dct1 = false; - for (i = 0; i < CHIPSELECT_COUNT; i++) { - csrow = &mci->csrows[i]; + if (pvt->fam != 0xf) + row_dct1 = !!csrow_enabled(i, 1, pvt); - if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) { - debugf1("----CSROW %d EMPTY for node %d\n", i, - pvt->mc_node_id); + if (!row_dct0 && !row_dct1) continue; + + csrow = mci->csrows[i]; + empty = 0; + + edac_dbg(1, "MC node: %d, csrow: %d\n", + pvt->mc_node_id, i); + + if (row_dct0) { + nr_pages = get_csrow_nr_pages(pvt, 0, i); + csrow->channels[0]->dimm->nr_pages = nr_pages; } - debugf1("----CSROW %d VALID for MC node %d\n", - i, pvt->mc_node_id); + /* K8 has only one DCT */ + if (pvt->fam != 0xf && row_dct1) { + int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i); - empty = 0; - csrow->nr_pages = amd64_csrow_nr_pages(i, pvt); - find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); - sys_addr = input_addr_to_sys_addr(mci, input_addr_min); - csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); - sys_addr = input_addr_to_sys_addr(mci, input_addr_max); - csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); - csrow->page_mask = ~mask_from_dct_mask(pvt, i); - /* 8 bytes of resolution */ - - csrow->mtype = amd64_determine_memory_type(pvt); - - debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); - debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", - (unsigned long)input_addr_min, - (unsigned long)input_addr_max); - debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", - (unsigned long)sys_addr, csrow->page_mask); - debugf1(" nr_pages: %u first_page: 0x%lx " - "last_page: 0x%lx\n", - (unsigned)csrow->nr_pages, - csrow->first_page, csrow->last_page); + csrow->channels[1]->dimm->nr_pages = row_dct1_pages; + nr_pages += row_dct1_pages; + } + + mtype = determine_memory_type(pvt, i); + + edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); /* * determine whether CHIPKILL or JUST ECC or NO ECC is operating */ - if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) - csrow->edac_mode = - (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? - EDAC_S4ECD4ED : EDAC_SECDED; + if (pvt->nbcfg & NBCFG_ECC_ENABLE) + edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ? + EDAC_S4ECD4ED : EDAC_SECDED; else - csrow->edac_mode = EDAC_NONE; + edac_mode = EDAC_NONE; + + for (j = 0; j < pvt->channel_count; j++) { + dimm = csrow->channels[j]->dimm; + dimm->mtype = mtype; + dimm->edac_mode = edac_mode; + } } return empty; } -/* - * Only if 'ecc_enable_override' is set AND BIOS had ECC disabled, do "we" - * enable it. - */ -static void amd64_enable_ecc_error_reporting(struct mem_ctl_info *mci) +/* get all cores on this DCT */ +static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) { - struct amd64_pvt *pvt = mci->pvt_info; - const cpumask_t *cpumask = cpumask_of_node(pvt->mc_node_id); - int cpu, idx = 0, err = 0; - struct msr msrs[cpumask_weight(cpumask)]; - u32 value; - u32 mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; + int cpu; - if (!ecc_enable_override) - return; + for_each_online_cpu(cpu) + if (amd_get_nb_id(cpu) == nid) + cpumask_set_cpu(cpu, mask); +} + +/* check MCG_CTL on all the cpus on this node */ +static bool nb_mce_bank_enabled_on_node(u16 nid) +{ + cpumask_var_t mask; + int cpu, nbe; + bool ret = false; - memset(msrs, 0, sizeof(msrs)); + if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { + amd64_warn("%s: Error allocating mask\n", __func__); + return false; + } - amd64_printk(KERN_WARNING, - "'ecc_enable_override' parameter is active, " - "Enabling AMD ECC hardware now: CAUTION\n"); + get_cpus_on_this_dct_cpumask(mask, nid); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCTL, &value); - if (err) - debugf0("Reading K8_NBCTL failed\n"); + rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs); - /* turn on UECCn and CECCEn bits */ - pvt->old_nbctl = value & mask; - pvt->nbctl_mcgctl_saved = 1; + for_each_cpu(cpu, mask) { + struct msr *reg = per_cpu_ptr(msrs, cpu); + nbe = reg->l & MSR_MCGCTL_NBE; - value |= mask; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); + edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", + cpu, reg->q, + (nbe ? "enabled" : "disabled")); - rdmsr_on_cpus(cpumask, K8_MSR_MCGCTL, msrs); + if (!nbe) + goto out; + } + ret = true; - for_each_cpu(cpu, cpumask) { - if (msrs[idx].l & K8_MSR_MCGCTL_NBE) - set_bit(idx, &pvt->old_mcgctl); +out: + free_cpumask_var(mask); + return ret; +} - msrs[idx].l |= K8_MSR_MCGCTL_NBE; - idx++; +static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) +{ + cpumask_var_t cmask; + int cpu; + + if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { + amd64_warn("%s: error allocating mask\n", __func__); + return false; } - wrmsr_on_cpus(cpumask, K8_MSR_MCGCTL, msrs); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &value); - if (err) - debugf0("Reading K8_NBCFG failed\n"); + get_cpus_on_this_dct_cpumask(cmask, nid); - debugf0("NBCFG(1)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, - (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); + rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); - if (!(value & K8_NBCFG_ECC_ENABLE)) { - amd64_printk(KERN_WARNING, - "This node reports that DRAM ECC is " - "currently Disabled; ENABLING now\n"); + for_each_cpu(cpu, cmask) { - /* Attempt to turn on DRAM ECC Enable */ - value |= K8_NBCFG_ECC_ENABLE; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value); + struct msr *reg = per_cpu_ptr(msrs, cpu); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &value); - if (err) - debugf0("Reading K8_NBCFG failed\n"); + if (on) { + if (reg->l & MSR_MCGCTL_NBE) + s->flags.nb_mce_enable = 1; - if (!(value & K8_NBCFG_ECC_ENABLE)) { - amd64_printk(KERN_WARNING, - "Hardware rejects Enabling DRAM ECC checking\n" - "Check memory DIMM configuration\n"); + reg->l |= MSR_MCGCTL_NBE; } else { - amd64_printk(KERN_DEBUG, - "Hardware accepted DRAM ECC Enable\n"); + /* + * Turn off NB MCE reporting only when it was off before + */ + if (!s->flags.nb_mce_enable) + reg->l &= ~MSR_MCGCTL_NBE; } } - debugf0("NBCFG(2)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, - (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); + wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); + + free_cpumask_var(cmask); - pvt->ctl_error_info.nbcfg = value; + return 0; } -static void amd64_restore_ecc_error_reporting(struct amd64_pvt *pvt) +static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, + struct pci_dev *F3) { - const cpumask_t *cpumask = cpumask_of_node(pvt->mc_node_id); - int cpu, idx = 0, err = 0; - struct msr msrs[cpumask_weight(cpumask)]; - u32 value; - u32 mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; + bool ret = true; + u32 value, mask = 0x3; /* UECC/CECC enable */ - if (!pvt->nbctl_mcgctl_saved) - return; + if (toggle_ecc_err_reporting(s, nid, ON)) { + amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); + return false; + } - memset(msrs, 0, sizeof(msrs)); + amd64_read_pci_cfg(F3, NBCTL, &value); - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCTL, &value); - if (err) - debugf0("Reading K8_NBCTL failed\n"); - value &= ~mask; - value |= pvt->old_nbctl; + s->old_nbctl = value & mask; + s->nbctl_valid = true; - /* restore the NB Enable MCGCTL bit */ - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); + value |= mask; + amd64_write_pci_cfg(F3, NBCTL, value); + + amd64_read_pci_cfg(F3, NBCFG, &value); - rdmsr_on_cpus(cpumask, K8_MSR_MCGCTL, msrs); + edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", + nid, value, !!(value & NBCFG_ECC_ENABLE)); - for_each_cpu(cpu, cpumask) { - msrs[idx].l &= ~K8_MSR_MCGCTL_NBE; - msrs[idx].l |= - test_bit(idx, &pvt->old_mcgctl) << K8_MSR_MCGCTL_NBE; - idx++; + if (!(value & NBCFG_ECC_ENABLE)) { + amd64_warn("DRAM ECC disabled on this node, enabling...\n"); + + s->flags.nb_ecc_prev = 0; + + /* Attempt to turn on DRAM ECC Enable */ + value |= NBCFG_ECC_ENABLE; + amd64_write_pci_cfg(F3, NBCFG, value); + + amd64_read_pci_cfg(F3, NBCFG, &value); + + if (!(value & NBCFG_ECC_ENABLE)) { + amd64_warn("Hardware rejected DRAM ECC enable," + "check memory DIMM configuration.\n"); + ret = false; + } else { + amd64_info("Hardware accepted DRAM ECC Enable\n"); + } + } else { + s->flags.nb_ecc_prev = 1; } - wrmsr_on_cpus(cpumask, K8_MSR_MCGCTL, msrs); + edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", + nid, value, !!(value & NBCFG_ECC_ENABLE)); + + return ret; } -static void check_mcg_ctl(void *ret) +static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, + struct pci_dev *F3) { - u64 msr_val = 0; - u8 nbe; + u32 value, mask = 0x3; /* UECC/CECC enable */ - rdmsrl(MSR_IA32_MCG_CTL, msr_val); - nbe = msr_val & K8_MSR_MCGCTL_NBE; - debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", - raw_smp_processor_id(), msr_val, - (nbe ? "enabled" : "disabled")); + if (!s->nbctl_valid) + return; - if (!nbe) - *(int *)ret = 0; -} + amd64_read_pci_cfg(F3, NBCTL, &value); + value &= ~mask; + value |= s->old_nbctl; -/* check MCG_CTL on all the cpus on this node */ -static int amd64_mcg_ctl_enabled_on_cpus(const cpumask_t *mask) -{ - int ret = 1; - preempt_disable(); - smp_call_function_many(mask, check_mcg_ctl, &ret, 1); - preempt_enable(); + amd64_write_pci_cfg(F3, NBCTL, value); - return ret; + /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ + if (!s->flags.nb_ecc_prev) { + amd64_read_pci_cfg(F3, NBCFG, &value); + value &= ~NBCFG_ECC_ENABLE; + amd64_write_pci_cfg(F3, NBCFG, value); + } + + /* restore the NB Enable MCGCTL bit */ + if (toggle_ecc_err_reporting(s, nid, OFF)) + amd64_warn("Error restoring NB MCGCTL settings!\n"); } /* - * EDAC requires that the BIOS have ECC enabled before taking over the - * processing of ECC errors. This is because the BIOS can properly initialize - * the memory system completely. A command line option allows to force-enable - * hardware ECC later in amd64_enable_ecc_error_reporting(). + * EDAC requires that the BIOS have ECC enabled before + * taking over the processing of ECC errors. A command line + * option allows to force-enable hardware ECC later in + * enable_ecc_error_reporting(). */ -static int amd64_check_ecc_enabled(struct amd64_pvt *pvt) +static const char *ecc_msg = + "ECC disabled in the BIOS or no ECC capability, module will not load.\n" + " Either enable ECC checking or force module loading by setting " + "'ecc_enable_override'.\n" + " (Note that use of the override may cause unknown side effects.)\n"; + +static bool ecc_enabled(struct pci_dev *F3, u16 nid) { u32 value; - int err = 0, ret = 0; - u8 ecc_enabled = 0; + u8 ecc_en = 0; + bool nb_mce_en = false; - err = pci_read_config_dword(pvt->misc_f3_ctl, K8_NBCFG, &value); - if (err) - debugf0("Reading K8_NBCTL failed\n"); + amd64_read_pci_cfg(F3, NBCFG, &value); - ecc_enabled = !!(value & K8_NBCFG_ECC_ENABLE); + ecc_en = !!(value & NBCFG_ECC_ENABLE); + amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); - ret = amd64_mcg_ctl_enabled_on_cpus(cpumask_of_node(pvt->mc_node_id)); + nb_mce_en = nb_mce_bank_enabled_on_node(nid); + if (!nb_mce_en) + amd64_notice("NB MCE bank disabled, set MSR " + "0x%08x[4] on node %d to enable.\n", + MSR_IA32_MCG_CTL, nid); - debugf0("K8_NBCFG=0x%x, DRAM ECC is %s\n", value, - (value & K8_NBCFG_ECC_ENABLE ? "enabled" : "disabled")); + if (!ecc_en || !nb_mce_en) { + amd64_notice("%s", ecc_msg); + return false; + } + return true; +} - if (!ecc_enabled || !ret) { - if (!ecc_enabled) { - amd64_printk(KERN_WARNING, "This node reports that " - "Memory ECC is currently " - "disabled.\n"); +static int set_mc_sysfs_attrs(struct mem_ctl_info *mci) +{ + struct amd64_pvt *pvt = mci->pvt_info; + int rc; - amd64_printk(KERN_WARNING, "bit 0x%lx in register " - "F3x%x of the MISC_CONTROL device (%s) " - "should be enabled\n", K8_NBCFG_ECC_ENABLE, - K8_NBCFG, pci_name(pvt->misc_f3_ctl)); - } - if (!ret) { - amd64_printk(KERN_WARNING, "bit 0x%016lx in MSR 0x%08x " - "of node %d should be enabled\n", - K8_MSR_MCGCTL_NBE, MSR_IA32_MCG_CTL, - pvt->mc_node_id); - } - if (!ecc_enable_override) { - amd64_printk(KERN_WARNING, "WARNING: ECC is NOT " - "currently enabled by the BIOS. Module " - "will NOT be loaded.\n" - " Either Enable ECC in the BIOS, " - "or use the 'ecc_enable_override' " - "parameter.\n" - " Might be a BIOS bug, if BIOS says " - "ECC is enabled\n" - " Use of the override can cause " - "unknown side effects.\n"); - ret = -ENODEV; - } - } else { - amd64_printk(KERN_INFO, - "ECC is enabled by BIOS, Proceeding " - "with EDAC module initialization\n"); + rc = amd64_create_sysfs_dbg_files(mci); + if (rc < 0) + return rc; - /* CLEAR the override, since BIOS controlled it */ - ecc_enable_override = 0; + if (pvt->fam >= 0x10) { + rc = amd64_create_sysfs_inject_files(mci); + if (rc < 0) + return rc; } - return ret; + return 0; } -struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + - ARRAY_SIZE(amd64_inj_attrs) + - 1]; - -struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; - -static void amd64_set_mc_sysfs_attributes(struct mem_ctl_info *mci) +static void del_mc_sysfs_attrs(struct mem_ctl_info *mci) { - unsigned int i = 0, j = 0; - - for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) - sysfs_attrs[i] = amd64_dbg_attrs[i]; - - for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) - sysfs_attrs[i] = amd64_inj_attrs[j]; + struct amd64_pvt *pvt = mci->pvt_info; - sysfs_attrs[i] = terminator; + amd64_remove_sysfs_dbg_files(mci); - mci->mc_driver_sysfs_attributes = sysfs_attrs; + if (pvt->fam >= 0x10) + amd64_remove_sysfs_inject_files(mci); } -static void amd64_setup_mci_misc_attributes(struct mem_ctl_info *mci) +static void setup_mci_misc_attrs(struct mem_ctl_info *mci, + struct amd64_family_type *fam) { struct amd64_pvt *pvt = mci->pvt_info; mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; mci->edac_ctl_cap = EDAC_FLAG_NONE; - mci->edac_cap = EDAC_FLAG_NONE; - if (pvt->nbcap & K8_NBCAP_SECDED) + if (pvt->nbcap & NBCAP_SECDED) mci->edac_ctl_cap |= EDAC_FLAG_SECDED; - if (pvt->nbcap & K8_NBCAP_CHIPKILL) + if (pvt->nbcap & NBCAP_CHIPKILL) mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; - mci->edac_cap = amd64_determine_edac_cap(pvt); + mci->edac_cap = determine_edac_cap(pvt); mci->mod_name = EDAC_MOD_STR; mci->mod_ver = EDAC_AMD64_VERSION; - mci->ctl_name = get_amd_family_name(pvt->mc_type_index); - mci->dev_name = pci_name(pvt->dram_f2_ctl); + mci->ctl_name = fam->ctl_name; + mci->dev_name = pci_name(pvt->F2); mci->ctl_page_to_phys = NULL; - /* IMPORTANT: Set the polling 'check' function in this module */ - mci->edac_check = amd64_check; - /* memory scrubber interface */ - mci->set_sdram_scrub_rate = amd64_set_scrub_rate; - mci->get_sdram_scrub_rate = amd64_get_scrub_rate; + mci->set_sdram_scrub_rate = set_scrub_rate; + mci->get_sdram_scrub_rate = get_scrub_rate; } /* - * Init stuff for this DRAM Controller device. - * - * Due to a hardware feature on Fam10h CPUs, the Enable Extended Configuration - * Space feature MUST be enabled on ALL Processors prior to actually reading - * from the ECS registers. Since the loading of the module can occur on any - * 'core', and cores don't 'see' all the other processors ECS data when the - * others are NOT enabled. Our solution is to first enable ECS access in this - * routine on all processors, gather some data in a amd64_pvt structure and - * later come back in a finish-setup function to perform that final - * initialization. See also amd64_init_2nd_stage() for that. + * returns a pointer to the family descriptor on success, NULL otherwise. */ -static int amd64_probe_one_instance(struct pci_dev *dram_f2_ctl, - int mc_type_index) +static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt) { - struct amd64_pvt *pvt = NULL; - int err = 0, ret; - - ret = -ENOMEM; - pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); - if (!pvt) - goto err_exit; + struct amd64_family_type *fam_type = NULL; - pvt->mc_node_id = get_mc_node_id_from_pdev(dram_f2_ctl); + pvt->ext_model = boot_cpu_data.x86_model >> 4; + pvt->stepping = boot_cpu_data.x86_mask; + pvt->model = boot_cpu_data.x86_model; + pvt->fam = boot_cpu_data.x86; - pvt->dram_f2_ctl = dram_f2_ctl; - pvt->ext_model = boot_cpu_data.x86_model >> 4; - pvt->mc_type_index = mc_type_index; - pvt->ops = family_ops(mc_type_index); - pvt->old_mcgctl = 0; - - /* - * We have the dram_f2_ctl device as an argument, now go reserve its - * sibling devices from the PCI system. - */ - ret = -ENODEV; - err = amd64_reserve_mc_sibling_devices(pvt, mc_type_index); - if (err) - goto err_free; - - ret = -EINVAL; - err = amd64_check_ecc_enabled(pvt); - if (err) - goto err_put; + switch (pvt->fam) { + case 0xf: + fam_type = &family_types[K8_CPUS]; + pvt->ops = &family_types[K8_CPUS].ops; + break; - /* - * Key operation here: setup of HW prior to performing ops on it. Some - * setup is required to access ECS data. After this is performed, the - * 'teardown' function must be called upon error and normal exit paths. - */ - if (boot_cpu_data.x86 >= 0x10) - amd64_setup(pvt); + case 0x10: + fam_type = &family_types[F10_CPUS]; + pvt->ops = &family_types[F10_CPUS].ops; + break; - /* - * Save the pointer to the private data for use in 2nd initialization - * stage - */ - pvt_lookup[pvt->mc_node_id] = pvt; + case 0x15: + if (pvt->model == 0x30) { + fam_type = &family_types[F15_M30H_CPUS]; + pvt->ops = &family_types[F15_M30H_CPUS].ops; + break; + } - return 0; + fam_type = &family_types[F15_CPUS]; + pvt->ops = &family_types[F15_CPUS].ops; + break; -err_put: - amd64_free_mc_sibling_devices(pvt); + case 0x16: + if (pvt->model == 0x30) { + fam_type = &family_types[F16_M30H_CPUS]; + pvt->ops = &family_types[F16_M30H_CPUS].ops; + break; + } + fam_type = &family_types[F16_CPUS]; + pvt->ops = &family_types[F16_CPUS].ops; + break; -err_free: - kfree(pvt); + default: + amd64_err("Unsupported family!\n"); + return NULL; + } -err_exit: - return ret; + amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, + (pvt->fam == 0xf ? + (pvt->ext_model >= K8_REV_F ? "revF or later " + : "revE or earlier ") + : ""), pvt->mc_node_id); + return fam_type; } -/* - * This is the finishing stage of the init code. Needs to be performed after all - * MCs' hardware have been prepped for accessing extended config space. - */ -static int amd64_init_2nd_stage(struct amd64_pvt *pvt) +static int init_one_instance(struct pci_dev *F2) { - int node_id = pvt->mc_node_id; - struct mem_ctl_info *mci; - int ret, err = 0; + struct amd64_pvt *pvt = NULL; + struct amd64_family_type *fam_type = NULL; + struct mem_ctl_info *mci = NULL; + struct edac_mc_layer layers[2]; + int err = 0, ret; + u16 nid = amd_get_node_id(F2); - amd64_read_mc_registers(pvt); + ret = -ENOMEM; + pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); + if (!pvt) + goto err_ret; + + pvt->mc_node_id = nid; + pvt->F2 = F2; + + ret = -EINVAL; + fam_type = per_family_init(pvt); + if (!fam_type) + goto err_free; ret = -ENODEV; - if (pvt->ops->probe_valid_hardware) { - err = pvt->ops->probe_valid_hardware(pvt); - if (err) - goto err_exit; - } + err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); + if (err) + goto err_free; + + read_mc_regs(pvt); /* * We need to determine how many memory channels there are. Then use * that information for calculating the size of the dynamic instance - * tables in the 'mci' structure + * tables in the 'mci' structure. */ + ret = -EINVAL; pvt->channel_count = pvt->ops->early_channel_count(pvt); if (pvt->channel_count < 0) - goto err_exit; + goto err_siblings; ret = -ENOMEM; - mci = edac_mc_alloc(0, CHIPSELECT_COUNT, pvt->channel_count, node_id); + layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; + layers[0].size = pvt->csels[0].b_cnt; + layers[0].is_virt_csrow = true; + layers[1].type = EDAC_MC_LAYER_CHANNEL; + + /* + * Always allocate two channels since we can have setups with DIMMs on + * only one channel. Also, this simplifies handling later for the price + * of a couple of KBs tops. + */ + layers[1].size = 2; + layers[1].is_virt_csrow = false; + + mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0); if (!mci) - goto err_exit; + goto err_siblings; mci->pvt_info = pvt; + mci->pdev = &pvt->F2->dev; - mci->dev = &pvt->dram_f2_ctl->dev; - amd64_setup_mci_misc_attributes(mci); + setup_mci_misc_attrs(mci, fam_type); - if (amd64_init_csrows(mci)) + if (init_csrows(mci)) mci->edac_cap = EDAC_FLAG_NONE; - amd64_enable_ecc_error_reporting(mci); - amd64_set_mc_sysfs_attributes(mci); - ret = -ENODEV; if (edac_mc_add_mc(mci)) { - debugf1("failed edac_mc_add_mc()\n"); + edac_dbg(1, "failed edac_mc_add_mc()\n"); goto err_add_mc; } + if (set_mc_sysfs_attrs(mci)) { + edac_dbg(1, "failed edac_mc_add_mc()\n"); + goto err_add_sysfs; + } - mci_lookup[node_id] = mci; - pvt_lookup[node_id] = NULL; - return 0; + /* register stuff with EDAC MCE */ + if (report_gart_errors) + amd_report_gart_errors(true); -err_add_mc: - edac_mc_free(mci); + amd_register_ecc_decoder(decode_bus_error); + + mcis[nid] = mci; -err_exit: - debugf0("failure to init 2nd stage: ret=%d\n", ret); + atomic_inc(&drv_instances); - amd64_restore_ecc_error_reporting(pvt); + return 0; - if (boot_cpu_data.x86 > 0xf) - amd64_teardown(pvt); +err_add_sysfs: + edac_mc_del_mc(mci->pdev); +err_add_mc: + edac_mc_free(mci); - amd64_free_mc_sibling_devices(pvt); +err_siblings: + free_mc_sibling_devs(pvt); - kfree(pvt_lookup[pvt->mc_node_id]); - pvt_lookup[node_id] = NULL; +err_free: + kfree(pvt); +err_ret: return ret; } - -static int __devinit amd64_init_one_instance(struct pci_dev *pdev, - const struct pci_device_id *mc_type) +static int probe_one_instance(struct pci_dev *pdev, + const struct pci_device_id *mc_type) { + u16 nid = amd_get_node_id(pdev); + struct pci_dev *F3 = node_to_amd_nb(nid)->misc; + struct ecc_settings *s; int ret = 0; - debugf0("(MC node=%d,mc_type='%s')\n", - get_mc_node_id_from_pdev(pdev), - get_amd_family_name(mc_type->driver_data)); - ret = pci_enable_device(pdev); - if (ret < 0) - ret = -EIO; - else - ret = amd64_probe_one_instance(pdev, mc_type->driver_data); + if (ret < 0) { + edac_dbg(0, "ret=%d\n", ret); + return -EIO; + } + + ret = -ENOMEM; + s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); + if (!s) + goto err_out; + + ecc_stngs[nid] = s; + + if (!ecc_enabled(F3, nid)) { + ret = -ENODEV; - if (ret < 0) - debugf0("ret=%d\n", ret); + if (!ecc_enable_override) + goto err_enable; + amd64_warn("Forcing ECC on!\n"); + + if (!enable_ecc_error_reporting(s, nid, F3)) + goto err_enable; + } + + ret = init_one_instance(pdev); + if (ret < 0) { + amd64_err("Error probing instance: %d\n", nid); + restore_ecc_error_reporting(s, nid, F3); + } + + return ret; + +err_enable: + kfree(s); + ecc_stngs[nid] = NULL; + +err_out: return ret; } -static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) +static void remove_one_instance(struct pci_dev *pdev) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; + u16 nid = amd_get_node_id(pdev); + struct pci_dev *F3 = node_to_amd_nb(nid)->misc; + struct ecc_settings *s = ecc_stngs[nid]; + mci = find_mci_by_dev(&pdev->dev); + WARN_ON(!mci); + + del_mc_sysfs_attrs(mci); /* Remove from EDAC CORE tracking list */ mci = edac_mc_del_mc(&pdev->dev); if (!mci) @@ -3207,19 +2789,22 @@ static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) pvt = mci->pvt_info; - amd64_restore_ecc_error_reporting(pvt); - - if (boot_cpu_data.x86 > 0xf) - amd64_teardown(pvt); + restore_ecc_error_reporting(s, nid, F3); - amd64_free_mc_sibling_devices(pvt); + free_mc_sibling_devs(pvt); - kfree(pvt); - mci->pvt_info = NULL; + /* unregister from EDAC MCE */ + amd_report_gart_errors(false); + amd_unregister_ecc_decoder(decode_bus_error); - mci_lookup[pvt->mc_node_id] = NULL; + kfree(ecc_stngs[nid]); + ecc_stngs[nid] = NULL; /* Free the EDAC CORE resources */ + mci->pvt_info = NULL; + mcis[nid] = NULL; + + kfree(pvt); edac_mc_free(mci); } @@ -3228,7 +2813,7 @@ static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) * PCI core identifies what devices are on a system during boot, and then * inquiry this table to see if this driver is for a given device found. */ -static const struct pci_device_id amd64_pci_table[] __devinitdata = { +static const struct pci_device_id amd64_pci_table[] = { { .vendor = PCI_VENDOR_ID_AMD, .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, @@ -3236,7 +2821,6 @@ static const struct pci_device_id amd64_pci_table[] __devinitdata = { .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = K8_CPUS }, { .vendor = PCI_VENDOR_ID_AMD, @@ -3245,100 +2829,136 @@ static const struct pci_device_id amd64_pci_table[] __devinitdata = { .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = F10_CPUS }, { .vendor = PCI_VENDOR_ID_AMD, - .device = PCI_DEVICE_ID_AMD_11H_NB_DRAM, + .device = PCI_DEVICE_ID_AMD_15H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_16H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2, .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = F11_CPUS }, + {0, } }; MODULE_DEVICE_TABLE(pci, amd64_pci_table); static struct pci_driver amd64_pci_driver = { .name = EDAC_MOD_STR, - .probe = amd64_init_one_instance, - .remove = __devexit_p(amd64_remove_one_instance), + .probe = probe_one_instance, + .remove = remove_one_instance, .id_table = amd64_pci_table, }; -static void amd64_setup_pci_device(void) +static void setup_pci_device(void) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; - if (amd64_ctl_pci) + if (pci_ctl) return; - mci = mci_lookup[0]; - if (mci) { - - pvt = mci->pvt_info; - amd64_ctl_pci = - edac_pci_create_generic_ctl(&pvt->dram_f2_ctl->dev, - EDAC_MOD_STR); - - if (!amd64_ctl_pci) { - pr_warning("%s(): Unable to create PCI control\n", - __func__); + mci = mcis[0]; + if (!mci) + return; - pr_warning("%s(): PCI error report via EDAC not set\n", - __func__); - } + pvt = mci->pvt_info; + pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); + if (!pci_ctl) { + pr_warn("%s(): Unable to create PCI control\n", __func__); + pr_warn("%s(): PCI error report via EDAC not set\n", __func__); } } static int __init amd64_edac_init(void) { - int nb, err = -ENODEV; + int err = -ENODEV; - edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n"); + printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); opstate_init(); - if (cache_k8_northbridges() < 0) - goto err_exit; + if (amd_cache_northbridges() < 0) + goto err_ret; - err = pci_register_driver(&amd64_pci_driver); - if (err) - return err; + err = -ENOMEM; + mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); + ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); + if (!(mcis && ecc_stngs)) + goto err_free; - /* - * At this point, the array 'pvt_lookup[]' contains pointers to alloc'd - * amd64_pvt structs. These will be used in the 2nd stage init function - * to finish initialization of the MC instances. - */ - for (nb = 0; nb < num_k8_northbridges; nb++) { - if (!pvt_lookup[nb]) - continue; + msrs = msrs_alloc(); + if (!msrs) + goto err_free; - err = amd64_init_2nd_stage(pvt_lookup[nb]); - if (err) - goto err_exit; - } + err = pci_register_driver(&amd64_pci_driver); + if (err) + goto err_pci; - amd64_setup_pci_device(); + err = -ENODEV; + if (!atomic_read(&drv_instances)) + goto err_no_instances; + setup_pci_device(); return 0; -err_exit: - debugf0("'finish_setup' stage failed\n"); +err_no_instances: pci_unregister_driver(&amd64_pci_driver); +err_pci: + msrs_free(msrs); + msrs = NULL; + +err_free: + kfree(mcis); + mcis = NULL; + + kfree(ecc_stngs); + ecc_stngs = NULL; + +err_ret: return err; } static void __exit amd64_edac_exit(void) { - if (amd64_ctl_pci) - edac_pci_release_generic_ctl(amd64_ctl_pci); + if (pci_ctl) + edac_pci_release_generic_ctl(pci_ctl); pci_unregister_driver(&amd64_pci_driver); + + kfree(ecc_stngs); + ecc_stngs = NULL; + + kfree(mcis); + mcis = NULL; + + msrs_free(msrs); + msrs = NULL; } module_init(amd64_edac_init); |