diff options
Diffstat (limited to 'drivers/edac/amd64_edac.c')
| -rw-r--r-- | drivers/edac/amd64_edac.c | 3143 |
1 files changed, 1567 insertions, 1576 deletions
diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 8521401bbd7..f8bf00010d4 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -1,7 +1,7 @@ #include "amd64_edac.h" #include <asm/amd_nb.h> -static struct edac_pci_ctl_info *amd64_ctl_pci; +static struct edac_pci_ctl_info *pci_ctl; static int report_gart_errors; module_param(report_gart_errors, int, 0644); @@ -15,55 +15,14 @@ module_param(ecc_enable_override, int, 0644); static struct msr __percpu *msrs; -/* Lookup table for all possible MC control instances */ -struct amd64_pvt; -static struct mem_ctl_info *mci_lookup[EDAC_MAX_NUMNODES]; -static struct amd64_pvt *pvt_lookup[EDAC_MAX_NUMNODES]; - /* - * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and - * later. + * count successfully initialized driver instances for setup_pci_device() */ -static int ddr2_dbam_revCG[] = { - [0] = 32, - [1] = 64, - [2] = 128, - [3] = 256, - [4] = 512, - [5] = 1024, - [6] = 2048, -}; - -static int ddr2_dbam_revD[] = { - [0] = 32, - [1] = 64, - [2 ... 3] = 128, - [4] = 256, - [5] = 512, - [6] = 256, - [7] = 512, - [8 ... 9] = 1024, - [10] = 2048, -}; - -static int ddr2_dbam[] = { [0] = 128, - [1] = 256, - [2 ... 4] = 512, - [5 ... 6] = 1024, - [7 ... 8] = 2048, - [9 ... 10] = 4096, - [11] = 8192, -}; +static atomic_t drv_instances = ATOMIC_INIT(0); -static int ddr3_dbam[] = { [0] = -1, - [1] = 256, - [2] = 512, - [3 ... 4] = -1, - [5 ... 6] = 1024, - [7 ... 8] = 2048, - [9 ... 10] = 4096, - [11] = 8192, -}; +/* Per-node driver instances */ +static struct mem_ctl_info **mcis; +static struct ecc_settings **ecc_stngs; /* * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing @@ -72,8 +31,10 @@ static int ddr3_dbam[] = { [0] = -1, * *FIXME: Produce a better mapping/linearisation. */ - -struct scrubrate scrubrates[] = { +static const struct scrubrate { + u32 scrubval; /* bit pattern for scrub rate */ + u32 bandwidth; /* bandwidth consumed (bytes/sec) */ +} scrubrates[] = { { 0x01, 1600000000UL}, { 0x02, 800000000UL}, { 0x03, 400000000UL}, @@ -99,6 +60,90 @@ struct scrubrate scrubrates[] = { { 0x00, 0UL}, /* scrubbing off */ }; +int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset, + u32 *val, const char *func) +{ + int err = 0; + + err = pci_read_config_dword(pdev, offset, val); + if (err) + amd64_warn("%s: error reading F%dx%03x.\n", + func, PCI_FUNC(pdev->devfn), offset); + + return err; +} + +int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset, + u32 val, const char *func) +{ + int err = 0; + + err = pci_write_config_dword(pdev, offset, val); + if (err) + amd64_warn("%s: error writing to F%dx%03x.\n", + func, PCI_FUNC(pdev->devfn), offset); + + return err; +} + +/* + * + * Depending on the family, F2 DCT reads need special handling: + * + * K8: has a single DCT only + * + * F10h: each DCT has its own set of regs + * DCT0 -> F2x040.. + * DCT1 -> F2x140.. + * + * F15h: we select which DCT we access using F1x10C[DctCfgSel] + * + * F16h: has only 1 DCT + */ +static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + if (addr >= 0x100) + return -EINVAL; + + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} + +static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} + +/* + * Select DCT to which PCI cfg accesses are routed + */ +static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct) +{ + u32 reg = 0; + + amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®); + reg &= (pvt->model >= 0x30) ? ~3 : ~1; + reg |= dct; + amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg); +} + +static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val, + const char *func) +{ + u8 dct = 0; + + /* For F15 M30h, the second dct is DCT 3, refer to BKDG Section 2.10 */ + if (addr >= 0x140 && addr <= 0x1a0) { + dct = (pvt->model >= 0x30) ? 3 : 1; + addr -= 0x100; + } + + f15h_select_dct(pvt, dct); + + return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func); +} + /* * Memory scrubber control interface. For K8, memory scrubbing is handled by * hardware and can involve L2 cache, dcache as well as the main memory. With @@ -117,8 +162,7 @@ struct scrubrate scrubrates[] = { * scan the scrub rate mapping table for a close or matching bandwidth value to * issue. If requested is too big, then use last maximum value found. */ -static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, - u32 min_scrubrate) +static int __set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate) { u32 scrubval; int i; @@ -128,143 +172,77 @@ static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, * memory controller and apply to register. Search for the first * bandwidth entry that is greater or equal than the setting requested * and program that. If at last entry, turn off DRAM scrubbing. + * + * If no suitable bandwidth is found, turn off DRAM scrubbing entirely + * by falling back to the last element in scrubrates[]. */ - for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { + for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) { /* * skip scrub rates which aren't recommended * (see F10 BKDG, F3x58) */ - if (scrubrates[i].scrubval < min_scrubrate) + if (scrubrates[i].scrubval < min_rate) continue; if (scrubrates[i].bandwidth <= new_bw) break; - - /* - * if no suitable bandwidth found, turn off DRAM scrubbing - * entirely by falling back to the last element in the - * scrubrates array. - */ } scrubval = scrubrates[i].scrubval; - if (scrubval) - edac_printk(KERN_DEBUG, EDAC_MC, - "Setting scrub rate bandwidth: %u\n", - scrubrates[i].bandwidth); - else - edac_printk(KERN_DEBUG, EDAC_MC, "Turning scrubbing off.\n"); - pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F); + pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F); + + if (scrubval) + return scrubrates[i].bandwidth; return 0; } -static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bandwidth) +static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw) { struct amd64_pvt *pvt = mci->pvt_info; - u32 min_scrubrate = 0x0; + u32 min_scrubrate = 0x5; - switch (boot_cpu_data.x86) { - case 0xf: - min_scrubrate = K8_MIN_SCRUB_RATE_BITS; - break; - case 0x10: - min_scrubrate = F10_MIN_SCRUB_RATE_BITS; - break; - case 0x11: - min_scrubrate = F11_MIN_SCRUB_RATE_BITS; - break; + if (pvt->fam == 0xf) + min_scrubrate = 0x0; - default: - amd64_printk(KERN_ERR, "Unsupported family!\n"); - return -EINVAL; - } - return amd64_search_set_scrub_rate(pvt->misc_f3_ctl, bandwidth, - min_scrubrate); + /* Erratum #505 */ + if (pvt->fam == 0x15 && pvt->model < 0x10) + f15h_select_dct(pvt, 0); + + return __set_scrub_rate(pvt->F3, bw, min_scrubrate); } -static int amd64_get_scrub_rate(struct mem_ctl_info *mci, u32 *bw) +static int get_scrub_rate(struct mem_ctl_info *mci) { struct amd64_pvt *pvt = mci->pvt_info; u32 scrubval = 0; - int status = -1, i; + int i, retval = -EINVAL; - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_SCRCTRL, &scrubval); + /* Erratum #505 */ + if (pvt->fam == 0x15 && pvt->model < 0x10) + f15h_select_dct(pvt, 0); - scrubval = scrubval & 0x001F; + amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval); - edac_printk(KERN_DEBUG, EDAC_MC, - "pci-read, sdram scrub control value: %d \n", scrubval); + scrubval = scrubval & 0x001F; for (i = 0; i < ARRAY_SIZE(scrubrates); i++) { if (scrubrates[i].scrubval == scrubval) { - *bw = scrubrates[i].bandwidth; - status = 0; + retval = scrubrates[i].bandwidth; break; } } - - return status; -} - -/* Map from a CSROW entry to the mask entry that operates on it */ -static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow) -{ - if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) - return csrow; - else - return csrow >> 1; -} - -/* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */ -static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow) -{ - if (dct == 0) - return pvt->dcsb0[csrow]; - else - return pvt->dcsb1[csrow]; + return retval; } /* - * Return the 'mask' address the i'th CS entry. This function is needed because - * there number of DCSM registers on Rev E and prior vs Rev F and later is - * different. + * returns true if the SysAddr given by sys_addr matches the + * DRAM base/limit associated with node_id */ -static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow) +static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid) { - if (dct == 0) - return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)]; - else - return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)]; -} - - -/* - * In *base and *limit, pass back the full 40-bit base and limit physical - * addresses for the node given by node_id. This information is obtained from - * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The - * base and limit addresses are of type SysAddr, as defined at the start of - * section 3.4.4 (p. 70). They are the lowest and highest physical addresses - * in the address range they represent. - */ -static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id, - u64 *base, u64 *limit) -{ - *base = pvt->dram_base[node_id]; - *limit = pvt->dram_limit[node_id]; -} - -/* - * Return 1 if the SysAddr given by sys_addr matches the base/limit associated - * with node_id - */ -static int amd64_base_limit_match(struct amd64_pvt *pvt, - u64 sys_addr, int node_id) -{ - u64 base, limit, addr; - - amd64_get_base_and_limit(pvt, node_id, &base, &limit); + u64 addr; /* The K8 treats this as a 40-bit value. However, bits 63-40 will be * all ones if the most significant implemented address bit is 1. @@ -274,7 +252,8 @@ static int amd64_base_limit_match(struct amd64_pvt *pvt, */ addr = sys_addr & 0x000000ffffffffffull; - return (addr >= base) && (addr <= limit); + return ((addr >= get_dram_base(pvt, nid)) && + (addr <= get_dram_limit(pvt, nid))); } /* @@ -287,7 +266,7 @@ static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, u64 sys_addr) { struct amd64_pvt *pvt; - int node_id; + u8 node_id; u32 intlv_en, bits; /* @@ -301,11 +280,11 @@ static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, * registers. Therefore we arbitrarily choose to read it from the * register for node 0. */ - intlv_en = pvt->dram_IntlvEn[0]; + intlv_en = dram_intlv_en(pvt, 0); if (intlv_en == 0) { - for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) { - if (amd64_base_limit_match(pvt, sys_addr, node_id)) + for (node_id = 0; node_id < DRAM_RANGES; node_id++) { + if (base_limit_match(pvt, sys_addr, node_id)) goto found; } goto err_no_match; @@ -314,74 +293,107 @@ static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci, if (unlikely((intlv_en != 0x01) && (intlv_en != 0x03) && (intlv_en != 0x07))) { - amd64_printk(KERN_WARNING, "junk value of 0x%x extracted from " - "IntlvEn field of DRAM Base Register for node 0: " - "this probably indicates a BIOS bug.\n", intlv_en); + amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en); return NULL; } bits = (((u32) sys_addr) >> 12) & intlv_en; for (node_id = 0; ; ) { - if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits) + if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits) break; /* intlv_sel field matches */ - if (++node_id >= DRAM_REG_COUNT) + if (++node_id >= DRAM_RANGES) goto err_no_match; } /* sanity test for sys_addr */ - if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) { - amd64_printk(KERN_WARNING, - "%s(): sys_addr 0x%llx falls outside base/limit " - "address range for node %d with node interleaving " - "enabled.\n", - __func__, sys_addr, node_id); + if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) { + amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address" + "range for node %d with node interleaving enabled.\n", + __func__, sys_addr, node_id); return NULL; } found: - return edac_mc_find(node_id); + return edac_mc_find((int)node_id); err_no_match: - debugf2("sys_addr 0x%lx doesn't match any node\n", - (unsigned long)sys_addr); + edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n", + (unsigned long)sys_addr); return NULL; } /* - * Extract the DRAM CS base address from selected csrow register. - */ -static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow) -{ - return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) << - pvt->dcs_shift; -} - -/* - * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way. + * compute the CS base address of the @csrow on the DRAM controller @dct. + * For details see F2x[5C:40] in the processor's BKDG */ -static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow) +static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct, + u64 *base, u64 *mask) { - u64 dcsm_bits, other_bits; - u64 mask; + u64 csbase, csmask, base_bits, mask_bits; + u8 addr_shift; - /* Extract bits from DRAM CS Mask. */ - dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask; - - other_bits = pvt->dcsm_mask; - other_bits = ~(other_bits << pvt->dcs_shift); + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow]; + base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9); + mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9); + addr_shift = 4; /* - * The extracted bits from DCSM belong in the spaces represented by - * the cleared bits in other_bits. + * F16h and F15h, models 30h and later need two addr_shift values: + * 8 for high and 6 for low (cf. F16h BKDG). */ - mask = (dcsm_bits << pvt->dcs_shift) | other_bits; + } else if (pvt->fam == 0x16 || + (pvt->fam == 0x15 && pvt->model >= 0x30)) { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow >> 1]; + + *base = (csbase & GENMASK_ULL(15, 5)) << 6; + *base |= (csbase & GENMASK_ULL(30, 19)) << 8; + + *mask = ~0ULL; + /* poke holes for the csmask */ + *mask &= ~((GENMASK_ULL(15, 5) << 6) | + (GENMASK_ULL(30, 19) << 8)); + + *mask |= (csmask & GENMASK_ULL(15, 5)) << 6; + *mask |= (csmask & GENMASK_ULL(30, 19)) << 8; + + return; + } else { + csbase = pvt->csels[dct].csbases[csrow]; + csmask = pvt->csels[dct].csmasks[csrow >> 1]; + addr_shift = 8; + + if (pvt->fam == 0x15) + base_bits = mask_bits = + GENMASK_ULL(30,19) | GENMASK_ULL(13,5); + else + base_bits = mask_bits = + GENMASK_ULL(28,19) | GENMASK_ULL(13,5); + } + + *base = (csbase & base_bits) << addr_shift; - return mask; + *mask = ~0ULL; + /* poke holes for the csmask */ + *mask &= ~(mask_bits << addr_shift); + /* OR them in */ + *mask |= (csmask & mask_bits) << addr_shift; } +#define for_each_chip_select(i, dct, pvt) \ + for (i = 0; i < pvt->csels[dct].b_cnt; i++) + +#define chip_select_base(i, dct, pvt) \ + pvt->csels[dct].csbases[i] + +#define for_each_chip_select_mask(i, dct, pvt) \ + for (i = 0; i < pvt->csels[dct].m_cnt; i++) + /* * @input_addr is an InputAddr associated with the node given by mci. Return the * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr). @@ -394,49 +406,29 @@ static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr) pvt = mci->pvt_info; - /* - * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS - * base/mask register pair, test the condition shown near the start of - * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E). - */ - for (csrow = 0; csrow < pvt->cs_count; csrow++) { - - /* This DRAM chip select is disabled on this node */ - if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0) + for_each_chip_select(csrow, 0, pvt) { + if (!csrow_enabled(csrow, 0, pvt)) continue; - base = base_from_dct_base(pvt, csrow); - mask = ~mask_from_dct_mask(pvt, csrow); + get_cs_base_and_mask(pvt, csrow, 0, &base, &mask); + + mask = ~mask; if ((input_addr & mask) == (base & mask)) { - debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n", - (unsigned long)input_addr, csrow, - pvt->mc_node_id); + edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n", + (unsigned long)input_addr, csrow, + pvt->mc_node_id); return csrow; } } - - debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n", - (unsigned long)input_addr, pvt->mc_node_id); + edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n", + (unsigned long)input_addr, pvt->mc_node_id); return -1; } /* - * Return the base value defined by the DRAM Base register for the node - * represented by mci. This function returns the full 40-bit value despite the - * fact that the register only stores bits 39-24 of the value. See section - * 3.4.4.1 (BKDG #26094, K8, revA-E) - */ -static inline u64 get_dram_base(struct mem_ctl_info *mci) -{ - struct amd64_pvt *pvt = mci->pvt_info; - - return pvt->dram_base[pvt->mc_node_id]; -} - -/* * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094) * for the node represented by mci. Info is passed back in *hole_base, * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if @@ -456,25 +448,23 @@ int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, u64 *hole_offset, u64 *hole_size) { struct amd64_pvt *pvt = mci->pvt_info; - u64 base; /* only revE and later have the DRAM Hole Address Register */ - if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) { - debugf1(" revision %d for node %d does not support DHAR\n", - pvt->ext_model, pvt->mc_node_id); + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) { + edac_dbg(1, " revision %d for node %d does not support DHAR\n", + pvt->ext_model, pvt->mc_node_id); return 1; } - /* only valid for Fam10h */ - if (boot_cpu_data.x86 == 0x10 && - (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) { - debugf1(" Dram Memory Hoisting is DISABLED on this system\n"); + /* valid for Fam10h and above */ + if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) { + edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n"); return 1; } - if ((pvt->dhar & DHAR_VALID) == 0) { - debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n", - pvt->mc_node_id); + if (!dhar_valid(pvt)) { + edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n", + pvt->mc_node_id); return 1; } @@ -496,19 +486,15 @@ int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base, * addresses in the hole so that they start at 0x100000000. */ - base = dhar_base(pvt->dhar); - - *hole_base = base; - *hole_size = (0x1ull << 32) - base; + *hole_base = dhar_base(pvt); + *hole_size = (1ULL << 32) - *hole_base; - if (boot_cpu_data.x86 > 0xf) - *hole_offset = f10_dhar_offset(pvt->dhar); - else - *hole_offset = k8_dhar_offset(pvt->dhar); + *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt) + : k8_dhar_offset(pvt); - debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", - pvt->mc_node_id, (unsigned long)*hole_base, - (unsigned long)*hole_offset, (unsigned long)*hole_size); + edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n", + pvt->mc_node_id, (unsigned long)*hole_base, + (unsigned long)*hole_offset, (unsigned long)*hole_size); return 0; } @@ -545,23 +531,23 @@ EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info); */ static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) { + struct amd64_pvt *pvt = mci->pvt_info; u64 dram_base, hole_base, hole_offset, hole_size, dram_addr; - int ret = 0; + int ret; - dram_base = get_dram_base(mci); + dram_base = get_dram_base(pvt, pvt->mc_node_id); ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size); if (!ret) { - if ((sys_addr >= (1ull << 32)) && - (sys_addr < ((1ull << 32) + hole_size))) { + if ((sys_addr >= (1ULL << 32)) && + (sys_addr < ((1ULL << 32) + hole_size))) { /* use DHAR to translate SysAddr to DramAddr */ dram_addr = sys_addr - hole_offset; - debugf2("using DHAR to translate SysAddr 0x%lx to " - "DramAddr 0x%lx\n", - (unsigned long)sys_addr, - (unsigned long)dram_addr); + edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n", + (unsigned long)sys_addr, + (unsigned long)dram_addr); return dram_addr; } @@ -576,11 +562,10 @@ static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr) * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture * Programmer's Manual Volume 1 Application Programming. */ - dram_addr = (sys_addr & 0xffffffffffull) - dram_base; + dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base; - debugf2("using DRAM Base register to translate SysAddr 0x%lx to " - "DramAddr 0x%lx\n", (unsigned long)sys_addr, - (unsigned long)dram_addr); + edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n", + (unsigned long)sys_addr, (unsigned long)dram_addr); return dram_addr; } @@ -612,13 +597,13 @@ static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr) * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) * concerning translating a DramAddr to an InputAddr. */ - intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); - input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) + - (dram_addr & 0xfff); + intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0)); + input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) + + (dram_addr & 0xfff); - debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", - intlv_shift, (unsigned long)dram_addr, - (unsigned long)input_addr); + edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n", + intlv_shift, (unsigned long)dram_addr, + (unsigned long)input_addr); return input_addr; } @@ -634,143 +619,18 @@ static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr) input_addr = dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr)); - debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n", - (unsigned long)sys_addr, (unsigned long)input_addr); + edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n", + (unsigned long)sys_addr, (unsigned long)input_addr); return input_addr; } - -/* - * @input_addr is an InputAddr associated with the node represented by mci. - * Translate @input_addr to a DramAddr and return the result. - */ -static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr) -{ - struct amd64_pvt *pvt; - int node_id, intlv_shift; - u64 bits, dram_addr; - u32 intlv_sel; - - /* - * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E) - * shows how to translate a DramAddr to an InputAddr. Here we reverse - * this procedure. When translating from a DramAddr to an InputAddr, the - * bits used for node interleaving are discarded. Here we recover these - * bits from the IntlvSel field of the DRAM Limit register (section - * 3.4.4.2) for the node that input_addr is associated with. - */ - pvt = mci->pvt_info; - node_id = pvt->mc_node_id; - BUG_ON((node_id < 0) || (node_id > 7)); - - intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]); - - if (intlv_shift == 0) { - debugf1(" InputAddr 0x%lx translates to DramAddr of " - "same value\n", (unsigned long)input_addr); - - return input_addr; - } - - bits = ((input_addr & 0xffffff000ull) << intlv_shift) + - (input_addr & 0xfff); - - intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1); - dram_addr = bits + (intlv_sel << 12); - - debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx " - "(%d node interleave bits)\n", (unsigned long)input_addr, - (unsigned long)dram_addr, intlv_shift); - - return dram_addr; -} - -/* - * @dram_addr is a DramAddr that maps to the node represented by mci. Convert - * @dram_addr to a SysAddr. - */ -static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr) -{ - struct amd64_pvt *pvt = mci->pvt_info; - u64 hole_base, hole_offset, hole_size, base, limit, sys_addr; - int ret = 0; - - ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset, - &hole_size); - if (!ret) { - if ((dram_addr >= hole_base) && - (dram_addr < (hole_base + hole_size))) { - sys_addr = dram_addr + hole_offset; - - debugf1("using DHAR to translate DramAddr 0x%lx to " - "SysAddr 0x%lx\n", (unsigned long)dram_addr, - (unsigned long)sys_addr); - - return sys_addr; - } - } - - amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit); - sys_addr = dram_addr + base; - - /* - * The sys_addr we have computed up to this point is a 40-bit value - * because the k8 deals with 40-bit values. However, the value we are - * supposed to return is a full 64-bit physical address. The AMD - * x86-64 architecture specifies that the most significant implemented - * address bit through bit 63 of a physical address must be either all - * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a - * 64-bit value below. See section 3.4.2 of AMD publication 24592: - * AMD x86-64 Architecture Programmer's Manual Volume 1 Application - * Programming. - */ - sys_addr |= ~((sys_addr & (1ull << 39)) - 1); - - debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n", - pvt->mc_node_id, (unsigned long)dram_addr, - (unsigned long)sys_addr); - - return sys_addr; -} - -/* - * @input_addr is an InputAddr associated with the node given by mci. Translate - * @input_addr to a SysAddr. - */ -static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci, - u64 input_addr) -{ - return dram_addr_to_sys_addr(mci, - input_addr_to_dram_addr(mci, input_addr)); -} - -/* - * Find the minimum and maximum InputAddr values that map to the given @csrow. - * Pass back these values in *input_addr_min and *input_addr_max. - */ -static void find_csrow_limits(struct mem_ctl_info *mci, int csrow, - u64 *input_addr_min, u64 *input_addr_max) -{ - struct amd64_pvt *pvt; - u64 base, mask; - - pvt = mci->pvt_info; - BUG_ON((csrow < 0) || (csrow >= pvt->cs_count)); - - base = base_from_dct_base(pvt, csrow); - mask = mask_from_dct_mask(pvt, csrow); - - *input_addr_min = base & ~mask; - *input_addr_max = base | mask | pvt->dcs_mask_notused; -} - /* Map the Error address to a PAGE and PAGE OFFSET. */ static inline void error_address_to_page_and_offset(u64 error_address, - u32 *page, u32 *offset) + struct err_info *err) { - *page = (u32) (error_address >> PAGE_SHIFT); - *offset = ((u32) error_address) & ~PAGE_MASK; + err->page = (u32) (error_address >> PAGE_SHIFT); + err->offset = ((u32) error_address) & ~PAGE_MASK; } /* @@ -788,44 +648,23 @@ static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr) csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr)); if (csrow == -1) - amd64_mc_printk(mci, KERN_ERR, - "Failed to translate InputAddr to csrow for " - "address 0x%lx\n", (unsigned long)sys_addr); + amd64_mc_err(mci, "Failed to translate InputAddr to csrow for " + "address 0x%lx\n", (unsigned long)sys_addr); return csrow; } static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16); -static u16 extract_syndrome(struct err_regs *err) -{ - return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00); -} - -static void amd64_cpu_display_info(struct amd64_pvt *pvt) -{ - if (boot_cpu_data.x86 == 0x11) - edac_printk(KERN_DEBUG, EDAC_MC, "F11h CPU detected\n"); - else if (boot_cpu_data.x86 == 0x10) - edac_printk(KERN_DEBUG, EDAC_MC, "F10h CPU detected\n"); - else if (boot_cpu_data.x86 == 0xf) - edac_printk(KERN_DEBUG, EDAC_MC, "%s detected\n", - (pvt->ext_model >= K8_REV_F) ? - "Rev F or later" : "Rev E or earlier"); - else - /* we'll hardly ever ever get here */ - edac_printk(KERN_ERR, EDAC_MC, "Unknown cpu!\n"); -} - /* * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs * are ECC capable. */ -static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) +static unsigned long determine_edac_cap(struct amd64_pvt *pvt) { - int bit; - enum dev_type edac_cap = EDAC_FLAG_NONE; + u8 bit; + unsigned long edac_cap = EDAC_FLAG_NONE; - bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F) + bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F) ? 19 : 17; @@ -835,198 +674,139 @@ static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt) return edac_cap; } +static void debug_display_dimm_sizes(struct amd64_pvt *, u8); -static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt); - -static void amd64_dump_dramcfg_low(u32 dclr, int chan) +static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan) { - debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); + edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr); - debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n", - (dclr & BIT(16)) ? "un" : "", - (dclr & BIT(19)) ? "yes" : "no"); + edac_dbg(1, " DIMM type: %sbuffered; all DIMMs support ECC: %s\n", + (dclr & BIT(16)) ? "un" : "", + (dclr & BIT(19)) ? "yes" : "no"); - debugf1(" PAR/ERR parity: %s\n", - (dclr & BIT(8)) ? "enabled" : "disabled"); + edac_dbg(1, " PAR/ERR parity: %s\n", + (dclr & BIT(8)) ? "enabled" : "disabled"); - debugf1(" DCT 128bit mode width: %s\n", - (dclr & BIT(11)) ? "128b" : "64b"); + if (pvt->fam == 0x10) + edac_dbg(1, " DCT 128bit mode width: %s\n", + (dclr & BIT(11)) ? "128b" : "64b"); - debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", - (dclr & BIT(12)) ? "yes" : "no", - (dclr & BIT(13)) ? "yes" : "no", - (dclr & BIT(14)) ? "yes" : "no", - (dclr & BIT(15)) ? "yes" : "no"); + edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n", + (dclr & BIT(12)) ? "yes" : "no", + (dclr & BIT(13)) ? "yes" : "no", + (dclr & BIT(14)) ? "yes" : "no", + (dclr & BIT(15)) ? "yes" : "no"); } /* Display and decode various NB registers for debug purposes. */ -static void amd64_dump_misc_regs(struct amd64_pvt *pvt) +static void dump_misc_regs(struct amd64_pvt *pvt) { - int ganged; + edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); - debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap); + edac_dbg(1, " NB two channel DRAM capable: %s\n", + (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no"); - debugf1(" NB two channel DRAM capable: %s\n", - (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no"); + edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n", + (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no", + (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no"); - debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n", - (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no", - (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no"); + debug_dump_dramcfg_low(pvt, pvt->dclr0, 0); - amd64_dump_dramcfg_low(pvt->dclr0, 0); + edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); - debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare); + edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n", + pvt->dhar, dhar_base(pvt), + (pvt->fam == 0xf) ? k8_dhar_offset(pvt) + : f10_dhar_offset(pvt)); - debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, " - "offset: 0x%08x\n", - pvt->dhar, - dhar_base(pvt->dhar), - (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar) - : f10_dhar_offset(pvt->dhar)); + edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no"); - debugf1(" DramHoleValid: %s\n", - (pvt->dhar & DHAR_VALID) ? "yes" : "no"); + debug_display_dimm_sizes(pvt, 0); /* everything below this point is Fam10h and above */ - if (boot_cpu_data.x86 == 0xf) { - amd64_debug_display_dimm_sizes(0, pvt); + if (pvt->fam == 0xf) return; - } - amd64_printk(KERN_INFO, "using %s syndromes.\n", - ((pvt->syn_type == 8) ? "x8" : "x4")); + debug_display_dimm_sizes(pvt, 1); + + amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4")); /* Only if NOT ganged does dclr1 have valid info */ if (!dct_ganging_enabled(pvt)) - amd64_dump_dramcfg_low(pvt->dclr1, 1); - - /* - * Determine if ganged and then dump memory sizes for first controller, - * and if NOT ganged dump info for 2nd controller. - */ - ganged = dct_ganging_enabled(pvt); - - amd64_debug_display_dimm_sizes(0, pvt); - - if (!ganged) - amd64_debug_display_dimm_sizes(1, pvt); -} - -/* Read in both of DBAM registers */ -static void amd64_read_dbam_reg(struct amd64_pvt *pvt) -{ - amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM0, &pvt->dbam0); - - if (boot_cpu_data.x86 >= 0x10) - amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM1, &pvt->dbam1); + debug_dump_dramcfg_low(pvt, pvt->dclr1, 1); } /* - * NOTE: CPU Revision Dependent code: Rev E and Rev F - * - * Set the DCSB and DCSM mask values depending on the CPU revision value. Also - * set the shift factor for the DCSB and DCSM values. - * - * ->dcs_mask_notused, RevE: - * - * To find the max InputAddr for the csrow, start with the base address and set - * all bits that are "don't care" bits in the test at the start of section - * 3.5.4 (p. 84). - * - * The "don't care" bits are all set bits in the mask and all bits in the gaps - * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS - * represents bits [24:20] and [12:0], which are all bits in the above-mentioned - * gaps. - * - * ->dcs_mask_notused, RevF and later: - * - * To find the max InputAddr for the csrow, start with the base address and set - * all bits that are "don't care" bits in the test at the start of NPT section - * 4.5.4 (p. 87). - * - * The "don't care" bits are all set bits in the mask and all bits in the gaps - * between bit ranges [36:27] and [21:13]. - * - * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0], - * which are all bits in the above-mentioned gaps. + * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60] */ -static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt) -{ - - if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) { - pvt->dcsb_base = REV_E_DCSB_BASE_BITS; - pvt->dcsm_mask = REV_E_DCSM_MASK_BITS; - pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS; - pvt->dcs_shift = REV_E_DCS_SHIFT; - pvt->cs_count = 8; - pvt->num_dcsm = 8; +static void prep_chip_selects(struct amd64_pvt *pvt) +{ + if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) { + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8; + } else if (pvt->fam == 0x15 && pvt->model >= 0x30) { + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2; } else { - pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS; - pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS; - pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS; - pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT; - - if (boot_cpu_data.x86 == 0x11) { - pvt->cs_count = 4; - pvt->num_dcsm = 2; - } else { - pvt->cs_count = 8; - pvt->num_dcsm = 4; - } + pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8; + pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4; } } /* - * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers + * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers */ -static void amd64_read_dct_base_mask(struct amd64_pvt *pvt) +static void read_dct_base_mask(struct amd64_pvt *pvt) { - int cs, reg; + int cs; - amd64_set_dct_base_and_mask(pvt); + prep_chip_selects(pvt); - for (cs = 0; cs < pvt->cs_count; cs++) { - reg = K8_DCSB0 + (cs * 4); - if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsb0[cs])) - debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsb0[cs], reg); + for_each_chip_select(cs, 0, pvt) { + int reg0 = DCSB0 + (cs * 4); + int reg1 = DCSB1 + (cs * 4); + u32 *base0 = &pvt->csels[0].csbases[cs]; + u32 *base1 = &pvt->csels[1].csbases[cs]; - /* If DCT are NOT ganged, then read in DCT1's base */ - if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { - reg = F10_DCSB1 + (cs * 4); - if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, - &pvt->dcsb1[cs])) - debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsb1[cs], reg); - } else { - pvt->dcsb1[cs] = 0; - } + if (!amd64_read_dct_pci_cfg(pvt, reg0, base0)) + edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n", + cs, *base0, reg0); + + if (pvt->fam == 0xf || dct_ganging_enabled(pvt)) + continue; + + if (!amd64_read_dct_pci_cfg(pvt, reg1, base1)) + edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n", + cs, *base1, reg1); } - for (cs = 0; cs < pvt->num_dcsm; cs++) { - reg = K8_DCSM0 + (cs * 4); - if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsm0[cs])) - debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsm0[cs], reg); + for_each_chip_select_mask(cs, 0, pvt) { + int reg0 = DCSM0 + (cs * 4); + int reg1 = DCSM1 + (cs * 4); + u32 *mask0 = &pvt->csels[0].csmasks[cs]; + u32 *mask1 = &pvt->csels[1].csmasks[cs]; - /* If DCT are NOT ganged, then read in DCT1's mask */ - if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) { - reg = F10_DCSM1 + (cs * 4); - if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, - &pvt->dcsm1[cs])) - debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n", - cs, pvt->dcsm1[cs], reg); - } else { - pvt->dcsm1[cs] = 0; - } + if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0)) + edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n", + cs, *mask0, reg0); + + if (pvt->fam == 0xf || dct_ganging_enabled(pvt)) + continue; + + if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1)) + edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n", + cs, *mask1, reg1); } } -static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt) +static enum mem_type determine_memory_type(struct amd64_pvt *pvt, int cs) { enum mem_type type; - if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) { + /* F15h supports only DDR3 */ + if (pvt->fam >= 0x15) + type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; + else if (pvt->fam == 0x10 || pvt->ext_model >= K8_REV_F) { if (pvt->dchr0 & DDR3_MODE) type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3; else @@ -1035,35 +815,22 @@ static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt) type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR; } - debugf1(" Memory type is: %s\n", edac_mem_types[type]); + amd64_info("CS%d: %s\n", cs, edac_mem_types[type]); return type; } -/* - * Read the DRAM Configuration Low register. It differs between CG, D & E revs - * and the later RevF memory controllers (DDR vs DDR2) - * - * Return: - * number of memory channels in operation - * Pass back: - * contents of the DCL0_LOW register - */ +/* Get the number of DCT channels the memory controller is using. */ static int k8_early_channel_count(struct amd64_pvt *pvt) { - int flag, err = 0; + int flag; - err = amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); - if (err) - return err; - - if ((boot_cpu_data.x86_model >> 4) >= K8_REV_F) { + if (pvt->ext_model >= K8_REV_F) /* RevF (NPT) and later */ - flag = pvt->dclr0 & F10_WIDTH_128; - } else { + flag = pvt->dclr0 & WIDTH_128; + else /* RevE and earlier */ flag = pvt->dclr0 & REVE_WIDTH_128; - } /* not used */ pvt->dclr1 = 0; @@ -1071,66 +838,173 @@ static int k8_early_channel_count(struct amd64_pvt *pvt) return (flag) ? 2 : 1; } -/* extract the ERROR ADDRESS for the K8 CPUs */ -static u64 k8_get_error_address(struct mem_ctl_info *mci, - struct err_regs *info) +/* On F10h and later ErrAddr is MC4_ADDR[47:1] */ +static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m) { - return (((u64) (info->nbeah & 0xff)) << 32) + - (info->nbeal & ~0x03); + u64 addr; + u8 start_bit = 1; + u8 end_bit = 47; + + if (pvt->fam == 0xf) { + start_bit = 3; + end_bit = 39; + } + + addr = m->addr & GENMASK_ULL(end_bit, start_bit); + + /* + * Erratum 637 workaround + */ + if (pvt->fam == 0x15) { + struct amd64_pvt *pvt; + u64 cc6_base, tmp_addr; + u32 tmp; + u16 mce_nid; + u8 intlv_en; + + if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7) + return addr; + + mce_nid = amd_get_nb_id(m->extcpu); + pvt = mcis[mce_nid]->pvt_info; + + amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp); + intlv_en = tmp >> 21 & 0x7; + + /* add [47:27] + 3 trailing bits */ + cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3; + + /* reverse and add DramIntlvEn */ + cc6_base |= intlv_en ^ 0x7; + + /* pin at [47:24] */ + cc6_base <<= 24; + + if (!intlv_en) + return cc6_base | (addr & GENMASK_ULL(23, 0)); + + amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp); + + /* faster log2 */ + tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1); + + /* OR DramIntlvSel into bits [14:12] */ + tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9; + + /* add remaining [11:0] bits from original MC4_ADDR */ + tmp_addr |= addr & GENMASK_ULL(11, 0); + + return cc6_base | tmp_addr; + } + + return addr; } -/* - * Read the Base and Limit registers for K8 based Memory controllers; extract - * fields from the 'raw' reg into separate data fields - * - * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN - */ -static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram) +static struct pci_dev *pci_get_related_function(unsigned int vendor, + unsigned int device, + struct pci_dev *related) { - u32 low; - u32 off = dram << 3; /* 8 bytes between DRAM entries */ + struct pci_dev *dev = NULL; - amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_BASE_LOW + off, &low); + while ((dev = pci_get_device(vendor, device, dev))) { + if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) && + (dev->bus->number == related->bus->number) && + (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) + break; + } - /* Extract parts into separate data entries */ - pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8; - pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7; - pvt->dram_rw_en[dram] = (low & 0x3); + return dev; +} - amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_LIMIT_LOW + off, &low); +static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range) +{ + struct amd_northbridge *nb; + struct pci_dev *f1 = NULL; + unsigned int pci_func; + int off = range << 3; + u32 llim; - /* - * Extract parts into separate data entries. Limit is the HIGHEST memory - * location of the region, so lower 24 bits need to be all ones - */ - pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF; - pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7; - pvt->dram_DstNode[dram] = (low & 0x7); + amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo); + amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo); + + if (pvt->fam == 0xf) + return; + + if (!dram_rw(pvt, range)) + return; + + amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi); + amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi); + + /* F15h: factor in CC6 save area by reading dst node's limit reg */ + if (pvt->fam != 0x15) + return; + + nb = node_to_amd_nb(dram_dst_node(pvt, range)); + if (WARN_ON(!nb)) + return; + + pci_func = (pvt->model == 0x30) ? PCI_DEVICE_ID_AMD_15H_M30H_NB_F1 + : PCI_DEVICE_ID_AMD_15H_NB_F1; + + f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc); + if (WARN_ON(!f1)) + return; + + amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim); + + pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0); + + /* {[39:27],111b} */ + pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16; + + pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0); + + /* [47:40] */ + pvt->ranges[range].lim.hi |= llim >> 13; + + pci_dev_put(f1); } -static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, - struct err_regs *err_info, u64 sys_addr) +static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, + struct err_info *err) { - struct mem_ctl_info *src_mci; - int channel, csrow; - u32 page, offset; - u16 syndrome; + struct amd64_pvt *pvt = mci->pvt_info; + + error_address_to_page_and_offset(sys_addr, err); + + /* + * Find out which node the error address belongs to. This may be + * different from the node that detected the error. + */ + err->src_mci = find_mc_by_sys_addr(mci, sys_addr); + if (!err->src_mci) { + amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n", + (unsigned long)sys_addr); + err->err_code = ERR_NODE; + return; + } - syndrome = extract_syndrome(err_info); + /* Now map the sys_addr to a CSROW */ + err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr); + if (err->csrow < 0) { + err->err_code = ERR_CSROW; + return; + } /* CHIPKILL enabled */ - if (err_info->nbcfg & K8_NBCFG_CHIPKILL) { - channel = get_channel_from_ecc_syndrome(mci, syndrome); - if (channel < 0) { + if (pvt->nbcfg & NBCFG_CHIPKILL) { + err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); + if (err->channel < 0) { /* * Syndrome didn't map, so we don't know which of the * 2 DIMMs is in error. So we need to ID 'both' of them * as suspect. */ - amd64_mc_printk(mci, KERN_WARNING, - "unknown syndrome 0x%04x - possible " - "error reporting race\n", syndrome); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); + amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - " + "possible error reporting race\n", + err->syndrome); + err->err_code = ERR_CHANNEL; return; } } else { @@ -1142,46 +1016,69 @@ static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, * was obtained from email communication with someone at AMD. * (Wish the email was placed in this comment - norsk) */ - channel = ((sys_addr & BIT(3)) != 0); + err->channel = ((sys_addr & BIT(3)) != 0); } +} - /* - * Find out which node the error address belongs to. This may be - * different from the node that detected the error. - */ - src_mci = find_mc_by_sys_addr(mci, sys_addr); - if (!src_mci) { - amd64_mc_printk(mci, KERN_ERR, - "failed to map error address 0x%lx to a node\n", - (unsigned long)sys_addr); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); - return; - } +static int ddr2_cs_size(unsigned i, bool dct_width) +{ + unsigned shift = 0; - /* Now map the sys_addr to a CSROW */ - csrow = sys_addr_to_csrow(src_mci, sys_addr); - if (csrow < 0) { - edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR); - } else { - error_address_to_page_and_offset(sys_addr, &page, &offset); + if (i <= 2) + shift = i; + else if (!(i & 0x1)) + shift = i >> 1; + else + shift = (i + 1) >> 1; - edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow, - channel, EDAC_MOD_STR); - } + return 128 << (shift + !!dct_width); } -static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) +static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - int *dbam_map; + u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; - if (pvt->ext_model >= K8_REV_F) - dbam_map = ddr2_dbam; - else if (pvt->ext_model >= K8_REV_D) - dbam_map = ddr2_dbam_revD; - else - dbam_map = ddr2_dbam_revCG; + if (pvt->ext_model >= K8_REV_F) { + WARN_ON(cs_mode > 11); + return ddr2_cs_size(cs_mode, dclr & WIDTH_128); + } + else if (pvt->ext_model >= K8_REV_D) { + unsigned diff; + WARN_ON(cs_mode > 10); - return dbam_map[cs_mode]; + /* + * the below calculation, besides trying to win an obfuscated C + * contest, maps cs_mode values to DIMM chip select sizes. The + * mappings are: + * + * cs_mode CS size (mb) + * ======= ============ + * 0 32 + * 1 64 + * 2 128 + * 3 128 + * 4 256 + * 5 512 + * 6 256 + * 7 512 + * 8 1024 + * 9 1024 + * 10 2048 + * + * Basically, it calculates a value with which to shift the + * smallest CS size of 32MB. + * + * ddr[23]_cs_size have a similar purpose. + */ + diff = cs_mode/3 + (unsigned)(cs_mode > 5); + + return 32 << (cs_mode - diff); + } + else { + WARN_ON(cs_mode > 6); + return 32 << cs_mode; + } } /* @@ -1192,17 +1089,13 @@ static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) * Pass back: * contents of the DCL0_LOW register */ -static int f10_early_channel_count(struct amd64_pvt *pvt) +static int f1x_early_channel_count(struct amd64_pvt *pvt) { - int dbams[] = { DBAM0, DBAM1 }; int i, j, channels = 0; - u32 dbam; - /* If we are in 128 bit mode, then we are using 2 channels */ - if (pvt->dclr0 & F10_WIDTH_128) { - channels = 2; - return channels; - } + /* On F10h, if we are in 128 bit mode, then we are using 2 channels */ + if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128)) + return 2; /* * Need to check if in unganged mode: In such, there are 2 channels, @@ -1212,16 +1105,15 @@ static int f10_early_channel_count(struct amd64_pvt *pvt) * Need to check DCT0[0] and DCT1[0] to see if only one of them has * their CSEnable bit on. If so, then SINGLE DIMM case. */ - debugf0("Data width is not 128 bits - need more decoding\n"); + edac_dbg(0, "Data width is not 128 bits - need more decoding\n"); /* * Check DRAM Bank Address Mapping values for each DIMM to see if there * is more than just one DIMM present in unganged mode. Need to check * both controllers since DIMMs can be placed in either one. */ - for (i = 0; i < ARRAY_SIZE(dbams); i++) { - if (amd64_read_pci_cfg(pvt->dram_f2_ctl, dbams[i], &dbam)) - goto err_reg; + for (i = 0; i < 2; i++) { + u32 dbam = (i ? pvt->dbam1 : pvt->dbam0); for (j = 0; j < 4; j++) { if (DBAM_DIMM(j, dbam) > 0) { @@ -1234,248 +1126,239 @@ static int f10_early_channel_count(struct amd64_pvt *pvt) if (channels > 2) channels = 2; - debugf0("MCT channel count: %d\n", channels); + amd64_info("MCT channel count: %d\n", channels); return channels; - -err_reg: - return -1; - } -static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode) +static int ddr3_cs_size(unsigned i, bool dct_width) { - int *dbam_map; + unsigned shift = 0; + int cs_size = 0; - if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) - dbam_map = ddr3_dbam; + if (i == 0 || i == 3 || i == 4) + cs_size = -1; + else if (i <= 2) + shift = i; + else if (i == 12) + shift = 7; + else if (!(i & 0x1)) + shift = i >> 1; else - dbam_map = ddr2_dbam; - - return dbam_map[cs_mode]; -} + shift = (i + 1) >> 1; -/* Enable extended configuration access via 0xCF8 feature */ -static void amd64_setup(struct amd64_pvt *pvt) -{ - u32 reg; - - amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); + if (cs_size != -1) + cs_size = (128 * (1 << !!dct_width)) << shift; - pvt->flags.cf8_extcfg = !!(reg & F10_NB_CFG_LOW_ENABLE_EXT_CFG); - reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; - pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); + return cs_size; } -/* Restore the extended configuration access via 0xCF8 feature */ -static void amd64_teardown(struct amd64_pvt *pvt) +static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - u32 reg; + u32 dclr = dct ? pvt->dclr1 : pvt->dclr0; - amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, ®); + WARN_ON(cs_mode > 11); - reg &= ~F10_NB_CFG_LOW_ENABLE_EXT_CFG; - if (pvt->flags.cf8_extcfg) - reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG; - pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg); + if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE) + return ddr3_cs_size(cs_mode, dclr & WIDTH_128); + else + return ddr2_cs_size(cs_mode, dclr & WIDTH_128); } -static u64 f10_get_error_address(struct mem_ctl_info *mci, - struct err_regs *info) +/* + * F15h supports only 64bit DCT interfaces + */ +static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - return (((u64) (info->nbeah & 0xffff)) << 32) + - (info->nbeal & ~0x01); + WARN_ON(cs_mode > 12); + + return ddr3_cs_size(cs_mode, false); } /* - * Read the Base and Limit registers for F10 based Memory controllers. Extract - * fields from the 'raw' reg into separate data fields. - * - * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN. + * F16h and F15h model 30h have only limited cs_modes. */ -static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram) +static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct, + unsigned cs_mode) { - u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit; - - low_offset = K8_DRAM_BASE_LOW + (dram << 3); - high_offset = F10_DRAM_BASE_HIGH + (dram << 3); + WARN_ON(cs_mode > 12); - /* read the 'raw' DRAM BASE Address register */ - amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_base); - - /* Read from the ECS data register */ - amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_base); + if (cs_mode == 6 || cs_mode == 8 || + cs_mode == 9 || cs_mode == 12) + return -1; + else + return ddr3_cs_size(cs_mode, false); +} - /* Extract parts into separate data entries */ - pvt->dram_rw_en[dram] = (low_base & 0x3); +static void read_dram_ctl_register(struct amd64_pvt *pvt) +{ - if (pvt->dram_rw_en[dram] == 0) + if (pvt->fam == 0xf) return; - pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7; - - pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) | - (((u64)low_base & 0xFFFF0000) << 8); + if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) { + edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n", + pvt->dct_sel_lo, dct_sel_baseaddr(pvt)); - low_offset = K8_DRAM_LIMIT_LOW + (dram << 3); - high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3); + edac_dbg(0, " DCTs operate in %s mode\n", + (dct_ganging_enabled(pvt) ? "ganged" : "unganged")); - /* read the 'raw' LIMIT registers */ - amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_limit); + if (!dct_ganging_enabled(pvt)) + edac_dbg(0, " Address range split per DCT: %s\n", + (dct_high_range_enabled(pvt) ? "yes" : "no")); - /* Read from the ECS data register for the HIGH portion */ - amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_limit); + edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n", + (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), + (dct_memory_cleared(pvt) ? "yes" : "no")); - pvt->dram_DstNode[dram] = (low_limit & 0x7); - pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7; + edac_dbg(0, " channel interleave: %s, " + "interleave bits selector: 0x%x\n", + (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), + dct_sel_interleave_addr(pvt)); + } - /* - * Extract address values and form a LIMIT address. Limit is the HIGHEST - * memory location of the region, so low 24 bits need to be all ones. - */ - pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) | - (((u64) low_limit & 0xFFFF0000) << 8) | - 0x00FFFFFF; + amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi); } -static void f10_read_dram_ctl_register(struct amd64_pvt *pvt) -{ - - if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_LOW, - &pvt->dram_ctl_select_low)) { - debugf0("F2x110 (DCTL Sel. Low): 0x%08x, " - "High range addresses at: 0x%x\n", - pvt->dram_ctl_select_low, - dct_sel_baseaddr(pvt)); - - debugf0(" DCT mode: %s, All DCTs on: %s\n", - (dct_ganging_enabled(pvt) ? "ganged" : "unganged"), - (dct_dram_enabled(pvt) ? "yes" : "no")); - - if (!dct_ganging_enabled(pvt)) - debugf0(" Address range split per DCT: %s\n", - (dct_high_range_enabled(pvt) ? "yes" : "no")); - - debugf0(" DCT data interleave for ECC: %s, " - "DRAM cleared since last warm reset: %s\n", - (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"), - (dct_memory_cleared(pvt) ? "yes" : "no")); - - debugf0(" DCT channel interleave: %s, " - "DCT interleave bits selector: 0x%x\n", - (dct_interleave_enabled(pvt) ? "enabled" : "disabled"), - dct_sel_interleave_addr(pvt)); +/* + * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG, + * 2.10.12 Memory Interleaving Modes). + */ +static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, + u8 intlv_en, int num_dcts_intlv, + u32 dct_sel) +{ + u8 channel = 0; + u8 select; + + if (!(intlv_en)) + return (u8)(dct_sel); + + if (num_dcts_intlv == 2) { + select = (sys_addr >> 8) & 0x3; + channel = select ? 0x3 : 0; + } else if (num_dcts_intlv == 4) { + u8 intlv_addr = dct_sel_interleave_addr(pvt); + switch (intlv_addr) { + case 0x4: + channel = (sys_addr >> 8) & 0x3; + break; + case 0x5: + channel = (sys_addr >> 9) & 0x3; + break; + } } - - amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_HIGH, - &pvt->dram_ctl_select_high); + return channel; } /* - * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory + * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory * Interleaving Modes. */ -static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, - int hi_range_sel, u32 intlv_en) +static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr, + bool hi_range_sel, u8 intlv_en) { - u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1; + u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1; if (dct_ganging_enabled(pvt)) - cs = 0; - else if (hi_range_sel) - cs = dct_sel_high; - else if (dct_interleave_enabled(pvt)) { - /* - * see F2x110[DctSelIntLvAddr] - channel interleave mode - */ - if (dct_sel_interleave_addr(pvt) == 0) - cs = sys_addr >> 6 & 1; - else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) { - temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; + return 0; - if (dct_sel_interleave_addr(pvt) & 1) - cs = (sys_addr >> 9 & 1) ^ temp; - else - cs = (sys_addr >> 6 & 1) ^ temp; - } else if (intlv_en & 4) - cs = sys_addr >> 15 & 1; - else if (intlv_en & 2) - cs = sys_addr >> 14 & 1; - else if (intlv_en & 1) - cs = sys_addr >> 13 & 1; - else - cs = sys_addr >> 12 & 1; - } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) - cs = ~dct_sel_high & 1; - else - cs = 0; + if (hi_range_sel) + return dct_sel_high; - return cs; -} + /* + * see F2x110[DctSelIntLvAddr] - channel interleave mode + */ + if (dct_interleave_enabled(pvt)) { + u8 intlv_addr = dct_sel_interleave_addr(pvt); -static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en) -{ - if (intlv_en == 1) - return 1; - else if (intlv_en == 3) - return 2; - else if (intlv_en == 7) - return 3; + /* return DCT select function: 0=DCT0, 1=DCT1 */ + if (!intlv_addr) + return sys_addr >> 6 & 1; + + if (intlv_addr & 0x2) { + u8 shift = intlv_addr & 0x1 ? 9 : 6; + u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2; + + return ((sys_addr >> shift) & 1) ^ temp; + } + + return (sys_addr >> (12 + hweight8(intlv_en))) & 1; + } + + if (dct_high_range_enabled(pvt)) + return ~dct_sel_high & 1; return 0; } -/* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */ -static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel, - u32 dct_sel_base_addr, - u64 dct_sel_base_off, - u32 hole_valid, u32 hole_off, - u64 dram_base) +/* Convert the sys_addr to the normalized DCT address */ +static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range, + u64 sys_addr, bool hi_rng, + u32 dct_sel_base_addr) { u64 chan_off; + u64 dram_base = get_dram_base(pvt, range); + u64 hole_off = f10_dhar_offset(pvt); + u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16; - if (hi_range_sel) { - if (!(dct_sel_base_addr & 0xFFFF0000) && - hole_valid && (sys_addr >= 0x100000000ULL)) - chan_off = hole_off << 16; + if (hi_rng) { + /* + * if + * base address of high range is below 4Gb + * (bits [47:27] at [31:11]) + * DRAM address space on this DCT is hoisted above 4Gb && + * sys_addr > 4Gb + * + * remove hole offset from sys_addr + * else + * remove high range offset from sys_addr + */ + if ((!(dct_sel_base_addr >> 16) || + dct_sel_base_addr < dhar_base(pvt)) && + dhar_valid(pvt) && + (sys_addr >= BIT_64(32))) + chan_off = hole_off; else chan_off = dct_sel_base_off; } else { - if (hole_valid && (sys_addr >= 0x100000000ULL)) - chan_off = hole_off << 16; + /* + * if + * we have a valid hole && + * sys_addr > 4Gb + * + * remove hole + * else + * remove dram base to normalize to DCT address + */ + if (dhar_valid(pvt) && (sys_addr >= BIT_64(32))) + chan_off = hole_off; else - chan_off = dram_base & 0xFFFFF8000000ULL; + chan_off = dram_base; } - return (sys_addr & 0x0000FFFFFFFFFFC0ULL) - - (chan_off & 0x0000FFFFFF800000ULL); + return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23)); } -/* Hack for the time being - Can we get this from BIOS?? */ -#define CH0SPARE_RANK 0 -#define CH1SPARE_RANK 1 - /* * checks if the csrow passed in is marked as SPARED, if so returns the new * spare row */ -static inline int f10_process_possible_spare(int csrow, - u32 cs, struct amd64_pvt *pvt) -{ - u32 swap_done; - u32 bad_dram_cs; - - /* Depending on channel, isolate respective SPARING info */ - if (cs) { - swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare); - bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare); - if (swap_done && (csrow == bad_dram_cs)) - csrow = CH1SPARE_RANK; - } else { - swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare); - bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare); - if (swap_done && (csrow == bad_dram_cs)) - csrow = CH0SPARE_RANK; +static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow) +{ + int tmp_cs; + + if (online_spare_swap_done(pvt, dct) && + csrow == online_spare_bad_dramcs(pvt, dct)) { + + for_each_chip_select(tmp_cs, dct, pvt) { + if (chip_select_base(tmp_cs, dct, pvt) & 0x2) { + csrow = tmp_cs; + break; + } + } } return csrow; } @@ -1488,94 +1371,114 @@ static inline int f10_process_possible_spare(int csrow, * -EINVAL: NOT FOUND * 0..csrow = Chip-Select Row */ -static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs) +static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; - u32 cs_base, cs_mask; + u64 cs_base, cs_mask; int cs_found = -EINVAL; int csrow; - mci = mci_lookup[nid]; + mci = mcis[nid]; if (!mci) return cs_found; pvt = mci->pvt_info; - debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs); - - for (csrow = 0; csrow < pvt->cs_count; csrow++) { + edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct); - cs_base = amd64_get_dct_base(pvt, cs, csrow); - if (!(cs_base & K8_DCSB_CS_ENABLE)) + for_each_chip_select(csrow, dct, pvt) { + if (!csrow_enabled(csrow, dct, pvt)) continue; - /* - * We have an ENABLED CSROW, Isolate just the MASK bits of the - * target: [28:19] and [13:5], which map to [36:27] and [21:13] - * of the actual address. - */ - cs_base &= REV_F_F1Xh_DCSB_BASE_BITS; - - /* - * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and - * [4:0] to become ON. Then mask off bits [28:0] ([36:8]) - */ - cs_mask = amd64_get_dct_mask(pvt, cs, csrow); + get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask); - debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n", - csrow, cs_base, cs_mask); + edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n", + csrow, cs_base, cs_mask); - cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF; + cs_mask = ~cs_mask; - debugf1(" Final CSMask=0x%x\n", cs_mask); - debugf1(" (InputAddr & ~CSMask)=0x%x " - "(CSBase & ~CSMask)=0x%x\n", - (in_addr & ~cs_mask), (cs_base & ~cs_mask)); + edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n", + (in_addr & cs_mask), (cs_base & cs_mask)); - if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) { - cs_found = f10_process_possible_spare(csrow, cs, pvt); + if ((in_addr & cs_mask) == (cs_base & cs_mask)) { + if (pvt->fam == 0x15 && pvt->model >= 0x30) { + cs_found = csrow; + break; + } + cs_found = f10_process_possible_spare(pvt, dct, csrow); - debugf1(" MATCH csrow=%d\n", cs_found); + edac_dbg(1, " MATCH csrow=%d\n", cs_found); break; } } return cs_found; } -/* For a given @dram_range, check if @sys_addr falls within it. */ -static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, - u64 sys_addr, int *nid, int *chan_sel) +/* + * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is + * swapped with a region located at the bottom of memory so that the GPU can use + * the interleaved region and thus two channels. + */ +static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr) { - int node_id, cs_found = -EINVAL, high_range = 0; - u32 intlv_en, intlv_sel, intlv_shift, hole_off; - u32 hole_valid, tmp, dct_sel_base, channel; - u64 dram_base, chan_addr, dct_sel_base_off; + u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr; - dram_base = pvt->dram_base[dram_range]; - intlv_en = pvt->dram_IntlvEn[dram_range]; + if (pvt->fam == 0x10) { + /* only revC3 and revE have that feature */ + if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3)) + return sys_addr; + } - node_id = pvt->dram_DstNode[dram_range]; - intlv_sel = pvt->dram_IntlvSel[dram_range]; + amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg); - debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n", - dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]); + if (!(swap_reg & 0x1)) + return sys_addr; - /* - * This assumes that one node's DHAR is the same as all the other - * nodes' DHAR. - */ - hole_off = (pvt->dhar & 0x0000FF80); - hole_valid = (pvt->dhar & 0x1); - dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16; + swap_base = (swap_reg >> 3) & 0x7f; + swap_limit = (swap_reg >> 11) & 0x7f; + rgn_size = (swap_reg >> 20) & 0x7f; + tmp_addr = sys_addr >> 27; + + if (!(sys_addr >> 34) && + (((tmp_addr >= swap_base) && + (tmp_addr <= swap_limit)) || + (tmp_addr < rgn_size))) + return sys_addr ^ (u64)swap_base << 27; - debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n", - hole_off, hole_valid, intlv_sel); + return sys_addr; +} + +/* For a given @dram_range, check if @sys_addr falls within it. */ +static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range, + u64 sys_addr, int *chan_sel) +{ + int cs_found = -EINVAL; + u64 chan_addr; + u32 dct_sel_base; + u8 channel; + bool high_range = false; + + u8 node_id = dram_dst_node(pvt, range); + u8 intlv_en = dram_intlv_en(pvt, range); + u32 intlv_sel = dram_intlv_sel(pvt, range); + + edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", + range, sys_addr, get_dram_limit(pvt, range)); + + if (dhar_valid(pvt) && + dhar_base(pvt) <= sys_addr && + sys_addr < BIT_64(32)) { + amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", + sys_addr); + return -EINVAL; + } - if (intlv_en || - (intlv_sel != ((sys_addr >> 12) & intlv_en))) + if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en))) return -EINVAL; + sys_addr = f1x_swap_interleaved_region(pvt, sys_addr); + dct_sel_base = dct_sel_baseaddr(pvt); /* @@ -1585,65 +1488,187 @@ static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range, if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt) && ((sys_addr >> 27) >= (dct_sel_base >> 11))) - high_range = 1; + high_range = true; - channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en); + channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en); - chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base, - dct_sel_base_off, hole_valid, - hole_off, dram_base); + chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr, + high_range, dct_sel_base); - intlv_shift = f10_map_intlv_en_to_shift(intlv_en); + /* Remove node interleaving, see F1x120 */ + if (intlv_en) + chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) | + (chan_addr & 0xfff); - /* remove Node ID (in case of memory interleaving) */ - tmp = chan_addr & 0xFC0; - - chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp; - - /* remove channel interleave and hash */ + /* remove channel interleave */ if (dct_interleave_enabled(pvt) && !dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt)) { - if (dct_sel_interleave_addr(pvt) != 1) - chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL; - else { - tmp = chan_addr & 0xFC0; - chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1) - | tmp; - } + + if (dct_sel_interleave_addr(pvt) != 1) { + if (dct_sel_interleave_addr(pvt) == 0x3) + /* hash 9 */ + chan_addr = ((chan_addr >> 10) << 9) | + (chan_addr & 0x1ff); + else + /* A[6] or hash 6 */ + chan_addr = ((chan_addr >> 7) << 6) | + (chan_addr & 0x3f); + } else + /* A[12] */ + chan_addr = ((chan_addr >> 13) << 12) | + (chan_addr & 0xfff); } - debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n", - chan_addr, (u32)(chan_addr >> 8)); + edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); - cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel); + cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel); - if (cs_found >= 0) { - *nid = node_id; + if (cs_found >= 0) *chan_sel = channel; - } + return cs_found; } -static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, - int *node, int *chan_sel) +static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range, + u64 sys_addr, int *chan_sel) { - int dram_range, cs_found = -EINVAL; - u64 dram_base, dram_limit; + int cs_found = -EINVAL; + int num_dcts_intlv = 0; + u64 chan_addr, chan_offset; + u64 dct_base, dct_limit; + u32 dct_cont_base_reg, dct_cont_limit_reg, tmp; + u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en; - for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) { + u64 dhar_offset = f10_dhar_offset(pvt); + u8 intlv_addr = dct_sel_interleave_addr(pvt); + u8 node_id = dram_dst_node(pvt, range); + u8 intlv_en = dram_intlv_en(pvt, range); - if (!pvt->dram_rw_en[dram_range]) - continue; + amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg); + amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg); + + dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0)); + dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7); + + edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n", + range, sys_addr, get_dram_limit(pvt, range)); + + if (!(get_dram_base(pvt, range) <= sys_addr) && + !(get_dram_limit(pvt, range) >= sys_addr)) + return -EINVAL; + + if (dhar_valid(pvt) && + dhar_base(pvt) <= sys_addr && + sys_addr < BIT_64(32)) { + amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n", + sys_addr); + return -EINVAL; + } + + /* Verify sys_addr is within DCT Range. */ + dct_base = (u64) dct_sel_baseaddr(pvt); + dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF; + + if (!(dct_cont_base_reg & BIT(0)) && + !(dct_base <= (sys_addr >> 27) && + dct_limit >= (sys_addr >> 27))) + return -EINVAL; + + /* Verify number of dct's that participate in channel interleaving. */ + num_dcts_intlv = (int) hweight8(intlv_en); + + if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4)) + return -EINVAL; + + channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en, + num_dcts_intlv, dct_sel); + + /* Verify we stay within the MAX number of channels allowed */ + if (channel > 3) + return -EINVAL; + + leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0)); + + /* Get normalized DCT addr */ + if (leg_mmio_hole && (sys_addr >= BIT_64(32))) + chan_offset = dhar_offset; + else + chan_offset = dct_base << 27; + + chan_addr = sys_addr - chan_offset; + + /* remove channel interleave */ + if (num_dcts_intlv == 2) { + if (intlv_addr == 0x4) + chan_addr = ((chan_addr >> 9) << 8) | + (chan_addr & 0xff); + else if (intlv_addr == 0x5) + chan_addr = ((chan_addr >> 10) << 9) | + (chan_addr & 0x1ff); + else + return -EINVAL; + + } else if (num_dcts_intlv == 4) { + if (intlv_addr == 0x4) + chan_addr = ((chan_addr >> 10) << 8) | + (chan_addr & 0xff); + else if (intlv_addr == 0x5) + chan_addr = ((chan_addr >> 11) << 9) | + (chan_addr & 0x1ff); + else + return -EINVAL; + } + + if (dct_offset_en) { + amd64_read_pci_cfg(pvt->F1, + DRAM_CONT_HIGH_OFF + (int) channel * 4, + &tmp); + chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27; + } + + f15h_select_dct(pvt, channel); + + edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr); + + /* + * Find Chip select: + * if channel = 3, then alias it to 1. This is because, in F15 M30h, + * there is support for 4 DCT's, but only 2 are currently functional. + * They are DCT0 and DCT3. But we have read all registers of DCT3 into + * pvt->csels[1]. So we need to use '1' here to get correct info. + * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications. + */ + alias_channel = (channel == 3) ? 1 : channel; + + cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel); + + if (cs_found >= 0) + *chan_sel = alias_channel; + + return cs_found; +} + +static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, + u64 sys_addr, + int *chan_sel) +{ + int cs_found = -EINVAL; + unsigned range; - dram_base = pvt->dram_base[dram_range]; - dram_limit = pvt->dram_limit[dram_range]; + for (range = 0; range < DRAM_RANGES; range++) { + if (!dram_rw(pvt, range)) + continue; - if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) { + if (pvt->fam == 0x15 && pvt->model >= 0x30) + cs_found = f15_m30h_match_to_this_node(pvt, range, + sys_addr, + chan_sel); - cs_found = f10_match_to_this_node(pvt, dram_range, - sys_addr, node, - chan_sel); + else if ((get_dram_base(pvt, range) <= sys_addr) && + (get_dram_limit(pvt, range) >= sys_addr)) { + cs_found = f1x_match_to_this_node(pvt, range, + sys_addr, chan_sel); if (cs_found >= 0) break; } @@ -1658,60 +1683,39 @@ static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr, * The @sys_addr is usually an error address received from the hardware * (MCX_ADDR). */ -static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci, - struct err_regs *err_info, - u64 sys_addr) +static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr, + struct err_info *err) { struct amd64_pvt *pvt = mci->pvt_info; - u32 page, offset; - int nid, csrow, chan = 0; - u16 syndrome; - csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan); + error_address_to_page_and_offset(sys_addr, err); - if (csrow < 0) { - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); + err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel); + if (err->csrow < 0) { + err->err_code = ERR_CSROW; return; } - error_address_to_page_and_offset(sys_addr, &page, &offset); - - syndrome = extract_syndrome(err_info); - /* * We need the syndromes for channel detection only when we're * ganged. Otherwise @chan should already contain the channel at * this point. */ - if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL)) - chan = get_channel_from_ecc_syndrome(mci, syndrome); - - if (chan >= 0) - edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan, - EDAC_MOD_STR); - else - /* - * Channel unknown, report all channels on this CSROW as failed. - */ - for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++) - edac_mc_handle_ce(mci, page, offset, syndrome, - csrow, chan, EDAC_MOD_STR); + if (dct_ganging_enabled(pvt)) + err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome); } /* * debug routine to display the memory sizes of all logical DIMMs and its - * CSROWs as well + * CSROWs */ -static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) +static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl) { - int dimm, size0, size1, factor = 0; - u32 dbam; - u32 *dcsb; - - if (boot_cpu_data.x86 == 0xf) { - if (pvt->dclr0 & F10_WIDTH_128) - factor = 1; + int dimm, size0, size1; + u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases; + u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0; + if (pvt->fam == 0xf) { /* K8 families < revF not supported yet */ if (pvt->ext_model < K8_REV_F) return; @@ -1719,11 +1723,12 @@ static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) WARN_ON(ctrl != 0); } - debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", - ctrl, ctrl ? pvt->dbam1 : pvt->dbam0); + dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0; + dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases + : pvt->csels[0].csbases; - dbam = ctrl ? pvt->dbam1 : pvt->dbam0; - dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0; + edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", + ctrl, dbam); edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl); @@ -1731,88 +1736,90 @@ static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt) for (dimm = 0; dimm < 4; dimm++) { size0 = 0; - if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE) - size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); + if (dcsb[dimm*2] & DCSB_CS_ENABLE) + size0 = pvt->ops->dbam_to_cs(pvt, ctrl, + DBAM_DIMM(dimm, dbam)); size1 = 0; - if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE) - size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam)); + if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE) + size1 = pvt->ops->dbam_to_cs(pvt, ctrl, + DBAM_DIMM(dimm, dbam)); - edac_printk(KERN_DEBUG, EDAC_MC, " %d: %5dMB %d: %5dMB\n", - dimm * 2, size0 << factor, - dimm * 2 + 1, size1 << factor); + amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n", + dimm * 2, size0, + dimm * 2 + 1, size1); } } -/* - * There currently are 3 types type of MC devices for AMD Athlon/Opterons - * (as per PCI DEVICE_IDs): - * - * Family K8: That is the Athlon64 and Opteron CPUs. They all have the same PCI - * DEVICE ID, even though there is differences between the different Revisions - * (CG,D,E,F). - * - * Family F10h and F11h. - * - */ -static struct amd64_family_type amd64_family_types[] = { +static struct amd64_family_type family_types[] = { [K8_CPUS] = { - .ctl_name = "RevF", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_K8_NB_MISC, + .ctl_name = "K8", + .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP, + .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC, .ops = { .early_channel_count = k8_early_channel_count, - .get_error_address = k8_get_error_address, - .read_dram_base_limit = k8_read_dram_base_limit, .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow, .dbam_to_cs = k8_dbam_to_chip_select, + .read_dct_pci_cfg = k8_read_dct_pci_cfg, } }, [F10_CPUS] = { - .ctl_name = "Family 10h", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_10H_NB_MAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_10H_NB_MISC, + .ctl_name = "F10h", + .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP, + .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC, .ops = { - .early_channel_count = f10_early_channel_count, - .get_error_address = f10_get_error_address, - .read_dram_base_limit = f10_read_dram_base_limit, - .read_dram_ctl_register = f10_read_dram_ctl_register, - .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, .dbam_to_cs = f10_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, } }, - [F11_CPUS] = { - .ctl_name = "Family 11h", - .addr_f1_ctl = PCI_DEVICE_ID_AMD_11H_NB_MAP, - .misc_f3_ctl = PCI_DEVICE_ID_AMD_11H_NB_MISC, + [F15_CPUS] = { + .ctl_name = "F15h", + .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3, .ops = { - .early_channel_count = f10_early_channel_count, - .get_error_address = f10_get_error_address, - .read_dram_base_limit = f10_read_dram_base_limit, - .read_dram_ctl_register = f10_read_dram_ctl_register, - .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow, - .dbam_to_cs = f10_dbam_to_chip_select, + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f15_dbam_to_chip_select, + .read_dct_pci_cfg = f15_read_dct_pci_cfg, + } + }, + [F15_M30H_CPUS] = { + .ctl_name = "F15h_M30h", + .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f15_read_dct_pci_cfg, + } + }, + [F16_CPUS] = { + .ctl_name = "F16h", + .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, + } + }, + [F16_M30H_CPUS] = { + .ctl_name = "F16h_M30h", + .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1, + .f3_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F3, + .ops = { + .early_channel_count = f1x_early_channel_count, + .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow, + .dbam_to_cs = f16_dbam_to_chip_select, + .read_dct_pci_cfg = f10_read_dct_pci_cfg, } }, }; -static struct pci_dev *pci_get_related_function(unsigned int vendor, - unsigned int device, - struct pci_dev *related) -{ - struct pci_dev *dev = NULL; - - dev = pci_get_device(vendor, device, dev); - while (dev) { - if ((dev->bus->number == related->bus->number) && - (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn))) - break; - dev = pci_get_device(vendor, device, dev); - } - - return dev; -} - /* * These are tables of eigenvectors (one per line) which can be used for the * construction of the syndrome tables. The modified syndrome search algorithm @@ -1820,7 +1827,7 @@ static struct pci_dev *pci_get_related_function(unsigned int vendor, * * Algorithm courtesy of Ross LaFetra from AMD. */ -static u16 x4_vectors[] = { +static const u16 x4_vectors[] = { 0x2f57, 0x1afe, 0x66cc, 0xdd88, 0x11eb, 0x3396, 0x7f4c, 0xeac8, 0x0001, 0x0002, 0x0004, 0x0008, @@ -1859,7 +1866,7 @@ static u16 x4_vectors[] = { 0x19a9, 0x2efe, 0xb5cc, 0x6f88, }; -static u16 x8_vectors[] = { +static const u16 x8_vectors[] = { 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480, 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80, 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80, @@ -1881,15 +1888,15 @@ static u16 x8_vectors[] = { 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000, }; -static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, - int v_dim) +static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs, + unsigned v_dim) { unsigned int i, err_sym; for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) { u16 s = syndrome; - int v_idx = err_sym * v_dim; - int v_end = (err_sym + 1) * v_dim; + unsigned v_idx = err_sym * v_dim; + unsigned v_end = (err_sym + 1) * v_dim; /* walk over all 16 bits of the syndrome */ for (i = 1; i < (1U << 16); i <<= 1) { @@ -1913,7 +1920,7 @@ static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs, } } - debugf0("syndrome(%x) not found\n", syndrome); + edac_dbg(0, "syndrome(%x) not found\n", syndrome); return -1; } @@ -1961,305 +1968,218 @@ static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome) struct amd64_pvt *pvt = mci->pvt_info; int err_sym = -1; - if (pvt->syn_type == 8) + if (pvt->ecc_sym_sz == 8) err_sym = decode_syndrome(syndrome, x8_vectors, ARRAY_SIZE(x8_vectors), - pvt->syn_type); - else if (pvt->syn_type == 4) + pvt->ecc_sym_sz); + else if (pvt->ecc_sym_sz == 4) err_sym = decode_syndrome(syndrome, x4_vectors, ARRAY_SIZE(x4_vectors), - pvt->syn_type); + pvt->ecc_sym_sz); else { - amd64_printk(KERN_WARNING, "%s: Illegal syndrome type: %u\n", - __func__, pvt->syn_type); + amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz); return err_sym; } - return map_err_sym_to_channel(err_sym, pvt->syn_type); + return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz); } -/* - * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR - * ADDRESS and process. - */ -static void amd64_handle_ce(struct mem_ctl_info *mci, - struct err_regs *info) +static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err, + u8 ecc_type) { - struct amd64_pvt *pvt = mci->pvt_info; - u64 sys_addr; + enum hw_event_mc_err_type err_type; + const char *string; - /* Ensure that the Error Address is VALID */ - if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { - amd64_mc_printk(mci, KERN_ERR, - "HW has no ERROR_ADDRESS available\n"); - edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR); + if (ecc_type == 2) + err_type = HW_EVENT_ERR_CORRECTED; + else if (ecc_type == 1) + err_type = HW_EVENT_ERR_UNCORRECTED; + else { + WARN(1, "Something is rotten in the state of Denmark.\n"); return; } - sys_addr = pvt->ops->get_error_address(mci, info); - - amd64_mc_printk(mci, KERN_ERR, - "CE ERROR_ADDRESS= 0x%llx\n", sys_addr); + switch (err->err_code) { + case DECODE_OK: + string = ""; + break; + case ERR_NODE: + string = "Failed to map error addr to a node"; + break; + case ERR_CSROW: + string = "Failed to map error addr to a csrow"; + break; + case ERR_CHANNEL: + string = "unknown syndrome - possible error reporting race"; + break; + default: + string = "WTF error"; + break; + } - pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr); + edac_mc_handle_error(err_type, mci, 1, + err->page, err->offset, err->syndrome, + err->csrow, err->channel, -1, + string, ""); } -/* Handle any Un-correctable Errors (UEs) */ -static void amd64_handle_ue(struct mem_ctl_info *mci, - struct err_regs *info) +static inline void decode_bus_error(int node_id, struct mce *m) { + struct mem_ctl_info *mci = mcis[node_id]; struct amd64_pvt *pvt = mci->pvt_info; - struct mem_ctl_info *log_mci, *src_mci = NULL; - int csrow; + u8 ecc_type = (m->status >> 45) & 0x3; + u8 xec = XEC(m->status, 0x1f); + u16 ec = EC(m->status); u64 sys_addr; - u32 page, offset; - - log_mci = mci; - - if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) { - amd64_mc_printk(mci, KERN_CRIT, - "HW has no ERROR_ADDRESS available\n"); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); - return; - } + struct err_info err; - sys_addr = pvt->ops->get_error_address(mci, info); - - /* - * Find out which node the error address belongs to. This may be - * different from the node that detected the error. - */ - src_mci = find_mc_by_sys_addr(mci, sys_addr); - if (!src_mci) { - amd64_mc_printk(mci, KERN_CRIT, - "ERROR ADDRESS (0x%lx) value NOT mapped to a MC\n", - (unsigned long)sys_addr); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); - return; - } - - log_mci = src_mci; - - csrow = sys_addr_to_csrow(log_mci, sys_addr); - if (csrow < 0) { - amd64_mc_printk(mci, KERN_CRIT, - "ERROR_ADDRESS (0x%lx) value NOT mapped to 'csrow'\n", - (unsigned long)sys_addr); - edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR); - } else { - error_address_to_page_and_offset(sys_addr, &page, &offset); - edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR); - } -} - -static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci, - struct err_regs *info) -{ - u32 ec = ERROR_CODE(info->nbsl); - u32 xec = EXT_ERROR_CODE(info->nbsl); - int ecc_type = (info->nbsh >> 13) & 0x3; - - /* Bail early out if this was an 'observed' error */ - if (PP(ec) == K8_NBSL_PP_OBS) + /* Bail out early if this was an 'observed' error */ + if (PP(ec) == NBSL_PP_OBS) return; /* Do only ECC errors */ if (xec && xec != F10_NBSL_EXT_ERR_ECC) return; - if (ecc_type == 2) - amd64_handle_ce(mci, info); - else if (ecc_type == 1) - amd64_handle_ue(mci, info); -} - -void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg) -{ - struct mem_ctl_info *mci = mci_lookup[node_id]; - struct err_regs regs; + memset(&err, 0, sizeof(err)); - regs.nbsl = (u32) m->status; - regs.nbsh = (u32)(m->status >> 32); - regs.nbeal = (u32) m->addr; - regs.nbeah = (u32)(m->addr >> 32); - regs.nbcfg = nbcfg; + sys_addr = get_error_address(pvt, m); - __amd64_decode_bus_error(mci, ®s); + if (ecc_type == 2) + err.syndrome = extract_syndrome(m->status); - /* - * Check the UE bit of the NB status high register, if set generate some - * logs. If NOT a GART error, then process the event as a NO-INFO event. - * If it was a GART error, skip that process. - * - * FIXME: this should go somewhere else, if at all. - */ - if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors) - edac_mc_handle_ue_no_info(mci, "UE bit is set"); + pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err); + __log_bus_error(mci, &err, ecc_type); } /* - * Input: - * 1) struct amd64_pvt which contains pvt->dram_f2_ctl pointer - * 2) AMD Family index value - * - * Ouput: - * Upon return of 0, the following filled in: - * - * struct pvt->addr_f1_ctl - * struct pvt->misc_f3_ctl - * - * Filled in with related device funcitions of 'dram_f2_ctl' - * These devices are "reserved" via the pci_get_device() - * - * Upon return of 1 (error status): - * - * Nothing reserved + * Use pvt->F2 which contains the F2 CPU PCI device to get the related + * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error. */ -static int amd64_reserve_mc_sibling_devices(struct amd64_pvt *pvt, int mc_idx) +static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id) { - const struct amd64_family_type *amd64_dev = &amd64_family_types[mc_idx]; - /* Reserve the ADDRESS MAP Device */ - pvt->addr_f1_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, - amd64_dev->addr_f1_ctl, - pvt->dram_f2_ctl); - - if (!pvt->addr_f1_ctl) { - amd64_printk(KERN_ERR, "error address map device not found: " - "vendor %x device 0x%x (broken BIOS?)\n", - PCI_VENDOR_ID_AMD, amd64_dev->addr_f1_ctl); - return 1; + pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2); + if (!pvt->F1) { + amd64_err("error address map device not found: " + "vendor %x device 0x%x (broken BIOS?)\n", + PCI_VENDOR_ID_AMD, f1_id); + return -ENODEV; } /* Reserve the MISC Device */ - pvt->misc_f3_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor, - amd64_dev->misc_f3_ctl, - pvt->dram_f2_ctl); + pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2); + if (!pvt->F3) { + pci_dev_put(pvt->F1); + pvt->F1 = NULL; - if (!pvt->misc_f3_ctl) { - pci_dev_put(pvt->addr_f1_ctl); - pvt->addr_f1_ctl = NULL; + amd64_err("error F3 device not found: " + "vendor %x device 0x%x (broken BIOS?)\n", + PCI_VENDOR_ID_AMD, f3_id); - amd64_printk(KERN_ERR, "error miscellaneous device not found: " - "vendor %x device 0x%x (broken BIOS?)\n", - PCI_VENDOR_ID_AMD, amd64_dev->misc_f3_ctl); - return 1; + return -ENODEV; } - - debugf1(" Addr Map device PCI Bus ID:\t%s\n", - pci_name(pvt->addr_f1_ctl)); - debugf1(" DRAM MEM-CTL PCI Bus ID:\t%s\n", - pci_name(pvt->dram_f2_ctl)); - debugf1(" Misc device PCI Bus ID:\t%s\n", - pci_name(pvt->misc_f3_ctl)); + edac_dbg(1, "F1: %s\n", pci_name(pvt->F1)); + edac_dbg(1, "F2: %s\n", pci_name(pvt->F2)); + edac_dbg(1, "F3: %s\n", pci_name(pvt->F3)); return 0; } -static void amd64_free_mc_sibling_devices(struct amd64_pvt *pvt) +static void free_mc_sibling_devs(struct amd64_pvt *pvt) { - pci_dev_put(pvt->addr_f1_ctl); - pci_dev_put(pvt->misc_f3_ctl); + pci_dev_put(pvt->F1); + pci_dev_put(pvt->F3); } /* * Retrieve the hardware registers of the memory controller (this includes the * 'Address Map' and 'Misc' device regs) */ -static void amd64_read_mc_registers(struct amd64_pvt *pvt) +static void read_mc_regs(struct amd64_pvt *pvt) { + unsigned range; u64 msr_val; u32 tmp; - int dram; /* * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since * those are Read-As-Zero */ rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem); - debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem); + edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem); /* check first whether TOP_MEM2 is enabled */ rdmsrl(MSR_K8_SYSCFG, msr_val); if (msr_val & (1U << 21)) { rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2); - debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2); + edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2); } else - debugf0(" TOP_MEM2 disabled.\n"); + edac_dbg(0, " TOP_MEM2 disabled\n"); - amd64_cpu_display_info(pvt); + amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap); - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCAP, &pvt->nbcap); + read_dram_ctl_register(pvt); - if (pvt->ops->read_dram_ctl_register) - pvt->ops->read_dram_ctl_register(pvt); + for (range = 0; range < DRAM_RANGES; range++) { + u8 rw; - for (dram = 0; dram < DRAM_REG_COUNT; dram++) { - /* - * Call CPU specific READ function to get the DRAM Base and - * Limit values from the DCT. - */ - pvt->ops->read_dram_base_limit(pvt, dram); + /* read settings for this DRAM range */ + read_dram_base_limit_regs(pvt, range); - /* - * Only print out debug info on rows with both R and W Enabled. - * Normal processing, compiler should optimize this whole 'if' - * debug output block away. - */ - if (pvt->dram_rw_en[dram] != 0) { - debugf1(" DRAM-BASE[%d]: 0x%016llx " - "DRAM-LIMIT: 0x%016llx\n", - dram, - pvt->dram_base[dram], - pvt->dram_limit[dram]); - - debugf1(" IntlvEn=%s %s %s " - "IntlvSel=%d DstNode=%d\n", - pvt->dram_IntlvEn[dram] ? - "Enabled" : "Disabled", - (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W", - (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R", - pvt->dram_IntlvSel[dram], - pvt->dram_DstNode[dram]); - } + rw = dram_rw(pvt, range); + if (!rw) + continue; + + edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n", + range, + get_dram_base(pvt, range), + get_dram_limit(pvt, range)); + + edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n", + dram_intlv_en(pvt, range) ? "Enabled" : "Disabled", + (rw & 0x1) ? "R" : "-", + (rw & 0x2) ? "W" : "-", + dram_intlv_sel(pvt, range), + dram_dst_node(pvt, range)); } - amd64_read_dct_base_mask(pvt); + read_dct_base_mask(pvt); - amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DHAR, &pvt->dhar); - amd64_read_dbam_reg(pvt); + amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar); + amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0); - amd64_read_pci_cfg(pvt->misc_f3_ctl, - F10_ONLINE_SPARE, &pvt->online_spare); + amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare); - amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0); - amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_0, &pvt->dchr0); + amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0); + amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0); - if (boot_cpu_data.x86 >= 0x10) { - if (!dct_ganging_enabled(pvt)) { - amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_1, &pvt->dclr1); - amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_1, &pvt->dchr1); - } - amd64_read_pci_cfg(pvt->misc_f3_ctl, EXT_NB_MCA_CFG, &tmp); + if (!dct_ganging_enabled(pvt)) { + amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1); + amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1); } - if (boot_cpu_data.x86 == 0x10 && - boot_cpu_data.x86_model > 7 && - /* F3x180[EccSymbolSize]=1 => x8 symbols */ - tmp & BIT(25)) - pvt->syn_type = 8; - else - pvt->syn_type = 4; + pvt->ecc_sym_sz = 4; + + if (pvt->fam >= 0x10) { + amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp); + if (pvt->fam != 0x16) + /* F16h has only DCT0 */ + amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1); - amd64_dump_misc_regs(pvt); + /* F10h, revD and later can do x8 ECC too */ + if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25)) + pvt->ecc_sym_sz = 8; + } + dump_misc_regs(pvt); } /* * NOTE: CPU Revision Dependent code * * Input: - * @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1) + * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1) * k8 private pointer to --> * DRAM Bank Address mapping register * node_id @@ -2289,9 +2209,11 @@ static void amd64_read_mc_registers(struct amd64_pvt *pvt) * encompasses * */ -static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) +static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr) { u32 cs_mode, nr_pages; + u32 dbam = dct ? pvt->dbam1 : pvt->dbam0; + /* * The math on this doesn't look right on the surface because x/2*4 can @@ -2300,19 +2222,13 @@ static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) * number of bits to shift the DBAM register to extract the proper CSROW * field. */ - cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF; - - nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT); + cs_mode = DBAM_DIMM(csrow_nr / 2, dbam); - /* - * If dual channel then double the memory size of single channel. - * Channel count is 1 or 2 - */ - nr_pages <<= (pvt->channel_count - 1); + nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT); - debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode); - debugf0(" nr_pages= %u channel-count = %d\n", - nr_pages, pvt->channel_count); + edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n", + csrow_nr, dct, cs_mode); + edac_dbg(0, "nr_pages/channel: %u\n", nr_pages); return nr_pages; } @@ -2321,73 +2237,82 @@ static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt) * Initialize the array of csrow attribute instances, based on the values * from pci config hardware registers. */ -static int amd64_init_csrows(struct mem_ctl_info *mci) +static int init_csrows(struct mem_ctl_info *mci) { + struct amd64_pvt *pvt = mci->pvt_info; struct csrow_info *csrow; - struct amd64_pvt *pvt; - u64 input_addr_min, input_addr_max, sys_addr; - int i, empty = 1; + struct dimm_info *dimm; + enum edac_type edac_mode; + enum mem_type mtype; + int i, j, empty = 1; + int nr_pages = 0; + u32 val; - pvt = mci->pvt_info; + amd64_read_pci_cfg(pvt->F3, NBCFG, &val); + + pvt->nbcfg = val; - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &pvt->nbcfg); + edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n", + pvt->mc_node_id, val, + !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE)); - debugf0("NBCFG= 0x%x CHIPKILL= %s DRAM ECC= %s\n", pvt->nbcfg, - (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled" - ); + /* + * We iterate over DCT0 here but we look at DCT1 in parallel, if needed. + */ + for_each_chip_select(i, 0, pvt) { + bool row_dct0 = !!csrow_enabled(i, 0, pvt); + bool row_dct1 = false; - for (i = 0; i < pvt->cs_count; i++) { - csrow = &mci->csrows[i]; + if (pvt->fam != 0xf) + row_dct1 = !!csrow_enabled(i, 1, pvt); - if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) { - debugf1("----CSROW %d EMPTY for node %d\n", i, - pvt->mc_node_id); + if (!row_dct0 && !row_dct1) continue; + + csrow = mci->csrows[i]; + empty = 0; + + edac_dbg(1, "MC node: %d, csrow: %d\n", + pvt->mc_node_id, i); + + if (row_dct0) { + nr_pages = get_csrow_nr_pages(pvt, 0, i); + csrow->channels[0]->dimm->nr_pages = nr_pages; } - debugf1("----CSROW %d VALID for MC node %d\n", - i, pvt->mc_node_id); + /* K8 has only one DCT */ + if (pvt->fam != 0xf && row_dct1) { + int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i); - empty = 0; - csrow->nr_pages = amd64_csrow_nr_pages(i, pvt); - find_csrow_limits(mci, i, &input_addr_min, &input_addr_max); - sys_addr = input_addr_to_sys_addr(mci, input_addr_min); - csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT); - sys_addr = input_addr_to_sys_addr(mci, input_addr_max); - csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT); - csrow->page_mask = ~mask_from_dct_mask(pvt, i); - /* 8 bytes of resolution */ - - csrow->mtype = amd64_determine_memory_type(pvt); - - debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i); - debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n", - (unsigned long)input_addr_min, - (unsigned long)input_addr_max); - debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n", - (unsigned long)sys_addr, csrow->page_mask); - debugf1(" nr_pages: %u first_page: 0x%lx " - "last_page: 0x%lx\n", - (unsigned)csrow->nr_pages, - csrow->first_page, csrow->last_page); + csrow->channels[1]->dimm->nr_pages = row_dct1_pages; + nr_pages += row_dct1_pages; + } + + mtype = determine_memory_type(pvt, i); + + edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages); /* * determine whether CHIPKILL or JUST ECC or NO ECC is operating */ - if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) - csrow->edac_mode = - (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? - EDAC_S4ECD4ED : EDAC_SECDED; + if (pvt->nbcfg & NBCFG_ECC_ENABLE) + edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ? + EDAC_S4ECD4ED : EDAC_SECDED; else - csrow->edac_mode = EDAC_NONE; + edac_mode = EDAC_NONE; + + for (j = 0; j < pvt->channel_count; j++) { + dimm = csrow->channels[j]->dimm; + dimm->mtype = mtype; + dimm->edac_mode = edac_mode; + } } return empty; } /* get all cores on this DCT */ -static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid) +static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid) { int cpu; @@ -2397,15 +2322,14 @@ static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid) } /* check MCG_CTL on all the cpus on this node */ -static bool amd64_nb_mce_bank_enabled_on_node(int nid) +static bool nb_mce_bank_enabled_on_node(u16 nid) { cpumask_var_t mask; int cpu, nbe; bool ret = false; if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { - amd64_printk(KERN_WARNING, "%s: error allocating mask\n", - __func__); + amd64_warn("%s: Error allocating mask\n", __func__); return false; } @@ -2415,11 +2339,11 @@ static bool amd64_nb_mce_bank_enabled_on_node(int nid) for_each_cpu(cpu, mask) { struct msr *reg = per_cpu_ptr(msrs, cpu); - nbe = reg->l & K8_MSR_MCGCTL_NBE; + nbe = reg->l & MSR_MCGCTL_NBE; - debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", - cpu, reg->q, - (nbe ? "enabled" : "disabled")); + edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n", + cpu, reg->q, + (nbe ? "enabled" : "disabled")); if (!nbe) goto out; @@ -2431,18 +2355,17 @@ out: return ret; } -static int amd64_toggle_ecc_err_reporting(struct amd64_pvt *pvt, bool on) +static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on) { cpumask_var_t cmask; int cpu; if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) { - amd64_printk(KERN_WARNING, "%s: error allocating mask\n", - __func__); + amd64_warn("%s: error allocating mask\n", __func__); return false; } - get_cpus_on_this_dct_cpumask(cmask, pvt->mc_node_id); + get_cpus_on_this_dct_cpumask(cmask, nid); rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); @@ -2451,16 +2374,16 @@ static int amd64_toggle_ecc_err_reporting(struct amd64_pvt *pvt, bool on) struct msr *reg = per_cpu_ptr(msrs, cpu); if (on) { - if (reg->l & K8_MSR_MCGCTL_NBE) - pvt->flags.nb_mce_enable = 1; + if (reg->l & MSR_MCGCTL_NBE) + s->flags.nb_mce_enable = 1; - reg->l |= K8_MSR_MCGCTL_NBE; + reg->l |= MSR_MCGCTL_NBE; } else { /* * Turn off NB MCE reporting only when it was off before */ - if (!pvt->flags.nb_mce_enable) - reg->l &= ~K8_MSR_MCGCTL_NBE; + if (!s->flags.nb_mce_enable) + reg->l &= ~MSR_MCGCTL_NBE; } } wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs); @@ -2470,92 +2393,90 @@ static int amd64_toggle_ecc_err_reporting(struct amd64_pvt *pvt, bool on) return 0; } -static void amd64_enable_ecc_error_reporting(struct mem_ctl_info *mci) +static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid, + struct pci_dev *F3) { - struct amd64_pvt *pvt = mci->pvt_info; - u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; + bool ret = true; + u32 value, mask = 0x3; /* UECC/CECC enable */ + + if (toggle_ecc_err_reporting(s, nid, ON)) { + amd64_warn("Error enabling ECC reporting over MCGCTL!\n"); + return false; + } - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value); + amd64_read_pci_cfg(F3, NBCTL, &value); - /* turn on UECCn and CECCEn bits */ - pvt->old_nbctl = value & mask; - pvt->nbctl_mcgctl_saved = 1; + s->old_nbctl = value & mask; + s->nbctl_valid = true; value |= mask; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); - - if (amd64_toggle_ecc_err_reporting(pvt, ON)) - amd64_printk(KERN_WARNING, "Error enabling ECC reporting over " - "MCGCTL!\n"); + amd64_write_pci_cfg(F3, NBCTL, value); - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); + amd64_read_pci_cfg(F3, NBCFG, &value); - debugf0("NBCFG(1)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, - (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); + edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", + nid, value, !!(value & NBCFG_ECC_ENABLE)); - if (!(value & K8_NBCFG_ECC_ENABLE)) { - amd64_printk(KERN_WARNING, - "This node reports that DRAM ECC is " - "currently Disabled; ENABLING now\n"); + if (!(value & NBCFG_ECC_ENABLE)) { + amd64_warn("DRAM ECC disabled on this node, enabling...\n"); - pvt->flags.nb_ecc_prev = 0; + s->flags.nb_ecc_prev = 0; /* Attempt to turn on DRAM ECC Enable */ - value |= K8_NBCFG_ECC_ENABLE; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value); + value |= NBCFG_ECC_ENABLE; + amd64_write_pci_cfg(F3, NBCFG, value); - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); + amd64_read_pci_cfg(F3, NBCFG, &value); - if (!(value & K8_NBCFG_ECC_ENABLE)) { - amd64_printk(KERN_WARNING, - "Hardware rejects Enabling DRAM ECC checking\n" - "Check memory DIMM configuration\n"); + if (!(value & NBCFG_ECC_ENABLE)) { + amd64_warn("Hardware rejected DRAM ECC enable," + "check memory DIMM configuration.\n"); + ret = false; } else { - amd64_printk(KERN_DEBUG, - "Hardware accepted DRAM ECC Enable\n"); + amd64_info("Hardware accepted DRAM ECC Enable\n"); } } else { - pvt->flags.nb_ecc_prev = 1; + s->flags.nb_ecc_prev = 1; } - debugf0("NBCFG(2)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value, - (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled", - (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"); + edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n", + nid, value, !!(value & NBCFG_ECC_ENABLE)); - pvt->ctl_error_info.nbcfg = value; + return ret; } -static void amd64_restore_ecc_error_reporting(struct amd64_pvt *pvt) +static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid, + struct pci_dev *F3) { - u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn; + u32 value, mask = 0x3; /* UECC/CECC enable */ - if (!pvt->nbctl_mcgctl_saved) + + if (!s->nbctl_valid) return; - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value); + amd64_read_pci_cfg(F3, NBCTL, &value); value &= ~mask; - value |= pvt->old_nbctl; + value |= s->old_nbctl; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value); + amd64_write_pci_cfg(F3, NBCTL, value); - /* restore previous BIOS DRAM ECC "off" setting which we force-enabled */ - if (!pvt->flags.nb_ecc_prev) { - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); - value &= ~K8_NBCFG_ECC_ENABLE; - pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value); + /* restore previous BIOS DRAM ECC "off" setting we force-enabled */ + if (!s->flags.nb_ecc_prev) { + amd64_read_pci_cfg(F3, NBCFG, &value); + value &= ~NBCFG_ECC_ENABLE; + amd64_write_pci_cfg(F3, NBCFG, value); } /* restore the NB Enable MCGCTL bit */ - if (amd64_toggle_ecc_err_reporting(pvt, OFF)) - amd64_printk(KERN_WARNING, "Error restoring NB MCGCTL settings!\n"); + if (toggle_ecc_err_reporting(s, nid, OFF)) + amd64_warn("Error restoring NB MCGCTL settings!\n"); } /* - * EDAC requires that the BIOS have ECC enabled before taking over the - * processing of ECC errors. This is because the BIOS can properly initialize - * the memory system completely. A command line option allows to force-enable - * hardware ECC later in amd64_enable_ecc_error_reporting(). + * EDAC requires that the BIOS have ECC enabled before + * taking over the processing of ECC errors. A command line + * option allows to force-enable hardware ECC later in + * enable_ecc_error_reporting(). */ static const char *ecc_msg = "ECC disabled in the BIOS or no ECC capability, module will not load.\n" @@ -2563,255 +2484,304 @@ static const char *ecc_msg = "'ecc_enable_override'.\n" " (Note that use of the override may cause unknown side effects.)\n"; -static int amd64_check_ecc_enabled(struct amd64_pvt *pvt) +static bool ecc_enabled(struct pci_dev *F3, u16 nid) { u32 value; - u8 ecc_enabled = 0; + u8 ecc_en = 0; bool nb_mce_en = false; - amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value); + amd64_read_pci_cfg(F3, NBCFG, &value); - ecc_enabled = !!(value & K8_NBCFG_ECC_ENABLE); - if (!ecc_enabled) - amd64_printk(KERN_NOTICE, "This node reports that Memory ECC " - "is currently disabled, set F3x%x[22] (%s).\n", - K8_NBCFG, pci_name(pvt->misc_f3_ctl)); - else - amd64_printk(KERN_INFO, "ECC is enabled by BIOS.\n"); + ecc_en = !!(value & NBCFG_ECC_ENABLE); + amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled")); - nb_mce_en = amd64_nb_mce_bank_enabled_on_node(pvt->mc_node_id); + nb_mce_en = nb_mce_bank_enabled_on_node(nid); if (!nb_mce_en) - amd64_printk(KERN_NOTICE, "NB MCE bank disabled, set MSR " + amd64_notice("NB MCE bank disabled, set MSR " "0x%08x[4] on node %d to enable.\n", - MSR_IA32_MCG_CTL, pvt->mc_node_id); + MSR_IA32_MCG_CTL, nid); - if (!ecc_enabled || !nb_mce_en) { - if (!ecc_enable_override) { - amd64_printk(KERN_NOTICE, "%s", ecc_msg); - return -ENODEV; - } else { - amd64_printk(KERN_WARNING, "Forcing ECC checking on!\n"); - } + if (!ecc_en || !nb_mce_en) { + amd64_notice("%s", ecc_msg); + return false; } - - return 0; + return true; } -struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) + - ARRAY_SIZE(amd64_inj_attrs) + - 1]; +static int set_mc_sysfs_attrs(struct mem_ctl_info *mci) +{ + struct amd64_pvt *pvt = mci->pvt_info; + int rc; -struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } }; + rc = amd64_create_sysfs_dbg_files(mci); + if (rc < 0) + return rc; -static void amd64_set_mc_sysfs_attributes(struct mem_ctl_info *mci) -{ - unsigned int i = 0, j = 0; + if (pvt->fam >= 0x10) { + rc = amd64_create_sysfs_inject_files(mci); + if (rc < 0) + return rc; + } - for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++) - sysfs_attrs[i] = amd64_dbg_attrs[i]; + return 0; +} - for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++) - sysfs_attrs[i] = amd64_inj_attrs[j]; +static void del_mc_sysfs_attrs(struct mem_ctl_info *mci) +{ + struct amd64_pvt *pvt = mci->pvt_info; - sysfs_attrs[i] = terminator; + amd64_remove_sysfs_dbg_files(mci); - mci->mc_driver_sysfs_attributes = sysfs_attrs; + if (pvt->fam >= 0x10) + amd64_remove_sysfs_inject_files(mci); } -static void amd64_setup_mci_misc_attributes(struct mem_ctl_info *mci) +static void setup_mci_misc_attrs(struct mem_ctl_info *mci, + struct amd64_family_type *fam) { struct amd64_pvt *pvt = mci->pvt_info; mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2; mci->edac_ctl_cap = EDAC_FLAG_NONE; - if (pvt->nbcap & K8_NBCAP_SECDED) + if (pvt->nbcap & NBCAP_SECDED) mci->edac_ctl_cap |= EDAC_FLAG_SECDED; - if (pvt->nbcap & K8_NBCAP_CHIPKILL) + if (pvt->nbcap & NBCAP_CHIPKILL) mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED; - mci->edac_cap = amd64_determine_edac_cap(pvt); + mci->edac_cap = determine_edac_cap(pvt); mci->mod_name = EDAC_MOD_STR; mci->mod_ver = EDAC_AMD64_VERSION; - mci->ctl_name = get_amd_family_name(pvt->mc_type_index); - mci->dev_name = pci_name(pvt->dram_f2_ctl); + mci->ctl_name = fam->ctl_name; + mci->dev_name = pci_name(pvt->F2); mci->ctl_page_to_phys = NULL; /* memory scrubber interface */ - mci->set_sdram_scrub_rate = amd64_set_scrub_rate; - mci->get_sdram_scrub_rate = amd64_get_scrub_rate; + mci->set_sdram_scrub_rate = set_scrub_rate; + mci->get_sdram_scrub_rate = get_scrub_rate; } /* - * Init stuff for this DRAM Controller device. - * - * Due to a hardware feature on Fam10h CPUs, the Enable Extended Configuration - * Space feature MUST be enabled on ALL Processors prior to actually reading - * from the ECS registers. Since the loading of the module can occur on any - * 'core', and cores don't 'see' all the other processors ECS data when the - * others are NOT enabled. Our solution is to first enable ECS access in this - * routine on all processors, gather some data in a amd64_pvt structure and - * later come back in a finish-setup function to perform that final - * initialization. See also amd64_init_2nd_stage() for that. + * returns a pointer to the family descriptor on success, NULL otherwise. */ -static int amd64_probe_one_instance(struct pci_dev *dram_f2_ctl, - int mc_type_index) +static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt) { - struct amd64_pvt *pvt = NULL; - int err = 0, ret; + struct amd64_family_type *fam_type = NULL; - ret = -ENOMEM; - pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); - if (!pvt) - goto err_exit; + pvt->ext_model = boot_cpu_data.x86_model >> 4; + pvt->stepping = boot_cpu_data.x86_mask; + pvt->model = boot_cpu_data.x86_model; + pvt->fam = boot_cpu_data.x86; - pvt->mc_node_id = get_node_id(dram_f2_ctl); + switch (pvt->fam) { + case 0xf: + fam_type = &family_types[K8_CPUS]; + pvt->ops = &family_types[K8_CPUS].ops; + break; - pvt->dram_f2_ctl = dram_f2_ctl; - pvt->ext_model = boot_cpu_data.x86_model >> 4; - pvt->mc_type_index = mc_type_index; - pvt->ops = family_ops(mc_type_index); + case 0x10: + fam_type = &family_types[F10_CPUS]; + pvt->ops = &family_types[F10_CPUS].ops; + break; - /* - * We have the dram_f2_ctl device as an argument, now go reserve its - * sibling devices from the PCI system. - */ - ret = -ENODEV; - err = amd64_reserve_mc_sibling_devices(pvt, mc_type_index); - if (err) - goto err_free; + case 0x15: + if (pvt->model == 0x30) { + fam_type = &family_types[F15_M30H_CPUS]; + pvt->ops = &family_types[F15_M30H_CPUS].ops; + break; + } - ret = -EINVAL; - err = amd64_check_ecc_enabled(pvt); - if (err) - goto err_put; + fam_type = &family_types[F15_CPUS]; + pvt->ops = &family_types[F15_CPUS].ops; + break; - /* - * Key operation here: setup of HW prior to performing ops on it. Some - * setup is required to access ECS data. After this is performed, the - * 'teardown' function must be called upon error and normal exit paths. - */ - if (boot_cpu_data.x86 >= 0x10) - amd64_setup(pvt); + case 0x16: + if (pvt->model == 0x30) { + fam_type = &family_types[F16_M30H_CPUS]; + pvt->ops = &family_types[F16_M30H_CPUS].ops; + break; + } + fam_type = &family_types[F16_CPUS]; + pvt->ops = &family_types[F16_CPUS].ops; + break; - /* - * Save the pointer to the private data for use in 2nd initialization - * stage - */ - pvt_lookup[pvt->mc_node_id] = pvt; + default: + amd64_err("Unsupported family!\n"); + return NULL; + } - return 0; + amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name, + (pvt->fam == 0xf ? + (pvt->ext_model >= K8_REV_F ? "revF or later " + : "revE or earlier ") + : ""), pvt->mc_node_id); + return fam_type; +} -err_put: - amd64_free_mc_sibling_devices(pvt); +static int init_one_instance(struct pci_dev *F2) +{ + struct amd64_pvt *pvt = NULL; + struct amd64_family_type *fam_type = NULL; + struct mem_ctl_info *mci = NULL; + struct edac_mc_layer layers[2]; + int err = 0, ret; + u16 nid = amd_get_node_id(F2); -err_free: - kfree(pvt); + ret = -ENOMEM; + pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL); + if (!pvt) + goto err_ret; -err_exit: - return ret; -} + pvt->mc_node_id = nid; + pvt->F2 = F2; -/* - * This is the finishing stage of the init code. Needs to be performed after all - * MCs' hardware have been prepped for accessing extended config space. - */ -static int amd64_init_2nd_stage(struct amd64_pvt *pvt) -{ - int node_id = pvt->mc_node_id; - struct mem_ctl_info *mci; - int ret = -ENODEV; + ret = -EINVAL; + fam_type = per_family_init(pvt); + if (!fam_type) + goto err_free; - amd64_read_mc_registers(pvt); + ret = -ENODEV; + err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id); + if (err) + goto err_free; + + read_mc_regs(pvt); /* * We need to determine how many memory channels there are. Then use * that information for calculating the size of the dynamic instance - * tables in the 'mci' structure + * tables in the 'mci' structure. */ + ret = -EINVAL; pvt->channel_count = pvt->ops->early_channel_count(pvt); if (pvt->channel_count < 0) - goto err_exit; + goto err_siblings; ret = -ENOMEM; - mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, node_id); + layers[0].type = EDAC_MC_LAYER_CHIP_SELECT; + layers[0].size = pvt->csels[0].b_cnt; + layers[0].is_virt_csrow = true; + layers[1].type = EDAC_MC_LAYER_CHANNEL; + + /* + * Always allocate two channels since we can have setups with DIMMs on + * only one channel. Also, this simplifies handling later for the price + * of a couple of KBs tops. + */ + layers[1].size = 2; + layers[1].is_virt_csrow = false; + + mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0); if (!mci) - goto err_exit; + goto err_siblings; mci->pvt_info = pvt; + mci->pdev = &pvt->F2->dev; - mci->dev = &pvt->dram_f2_ctl->dev; - amd64_setup_mci_misc_attributes(mci); + setup_mci_misc_attrs(mci, fam_type); - if (amd64_init_csrows(mci)) + if (init_csrows(mci)) mci->edac_cap = EDAC_FLAG_NONE; - amd64_enable_ecc_error_reporting(mci); - amd64_set_mc_sysfs_attributes(mci); - ret = -ENODEV; if (edac_mc_add_mc(mci)) { - debugf1("failed edac_mc_add_mc()\n"); + edac_dbg(1, "failed edac_mc_add_mc()\n"); goto err_add_mc; } - - mci_lookup[node_id] = mci; - pvt_lookup[node_id] = NULL; + if (set_mc_sysfs_attrs(mci)) { + edac_dbg(1, "failed edac_mc_add_mc()\n"); + goto err_add_sysfs; + } /* register stuff with EDAC MCE */ if (report_gart_errors) amd_report_gart_errors(true); - amd_register_ecc_decoder(amd64_decode_bus_error); + amd_register_ecc_decoder(decode_bus_error); + + mcis[nid] = mci; + + atomic_inc(&drv_instances); return 0; +err_add_sysfs: + edac_mc_del_mc(mci->pdev); err_add_mc: edac_mc_free(mci); -err_exit: - debugf0("failure to init 2nd stage: ret=%d\n", ret); - - amd64_restore_ecc_error_reporting(pvt); +err_siblings: + free_mc_sibling_devs(pvt); - if (boot_cpu_data.x86 > 0xf) - amd64_teardown(pvt); - - amd64_free_mc_sibling_devices(pvt); - - kfree(pvt_lookup[pvt->mc_node_id]); - pvt_lookup[node_id] = NULL; +err_free: + kfree(pvt); +err_ret: return ret; } - -static int __devinit amd64_init_one_instance(struct pci_dev *pdev, - const struct pci_device_id *mc_type) +static int probe_one_instance(struct pci_dev *pdev, + const struct pci_device_id *mc_type) { + u16 nid = amd_get_node_id(pdev); + struct pci_dev *F3 = node_to_amd_nb(nid)->misc; + struct ecc_settings *s; int ret = 0; - debugf0("(MC node=%d,mc_type='%s')\n", get_node_id(pdev), - get_amd_family_name(mc_type->driver_data)); - ret = pci_enable_device(pdev); - if (ret < 0) - ret = -EIO; - else - ret = amd64_probe_one_instance(pdev, mc_type->driver_data); + if (ret < 0) { + edac_dbg(0, "ret=%d\n", ret); + return -EIO; + } - if (ret < 0) - debugf0("ret=%d\n", ret); + ret = -ENOMEM; + s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL); + if (!s) + goto err_out; + + ecc_stngs[nid] = s; + + if (!ecc_enabled(F3, nid)) { + ret = -ENODEV; + if (!ecc_enable_override) + goto err_enable; + + amd64_warn("Forcing ECC on!\n"); + + if (!enable_ecc_error_reporting(s, nid, F3)) + goto err_enable; + } + + ret = init_one_instance(pdev); + if (ret < 0) { + amd64_err("Error probing instance: %d\n", nid); + restore_ecc_error_reporting(s, nid, F3); + } + + return ret; + +err_enable: + kfree(s); + ecc_stngs[nid] = NULL; + +err_out: return ret; } -static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) +static void remove_one_instance(struct pci_dev *pdev) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; + u16 nid = amd_get_node_id(pdev); + struct pci_dev *F3 = node_to_amd_nb(nid)->misc; + struct ecc_settings *s = ecc_stngs[nid]; + mci = find_mci_by_dev(&pdev->dev); + WARN_ON(!mci); + + del_mc_sysfs_attrs(mci); /* Remove from EDAC CORE tracking list */ mci = edac_mc_del_mc(&pdev->dev); if (!mci) @@ -2819,20 +2789,20 @@ static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) pvt = mci->pvt_info; - amd64_restore_ecc_error_reporting(pvt); - - if (boot_cpu_data.x86 > 0xf) - amd64_teardown(pvt); + restore_ecc_error_reporting(s, nid, F3); - amd64_free_mc_sibling_devices(pvt); + free_mc_sibling_devs(pvt); /* unregister from EDAC MCE */ amd_report_gart_errors(false); - amd_unregister_ecc_decoder(amd64_decode_bus_error); + amd_unregister_ecc_decoder(decode_bus_error); + + kfree(ecc_stngs[nid]); + ecc_stngs[nid] = NULL; /* Free the EDAC CORE resources */ mci->pvt_info = NULL; - mci_lookup[pvt->mc_node_id] = NULL; + mcis[nid] = NULL; kfree(pvt); edac_mc_free(mci); @@ -2843,7 +2813,7 @@ static void __devexit amd64_remove_one_instance(struct pci_dev *pdev) * PCI core identifies what devices are on a system during boot, and then * inquiry this table to see if this driver is for a given device found. */ -static const struct pci_device_id amd64_pci_table[] __devinitdata = { +static const struct pci_device_id amd64_pci_table[] = { { .vendor = PCI_VENDOR_ID_AMD, .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL, @@ -2851,7 +2821,6 @@ static const struct pci_device_id amd64_pci_table[] __devinitdata = { .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = K8_CPUS }, { .vendor = PCI_VENDOR_ID_AMD, @@ -2860,112 +2829,134 @@ static const struct pci_device_id amd64_pci_table[] __devinitdata = { .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = F10_CPUS }, { .vendor = PCI_VENDOR_ID_AMD, - .device = PCI_DEVICE_ID_AMD_11H_NB_DRAM, + .device = PCI_DEVICE_ID_AMD_15H_NB_F2, .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, .class = 0, .class_mask = 0, - .driver_data = F11_CPUS }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_16H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + { + .vendor = PCI_VENDOR_ID_AMD, + .device = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2, + .subvendor = PCI_ANY_ID, + .subdevice = PCI_ANY_ID, + .class = 0, + .class_mask = 0, + }, + {0, } }; MODULE_DEVICE_TABLE(pci, amd64_pci_table); static struct pci_driver amd64_pci_driver = { .name = EDAC_MOD_STR, - .probe = amd64_init_one_instance, - .remove = __devexit_p(amd64_remove_one_instance), + .probe = probe_one_instance, + .remove = remove_one_instance, .id_table = amd64_pci_table, }; -static void amd64_setup_pci_device(void) +static void setup_pci_device(void) { struct mem_ctl_info *mci; struct amd64_pvt *pvt; - if (amd64_ctl_pci) + if (pci_ctl) return; - mci = mci_lookup[0]; - if (mci) { - - pvt = mci->pvt_info; - amd64_ctl_pci = - edac_pci_create_generic_ctl(&pvt->dram_f2_ctl->dev, - EDAC_MOD_STR); - - if (!amd64_ctl_pci) { - pr_warning("%s(): Unable to create PCI control\n", - __func__); + mci = mcis[0]; + if (!mci) + return; - pr_warning("%s(): PCI error report via EDAC not set\n", - __func__); - } + pvt = mci->pvt_info; + pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR); + if (!pci_ctl) { + pr_warn("%s(): Unable to create PCI control\n", __func__); + pr_warn("%s(): PCI error report via EDAC not set\n", __func__); } } static int __init amd64_edac_init(void) { - int nb, err = -ENODEV; - bool load_ok = false; + int err = -ENODEV; - edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n"); + printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION); opstate_init(); - if (cache_k8_northbridges() < 0) + if (amd_cache_northbridges() < 0) goto err_ret; + err = -ENOMEM; + mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL); + ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL); + if (!(mcis && ecc_stngs)) + goto err_free; + msrs = msrs_alloc(); if (!msrs) - goto err_ret; + goto err_free; err = pci_register_driver(&amd64_pci_driver); if (err) goto err_pci; - /* - * At this point, the array 'pvt_lookup[]' contains pointers to alloc'd - * amd64_pvt structs. These will be used in the 2nd stage init function - * to finish initialization of the MC instances. - */ err = -ENODEV; - for (nb = 0; nb < k8_northbridges.num; nb++) { - if (!pvt_lookup[nb]) - continue; - - err = amd64_init_2nd_stage(pvt_lookup[nb]); - if (err) - goto err_2nd_stage; + if (!atomic_read(&drv_instances)) + goto err_no_instances; - load_ok = true; - } - - if (load_ok) { - amd64_setup_pci_device(); - return 0; - } + setup_pci_device(); + return 0; -err_2nd_stage: +err_no_instances: pci_unregister_driver(&amd64_pci_driver); + err_pci: msrs_free(msrs); msrs = NULL; + +err_free: + kfree(mcis); + mcis = NULL; + + kfree(ecc_stngs); + ecc_stngs = NULL; + err_ret: return err; } static void __exit amd64_edac_exit(void) { - if (amd64_ctl_pci) - edac_pci_release_generic_ctl(amd64_ctl_pci); + if (pci_ctl) + edac_pci_release_generic_ctl(pci_ctl); pci_unregister_driver(&amd64_pci_driver); + kfree(ecc_stngs); + ecc_stngs = NULL; + + kfree(mcis); + mcis = NULL; + msrs_free(msrs); msrs = NULL; } |