// SPDX-License-Identifier: GPL-2.0-only

  #include "amd64_edac.h"

  #include <asm/amd_nb.h>

  static struct edac_pci_ctl_info *pci_ctl;

  /*
   * Set by command line parameter. If BIOS has enabled the ECC, this override is
   * cleared to prevent re-enabling the hardware by this driver.
   */
  static int ecc_enable_override;
  module_param(ecc_enable_override, int, 0644);

  static struct msr __percpu *msrs;

  static struct amd64_family_type *fam_type;

  /* Per-node stuff */

  static struct ecc_settings **ecc_stngs;

  /* Device for the PCI component */
  static struct device *pci_ctl_dev;

  /*

   * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
   * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
   * or higher value'.
   *
   *FIXME: Produce a better mapping/linearisation.
   */

  static const struct scrubrate {

         u32 scrubval;           /* bit pattern for scrub rate */
         u32 bandwidth;          /* bandwidth consumed (bytes/sec) */
  } scrubrates[] = {

  	{ 0x01, 1600000000UL},
  	{ 0x02, 800000000UL},
  	{ 0x03, 400000000UL},
  	{ 0x04, 200000000UL},
  	{ 0x05, 100000000UL},
  	{ 0x06, 50000000UL},
  	{ 0x07, 25000000UL},
  	{ 0x08, 12284069UL},
  	{ 0x09, 6274509UL},
  	{ 0x0A, 3121951UL},
  	{ 0x0B, 1560975UL},
  	{ 0x0C, 781440UL},
  	{ 0x0D, 390720UL},
  	{ 0x0E, 195300UL},
  	{ 0x0F, 97650UL},
  	{ 0x10, 48854UL},
  	{ 0x11, 24427UL},
  	{ 0x12, 12213UL},
  	{ 0x13, 6101UL},
  	{ 0x14, 3051UL},
  	{ 0x15, 1523UL},
  	{ 0x16, 761UL},
  	{ 0x00, 0UL},        /* scrubbing off */
  };

  int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
  			       u32 *val, const char *func)

  {
  	int err = 0;
  
  	err = pci_read_config_dword(pdev, offset, val);
  	if (err)
  		amd64_warn("%s: error reading F%dx%03x.
  ",
  			   func, PCI_FUNC(pdev->devfn), offset);
  
  	return err;
  }
  
  int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
  				u32 val, const char *func)
  {
  	int err = 0;
  
  	err = pci_write_config_dword(pdev, offset, val);
  	if (err)
  		amd64_warn("%s: error writing to F%dx%03x.
  ",
  			   func, PCI_FUNC(pdev->devfn), offset);
  
  	return err;
  }
  
  /*

   * Select DCT to which PCI cfg accesses are routed
   */
  static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
  {
  	u32 reg = 0;
  
  	amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
  	reg &= (pvt->model == 0x30) ? ~3 : ~1;
  	reg |= dct;
  	amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
  }
  
  /*

   *
   * Depending on the family, F2 DCT reads need special handling:
   *

   * K8: has a single DCT only and no address offsets >= 0x100

   *
   * F10h: each DCT has its own set of regs
   *	DCT0 -> F2x040..
   *	DCT1 -> F2x140..
   *

   * F16h: has only 1 DCT

   *
   * F15h: we select which DCT we access using F1x10C[DctCfgSel]

   */

  static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
  					 int offset, u32 *val)

  {

  	switch (pvt->fam) {
  	case 0xf:
  		if (dct || offset >= 0x100)
  			return -EINVAL;
  		break;

  	case 0x10:
  		if (dct) {
  			/*
  			 * Note: If ganging is enabled, barring the regs
  			 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
  			 * return 0. (cf. Section 2.8.1 F10h BKDG)
  			 */
  			if (dct_ganging_enabled(pvt))
  				return 0;

  			offset += 0x100;
  		}
  		break;

  	case 0x15:
  		/*
  		 * F15h: F2x1xx addresses do not map explicitly to DCT1.
  		 * We should select which DCT we access using F1x10C[DctCfgSel]
  		 */
  		dct = (dct && pvt->model == 0x30) ? 3 : dct;
  		f15h_select_dct(pvt, dct);
  		break;

  	case 0x16:
  		if (dct)
  			return -EINVAL;
  		break;

  	default:
  		break;

  	}

  	return amd64_read_pci_cfg(pvt->F2, offset, val);

  }

  /*

   * Memory scrubber control interface. For K8, memory scrubbing is handled by
   * hardware and can involve L2 cache, dcache as well as the main memory. With
   * F10, this is extended to L3 cache scrubbing on CPU models sporting that
   * functionality.
   *
   * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
   * (dram) over to cache lines. This is nasty, so we will use bandwidth in
   * bytes/sec for the setting.
   *
   * Currently, we only do dram scrubbing. If the scrubbing is done in software on
   * other archs, we might not have access to the caches directly.
   */

  static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
  {
  	/*
  	 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
  	 * are shifted down by 0x5, so scrubval 0x5 is written to the register
  	 * as 0x0, scrubval 0x6 as 0x1, etc.
  	 */
  	if (scrubval >= 0x5 && scrubval <= 0x14) {
  		scrubval -= 0x5;
  		pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
  		pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
  	} else {
  		pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
  	}
  }

  /*

   * Scan the scrub rate mapping table for a close or matching bandwidth value to

   * issue. If requested is too big, then use last maximum value found.
   */

  static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)

  {
  	u32 scrubval;
  	int i;
  
  	/*
  	 * map the configured rate (new_bw) to a value specific to the AMD64
  	 * memory controller and apply to register. Search for the first
  	 * bandwidth entry that is greater or equal than the setting requested
  	 * and program that. If at last entry, turn off DRAM scrubbing.

  	 *
  	 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
  	 * by falling back to the last element in scrubrates[].

  	 */

  	for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {

  		/*
  		 * skip scrub rates which aren't recommended
  		 * (see F10 BKDG, F3x58)
  		 */

  		if (scrubrates[i].scrubval < min_rate)

  			continue;
  
  		if (scrubrates[i].bandwidth <= new_bw)
  			break;

  	}
  
  	scrubval = scrubrates[i].scrubval;

  	if (pvt->umc) {

  		__f17h_set_scrubval(pvt, scrubval);
  	} else if (pvt->fam == 0x15 && pvt->model == 0x60) {

  		f15h_select_dct(pvt, 0);
  		pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
  		f15h_select_dct(pvt, 1);
  		pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
  	} else {
  		pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
  	}

  	if (scrubval)
  		return scrubrates[i].bandwidth;

  	return 0;
  }

  static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)

  {
  	struct amd64_pvt *pvt = mci->pvt_info;

  	u32 min_scrubrate = 0x5;

  	if (pvt->fam == 0xf)

  		min_scrubrate = 0x0;

  	if (pvt->fam == 0x15) {
  		/* Erratum #505 */
  		if (pvt->model < 0x10)
  			f15h_select_dct(pvt, 0);

  		if (pvt->model == 0x60)
  			min_scrubrate = 0x6;
  	}
  	return __set_scrub_rate(pvt, bw, min_scrubrate);

  }

  static int get_scrub_rate(struct mem_ctl_info *mci)

  {
  	struct amd64_pvt *pvt = mci->pvt_info;

  	int i, retval = -EINVAL;

  	u32 scrubval = 0;

  	if (pvt->umc) {

  		amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
  		if (scrubval & BIT(0)) {
  			amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
  			scrubval &= 0xF;
  			scrubval += 0x5;
  		} else {
  			scrubval = 0;
  		}

  	} else if (pvt->fam == 0x15) {
  		/* Erratum #505 */
  		if (pvt->model < 0x10)
  			f15h_select_dct(pvt, 0);

  		if (pvt->model == 0x60)
  			amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);

  		else
  			amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);

  	} else {

  		amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);

  	}

  
  	scrubval = scrubval & 0x001F;

  	for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {

  		if (scrubrates[i].scrubval == scrubval) {

  			retval = scrubrates[i].bandwidth;

  			break;
  		}
  	}

  	return retval;

  }

  /*

   * returns true if the SysAddr given by sys_addr matches the
   * DRAM base/limit associated with node_id

   */

  static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)

  {

  	u64 addr;

  
  	/* The K8 treats this as a 40-bit value.  However, bits 63-40 will be
  	 * all ones if the most significant implemented address bit is 1.
  	 * Here we discard bits 63-40.  See section 3.4.2 of AMD publication
  	 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
  	 * Application Programming.
  	 */
  	addr = sys_addr & 0x000000ffffffffffull;

  	return ((addr >= get_dram_base(pvt, nid)) &&
  		(addr <= get_dram_limit(pvt, nid)));

  }
  
  /*
   * Attempt to map a SysAddr to a node. On success, return a pointer to the
   * mem_ctl_info structure for the node that the SysAddr maps to.
   *
   * On failure, return NULL.
   */
  static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
  						u64 sys_addr)
  {
  	struct amd64_pvt *pvt;

  	u8 node_id;

  	u32 intlv_en, bits;
  
  	/*
  	 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
  	 * 3.4.4.2) registers to map the SysAddr to a node ID.
  	 */
  	pvt = mci->pvt_info;
  
  	/*
  	 * The value of this field should be the same for all DRAM Base
  	 * registers.  Therefore we arbitrarily choose to read it from the
  	 * register for node 0.
  	 */

  	intlv_en = dram_intlv_en(pvt, 0);

  
  	if (intlv_en == 0) {

  		for (node_id = 0; node_id < DRAM_RANGES; node_id++) {

  			if (base_limit_match(pvt, sys_addr, node_id))

  				goto found;

  		}

  		goto err_no_match;

  	}

  	if (unlikely((intlv_en != 0x01) &&
  		     (intlv_en != 0x03) &&
  		     (intlv_en != 0x07))) {

  		amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?
  ", intlv_en);

  		return NULL;
  	}
  
  	bits = (((u32) sys_addr) >> 12) & intlv_en;
  
  	for (node_id = 0; ; ) {

  		if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)

  			break;	/* intlv_sel field matches */

  		if (++node_id >= DRAM_RANGES)

  			goto err_no_match;
  	}
  
  	/* sanity test for sys_addr */

  	if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {

  		amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
  			   "range for node %d with node interleaving enabled.
  ",
  			   __func__, sys_addr, node_id);

  		return NULL;
  	}
  
  found:

  	return edac_mc_find((int)node_id);

  
  err_no_match:

  	edac_dbg(2, "sys_addr 0x%lx doesn't match any node
  ",
  		 (unsigned long)sys_addr);

  
  	return NULL;
  }

  
  /*

   * compute the CS base address of the @csrow on the DRAM controller @dct.
   * For details see F2x[5C:40] in the processor's BKDG

   */

  static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
  				 u64 *base, u64 *mask)

  {

  	u64 csbase, csmask, base_bits, mask_bits;
  	u8 addr_shift;

  	if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {

  		csbase		= pvt->csels[dct].csbases[csrow];
  		csmask		= pvt->csels[dct].csmasks[csrow];

  		base_bits	= GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
  		mask_bits	= GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);

  		addr_shift	= 4;

  
  	/*

  	 * F16h and F15h, models 30h and later need two addr_shift values:
  	 * 8 for high and 6 for low (cf. F16h BKDG).
  	 */
  	} else if (pvt->fam == 0x16 ||
  		  (pvt->fam == 0x15 && pvt->model >= 0x30)) {

  		csbase          = pvt->csels[dct].csbases[csrow];
  		csmask          = pvt->csels[dct].csmasks[csrow >> 1];

  		*base  = (csbase & GENMASK_ULL(15,  5)) << 6;
  		*base |= (csbase & GENMASK_ULL(30, 19)) << 8;

  
  		*mask = ~0ULL;
  		/* poke holes for the csmask */

  		*mask &= ~((GENMASK_ULL(15, 5)  << 6) |
  			   (GENMASK_ULL(30, 19) << 8));

  		*mask |= (csmask & GENMASK_ULL(15, 5))  << 6;
  		*mask |= (csmask & GENMASK_ULL(30, 19)) << 8;

  
  		return;

  	} else {
  		csbase		= pvt->csels[dct].csbases[csrow];
  		csmask		= pvt->csels[dct].csmasks[csrow >> 1];
  		addr_shift	= 8;

  		if (pvt->fam == 0x15)

  			base_bits = mask_bits =
  				GENMASK_ULL(30,19) | GENMASK_ULL(13,5);

  		else

  			base_bits = mask_bits =
  				GENMASK_ULL(28,19) | GENMASK_ULL(13,5);

  	}

  	*base  = (csbase & base_bits) << addr_shift;

  	*mask  = ~0ULL;
  	/* poke holes for the csmask */
  	*mask &= ~(mask_bits << addr_shift);
  	/* OR them in */
  	*mask |= (csmask & mask_bits) << addr_shift;

  }

  #define for_each_chip_select(i, dct, pvt) \
  	for (i = 0; i < pvt->csels[dct].b_cnt; i++)

  #define chip_select_base(i, dct, pvt) \
  	pvt->csels[dct].csbases[i]

  #define for_each_chip_select_mask(i, dct, pvt) \
  	for (i = 0; i < pvt->csels[dct].m_cnt; i++)

  #define for_each_umc(i) \

  	for (i = 0; i < fam_type->max_mcs; i++)

  /*
   * @input_addr is an InputAddr associated with the node given by mci. Return the
   * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
   */
  static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
  {
  	struct amd64_pvt *pvt;
  	int csrow;
  	u64 base, mask;
  
  	pvt = mci->pvt_info;

  	for_each_chip_select(csrow, 0, pvt) {
  		if (!csrow_enabled(csrow, 0, pvt))

  			continue;

  		get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
  
  		mask = ~mask;

  
  		if ((input_addr & mask) == (base & mask)) {

  			edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)
  ",
  				 (unsigned long)input_addr, csrow,
  				 pvt->mc_node_id);

  
  			return csrow;
  		}
  	}

  	edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)
  ",
  		 (unsigned long)input_addr, pvt->mc_node_id);

  
  	return -1;
  }
  
  /*

   * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
   * for the node represented by mci. Info is passed back in *hole_base,
   * *hole_offset, and *hole_size.  Function returns 0 if info is valid or 1 if
   * info is invalid. Info may be invalid for either of the following reasons:
   *
   * - The revision of the node is not E or greater.  In this case, the DRAM Hole
   *   Address Register does not exist.
   *
   * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
   *   indicating that its contents are not valid.
   *
   * The values passed back in *hole_base, *hole_offset, and *hole_size are
   * complete 32-bit values despite the fact that the bitfields in the DHAR
   * only represent bits 31-24 of the base and offset values.
   */
  int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
  			     u64 *hole_offset, u64 *hole_size)
  {
  	struct amd64_pvt *pvt = mci->pvt_info;

  
  	/* only revE and later have the DRAM Hole Address Register */

  	if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {

  		edac_dbg(1, "  revision %d for node %d does not support DHAR
  ",
  			 pvt->ext_model, pvt->mc_node_id);

  		return 1;
  	}

  	/* valid for Fam10h and above */

  	if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {

  		edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this system
  ");

  		return 1;
  	}

  	if (!dhar_valid(pvt)) {

  		edac_dbg(1, "  Dram Memory Hoisting is DISABLED on this node %d
  ",
  			 pvt->mc_node_id);

  		return 1;
  	}
  
  	/* This node has Memory Hoisting */
  
  	/* +------------------+--------------------+--------------------+-----
  	 * | memory           | DRAM hole          | relocated          |
  	 * | [0, (x - 1)]     | [x, 0xffffffff]    | addresses from     |
  	 * |                  |                    | DRAM hole          |
  	 * |                  |                    | [0x100000000,      |
  	 * |                  |                    |  (0x100000000+     |
  	 * |                  |                    |   (0xffffffff-x))] |
  	 * +------------------+--------------------+--------------------+-----
  	 *
  	 * Above is a diagram of physical memory showing the DRAM hole and the
  	 * relocated addresses from the DRAM hole.  As shown, the DRAM hole
  	 * starts at address x (the base address) and extends through address
  	 * 0xffffffff.  The DRAM Hole Address Register (DHAR) relocates the
  	 * addresses in the hole so that they start at 0x100000000.
  	 */

  	*hole_base = dhar_base(pvt);
  	*hole_size = (1ULL << 32) - *hole_base;

  	*hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
  					: k8_dhar_offset(pvt);

  	edac_dbg(1, "  DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx
  ",
  		 pvt->mc_node_id, (unsigned long)*hole_base,
  		 (unsigned long)*hole_offset, (unsigned long)*hole_size);

  
  	return 0;
  }
  EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);

  /*
   * Return the DramAddr that the SysAddr given by @sys_addr maps to.  It is
   * assumed that sys_addr maps to the node given by mci.
   *
   * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
   * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
   * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
   * then it is also involved in translating a SysAddr to a DramAddr. Sections
   * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
   * These parts of the documentation are unclear. I interpret them as follows:
   *
   * When node n receives a SysAddr, it processes the SysAddr as follows:
   *
   * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
   *    Limit registers for node n. If the SysAddr is not within the range
   *    specified by the base and limit values, then node n ignores the Sysaddr
   *    (since it does not map to node n). Otherwise continue to step 2 below.
   *
   * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
   *    disabled so skip to step 3 below. Otherwise see if the SysAddr is within
   *    the range of relocated addresses (starting at 0x100000000) from the DRAM
   *    hole. If not, skip to step 3 below. Else get the value of the
   *    DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
   *    offset defined by this value from the SysAddr.
   *
   * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
   *    Base register for node n. To obtain the DramAddr, subtract the base
   *    address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
   */
  static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
  {

  	struct amd64_pvt *pvt = mci->pvt_info;

  	u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;

  	int ret;

  	dram_base = get_dram_base(pvt, pvt->mc_node_id);

  
  	ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
  				      &hole_size);
  	if (!ret) {

  		if ((sys_addr >= (1ULL << 32)) &&
  		    (sys_addr < ((1ULL << 32) + hole_size))) {

  			/* use DHAR to translate SysAddr to DramAddr */
  			dram_addr = sys_addr - hole_offset;

  			edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx
  ",
  				 (unsigned long)sys_addr,
  				 (unsigned long)dram_addr);

  
  			return dram_addr;
  		}
  	}
  
  	/*
  	 * Translate the SysAddr to a DramAddr as shown near the start of
  	 * section 3.4.4 (p. 70).  Although sys_addr is a 64-bit value, the k8
  	 * only deals with 40-bit values.  Therefore we discard bits 63-40 of
  	 * sys_addr below.  If bit 39 of sys_addr is 1 then the bits we
  	 * discard are all 1s.  Otherwise the bits we discard are all 0s.  See
  	 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
  	 * Programmer's Manual Volume 1 Application Programming.
  	 */

  	dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;

  	edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx
  ",
  		 (unsigned long)sys_addr, (unsigned long)dram_addr);

  	return dram_addr;
  }
  
  /*
   * @intlv_en is the value of the IntlvEn field from a DRAM Base register
   * (section 3.4.4.1).  Return the number of bits from a SysAddr that are used
   * for node interleaving.
   */
  static int num_node_interleave_bits(unsigned intlv_en)
  {
  	static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
  	int n;
  
  	BUG_ON(intlv_en > 7);
  	n = intlv_shift_table[intlv_en];
  	return n;
  }
  
  /* Translate the DramAddr given by @dram_addr to an InputAddr. */
  static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
  {
  	struct amd64_pvt *pvt;
  	int intlv_shift;
  	u64 input_addr;
  
  	pvt = mci->pvt_info;
  
  	/*
  	 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
  	 * concerning translating a DramAddr to an InputAddr.
  	 */

  	intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));

  	input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +

  		      (dram_addr & 0xfff);

  	edac_dbg(2, "  Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx
  ",
  		 intlv_shift, (unsigned long)dram_addr,
  		 (unsigned long)input_addr);

  
  	return input_addr;
  }
  
  /*
   * Translate the SysAddr represented by @sys_addr to an InputAddr.  It is
   * assumed that @sys_addr maps to the node given by mci.
   */
  static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
  {
  	u64 input_addr;
  
  	input_addr =
  	    dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));

  	edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx
  ",

  		 (unsigned long)sys_addr, (unsigned long)input_addr);

  
  	return input_addr;
  }

  /* Map the Error address to a PAGE and PAGE OFFSET. */
  static inline void error_address_to_page_and_offset(u64 error_address,

  						    struct err_info *err)

  {

  	err->page = (u32) (error_address >> PAGE_SHIFT);
  	err->offset = ((u32) error_address) & ~PAGE_MASK;

  }
  
  /*
   * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
   * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
   * of a node that detected an ECC memory error.  mci represents the node that
   * the error address maps to (possibly different from the node that detected
   * the error).  Return the number of the csrow that sys_addr maps to, or -1 on
   * error.
   */
  static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
  {
  	int csrow;
  
  	csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
  
  	if (csrow == -1)

  		amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
  				  "address 0x%lx
  ", (unsigned long)sys_addr);

  	return csrow;
  }

  static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);

  /*
   * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
   * are ECC capable.
   */

  static unsigned long determine_edac_cap(struct amd64_pvt *pvt)

  {

  	unsigned long edac_cap = EDAC_FLAG_NONE;

  	u8 bit;
  
  	if (pvt->umc) {
  		u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;

  		for_each_umc(i) {

  			if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
  				continue;

  			umc_en_mask |= BIT(i);
  
  			/* UMC Configuration bit 12 (DimmEccEn) */
  			if (pvt->umc[i].umc_cfg & BIT(12))
  				dimm_ecc_en_mask |= BIT(i);
  		}
  
  		if (umc_en_mask == dimm_ecc_en_mask)
  			edac_cap = EDAC_FLAG_SECDED;
  	} else {
  		bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
  			? 19
  			: 17;
  
  		if (pvt->dclr0 & BIT(bit))
  			edac_cap = EDAC_FLAG_SECDED;
  	}

  
  	return edac_cap;
  }

  static void debug_display_dimm_sizes(struct amd64_pvt *, u8);

  static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)

  {

  	edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x
  ", chan, dclr);

  	if (pvt->dram_type == MEM_LRDDR3) {
  		u32 dcsm = pvt->csels[chan].csmasks[0];
  		/*
  		 * It's assumed all LRDIMMs in a DCT are going to be of
  		 * same 'type' until proven otherwise. So, use a cs
  		 * value of '0' here to get dcsm value.
  		 */
  		edac_dbg(1, " LRDIMM %dx rank multiply
  ", (dcsm & 0x3));
  	}
  
  	edac_dbg(1, "All DIMMs support ECC:%s
  ",
  		    (dclr & BIT(19)) ? "yes" : "no");

  	edac_dbg(1, "  PAR/ERR parity: %s
  ",
  		 (dclr & BIT(8)) ?  "enabled" : "disabled");

  	if (pvt->fam == 0x10)

  		edac_dbg(1, "  DCT 128bit mode width: %s
  ",
  			 (dclr & BIT(11)) ?  "128b" : "64b");

  	edac_dbg(1, "  x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s
  ",
  		 (dclr & BIT(12)) ?  "yes" : "no",
  		 (dclr & BIT(13)) ?  "yes" : "no",
  		 (dclr & BIT(14)) ?  "yes" : "no",
  		 (dclr & BIT(15)) ?  "yes" : "no");

  }

  #define CS_EVEN_PRIMARY		BIT(0)
  #define CS_ODD_PRIMARY		BIT(1)

  #define CS_EVEN_SECONDARY	BIT(2)
  #define CS_ODD_SECONDARY	BIT(3)

  #define CS_EVEN			(CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
  #define CS_ODD			(CS_ODD_PRIMARY | CS_ODD_SECONDARY)

  
  static int f17_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)

  {

  	int cs_mode = 0;

  	if (csrow_enabled(2 * dimm, ctrl, pvt))
  		cs_mode |= CS_EVEN_PRIMARY;

  	if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
  		cs_mode |= CS_ODD_PRIMARY;

  	/* Asymmetric dual-rank DIMM support. */
  	if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
  		cs_mode |= CS_ODD_SECONDARY;

  	return cs_mode;

  }

  static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
  {

  	int dimm, size0, size1, cs0, cs1, cs_mode;

  
  	edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:
  ", ctrl);

  	for (dimm = 0; dimm < 2; dimm++) {

  		cs0 = dimm * 2;

  		cs1 = dimm * 2 + 1;

  		cs_mode = f17_get_cs_mode(dimm, ctrl, pvt);
  
  		size0 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs0);
  		size1 = pvt->ops->dbam_to_cs(pvt, ctrl, cs_mode, cs1);

  
  		amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB
  ",

  				cs0,	size0,
  				cs1,	size1);

  	}
  }
  
  static void __dump_misc_regs_df(struct amd64_pvt *pvt)
  {
  	struct amd64_umc *umc;
  	u32 i, tmp, umc_base;

  	for_each_umc(i) {

  		umc_base = get_umc_base(i);
  		umc = &pvt->umc[i];
  
  		edac_dbg(1, "UMC%d DIMM cfg: 0x%x
  ", i, umc->dimm_cfg);
  		edac_dbg(1, "UMC%d UMC cfg: 0x%x
  ", i, umc->umc_cfg);
  		edac_dbg(1, "UMC%d SDP ctrl: 0x%x
  ", i, umc->sdp_ctrl);
  		edac_dbg(1, "UMC%d ECC ctrl: 0x%x
  ", i, umc->ecc_ctrl);
  
  		amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
  		edac_dbg(1, "UMC%d ECC bad symbol: 0x%x
  ", i, tmp);
  
  		amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
  		edac_dbg(1, "UMC%d UMC cap: 0x%x
  ", i, tmp);
  		edac_dbg(1, "UMC%d UMC cap high: 0x%x
  ", i, umc->umc_cap_hi);
  
  		edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s
  ",
  				i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
  				    (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
  		edac_dbg(1, "UMC%d All DIMMs support ECC: %s
  ",
  				i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
  		edac_dbg(1, "UMC%d x4 DIMMs present: %s
  ",
  				i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
  		edac_dbg(1, "UMC%d x16 DIMMs present: %s
  ",
  				i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
  
  		if (pvt->dram_type == MEM_LRDDR4) {
  			amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
  			edac_dbg(1, "UMC%d LRDIMM %dx rank multiply
  ",
  					i, 1 << ((tmp >> 4) & 0x3));
  		}
  
  		debug_display_dimm_sizes_df(pvt, i);
  	}
  
  	edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x
  ",
  		 pvt->dhar, dhar_base(pvt));
  }

  /* Display and decode various NB registers for debug purposes. */

  static void __dump_misc_regs(struct amd64_pvt *pvt)

  {

  	edac_dbg(1, "F3xE8 (NB Cap): 0x%08x
  ", pvt->nbcap);

  	edac_dbg(1, "  NB two channel DRAM capable: %s
  ",
  		 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");

  	edac_dbg(1, "  ECC capable: %s, ChipKill ECC capable: %s
  ",
  		 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
  		 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");

  	debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);

  	edac_dbg(1, "F3xB0 (Online Spare): 0x%08x
  ", pvt->online_spare);

  	edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x
  ",
  		 pvt->dhar, dhar_base(pvt),

  		 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
  				   : f10_dhar_offset(pvt));

  	debug_display_dimm_sizes(pvt, 0);

  	/* everything below this point is Fam10h and above */

  	if (pvt->fam == 0xf)

  		return;

  	debug_display_dimm_sizes(pvt, 1);

  	/* Only if NOT ganged does dclr1 have valid info */

  	if (!dct_ganging_enabled(pvt))

  		debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);

  }

  /* Display and decode various NB registers for debug purposes. */
  static void dump_misc_regs(struct amd64_pvt *pvt)
  {
  	if (pvt->umc)
  		__dump_misc_regs_df(pvt);
  	else
  		__dump_misc_regs(pvt);
  
  	edac_dbg(1, "  DramHoleValid: %s
  ", dhar_valid(pvt) ? "yes" : "no");

  	amd64_info("using x%u syndromes.
  ", pvt->ecc_sym_sz);

  }

  /*

   * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]

   */

  static void prep_chip_selects(struct amd64_pvt *pvt)

  {

  	if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {

  		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
  		pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;

  	} else if (pvt->fam == 0x15 && pvt->model == 0x30) {

  		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
  		pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;

  	} else if (pvt->fam >= 0x17) {
  		int umc;
  
  		for_each_umc(umc) {
  			pvt->csels[umc].b_cnt = 4;
  			pvt->csels[umc].m_cnt = 2;
  		}

  	} else {

  		pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
  		pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;

  	}
  }

  static void read_umc_base_mask(struct amd64_pvt *pvt)
  {

  	u32 umc_base_reg, umc_base_reg_sec;
  	u32 umc_mask_reg, umc_mask_reg_sec;
  	u32 base_reg, base_reg_sec;
  	u32 mask_reg, mask_reg_sec;
  	u32 *base, *base_sec;
  	u32 *mask, *mask_sec;

  	int cs, umc;
  
  	for_each_umc(umc) {
  		umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;

  		umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;

  
  		for_each_chip_select(cs, umc, pvt) {
  			base = &pvt->csels[umc].csbases[cs];

  			base_sec = &pvt->csels[umc].csbases_sec[cs];

  
  			base_reg = umc_base_reg + (cs * 4);

  			base_reg_sec = umc_base_reg_sec + (cs * 4);

  
  			if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
  				edac_dbg(0, "  DCSB%d[%d]=0x%08x reg: 0x%x
  ",
  					 umc, cs, *base, base_reg);

  
  			if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
  				edac_dbg(0, "    DCSB_SEC%d[%d]=0x%08x reg: 0x%x
  ",
  					 umc, cs, *base_sec, base_reg_sec);

  		}
  
  		umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;

  		umc_mask_reg_sec = get_umc_base(umc) + UMCCH_ADDR_MASK_SEC;

  
  		for_each_chip_select_mask(cs, umc, pvt) {
  			mask = &pvt->csels[umc].csmasks[cs];

  			mask_sec = &pvt->csels[umc].csmasks_sec[cs];

  
  			mask_reg = umc_mask_reg + (cs * 4);

  			mask_reg_sec = umc_mask_reg_sec + (cs * 4);

  
  			if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
  				edac_dbg(0, "  DCSM%d[%d]=0x%08x reg: 0x%x
  ",
  					 umc, cs, *mask, mask_reg);

  
  			if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
  				edac_dbg(0, "    DCSM_SEC%d[%d]=0x%08x reg: 0x%x
  ",
  					 umc, cs, *mask_sec, mask_reg_sec);

  		}
  	}
  }

  /*

   * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers

   */

  static void read_dct_base_mask(struct amd64_pvt *pvt)

  {

  	int cs;

  	prep_chip_selects(pvt);

  	if (pvt->umc)
  		return read_umc_base_mask(pvt);

  	for_each_chip_select(cs, 0, pvt) {

  		int reg0   = DCSB0 + (cs * 4);
  		int reg1   = DCSB1 + (cs * 4);

  		u32 *base0 = &pvt->csels[0].csbases[cs];
  		u32 *base1 = &pvt->csels[1].csbases[cs];

  		if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
  			edac_dbg(0, "  DCSB0[%d]=0x%08x reg: F2x%x
  ",
  				 cs, *base0, reg0);

  		if (pvt->fam == 0xf)
  			continue;

  		if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
  			edac_dbg(0, "  DCSB1[%d]=0x%08x reg: F2x%x
  ",
  				 cs, *base1, (pvt->fam == 0x10) ? reg1
  							: reg0);

  	}

  	for_each_chip_select_mask(cs, 0, pvt) {

  		int reg0   = DCSM0 + (cs * 4);
  		int reg1   = DCSM1 + (cs * 4);

  		u32 *mask0 = &pvt->csels[0].csmasks[cs];
  		u32 *mask1 = &pvt->csels[1].csmasks[cs];

  		if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
  			edac_dbg(0, "    DCSM0[%d]=0x%08x reg: F2x%x
  ",
  				 cs, *mask0, reg0);

  		if (pvt->fam == 0xf)
  			continue;

  		if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
  			edac_dbg(0, "    DCSM1[%d]=0x%08x reg: F2x%x
  ",
  				 cs, *mask1, (pvt->fam == 0x10) ? reg1
  							: reg0);

  	}
  }

  static void determine_memory_type(struct amd64_pvt *pvt)

  {

  	u32 dram_ctrl, dcsm;

  	if (pvt->umc) {
  		if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
  			pvt->dram_type = MEM_LRDDR4;
  		else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
  			pvt->dram_type = MEM_RDDR4;
  		else
  			pvt->dram_type = MEM_DDR4;
  		return;
  	}

  	switch (pvt->fam) {
  	case 0xf:
  		if (pvt->ext_model >= K8_REV_F)
  			goto ddr3;
  
  		pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
  		return;
  
  	case 0x10:

  		if (pvt->dchr0 & DDR3_MODE)

  			goto ddr3;
  
  		pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
  		return;
  
  	case 0x15:
  		if (pvt->model < 0x60)
  			goto ddr3;
  
  		/*
  		 * Model 0x60h needs special handling:
  		 *
  		 * We use a Chip Select value of '0' to obtain dcsm.
  		 * Theoretically, it is possible to populate LRDIMMs of different
  		 * 'Rank' value on a DCT. But this is not the common case. So,
  		 * it's reasonable to assume all DIMMs are going to be of same
  		 * 'type' until proven otherwise.
  		 */
  		amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
  		dcsm = pvt->csels[0].csmasks[0];
  
  		if (((dram_ctrl >> 8) & 0x7) == 0x2)
  			pvt->dram_type = MEM_DDR4;
  		else if (pvt->dclr0 & BIT(16))
  			pvt->dram_type = MEM_DDR3;
  		else if (dcsm & 0x3)
  			pvt->dram_type = MEM_LRDDR3;

  		else

  			pvt->dram_type = MEM_RDDR3;

  		return;
  
  	case 0x16:
  		goto ddr3;
  
  	default:
  		WARN(1, KERN_ERR "%s: Family??? 0x%x
  ", __func__, pvt->fam);
  		pvt->dram_type = MEM_EMPTY;
  	}
  	return;

  ddr3:
  	pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;

  }

  /* Get the number of DCT channels the memory controller is using. */

  static int k8_early_channel_count(struct amd64_pvt *pvt)
  {

  	int flag;

  	if (pvt->ext_model >= K8_REV_F)

  		/* RevF (NPT) and later */

  		flag = pvt->dclr0 & WIDTH_128;

  	else

  		/* RevE and earlier */
  		flag = pvt->dclr0 & REVE_WIDTH_128;

  
  	/* not used */
  	pvt->dclr1 = 0;
  
  	return (flag) ? 2 : 1;
  }

  /* On F10h and later ErrAddr is MC4_ADDR[47:1] */

  static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)

  {

  	u16 mce_nid = amd_get_nb_id(m->extcpu);
  	struct mem_ctl_info *mci;

  	u8 start_bit = 1;
  	u8 end_bit   = 47;

  	u64 addr;
  
  	mci = edac_mc_find(mce_nid);
  	if (!mci)
  		return 0;
  
  	pvt = mci->pvt_info;

  	if (pvt->fam == 0xf) {

  		start_bit = 3;
  		end_bit   = 39;
  	}

  	addr = m->addr & GENMASK_ULL(end_bit, start_bit);

  
  	/*
  	 * Erratum 637 workaround
  	 */

  	if (pvt->fam == 0x15) {

  		u64 cc6_base, tmp_addr;
  		u32 tmp;

  		u8 intlv_en;

  		if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)

  			return addr;

  
  		amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
  		intlv_en = tmp >> 21 & 0x7;
  
  		/* add [47:27] + 3 trailing bits */

  		cc6_base  = (tmp & GENMASK_ULL(20, 0)) << 3;

  
  		/* reverse and add DramIntlvEn */
  		cc6_base |= intlv_en ^ 0x7;
  
  		/* pin at [47:24] */
  		cc6_base <<= 24;
  
  		if (!intlv_en)

  			return cc6_base | (addr & GENMASK_ULL(23, 0));

  
  		amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
  
  							/* faster log2 */

  		tmp_addr  = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);

  
  		/* OR DramIntlvSel into bits [14:12] */

  		tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;

  
  		/* add remaining [11:0] bits from original MC4_ADDR */

  		tmp_addr |= addr & GENMASK_ULL(11, 0);

  
  		return cc6_base | tmp_addr;
  	}
  
  	return addr;

  }

  static struct pci_dev *pci_get_related_function(unsigned int vendor,
  						unsigned int device,
  						struct pci_dev *related)
  {
  	struct pci_dev *dev = NULL;
  
  	while ((dev = pci_get_device(vendor, device, dev))) {
  		if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
  		    (dev->bus->number == related->bus->number) &&
  		    (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
  			break;
  	}
  
  	return dev;
  }

  static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)

  {

  	struct amd_northbridge *nb;

  	struct pci_dev *f1 = NULL;
  	unsigned int pci_func;

  	int off = range << 3;

  	u32 llim;

  	amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off,  &pvt->ranges[range].base.lo);
  	amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);

  	if (pvt->fam == 0xf)

  		return;

  	if (!dram_rw(pvt, range))
  		return;

  	amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off,  &pvt->ranges[range].base.hi);
  	amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);

  	/* F15h: factor in CC6 save area by reading dst node's limit reg */

  	if (pvt->fam != 0x15)

  		return;

  	nb = node_to_amd_nb(dram_dst_node(pvt, range));
  	if (WARN_ON(!nb))
  		return;

  	if (pvt->model == 0x60)
  		pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
  	else if (pvt->model == 0x30)
  		pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
  	else
  		pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;

  
  	f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);

  	if (WARN_ON(!f1))
  		return;

  	amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);

  	pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);

  				    /* {[39:27],111b} */
  	pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;

  	pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);

  				    /* [47:40] */
  	pvt->ranges[range].lim.hi |= llim >> 13;
  
  	pci_dev_put(f1);

  }

  static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,

  				    struct err_info *err)

  {

  	struct amd64_pvt *pvt = mci->pvt_info;

  	error_address_to_page_and_offset(sys_addr, err);

  
  	/*
  	 * Find out which node the error address belongs to. This may be
  	 * different from the node that detected the error.
  	 */

  	err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
  	if (!err->src_mci) {

  		amd64_mc_err(mci, "failed to map error addr 0x%lx to a node
  ",
  			     (unsigned long)sys_addr);

  		err->err_code = ERR_NODE;

  		return;
  	}
  
  	/* Now map the sys_addr to a CSROW */

  	err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
  	if (err->csrow < 0) {
  		err->err_code = ERR_CSROW;

  		return;
  	}

  	/* CHIPKILL enabled */

  	if (pvt->nbcfg & NBCFG_CHIPKILL) {

  		err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
  		if (err->channel < 0) {

  			/*
  			 * Syndrome didn't map, so we don't know which of the
  			 * 2 DIMMs is in error. So we need to ID 'both' of them
  			 * as suspect.
  			 */

  			amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "

  				      "possible error reporting race
  ",

  				      err->syndrome);
  			err->err_code = ERR_CHANNEL;

  			return;
  		}
  	} else {
  		/*
  		 * non-chipkill ecc mode
  		 *
  		 * The k8 documentation is unclear about how to determine the
  		 * channel number when using non-chipkill memory.  This method
  		 * was obtained from email communication with someone at AMD.
  		 * (Wish the email was placed in this comment - norsk)
  		 */

  		err->channel = ((sys_addr & BIT(3)) != 0);

  	}

  }

  static int ddr2_cs_size(unsigned i, bool dct_width)

  {

  	unsigned shift = 0;

  	if (i <= 2)
  		shift = i;
  	else if (!(i & 0x1))
  		shift = i >> 1;

  	else

  		shift = (i + 1) >> 1;

  	return 128 << (shift + !!dct_width);
  }
  
  static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,

  				  unsigned cs_mode, int cs_mask_nr)

  {
  	u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
  
  	if (pvt->ext_model >= K8_REV_F) {
  		WARN_ON(cs_mode > 11);
  		return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
  	}
  	else if (pvt->ext_model >= K8_REV_D) {

  		unsigned diff;

  		WARN_ON(cs_mode > 10);

  		/*
  		 * the below calculation, besides trying to win an obfuscated C
  		 * contest, maps cs_mode values to DIMM chip select sizes. The
  		 * mappings are:
  		 *
  		 * cs_mode	CS size (mb)
  		 * =======	============
  		 * 0		32
  		 * 1		64
  		 * 2		128
  		 * 3		128
  		 * 4		256
  		 * 5		512
  		 * 6		256
  		 * 7		512
  		 * 8		1024
  		 * 9		1024
  		 * 10		2048
  		 *
  		 * Basically, it calculates a value with which to shift the
  		 * smallest CS size of 32MB.
  		 *
  		 * ddr[23]_cs_size have a similar purpose.
  		 */
  		diff = cs_mode/3 + (unsigned)(cs_mode > 5);
  
  		return 32 << (cs_mode - diff);

  	}
  	else {
  		WARN_ON(cs_mode > 6);
  		return 32 << cs_mode;
  	}

  }

  /*
   * Get the number of DCT channels in use.
   *
   * Return:
   *	number of Memory Channels in operation
   * Pass back:
   *	contents of the DCL0_LOW register
   */

  static int f1x_early_channel_count(struct amd64_pvt *pvt)

  {

  	int i, j, channels = 0;

  	/* On F10h, if we are in 128 bit mode, then we are using 2 channels */

  	if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))

  		return 2;

  
  	/*

  	 * Need to check if in unganged mode: In such, there are 2 channels,
  	 * but they are not in 128 bit mode and thus the above 'dclr0' status
  	 * bit will be OFF.

  	 *
  	 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
  	 * their CSEnable bit on. If so, then SINGLE DIMM case.
  	 */

  	edac_dbg(0, "Data width is not 128 bits - need more decoding
  ");

  	/*
  	 * Check DRAM Bank Address Mapping values for each DIMM to see if there
  	 * is more than just one DIMM present in unganged mode. Need to check
  	 * both controllers since DIMMs can be placed in either one.
  	 */

  	for (i = 0; i < 2; i++) {
  		u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);

  		for (j = 0; j < 4; j++) {
  			if (DBAM_DIMM(j, dbam) > 0) {
  				channels++;
  				break;
  			}
  		}

  	}

  	if (channels > 2)
  		channels = 2;

  	amd64_info("MCT channel count: %d
  ", channels);

  
  	return channels;

  }

  static int f17_early_channel_count(struct amd64_pvt *pvt)
  {
  	int i, channels = 0;
  
  	/* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */

  	for_each_umc(i)

  		channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
  
  	amd64_info("MCT channel count: %d
  ", channels);
  
  	return channels;
  }

  static int ddr3_cs_size(unsigned i, bool dct_width)

  {

  	unsigned shift = 0;
  	int cs_size = 0;
  
  	if (i == 0 || i == 3 || i == 4)
  		cs_size = -1;
  	else if (i <= 2)
  		shift = i;
  	else if (i == 12)
  		shift = 7;
  	else if (!(i & 0x1))
  		shift = i >> 1;
  	else
  		shift = (i + 1) >> 1;
  
  	if (cs_size != -1)
  		cs_size = (128 * (1 << !!dct_width)) << shift;
  
  	return cs_size;
  }

  static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
  {
  	unsigned shift = 0;
  	int cs_size = 0;
  
  	if (i < 4 || i == 6)
  		cs_size = -1;
  	else if (i == 12)
  		shift = 7;
  	else if (!(i & 0x1))
  		shift = i >> 1;
  	else
  		shift = (i + 1) >> 1;
  
  	if (cs_size != -1)
  		cs_size = rank_multiply * (128 << shift);
  
  	return cs_size;
  }
  
  static int ddr4_cs_size(unsigned i)
  {
  	int cs_size = 0;
  
  	if (i == 0)
  		cs_size = -1;
  	else if (i == 1)
  		cs_size = 1024;
  	else
  		/* Min cs_size = 1G */
  		cs_size = 1024 * (1 << (i >> 1));
  
  	return cs_size;
  }

  static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,

  				   unsigned cs_mode, int cs_mask_nr)

  {
  	u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
  
  	WARN_ON(cs_mode > 11);

  
  	if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)

  		return ddr3_cs_size(cs_mode, dclr & WIDTH_128);

  	else

  		return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
  }
  
  /*
   * F15h supports only 64bit DCT interfaces
   */
  static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,

  				   unsigned cs_mode, int cs_mask_nr)

  {
  	WARN_ON(cs_mode > 12);

  	return ddr3_cs_size(cs_mode, false);

  }

  /* F15h M60h supports DDR4 mapping as well.. */
  static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
  					unsigned cs_mode, int cs_mask_nr)
  {
  	int cs_size;
  	u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
  
  	WARN_ON(cs_mode > 12);
  
  	if (pvt->dram_type == MEM_DDR4) {
  		if (cs_mode > 9)
  			return -1;
  
  		cs_size = ddr4_cs_size(cs_mode);
  	} else if (pvt->dram_type == MEM_LRDDR3) {
  		unsigned rank_multiply = dcsm & 0xf;
  
  		if (rank_multiply == 3)
  			rank_multiply = 4;
  		cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
  	} else {
  		/* Minimum cs size is 512mb for F15hM60h*/
  		if (cs_mode == 0x1)
  			return -1;
  
  		cs_size = ddr3_cs_size(cs_mode, false);
  	}
  
  	return cs_size;
  }

  /*

   * F16h and F15h model 30h have only limited cs_modes.

   */
  static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,

  				unsigned cs_mode, int cs_mask_nr)

  {
  	WARN_ON(cs_mode > 12);
  
  	if (cs_mode == 6 || cs_mode == 8 ||
  	    cs_mode == 9 || cs_mode == 12)
  		return -1;
  	else
  		return ddr3_cs_size(cs_mode, false);
  }

  static int f17_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,

  				    unsigned int cs_mode, int csrow_nr)
  {

  	u32 addr_mask_orig, addr_mask_deinterleaved;
  	u32 msb, weight, num_zero_bits;
  	int dimm, size = 0;

  	/* No Chip Selects are enabled. */
  	if (!cs_mode)
  		return size;

  	/* Requested size of an even CS but none are enabled. */
  	if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
  		return size;

  	/* Requested size of an odd CS but none are enabled. */
  	if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
  		return size;
  
  	/*
  	 * There is one mask per DIMM, and two Chip Selects per DIMM.
  	 *	CS0 and CS1 -> DIMM0
  	 *	CS2 and CS3 -> DIMM1
  	 */
  	dimm = csrow_nr >> 1;

  	/* Asymmetric dual-rank DIMM support. */
  	if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
  		addr_mask_orig = pvt->csels[umc].csmasks_sec[dimm];
  	else
  		addr_mask_orig = pvt->csels[umc].csmasks[dimm];

  
  	/*
  	 * The number of zero bits in the mask is equal to the number of bits
  	 * in a full mask minus the number of bits in the current mask.
  	 *
  	 * The MSB is the number of bits in the full mask because BIT[0] is
  	 * always 0.
  	 */
  	msb = fls(addr_mask_orig) - 1;
  	weight = hweight_long(addr_mask_orig);
  	num_zero_bits = msb - weight;
  
  	/* Take the number of zero bits off from the top of the mask. */
  	addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
  
  	edac_dbg(1, "CS%d DIMM%d AddrMasks:
  ", csrow_nr, dimm);
  	edac_dbg(1, "  Original AddrMask: 0x%x
  ", addr_mask_orig);
  	edac_dbg(1, "  Deinterleaved AddrMask: 0x%x
  ", addr_mask_deinterleaved);
  
  	/* Register [31:1] = Address [39:9]. Size is in kBs here. */
  	size = (addr_mask_deinterleaved >> 2) + 1;

  
  	/* Return size in MBs. */
  	return size >> 10;
  }

  static void read_dram_ctl_register(struct amd64_pvt *pvt)

  {

  	if (pvt->fam == 0xf)

  		return;

  	if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {

  		edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x
  ",
  			 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));

  		edac_dbg(0, "  DCTs operate in %s mode
  ",
  			 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));

  
  		if (!dct_ganging_enabled(pvt))

  			edac_dbg(0, "  Address range split per DCT: %s
  ",
  				 (dct_high_range_enabled(pvt) ? "yes" : "no"));

  		edac_dbg(0, "  data interleave for ECC: %s, DRAM cleared since last warm reset: %s
  ",
  			 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
  			 (dct_memory_cleared(pvt) ? "yes" : "no"));

  		edac_dbg(0, "  channel interleave: %s, "
  			 "interleave bits selector: 0x%x
  ",
  			 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
  			 dct_sel_interleave_addr(pvt));

  	}

  	amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);

  }

  /*

   * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
   * 2.10.12 Memory Interleaving Modes).
   */
  static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
  				     u8 intlv_en, int num_dcts_intlv,
  				     u32 dct_sel)
  {
  	u8 channel = 0;
  	u8 select;
  
  	if (!(intlv_en))
  		return (u8)(dct_sel);
  
  	if (num_dcts_intlv == 2) {
  		select = (sys_addr >> 8) & 0x3;
  		channel = select ? 0x3 : 0;

  	} else if (num_dcts_intlv == 4) {
  		u8 intlv_addr = dct_sel_interleave_addr(pvt);
  		switch (intlv_addr) {
  		case 0x4:
  			channel = (sys_addr >> 8) & 0x3;
  			break;
  		case 0x5:
  			channel = (sys_addr >> 9) & 0x3;
  			break;
  		}
  	}

  	return channel;
  }
  
  /*

   * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory

   * Interleaving Modes.
   */

  static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,

  				bool hi_range_sel, u8 intlv_en)

  {

  	u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;

  
  	if (dct_ganging_enabled(pvt))

  		return 0;

  	if (hi_range_sel)
  		return dct_sel_high;

  	/*
  	 * see F2x110[DctSelIntLvAddr] - channel interleave mode
  	 */
  	if (dct_interleave_enabled(pvt)) {
  		u8 intlv_addr = dct_sel_interleave_addr(pvt);
  
  		/* return DCT select function: 0=DCT0, 1=DCT1 */
  		if (!intlv_addr)
  			return sys_addr >> 6 & 1;
  
  		if (intlv_addr & 0x2) {
  			u8 shift = intlv_addr & 0x1 ? 9 : 6;

  			u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;

  
  			return ((sys_addr >> shift) & 1) ^ temp;
  		}

  		if (intlv_addr & 0x4) {
  			u8 shift = intlv_addr & 0x1 ? 9 : 8;
  
  			return (sys_addr >> shift) & 1;
  		}

  		return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
  	}
  
  	if (dct_high_range_enabled(pvt))
  		return ~dct_sel_high & 1;

  
  	return 0;
  }

  /* Convert the sys_addr to the normalized DCT address */

  static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,

  				 u64 sys_addr, bool hi_rng,
  				 u32 dct_sel_base_addr)

  {
  	u64 chan_off;

  	u64 dram_base		= get_dram_base(pvt, range);
  	u64 hole_off		= f10_dhar_offset(pvt);

  	u64 dct_sel_base_off	= (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;

  	if (hi_rng) {
  		/*
  		 * if
  		 * base address of high range is below 4Gb
  		 * (bits [47:27] at [31:11])
  		 * DRAM address space on this DCT is hoisted above 4Gb	&&
  		 * sys_addr > 4Gb
  		 *
  		 *	remove hole offset from sys_addr
  		 * else
  		 *	remove high range offset from sys_addr
  		 */
  		if ((!(dct_sel_base_addr >> 16) ||
  		     dct_sel_base_addr < dhar_base(pvt)) &&

  		    dhar_valid(pvt) &&

  		    (sys_addr >= BIT_64(32)))

  			chan_off = hole_off;

  		else
  			chan_off = dct_sel_base_off;
  	} else {

  		/*
  		 * if
  		 * we have a valid hole		&&
  		 * sys_addr > 4Gb
  		 *
  		 *	remove hole
  		 * else
  		 *	remove dram base to normalize to DCT address
  		 */

  		if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))

  			chan_off = hole_off;

  		else

  			chan_off = dram_base;

  	}

  	return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));

  }

  /*
   * checks if the csrow passed in is marked as SPARED, if so returns the new
   * spare row
   */

  static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)

  {

  	int tmp_cs;
  
  	if (online_spare_swap_done(pvt, dct) &&
  	    csrow == online_spare_bad_dramcs(pvt, dct)) {
  
  		for_each_chip_select(tmp_cs, dct, pvt) {
  			if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
  				csrow = tmp_cs;
  				break;
  			}
  		}

  	}
  	return csrow;
  }
  
  /*
   * Iterate over the DRAM DCT "base" and "mask" registers looking for a
   * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
   *
   * Return:
   *	-EINVAL:  NOT FOUND
   *	0..csrow = Chip-Select Row
   */

  static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)

  {
  	struct mem_ctl_info *mci;
  	struct amd64_pvt *pvt;

  	u64 cs_base, cs_mask;

  	int cs_found = -EINVAL;
  	int csrow;

  	mci = edac_mc_find(nid);

  	if (!mci)
  		return cs_found;
  
  	pvt = mci->pvt_info;

  	edac_dbg(1, "input addr: 0x%llx, DCT: %d
  ", in_addr, dct);

  	for_each_chip_select(csrow, dct, pvt) {
  		if (!csrow_enabled(csrow, dct, pvt))

  			continue;

  		get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);

  		edac_dbg(1, "    CSROW=%d CSBase=0x%llx CSMask=0x%llx
  ",
  			 csrow, cs_base, cs_mask);

  		cs_mask = ~cs_mask;

  		edac_dbg(1, "    (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx
  ",
  			 (in_addr & cs_mask), (cs_base & cs_mask));

  		if ((in_addr & cs_mask) == (cs_base & cs_mask)) {

  			if (pvt->fam == 0x15 && pvt->model >= 0x30) {
  				cs_found =  csrow;
  				break;
  			}

  			cs_found = f10_process_possible_spare(pvt, dct, csrow);

  			edac_dbg(1, " MATCH csrow=%d
  ", cs_found);

  			break;
  		}
  	}
  	return cs_found;
  }

  /*
   * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
   * swapped with a region located at the bottom of memory so that the GPU can use
   * the interleaved region and thus two channels.
   */

  static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)

  {
  	u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;

  	if (pvt->fam == 0x10) {

  		/* only revC3 and revE have that feature */

  		if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))

  			return sys_addr;
  	}

  	amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);

  
  	if (!(swap_reg & 0x1))
  		return sys_addr;
  
  	swap_base	= (swap_reg >> 3) & 0x7f;
  	swap_limit	= (swap_reg >> 11) & 0x7f;
  	rgn_size	= (swap_reg >> 20) & 0x7f;
  	tmp_addr	= sys_addr >> 27;
  
  	if (!(sys_addr >> 34) &&
  	    (((tmp_addr >= swap_base) &&
  	     (tmp_addr <= swap_limit)) ||
  	     (tmp_addr < rgn_size)))
  		return sys_addr ^ (u64)swap_base << 27;
  
  	return sys_addr;
  }

  /* For a given @dram_range, check if @sys_addr falls within it. */