ddr: vybrid: Provide code to perform on-boot calibration

This patch provides the code to calibrate the DDR's DQS to DQ signals (RDLVL). It is based on: VFxxx Controller Reference Manual, Rev. 0, 10/2016, page 1600 10.1.6.16.4.1 "Software Read Leveling in MC Evaluation Mode" and NXP's community thread: "Vybrid: About DDR leveling feature on DDRMC." https://community.nxp.com/thread/395323 Signed-off-by: Lukasz Majewski <lukma@denx.de>

ddr: vybrid: Provide code to perform on-boot calibration
This patch provides the code to calibrate the DDR's DQS to DQ signals (RDLVL). It is based on: VFxxx Controller Reference Manual, Rev. 0, 10/2016, page 1600 10.1.6.16.4.1 "Software Read Leveling in MC Evaluation Mode" and NXP's community thread: "Vybrid: About DDR leveling feature on DDRMC." https://community.nxp.com/thread/395323 Signed-off-by: Lukasz Majewski <lukma@denx.de>
Lukasz Majewski · Stefano Babic
1 parent c5b22a5360
Showing 4 changed files with 400 additions and 0 deletions Side-by-side Diff
arch/arm/mach-imx/Kconfig
arch/arm/mach-imx/Makefile
arch/arm/mach-imx/ddrmc-vf610-calibration.c
arch/arm/mach-imx/ddrmc-vf610-calibration.h
@@ -78,4 +78,16 @@
 	  NXP boards based on i.MX6/7 contain the board revision information
 	  stored in the fuses. Select this option if you want to be able to
 	  retrieve the board revision information.
+
+config DDRMC_VF610_CALIBRATION
+	bool "Enable DDRMC (DDR3) on-chip calibration"
+	depends on ARCH_VF610
+	help
+	  Vybrid (vf610) SoC provides some on-chip facility to tune the DDR3
+	  memory parameters. Select this option if you want to calculate them
+	  at boot time.
+	  NOTE:
+	  NXP does NOT recommend to perform this calibration at each boot. One
+	  shall perform it on a new PCB and then use those values to program
+	  the ddrmc_cr_setting on relevant board file.
@@ -53,6 +53,7 @@
 endif
 ifeq ($(SOC),$(filter $(SOC),vf610))
 obj-y += ddrmc-vf610.o
+obj-$(CONFIG_DDRMC_VF610_CALIBRATION) += ddrmc-vf610-calibration.o
 endif
 ifneq ($(CONFIG_SPL_BUILD),y)
 obj-$(CONFIG_CMD_BMODE) += cmd_bmode.o
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * ddrmc DDR3 calibration code for NXP's VF610
+ *
+ * Copyright (C) 2018 DENX Software Engineering
+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
+ *
+ */
+/* #define DEBUG */
+#include <common.h>
+#include <asm/io.h>
+#include <asm/arch/imx-regs.h>
+#include <linux/bitmap.h>
+
+#include "ddrmc-vf610-calibration.h"
+
+/*
+ * Documents:
+ *
+ * [1] "Vybrid: About DDR leveling feature on DDRMC."
+ * https://community.nxp.com/thread/395323
+ *
+ * [2] VFxxx Controller Reference Manual, Rev. 0, 10/2016
+ *
+ *
+ * NOTE
+ * ====
+ *
+ * NXP recommends setting 'fixed' parameters instead of performing the
+ * training at each boot.
+ *
+ * Use those functions to determine those values on new HW, read the
+ * calculated value from registers and add them to the board specific
+ * struct ddrmc_cr_setting.
+ *
+ * SW leveling supported operations - CR93[SW_LVL_MODE]:
+ *
+ * - 0x0 (b'00) - No leveling
+ *
+ * - 0x1 (b'01) - WRLVL_DL_X - It is not recommended to perform this tuning
+ *                             on HW designs utilizing non-flyback topology
+ *                             (Single DDR3 with x16).
+ *                             Instead the WRLVL_DL_0/1 fields shall be set
+ *                             based on trace length differences from their
+ *                             layout.
+ *                             Mismatches up to 25% or tCK (clock period) are
+ *                             allowed, so the value in the filed doesn’t have
+ *                             to be very accurate.
+ *
+ * - 0x2 (b'10) - RDLVL_DL_0/1 - refers to adjusting the DQS strobe in relation
+ *                             to the DQ signals so that the strobe edge is
+ *                             centered in the window of valid read data.
+ *
+ * - 0x3 (b'11) - RDLVL_GTDL_0/1 - refers to the delay the PHY uses to un-gate
+ *                             the Read DQS strobe pad from the time that the
+ *                             PHY enables the pad to input the strobe signal.
+ *
+ */
+static int ddr_cal_get_first_edge_index(unsigned long *bmap, enum edge e,
+					int samples, int start, int max)
+{
+	int i, ret = -1;
+
+	/*
+	 * We look only for the first value (and filter out
+	 * some wrong data)
+	 */
+	switch (e) {
+	case RISING_EDGE:
+		for (i = start; i <= max - samples; i++) {
+			if (test_bit(i, bmap)) {
+				if (!test_bit(i - 1, bmap) &&
+				    test_bit(i + 1, bmap) &&
+				    test_bit(i + 2, bmap) &&
+				    test_bit(i + 3, bmap)) {
+					return i;
+				}
+			}
+		}
+		break;
+	case FALLING_EDGE:
+		for (i = start; i <= max - samples; i++) {
+			if (!test_bit(i, bmap)) {
+				if (test_bit(i - 1, bmap) &&
+				    test_bit(i - 2, bmap) &&
+				    test_bit(i - 3, bmap)) {
+					return i;
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+static void bitmap_print(unsigned long *bmap, int max)
+{
+	int i;
+
+	debug("BITMAP [0x%p]:\n", bmap);
+	for (i = 0; i <= max; i++) {
+		debug("%d ", test_bit(i, bmap) ? 1 : 0);
+		if (i && (i % 32) == (32 - 1))
+			debug("\n");
+	}
+	debug("\n");
+}
+
+#define sw_leveling_op_done \
+	while (!(readl(&ddrmr->cr[94]) & DDRMC_CR94_SWLVL_OP_DONE))
+
+#define sw_leveling_load_value \
+	do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_LOAD, \
+			     DDRMC_CR93_SWLVL_LOAD); } while (0)
+
+#define sw_leveling_start \
+	do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_START, \
+			     DDRMC_CR93_SWLVL_START); } while (0)
+
+#define sw_leveling_exit \
+	do { clrsetbits_le32(&ddrmr->cr[94], DDRMC_CR94_SWLVL_EXIT, \
+			     DDRMC_CR94_SWLVL_EXIT); } while (0)
+
+/*
+ * RDLVL_DL calibration:
+ *
+ * NXP is _NOT_ recommending performing the leveling at each
+ * boot. Instead - one shall run this procedure on new boards
+ * and then use hardcoded values.
+ *
+ */
+static int ddrmc_cal_dqs_to_dq(struct ddrmr_regs *ddrmr)
+{
+	DECLARE_BITMAP(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
+	int rdlvl_dl_0_min = -1, rdlvl_dl_0_max = -1;
+	int rdlvl_dl_1_min = -1, rdlvl_dl_1_max = -1;
+	int rdlvl_dl_0, rdlvl_dl_1;
+	u8 swlvl_rsp;
+	u32 tmp;
+	int i;
+
+	/* Read defaults */
+	u16 rdlvl_dl_0_def =
+		(readl(&ddrmr->cr[105]) >> DDRMC_CR105_RDLVL_DL_0_OFF) & 0xFFFF;
+	u16 rdlvl_dl_1_def = readl(&ddrmr->cr[110]) & 0xFFFF;
+
+	debug("\nRDLVL: ======================\n");
+	debug("RDLVL: DQS to DQ (RDLVL)\n");
+
+	debug("RDLVL: RDLVL_DL_0_DFL:\t 0x%x\n", rdlvl_dl_0_def);
+	debug("RDLVL: RDLVL_DL_1_DFL:\t 0x%x\n", rdlvl_dl_1_def);
+
+	/*
+	 * Set/Read setup for calibration
+	 *
+	 * Values necessary for leveling from Vybrid RM [2] - page 1600
+	 */
+	writel(0x40703030, &ddrmr->cr[144]);
+	writel(0x40, &ddrmr->cr[145]);
+	writel(0x40, &ddrmr->cr[146]);
+
+	tmp = readl(&ddrmr->cr[144]);
+	debug("RDLVL: PHY_RDLVL_RES:\t 0x%x\n", (tmp >> 24) & 0xFF);// set 0x40
+	debug("RDLVL: PHY_RDLV_LOAD:\t 0x%x\n", (tmp >> 16) & 0xFF);// set 0x70
+	debug("RDLVL: PHY_RDLV_DLL:\t 0x%x\n", (tmp >> 8) & 0xFF); // set 0x30
+	debug("RDLVL: PHY_RDLV_EN:\t 0x%x\n", tmp & 0xFF); //set 0x30
+
+	tmp = readl(&ddrmr->cr[145]);
+	debug("RDLVL: PHY_RDLV_RR:\t 0x%x\n", tmp & 0x3FF); //set 0x40
+
+	tmp = readl(&ddrmr->cr[146]);
+	debug("RDLVL: PHY_RDLV_RESP:\t 0x%x\n", tmp); //set 0x40
+
+	/*
+	 * Program/read the leveling edge RDLVL_EDGE = 0
+	 *
+	 * 0x00 is the correct output on SWLVL_RSP_X
+	 * If by any chance 1s are visible -> wrong number read
+	 */
+	clrbits_le32(&ddrmr->cr[101], DDRMC_CR101_PHY_RDLVL_EDGE);
+
+	tmp = readl(&ddrmr->cr[101]);
+	debug("RDLVL: PHY_RDLVL_EDGE:\t 0x%x\n",
+	      (tmp >> DDRMC_CR101_PHY_RDLVL_EDGE_OFF) & 0x1); //set 0
+
+	/* Program Leveling mode - CR93[SW_LVL_MODE] to ’b10 */
+	clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SW_LVL_MODE(0x3),
+			DDRMC_CR93_SW_LVL_MODE(0x2));
+	tmp = readl(&ddrmr->cr[93]);
+	debug("RDLVL: SW_LVL_MODE:\t 0x%x\n",
+	      (tmp >> DDRMC_CR93_SW_LVL_MODE_OFF) & 0x3);
+
+	/* Start procedure - CR93[SWLVL_START] to ’b1 */
+	sw_leveling_start;
+
+	/* Poll CR94[SWLVL_OP_DONE] */
+	sw_leveling_op_done;
+
+	/*
+	 * Program delays for RDLVL_DL_0
+	 *
+	 * The procedure is to increase the delay values from 0 to 0xFF
+	 * and read the response from the DDRMC
+	 */
+	debug("\nRDLVL: ---> RDLVL_DL_0\n");
+	bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
+
+	for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
+		clrsetbits_le32(&ddrmr->cr[105],
+				0xFFFF << DDRMC_CR105_RDLVL_DL_0_OFF,
+				i << DDRMC_CR105_RDLVL_DL_0_OFF);
+
+		/* Load values CR93[SWLVL_LOAD] to ’b1 */
+		sw_leveling_load_value;
+
+		/* Poll CR94[SWLVL_OP_DONE] */
+		sw_leveling_op_done;
+
+		/*
+		 * Read Responses - SWLVL_RESP_0
+		 *
+		 * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
+		 * -> 1 in the bit vector
+		 */
+		swlvl_rsp = (readl(&ddrmr->cr[94]) >>
+			     DDRMC_CR94_SWLVL_RESP_0_OFF) & 0xF;
+		if (swlvl_rsp == 0)
+			generic_set_bit(i, rdlvl_rsp);
+	}
+
+	bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
+
+	/*
+	 * First test for rising edge 0x0 -> 0x1 in bitmap
+	 */
+	rdlvl_dl_0_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
+						      N_SAMPLES, N_SAMPLES,
+						      DDRMC_DQS_DQ_MAX_DELAY);
+
+	/*
+	 * Secondly test for falling edge 0x1 -> 0x0 in bitmap
+	 */
+	rdlvl_dl_0_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
+						      N_SAMPLES, rdlvl_dl_0_min,
+						      DDRMC_DQS_DQ_MAX_DELAY);
+
+	debug("RDLVL: DL_0 min: %d [0x%x] DL_0 max: %d [0x%x]\n",
+	      rdlvl_dl_0_min, rdlvl_dl_0_min, rdlvl_dl_0_max, rdlvl_dl_0_max);
+	rdlvl_dl_0 = (rdlvl_dl_0_max - rdlvl_dl_0_min) / 2;
+
+	if (rdlvl_dl_0_max == -1 || rdlvl_dl_0_min == -1 || rdlvl_dl_0 <= 0) {
+		debug("RDLVL: The DQS to DQ delay cannot be found!\n");
+		debug("RDLVL: Using default - slice 0: %d!\n", rdlvl_dl_0_def);
+		rdlvl_dl_0 = rdlvl_dl_0_def;
+	}
+
+	debug("\nRDLVL: ---> RDLVL_DL_1\n");
+	bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
+
+	for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
+		clrsetbits_le32(&ddrmr->cr[110],
+				0xFFFF << DDRMC_CR110_RDLVL_DL_1_OFF,
+				i << DDRMC_CR110_RDLVL_DL_1_OFF);
+
+		/* Load values CR93[SWLVL_LOAD] to ’b1 */
+		sw_leveling_load_value;
+
+		/* Poll CR94[SWLVL_OP_DONE] */
+		sw_leveling_op_done;
+
+		/*
+		 * Read Responses - SWLVL_RESP_1
+		 *
+		 * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
+		 * -> 1 in the bit vector
+		 */
+		swlvl_rsp = (readl(&ddrmr->cr[95]) >>
+			     DDRMC_CR95_SWLVL_RESP_1_OFF) & 0xF;
+		if (swlvl_rsp == 0)
+			generic_set_bit(i, rdlvl_rsp);
+	}
+
+	bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
+
+	/*
+	 * First test for rising edge 0x0 -> 0x1 in bitmap
+	 */
+	rdlvl_dl_1_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
+						      N_SAMPLES, N_SAMPLES,
+						      DDRMC_DQS_DQ_MAX_DELAY);
+
+	/*
+	 * Secondly test for falling edge 0x1 -> 0x0 in bitmap
+	 */
+	rdlvl_dl_1_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
+						      N_SAMPLES, rdlvl_dl_1_min,
+						      DDRMC_DQS_DQ_MAX_DELAY);
+
+	debug("RDLVL: DL_1 min: %d [0x%x] DL_1 max: %d [0x%x]\n",
+	      rdlvl_dl_1_min, rdlvl_dl_1_min, rdlvl_dl_1_max, rdlvl_dl_1_max);
+	rdlvl_dl_1 = (rdlvl_dl_1_max - rdlvl_dl_1_min) / 2;
+
+	if (rdlvl_dl_1_max == -1 || rdlvl_dl_1_min == -1 || rdlvl_dl_1 <= 0) {
+		debug("RDLVL: The DQS to DQ delay cannot be found!\n");
+		debug("RDLVL: Using default - slice 1: %d!\n", rdlvl_dl_1_def);
+		rdlvl_dl_1 = rdlvl_dl_1_def;
+	}
+
+	debug("RDLVL: CALIBRATED: rdlvl_dl_0: 0x%x\t rdlvl_dl_1: 0x%x\n",
+	      rdlvl_dl_0, rdlvl_dl_1);
+
+	/* Write new delay values */
+	writel(DDRMC_CR105_RDLVL_DL_0(rdlvl_dl_0), &ddrmr->cr[105]);
+	writel(DDRMC_CR110_RDLVL_DL_1(rdlvl_dl_1), &ddrmr->cr[110]);
+
+	sw_leveling_load_value;
+	sw_leveling_op_done;
+
+	/* Exit procedure - CR94[SWLVL_EXIT] to ’b1 */
+	sw_leveling_exit;
+
+	/* Poll CR94[SWLVL_OP_DONE] */
+	sw_leveling_op_done;
+
+	return 0;
+}
+
+/*
+ * WRLVL_DL calibration:
+ *
+ * For non-flyback memory architecture - where one have a single DDR3 x16
+ * memory - it is NOT necessary to perform "Write Leveling"
+ * [3] 'Vybrid DDR3 write leveling' https://community.nxp.com/thread/429362
+ *
+ */
+
+int ddrmc_calibration(struct ddrmr_regs *ddrmr)
+{
+	ddrmc_cal_dqs_to_dq(ddrmr);
+
+	return 0;
+}
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * ddrmc DDR3 calibration code for NXP's VF610
+ *
+ * Copyright (C) 2018 DENX Software Engineering
+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
+ *
+ */
+
+#ifndef __DDRMC_VF610_CALIBRATOIN_H_
+#define __DDRMC_VF610_CALIBRATOIN_H_
+
+/*
+ * Number of "samples" in the calibration bitmap
+ * to be considered during calibration.
+ */
+#define N_SAMPLES 3
+
+/*
+ * Constants to indicate if we are looking for a rising or
+ * falling edge in the calibration bitmap
+ */
+enum edge {
+	FALLING_EDGE = 1,
+	RISING_EDGE
+};
+
+/*
+ * The max number of delay elements when DQS to DQ setting
+ */
+#define DDRMC_DQS_DQ_MAX_DELAY 0xFF
+
+/**
+ * ddrmc_calibration - Vybrid's (VF610) DDR3 calibration code
+ *
+ * This function is calculating proper memory controller values
+ * during run time.
+ *
+ * @param ddrmr_regs - memory controller registers
+ *
+ * @return 0 on success, otherwise error code
+ */
+int ddrmc_calibration(struct ddrmr_regs *ddrmr);
+
+#endif /* __DDRMC_VF610_CALIBRATOIN_H_ */
...	...	@@ -78,4 +78,16 @@
78	78	NXP boards based on i.MX6/7 contain the board revision information
79	79	stored in the fuses. Select this option if you want to be able to
80	80	retrieve the board revision information.
	81	+
	82	+config DDRMC_VF610_CALIBRATION
	83	+ bool "Enable DDRMC (DDR3) on-chip calibration"
	84	+ depends on ARCH_VF610
	85	+ help
	86	+ Vybrid (vf610) SoC provides some on-chip facility to tune the DDR3
	87	+ memory parameters. Select this option if you want to calculate them
	88	+ at boot time.
	89	+ NOTE:
	90	+ NXP does NOT recommend to perform this calibration at each boot. One
	91	+ shall perform it on a new PCB and then use those values to program
	92	+ the ddrmc_cr_setting on relevant board file.
...	...	@@ -53,6 +53,7 @@
53	53	endif
54	54	ifeq ($(SOC),$(filter $(SOC),vf610))
55	55	obj-y += ddrmc-vf610.o
	56	+obj-$(CONFIG_DDRMC_VF610_CALIBRATION) += ddrmc-vf610-calibration.o
56	57	endif
57	58	ifneq ($(CONFIG_SPL_BUILD),y)
58	59	obj-$(CONFIG_CMD_BMODE) += cmd_bmode.o
	1	+// SPDX-License-Identifier: GPL-2.0+
	2	+/*
	3	+ * ddrmc DDR3 calibration code for NXP's VF610
	4	+ *
	5	+ * Copyright (C) 2018 DENX Software Engineering
	6	+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
	7	+ *
	8	+ */
	9	+/* #define DEBUG */
	10	+#include <common.h>
	11	+#include <asm/io.h>
	12	+#include <asm/arch/imx-regs.h>
	13	+#include <linux/bitmap.h>
	14	+
	15	+#include "ddrmc-vf610-calibration.h"
	16	+
	17	+/*
	18	+ * Documents:
	19	+ *
	20	+ * [1] "Vybrid: About DDR leveling feature on DDRMC."
	21	+ * https://community.nxp.com/thread/395323
	22	+ *
	23	+ * [2] VFxxx Controller Reference Manual, Rev. 0, 10/2016
	24	+ *
	25	+ *
	26	+ * NOTE
	27	+ * ====
	28	+ *
	29	+ * NXP recommends setting 'fixed' parameters instead of performing the
	30	+ * training at each boot.
	31	+ *
	32	+ * Use those functions to determine those values on new HW, read the
	33	+ * calculated value from registers and add them to the board specific
	34	+ * struct ddrmc_cr_setting.
	35	+ *
	36	+ * SW leveling supported operations - CR93[SW_LVL_MODE]:
	37	+ *
	38	+ * - 0x0 (b'00) - No leveling
	39	+ *
	40	+ * - 0x1 (b'01) - WRLVL_DL_X - It is not recommended to perform this tuning
	41	+ * on HW designs utilizing non-flyback topology
	42	+ * (Single DDR3 with x16).
	43	+ * Instead the WRLVL_DL_0/1 fields shall be set
	44	+ * based on trace length differences from their
	45	+ * layout.
	46	+ * Mismatches up to 25% or tCK (clock period) are
	47	+ * allowed, so the value in the filed doesn’t have
	48	+ * to be very accurate.
	49	+ *
	50	+ * - 0x2 (b'10) - RDLVL_DL_0/1 - refers to adjusting the DQS strobe in relation
	51	+ * to the DQ signals so that the strobe edge is
	52	+ * centered in the window of valid read data.
	53	+ *
	54	+ * - 0x3 (b'11) - RDLVL_GTDL_0/1 - refers to the delay the PHY uses to un-gate
	55	+ * the Read DQS strobe pad from the time that the
	56	+ * PHY enables the pad to input the strobe signal.
	57	+ *
	58	+ */
	59	+static int ddr_cal_get_first_edge_index(unsigned long *bmap, enum edge e,
	60	+ int samples, int start, int max)
	61	+{
	62	+ int i, ret = -1;
	63	+
	64	+ /*
	65	+ * We look only for the first value (and filter out
	66	+ * some wrong data)
	67	+ */
	68	+ switch (e) {
	69	+ case RISING_EDGE:
	70	+ for (i = start; i <= max - samples; i++) {
	71	+ if (test_bit(i, bmap)) {
	72	+ if (!test_bit(i - 1, bmap) &&
	73	+ test_bit(i + 1, bmap) &&
	74	+ test_bit(i + 2, bmap) &&
	75	+ test_bit(i + 3, bmap)) {
	76	+ return i;
	77	+ }
	78	+ }
	79	+ }
	80	+ break;
	81	+ case FALLING_EDGE:
	82	+ for (i = start; i <= max - samples; i++) {
	83	+ if (!test_bit(i, bmap)) {
	84	+ if (test_bit(i - 1, bmap) &&
	85	+ test_bit(i - 2, bmap) &&
	86	+ test_bit(i - 3, bmap)) {
	87	+ return i;
	88	+ }
	89	+ }
	90	+ }
	91	+ }
	92	+
	93	+ return ret;
	94	+}
	95	+
	96	+static void bitmap_print(unsigned long *bmap, int max)
	97	+{
	98	+ int i;
	99	+
	100	+ debug("BITMAP [0x%p]:\n", bmap);
	101	+ for (i = 0; i <= max; i++) {
	102	+ debug("%d ", test_bit(i, bmap) ? 1 : 0);
	103	+ if (i && (i % 32) == (32 - 1))
	104	+ debug("\n");
	105	+ }
	106	+ debug("\n");
	107	+}
	108	+
	109	+#define sw_leveling_op_done \
	110	+ while (!(readl(&ddrmr->cr[94]) & DDRMC_CR94_SWLVL_OP_DONE))
	111	+
	112	+#define sw_leveling_load_value \
	113	+ do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_LOAD, \
	114	+ DDRMC_CR93_SWLVL_LOAD); } while (0)
	115	+
	116	+#define sw_leveling_start \
	117	+ do { clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SWLVL_START, \
	118	+ DDRMC_CR93_SWLVL_START); } while (0)
	119	+
	120	+#define sw_leveling_exit \
	121	+ do { clrsetbits_le32(&ddrmr->cr[94], DDRMC_CR94_SWLVL_EXIT, \
	122	+ DDRMC_CR94_SWLVL_EXIT); } while (0)
	123	+
	124	+/*
	125	+ * RDLVL_DL calibration:
	126	+ *
	127	+ * NXP is _NOT_ recommending performing the leveling at each
	128	+ * boot. Instead - one shall run this procedure on new boards
	129	+ * and then use hardcoded values.
	130	+ *
	131	+ */
	132	+static int ddrmc_cal_dqs_to_dq(struct ddrmr_regs *ddrmr)
	133	+{
	134	+ DECLARE_BITMAP(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
	135	+ int rdlvl_dl_0_min = -1, rdlvl_dl_0_max = -1;
	136	+ int rdlvl_dl_1_min = -1, rdlvl_dl_1_max = -1;
	137	+ int rdlvl_dl_0, rdlvl_dl_1;
	138	+ u8 swlvl_rsp;
	139	+ u32 tmp;
	140	+ int i;
	141	+
	142	+ /* Read defaults */
	143	+ u16 rdlvl_dl_0_def =
	144	+ (readl(&ddrmr->cr[105]) >> DDRMC_CR105_RDLVL_DL_0_OFF) & 0xFFFF;
	145	+ u16 rdlvl_dl_1_def = readl(&ddrmr->cr[110]) & 0xFFFF;
	146	+
	147	+ debug("\nRDLVL: ======================\n");
	148	+ debug("RDLVL: DQS to DQ (RDLVL)\n");
	149	+
	150	+ debug("RDLVL: RDLVL_DL_0_DFL:\t 0x%x\n", rdlvl_dl_0_def);
	151	+ debug("RDLVL: RDLVL_DL_1_DFL:\t 0x%x\n", rdlvl_dl_1_def);
	152	+
	153	+ /*
	154	+ * Set/Read setup for calibration
	155	+ *
	156	+ * Values necessary for leveling from Vybrid RM [2] - page 1600
	157	+ */
	158	+ writel(0x40703030, &ddrmr->cr[144]);
	159	+ writel(0x40, &ddrmr->cr[145]);
	160	+ writel(0x40, &ddrmr->cr[146]);
	161	+
	162	+ tmp = readl(&ddrmr->cr[144]);
	163	+ debug("RDLVL: PHY_RDLVL_RES:\t 0x%x\n", (tmp >> 24) & 0xFF);// set 0x40
	164	+ debug("RDLVL: PHY_RDLV_LOAD:\t 0x%x\n", (tmp >> 16) & 0xFF);// set 0x70
	165	+ debug("RDLVL: PHY_RDLV_DLL:\t 0x%x\n", (tmp >> 8) & 0xFF); // set 0x30
	166	+ debug("RDLVL: PHY_RDLV_EN:\t 0x%x\n", tmp & 0xFF); //set 0x30
	167	+
	168	+ tmp = readl(&ddrmr->cr[145]);
	169	+ debug("RDLVL: PHY_RDLV_RR:\t 0x%x\n", tmp & 0x3FF); //set 0x40
	170	+
	171	+ tmp = readl(&ddrmr->cr[146]);
	172	+ debug("RDLVL: PHY_RDLV_RESP:\t 0x%x\n", tmp); //set 0x40
	173	+
	174	+ /*
	175	+ * Program/read the leveling edge RDLVL_EDGE = 0
	176	+ *
	177	+ * 0x00 is the correct output on SWLVL_RSP_X
	178	+ * If by any chance 1s are visible -> wrong number read
	179	+ */
	180	+ clrbits_le32(&ddrmr->cr[101], DDRMC_CR101_PHY_RDLVL_EDGE);
	181	+
	182	+ tmp = readl(&ddrmr->cr[101]);
	183	+ debug("RDLVL: PHY_RDLVL_EDGE:\t 0x%x\n",
	184	+ (tmp >> DDRMC_CR101_PHY_RDLVL_EDGE_OFF) & 0x1); //set 0
	185	+
	186	+ /* Program Leveling mode - CR93[SW_LVL_MODE] to ’b10 */
	187	+ clrsetbits_le32(&ddrmr->cr[93], DDRMC_CR93_SW_LVL_MODE(0x3),
	188	+ DDRMC_CR93_SW_LVL_MODE(0x2));
	189	+ tmp = readl(&ddrmr->cr[93]);
	190	+ debug("RDLVL: SW_LVL_MODE:\t 0x%x\n",
	191	+ (tmp >> DDRMC_CR93_SW_LVL_MODE_OFF) & 0x3);
	192	+
	193	+ /* Start procedure - CR93[SWLVL_START] to ’b1 */
	194	+ sw_leveling_start;
	195	+
	196	+ /* Poll CR94[SWLVL_OP_DONE] */
	197	+ sw_leveling_op_done;
	198	+
	199	+ /*
	200	+ * Program delays for RDLVL_DL_0
	201	+ *
	202	+ * The procedure is to increase the delay values from 0 to 0xFF
	203	+ * and read the response from the DDRMC
	204	+ */
	205	+ debug("\nRDLVL: ---> RDLVL_DL_0\n");
	206	+ bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
	207	+
	208	+ for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
	209	+ clrsetbits_le32(&ddrmr->cr[105],
	210	+ 0xFFFF << DDRMC_CR105_RDLVL_DL_0_OFF,
	211	+ i << DDRMC_CR105_RDLVL_DL_0_OFF);
	212	+
	213	+ /* Load values CR93[SWLVL_LOAD] to ’b1 */
	214	+ sw_leveling_load_value;
	215	+
	216	+ /* Poll CR94[SWLVL_OP_DONE] */
	217	+ sw_leveling_op_done;
	218	+
	219	+ /*
	220	+ * Read Responses - SWLVL_RESP_0
	221	+ *
	222	+ * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
	223	+ * -> 1 in the bit vector
	224	+ */
	225	+ swlvl_rsp = (readl(&ddrmr->cr[94]) >>
	226	+ DDRMC_CR94_SWLVL_RESP_0_OFF) & 0xF;
	227	+ if (swlvl_rsp == 0)
	228	+ generic_set_bit(i, rdlvl_rsp);
	229	+ }
	230	+
	231	+ bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
	232	+
	233	+ /*
	234	+ * First test for rising edge 0x0 -> 0x1 in bitmap
	235	+ */
	236	+ rdlvl_dl_0_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
	237	+ N_SAMPLES, N_SAMPLES,
	238	+ DDRMC_DQS_DQ_MAX_DELAY);
	239	+
	240	+ /*
	241	+ * Secondly test for falling edge 0x1 -> 0x0 in bitmap
	242	+ */
	243	+ rdlvl_dl_0_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
	244	+ N_SAMPLES, rdlvl_dl_0_min,
	245	+ DDRMC_DQS_DQ_MAX_DELAY);
	246	+
	247	+ debug("RDLVL: DL_0 min: %d [0x%x] DL_0 max: %d [0x%x]\n",
	248	+ rdlvl_dl_0_min, rdlvl_dl_0_min, rdlvl_dl_0_max, rdlvl_dl_0_max);
	249	+ rdlvl_dl_0 = (rdlvl_dl_0_max - rdlvl_dl_0_min) / 2;
	250	+
	251	+ if (rdlvl_dl_0_max == -1 \|\| rdlvl_dl_0_min == -1 \|\| rdlvl_dl_0 <= 0) {
	252	+ debug("RDLVL: The DQS to DQ delay cannot be found!\n");
	253	+ debug("RDLVL: Using default - slice 0: %d!\n", rdlvl_dl_0_def);
	254	+ rdlvl_dl_0 = rdlvl_dl_0_def;
	255	+ }
	256	+
	257	+ debug("\nRDLVL: ---> RDLVL_DL_1\n");
	258	+ bitmap_zero(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY + 1);
	259	+
	260	+ for (i = 0; i <= DDRMC_DQS_DQ_MAX_DELAY; i++) {
	261	+ clrsetbits_le32(&ddrmr->cr[110],
	262	+ 0xFFFF << DDRMC_CR110_RDLVL_DL_1_OFF,
	263	+ i << DDRMC_CR110_RDLVL_DL_1_OFF);
	264	+
	265	+ /* Load values CR93[SWLVL_LOAD] to ’b1 */
	266	+ sw_leveling_load_value;
	267	+
	268	+ /* Poll CR94[SWLVL_OP_DONE] */
	269	+ sw_leveling_op_done;
	270	+
	271	+ /*
	272	+ * Read Responses - SWLVL_RESP_1
	273	+ *
	274	+ * The 0x00 (correct response when PHY_RDLVL_EDGE = 0)
	275	+ * -> 1 in the bit vector
	276	+ */
	277	+ swlvl_rsp = (readl(&ddrmr->cr[95]) >>
	278	+ DDRMC_CR95_SWLVL_RESP_1_OFF) & 0xF;
	279	+ if (swlvl_rsp == 0)
	280	+ generic_set_bit(i, rdlvl_rsp);
	281	+ }
	282	+
	283	+ bitmap_print(rdlvl_rsp, DDRMC_DQS_DQ_MAX_DELAY);
	284	+
	285	+ /*
	286	+ * First test for rising edge 0x0 -> 0x1 in bitmap
	287	+ */
	288	+ rdlvl_dl_1_min = ddr_cal_get_first_edge_index(rdlvl_rsp, RISING_EDGE,
	289	+ N_SAMPLES, N_SAMPLES,
	290	+ DDRMC_DQS_DQ_MAX_DELAY);
	291	+
	292	+ /*
	293	+ * Secondly test for falling edge 0x1 -> 0x0 in bitmap
	294	+ */
	295	+ rdlvl_dl_1_max = ddr_cal_get_first_edge_index(rdlvl_rsp, FALLING_EDGE,
	296	+ N_SAMPLES, rdlvl_dl_1_min,
	297	+ DDRMC_DQS_DQ_MAX_DELAY);
	298	+
	299	+ debug("RDLVL: DL_1 min: %d [0x%x] DL_1 max: %d [0x%x]\n",
	300	+ rdlvl_dl_1_min, rdlvl_dl_1_min, rdlvl_dl_1_max, rdlvl_dl_1_max);
	301	+ rdlvl_dl_1 = (rdlvl_dl_1_max - rdlvl_dl_1_min) / 2;
	302	+
	303	+ if (rdlvl_dl_1_max == -1 \|\| rdlvl_dl_1_min == -1 \|\| rdlvl_dl_1 <= 0) {
	304	+ debug("RDLVL: The DQS to DQ delay cannot be found!\n");
	305	+ debug("RDLVL: Using default - slice 1: %d!\n", rdlvl_dl_1_def);
	306	+ rdlvl_dl_1 = rdlvl_dl_1_def;
	307	+ }
	308	+
	309	+ debug("RDLVL: CALIBRATED: rdlvl_dl_0: 0x%x\t rdlvl_dl_1: 0x%x\n",
	310	+ rdlvl_dl_0, rdlvl_dl_1);
	311	+
	312	+ /* Write new delay values */
	313	+ writel(DDRMC_CR105_RDLVL_DL_0(rdlvl_dl_0), &ddrmr->cr[105]);
	314	+ writel(DDRMC_CR110_RDLVL_DL_1(rdlvl_dl_1), &ddrmr->cr[110]);
	315	+
	316	+ sw_leveling_load_value;
	317	+ sw_leveling_op_done;
	318	+
	319	+ /* Exit procedure - CR94[SWLVL_EXIT] to ’b1 */
	320	+ sw_leveling_exit;
	321	+
	322	+ /* Poll CR94[SWLVL_OP_DONE] */
	323	+ sw_leveling_op_done;
	324	+
	325	+ return 0;
	326	+}
	327	+
	328	+/*
	329	+ * WRLVL_DL calibration:
	330	+ *
	331	+ * For non-flyback memory architecture - where one have a single DDR3 x16
	332	+ * memory - it is NOT necessary to perform "Write Leveling"
	333	+ * [3] 'Vybrid DDR3 write leveling' https://community.nxp.com/thread/429362
	334	+ *
	335	+ */
	336	+
	337	+int ddrmc_calibration(struct ddrmr_regs *ddrmr)
	338	+{
	339	+ ddrmc_cal_dqs_to_dq(ddrmr);
	340	+
	341	+ return 0;
	342	+}
	1	+/* SPDX-License-Identifier: GPL-2.0+ */
	2	+/*
	3	+ * ddrmc DDR3 calibration code for NXP's VF610
	4	+ *
	5	+ * Copyright (C) 2018 DENX Software Engineering
	6	+ * Lukasz Majewski, DENX Software Engineering, lukma@denx.de
	7	+ *
	8	+ */
	9	+
	10	+#ifndef __DDRMC_VF610_CALIBRATOIN_H_
	11	+#define __DDRMC_VF610_CALIBRATOIN_H_
	12	+
	13	+/*
	14	+ * Number of "samples" in the calibration bitmap
	15	+ * to be considered during calibration.
	16	+ */
	17	+#define N_SAMPLES 3
	18	+
	19	+/*
	20	+ * Constants to indicate if we are looking for a rising or
	21	+ * falling edge in the calibration bitmap
	22	+ */
	23	+enum edge {
	24	+ FALLING_EDGE = 1,
	25	+ RISING_EDGE
	26	+};
	27	+
	28	+/*
	29	+ * The max number of delay elements when DQS to DQ setting
	30	+ */
	31	+#define DDRMC_DQS_DQ_MAX_DELAY 0xFF
	32	+
	33	+/**
	34	+ * ddrmc_calibration - Vybrid's (VF610) DDR3 calibration code
	35	+ *
	36	+ * This function is calculating proper memory controller values
	37	+ * during run time.
	38	+ *
	39	+ * @param ddrmr_regs - memory controller registers
	40	+ *
	41	+ * @return 0 on success, otherwise error code
	42	+ */
	43	+int ddrmc_calibration(struct ddrmr_regs *ddrmr);
	44	+
	45	+#endif /* __DDRMC_VF610_CALIBRATOIN_H_ */