Merge tag 'ntb-5.7' of git://github.com/jonmason/ntb

Pull NTB updates from Jon Mason: "Bug fixes for a few printing issues, link status detection bug on AMD hardware, and a DMA address issue with ntb_perf. Also, large series of AMD NTB patches" * tag 'ntb-5.7' of git://github.com/jonmason/ntb: (21 commits) NTB: add pci shutdown handler for AMD NTB NTB: send DB event when driver is loaded or un-loaded NTB: remove redundant setting of DB valid mask NTB: return link up status correctly for PRI and SEC NTB: add helper functions to set and clear sideinfo NTB: move ntb_ctrl handling to init and deinit NTB: handle link up, D0 and D3 events correctly NTB: handle link down event correctly NTB: remove handling of peer_sta from amd_link_is_up NTB: set peer_sta within event handler itself NTB: return the side info status from amd_poll_link NTB: define a new function to get link status NTB: Enable link up and down event notification NTB: clear interrupt status register NTB: Fix access to link status and control register MAINTAINERS: update maintainer list for AMD NTB driver NTB: ntb_transport: Use scnprintf() for avoiding potential buffer overflow ntb_hw_switchtec: Fix ntb_mw_clear_trans error if size == 0 ntb_tool: Fix printk format NTB: ntb_perf: Fix address err in perf_copy_chunk ...

Merge tag 'ntb-5.7' of git://github.com/jonmason/ntb
Pull NTB updates from Jon Mason: "Bug fixes for a few printing issues, link status detection bug on AMD hardware, and a DMA address issue with ntb_perf. Also, large series of AMD NTB patches" * tag 'ntb-5.7' of git://github.com/jonmason/ntb: (21 commits) NTB: add pci shutdown handler for AMD NTB NTB: send DB event when driver is loaded or un-loaded NTB: remove redundant setting of DB valid mask NTB: return link up status correctly for PRI and SEC NTB: add helper functions to set and clear sideinfo NTB: move ntb_ctrl handling to init and deinit NTB: handle link up, D0 and D3 events correctly NTB: handle link down event correctly NTB: remove handling of peer_sta from amd_link_is_up NTB: set peer_sta within event handler itself NTB: return the side info status from amd_poll_link NTB: define a new function to get link status NTB: Enable link up and down event notification NTB: clear interrupt status register NTB: Fix access to link status and control register MAINTAINERS: update maintainer list for AMD NTB driver NTB: ntb_transport: Use scnprintf() for avoiding potential buffer overflow ntb_hw_switchtec: Fix ntb_mw_clear_trans error if size == 0 ntb_tool: Fix printk format NTB: ntb_perf: Fix address err in perf_copy_chunk ...
Linus Torvalds
2 parents dba43fc4ba b350f0a3eb
Showing 7 changed files Side-by-side Diff
MAINTAINERS
drivers/ntb/hw/amd/ntb_hw_amd.c
drivers/ntb/hw/amd/ntb_hw_amd.h
drivers/ntb/hw/mscc/ntb_hw_switchtec.c
drivers/ntb/ntb_transport.c
drivers/ntb/test/ntb_perf.c
drivers/ntb/test/ntb_tool.c
@@ -11896,6 +11896,7 @@
 F:	Documentation/core-api/symbol-namespaces.rst
  
 NTB AMD DRIVER
+M:	Sanjay R Mehta <sanju.mehta@amd.com>
 M:	Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
 L:	linux-ntb@googlegroups.com
 S:	Supported
@@ -195,27 +195,118 @@
 	return 0;
 }
  
-static int amd_link_is_up(struct amd_ntb_dev *ndev)
+static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev)
 {
-	if (!ndev->peer_sta)
-		return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
+	struct pci_dev *pdev = NULL;
+	struct pci_dev *pci_swds = NULL;
+	struct pci_dev *pci_swus = NULL;
+	u32 stat;
+	int rc;
  
-	if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
-		ndev->peer_sta = 0;
-		return 1;
+	if (ndev->ntb.topo == NTB_TOPO_SEC) {
+		/* Locate the pointer to Downstream Switch for this device */
+		pci_swds = pci_upstream_bridge(ndev->ntb.pdev);
+		if (pci_swds) {
+			/*
+			 * Locate the pointer to Upstream Switch for
+			 * the Downstream Switch.
+			 */
+			pci_swus = pci_upstream_bridge(pci_swds);
+			if (pci_swus) {
+				rc = pcie_capability_read_dword(pci_swus,
+								PCI_EXP_LNKCTL,
+								&stat);
+				if (rc)
+					return 0;
+			} else {
+				return 0;
+			}
+		} else {
+			return 0;
+		}
+	} else if (ndev->ntb.topo == NTB_TOPO_PRI) {
+		/*
+		 * For NTB primary, we simply read the Link Status and control
+		 * register of the NTB device itself.
+		 */
+		pdev = ndev->ntb.pdev;
+		rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat);
+		if (rc)
+			return 0;
+	} else {
+		/* Catch all for everything else */
+		return 0;
 	}
  
-	/* If peer_sta is reset or D0 event, the ISR has
-	 * started a timer to check link status of hardware.
-	 * So here just clear status bit. And if peer_sta is
-	 * D3 or PME_TO, D0/reset event will be happened when
-	 * system wakeup/poweron, so do nothing here.
+	ndev->lnk_sta = stat;
+
+	return 1;
+}
+
+static int amd_link_is_up(struct amd_ntb_dev *ndev)
+{
+	int ret;
+
+	/*
+	 * We consider the link to be up under two conditions:
+	 *
+	 *   - When a link-up event is received. This is indicated by
+	 *     AMD_LINK_UP_EVENT set in peer_sta.
+	 *   - When driver on both sides of the link have been loaded.
+	 *     This is indicated by bit 1 being set in the peer
+	 *     SIDEINFO register.
+	 *
+	 * This function should return 1 when the latter of the above
+	 * two conditions is true.
+	 *
+	 * Now consider the sequence of events - Link-Up event occurs,
+	 * then the peer side driver loads. In this case, we would have
+	 * received LINK_UP event and bit 1 of peer SIDEINFO is also
+	 * set. What happens now if the link goes down? Bit 1 of
+	 * peer SIDEINFO remains set, but LINK_DOWN bit is set in
+	 * peer_sta. So we should return 0 from this function. Not only
+	 * that, we clear bit 1 of peer SIDEINFO to 0, since the peer
+	 * side driver did not even get a chance to clear it before
+	 * the link went down. This can be the case of surprise link
+	 * removal.
+	 *
+	 * LINK_UP event will always occur before the peer side driver
+	 * gets loaded the very first time. So there can be a case when
+	 * the LINK_UP event has occurred, but the peer side driver hasn't
+	 * yet loaded. We return 0 in that case.
+	 *
+	 * There is also a special case when the primary side driver is
+	 * unloaded and then loaded again. Since there is no change in
+	 * the status of NTB secondary in this case, there is no Link-Up
+	 * or Link-Down notification received. We recognize this condition
+	 * with peer_sta being set to 0.
+	 *
+	 * If bit 1 of peer SIDEINFO register is not set, then we
+	 * simply return 0 irrespective of the link up or down status
+	 * set in peer_sta.
 	 */
-	if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
-		ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
-	else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
-		ndev->peer_sta = 0;
+	ret = amd_poll_link(ndev);
+	if (ret) {
+		/*
+		 * We need to check the below only for NTB primary. For NTB
+		 * secondary, simply checking the result of PSIDE_INFO
+		 * register will suffice.
+		 */
+		if (ndev->ntb.topo == NTB_TOPO_PRI) {
+			if ((ndev->peer_sta & AMD_LINK_UP_EVENT) ||
+			    (ndev->peer_sta == 0))
+				return ret;
+			else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) {
+				/* Clear peer sideinfo register */
+				amd_clear_side_info_reg(ndev, true);
  
+				return 0;
+			}
+		} else { /* NTB_TOPO_SEC */
+			return ret;
+		}
+	}
+
 	return 0;
 }
  
@@ -253,7 +344,6 @@
 {
 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 	void __iomem *mmio = ndev->self_mmio;
-	u32 ntb_ctl;
  
 	/* Enable event interrupt */
 	ndev->int_mask &= ~AMD_EVENT_INTMASK;
@@ -263,10 +353,6 @@
 		return -EINVAL;
 	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
  
-	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
-	ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
-	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
-
 	return 0;
 }
  
@@ -274,7 +360,6 @@
 {
 	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
 	void __iomem *mmio = ndev->self_mmio;
-	u32 ntb_ctl;
  
 	/* Disable event interrupt */
 	ndev->int_mask |= AMD_EVENT_INTMASK;
@@ -284,10 +369,6 @@
 		return -EINVAL;
 	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
  
-	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
-	ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
-	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
-
 	return 0;
 }
  
@@ -493,8 +574,6 @@
 	reg = readl(mmio + AMD_SMUACK_OFFSET);
 	reg |= bit;
 	writel(reg, mmio + AMD_SMUACK_OFFSET);
-
-	ndev->peer_sta |= bit;
 }
  
 static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
  
  
@@ -512,11 +591,17 @@
 	status &= AMD_EVENT_INTMASK;
 	switch (status) {
 	case AMD_PEER_FLUSH_EVENT:
+		ndev->peer_sta |= AMD_PEER_FLUSH_EVENT;
 		dev_info(dev, "Flush is done.\n");
 		break;
 	case AMD_PEER_RESET_EVENT:
-		amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
+	case AMD_LINK_DOWN_EVENT:
+		ndev->peer_sta |= status;
+		if (status == AMD_LINK_DOWN_EVENT)
+			ndev->peer_sta &= ~AMD_LINK_UP_EVENT;
  
+		amd_ack_smu(ndev, status);
+
 		/* link down first */
 		ntb_link_event(&ndev->ntb);
 		/* polling peer status */
@@ -526,7 +611,12 @@
 	case AMD_PEER_D3_EVENT:
 	case AMD_PEER_PMETO_EVENT:
 	case AMD_LINK_UP_EVENT:
-	case AMD_LINK_DOWN_EVENT:
+		ndev->peer_sta |= status;
+		if (status == AMD_LINK_UP_EVENT)
+			ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT;
+		else if (status == AMD_PEER_D3_EVENT)
+			ndev->peer_sta &= ~AMD_PEER_D0_EVENT;
+
 		amd_ack_smu(ndev, status);
  
 		/* link down */
@@ -540,6 +630,8 @@
 		if (status & 0x1)
 			dev_info(dev, "Wakeup is done.\n");
  
+		ndev->peer_sta |= AMD_PEER_D0_EVENT;
+		ndev->peer_sta &= ~AMD_PEER_D3_EVENT;
 		amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
  
 		/* start a timer to poll link status */
  
@@ -550,8 +642,41 @@
 		dev_info(dev, "event status = 0x%x.\n", status);
 		break;
 	}
+
+	/* Clear the interrupt status */
+	writel(status, mmio + AMD_INTSTAT_OFFSET);
 }
  
+static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec)
+{
+	struct device *dev = &ndev->ntb.pdev->dev;
+	u64 status;
+
+	status = amd_ntb_db_read(&ndev->ntb);
+
+	dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec);
+
+	/*
+	 * Since we had reserved highest order bit of DB for signaling peer of
+	 * a special event, this is the only status bit we should be concerned
+	 * here now.
+	 */
+	if (status & BIT(ndev->db_last_bit)) {
+		ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit));
+		/* send link down event notification */
+		ntb_link_event(&ndev->ntb);
+
+		/*
+		 * If we are here, that means the peer has signalled a special
+		 * event which notifies that the peer driver has been
+		 * un-loaded for some reason. Since there is a chance that the
+		 * peer will load its driver again sometime, we schedule link
+		 * polling routine.
+		 */
+		schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
+	}
+}
+
 static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
 {
 	dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
  
@@ -559,8 +684,10 @@
 	if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
 		amd_handle_event(ndev, vec);
  
-	if (vec < AMD_DB_CNT)
+	if (vec < AMD_DB_CNT) {
+		amd_handle_db_event(ndev, vec);
 		ntb_db_event(&ndev->ntb, vec);
+	}
  
 	return IRQ_HANDLED;
 }
  
  
  
  
@@ -842,26 +969,18 @@
 static int amd_poll_link(struct amd_ntb_dev *ndev)
 {
 	void __iomem *mmio = ndev->peer_mmio;
-	u32 reg, stat;
-	int rc;
+	u32 reg;
  
 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
-	reg &= NTB_LIN_STA_ACTIVE_BIT;
+	reg &= AMD_SIDE_READY;
  
 	dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
  
-	if (reg == ndev->cntl_sta)
-		return 0;
-
 	ndev->cntl_sta = reg;
  
-	rc = pci_read_config_dword(ndev->ntb.pdev,
-				   AMD_LINK_STATUS_OFFSET, &stat);
-	if (rc)
-		return 0;
-	ndev->lnk_sta = stat;
+	amd_ntb_get_link_status(ndev);
  
-	return 1;
+	return ndev->cntl_sta;
 }
  
 static void amd_link_hb(struct work_struct *work)
  
  
@@ -880,11 +999,16 @@
 	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
 }
  
-static void amd_init_side_info(struct amd_ntb_dev *ndev)
+static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
 {
-	void __iomem *mmio = ndev->self_mmio;
+	void __iomem *mmio = NULL;
 	unsigned int reg;
  
+	if (peer)
+		mmio = ndev->peer_mmio;
+	else
+		mmio = ndev->self_mmio;
+
 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 	if (!(reg & AMD_SIDE_READY)) {
 		reg |= AMD_SIDE_READY;
  
  
@@ -892,11 +1016,16 @@
 	}
 }
  
-static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
+static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
 {
-	void __iomem *mmio = ndev->self_mmio;
+	void __iomem *mmio = NULL;
 	unsigned int reg;
  
+	if (peer)
+		mmio = ndev->peer_mmio;
+	else
+		mmio = ndev->self_mmio;
+
 	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
 	if (reg & AMD_SIDE_READY) {
 		reg &= ~AMD_SIDE_READY;
@@ -905,6 +1034,30 @@
 	}
 }
  
+static void amd_init_side_info(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	u32 ntb_ctl;
+
+	amd_set_side_info_reg(ndev, false);
+
+	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+	ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
+	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+}
+
+static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
+{
+	void __iomem *mmio = ndev->self_mmio;
+	u32 ntb_ctl;
+
+	amd_clear_side_info_reg(ndev, false);
+
+	ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
+	ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
+	writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
+}
+
 static int amd_init_ntb(struct amd_ntb_dev *ndev)
 {
 	void __iomem *mmio = ndev->self_mmio;
@@ -935,8 +1088,6 @@
 		return -EINVAL;
 	}
  
-	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
-
 	/* Mask event interrupts */
 	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
  
@@ -957,6 +1108,7 @@
  
 static int amd_init_dev(struct amd_ntb_dev *ndev)
 {
+	void __iomem *mmio = ndev->self_mmio;
 	struct pci_dev *pdev;
 	int rc = 0;
  
  
@@ -977,7 +1129,26 @@
 	}
  
 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+	/*
+	 * We reserve the highest order bit of the DB register which will
+	 * be used to notify peer when the driver on this side is being
+	 * un-loaded.
+	 */
+	ndev->db_last_bit =
+			find_last_bit((unsigned long *)&ndev->db_valid_mask,
+				      hweight64(ndev->db_valid_mask));
+	writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET);
+	/*
+	 * Since now there is one less bit to account for, the DB count
+	 * and DB mask should be adjusted accordingly.
+	 */
+	ndev->db_count -= 1;
+	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
  
+	/* Enable Link-Up and Link-Down event interrupts */
+	ndev->int_mask &= ~(AMD_LINK_UP_EVENT | AMD_LINK_DOWN_EVENT);
+	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
+
 	return 0;
 }
  
  
  
@@ -1111,9 +1282,31 @@
 {
 	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
  
+	/*
+	 * Clear the READY bit in SIDEINFO register before sending DB event
+	 * to the peer. This will make sure that when the peer handles the
+	 * DB event, it correctly reads this bit as being 0.
+	 */
+	amd_deinit_side_info(ndev);
+	ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
 	ntb_unregister_device(&ndev->ntb);
 	ndev_deinit_debugfs(ndev);
+	amd_deinit_dev(ndev);
+	amd_ntb_deinit_pci(ndev);
+	kfree(ndev);
+}
+
+static void amd_ntb_pci_shutdown(struct pci_dev *pdev)
+{
+	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
+
+	/* Send link down notification */
+	ntb_link_event(&ndev->ntb);
+
 	amd_deinit_side_info(ndev);
+	ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
+	ntb_unregister_device(&ndev->ntb);
+	ndev_deinit_debugfs(ndev);
 	amd_deinit_dev(ndev);
 	amd_ntb_deinit_pci(ndev);
 	kfree(ndev);
@@ -1149,6 +1342,7 @@
 	.id_table	= amd_ntb_pci_tbl,
 	.probe		= amd_ntb_pci_probe,
 	.remove		= amd_ntb_pci_remove,
+	.shutdown	= amd_ntb_pci_shutdown,
 };
  
 static int __init amd_ntb_pci_driver_init(void)
@@ -53,11 +53,8 @@
 #include <linux/pci.h>
  
 #define AMD_LINK_HB_TIMEOUT	msecs_to_jiffies(1000)
-#define AMD_LINK_STATUS_OFFSET	0x68
-#define NTB_LIN_STA_ACTIVE_BIT	0x00000002
 #define NTB_LNK_STA_SPEED_MASK	0x000F0000
 #define NTB_LNK_STA_WIDTH_MASK	0x03F00000
-#define NTB_LNK_STA_ACTIVE(x)	(!!((x) & NTB_LIN_STA_ACTIVE_BIT))
 #define NTB_LNK_STA_SPEED(x)	(((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
 #define NTB_LNK_STA_WIDTH(x)	(((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
  
@@ -196,6 +193,7 @@
  
 	u64 db_valid_mask;
 	u64 db_mask;
+	u64 db_last_bit;
 	u32 int_mask;
  
 	struct msix_entry *msix;
@@ -217,6 +215,10 @@
  
 #define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb)
 #define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work)
+
+static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer);
+static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer);
+static int amd_poll_link(struct amd_ntb_dev *ndev);
  
 #endif
@@ -285,7 +285,7 @@
 	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
 		return -EINVAL;
  
-	if (xlate_pos < 12)
+	if (size != 0 && xlate_pos < 12)
 		return -EINVAL;
  
 	if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
@@ -481,70 +481,70 @@
 		return -ENOMEM;
  
 	out_offset = 0;
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "\nNTB QP stats:\n\n");
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_bytes - \t%llu\n", qp->rx_bytes);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_pkts - \t%llu\n", qp->rx_pkts);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_memcpy - \t%llu\n", qp->rx_memcpy);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_async - \t%llu\n", qp->rx_async);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_ring_empty - %llu\n", qp->rx_ring_empty);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_err_ver - \t%llu\n", qp->rx_err_ver);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_buff - \t0x%p\n", qp->rx_buff);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_index - \t%u\n", qp->rx_index);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_max_entry - \t%u\n", qp->rx_max_entry);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
  
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_bytes - \t%llu\n", qp->tx_bytes);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_pkts - \t%llu\n", qp->tx_pkts);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_memcpy - \t%llu\n", qp->tx_memcpy);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_async - \t%llu\n", qp->tx_async);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_ring_full - \t%llu\n", qp->tx_ring_full);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_mw - \t0x%p\n", qp->tx_mw);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_index (H) - \t%u\n", qp->tx_index);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "RRI (T) - \t%u\n",
 			       qp->remote_rx_info->entry);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "tx_max_entry - \t%u\n", qp->tx_max_entry);
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "free tx - \t%u\n",
 			       ntb_transport_tx_free_entry(qp));
  
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "\n");
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "Using TX DMA - \t%s\n",
 			       qp->tx_dma_chan ? "Yes" : "No");
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "Using RX DMA - \t%s\n",
 			       qp->rx_dma_chan ? "Yes" : "No");
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "QP Link - \t%s\n",
 			       qp->link_is_up ? "Up" : "Down");
-	out_offset += snprintf(buf + out_offset, out_count - out_offset,
+	out_offset += scnprintf(buf + out_offset, out_count - out_offset,
 			       "\n");
  
 	if (out_offset > out_count)
@@ -149,7 +149,8 @@
 	u64 outbuf_xlat;
 	resource_size_t outbuf_size;
 	void __iomem *outbuf;
-
+	phys_addr_t out_phys_addr;
+	dma_addr_t dma_dst_addr;
 	/* Inbound MW params */
 	dma_addr_t inbuf_xlat;
 	resource_size_t inbuf_size;
@@ -782,6 +783,10 @@
 	struct dmaengine_unmap_data *unmap;
 	struct device *dma_dev;
 	int try = 0, ret = 0;
+	struct perf_peer *peer = pthr->perf->test_peer;
+	void __iomem *vbase;
+	void __iomem *dst_vaddr;
+	dma_addr_t dst_dma_addr;
  
 	if (!use_dma) {
 		memcpy_toio(dst, src, len);
@@ -794,6 +799,10 @@
 				 offset_in_page(dst), len))
 		return -EIO;
  
+	vbase = peer->outbuf;
+	dst_vaddr = dst;
+	dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);
+
 	unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
 	if (!unmap)
 		return -ENOMEM;
@@ -807,8 +816,7 @@
 	}
 	unmap->to_cnt = 1;
  
-	unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
-		offset_in_page(dst), len, DMA_FROM_DEVICE);
+	unmap->addr[1] = dst_dma_addr;
 	if (dma_mapping_error(dma_dev, unmap->addr[1])) {
 		ret = -EIO;
 		goto err_free_resource;
@@ -865,6 +873,7 @@
 {
 	struct perf_ctx *perf = pthr->perf;
 	dma_cap_mask_t dma_mask;
+	struct perf_peer *peer = pthr->perf->test_peer;
  
 	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
 				 dev_to_node(&perf->ntb->dev));
  
  
  
@@ -882,15 +891,33 @@
 	if (!pthr->dma_chan) {
 		dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
 			pthr->tidx);
-		atomic_dec(&perf->tsync);
-		wake_up(&perf->twait);
-		kfree(pthr->src);
-		return -ENODEV;
+		goto err_free;
 	}
+	peer->dma_dst_addr =
+		dma_map_resource(pthr->dma_chan->device->dev,
+				 peer->out_phys_addr, peer->outbuf_size,
+				 DMA_FROM_DEVICE, 0);
+	if (dma_mapping_error(pthr->dma_chan->device->dev,
+			      peer->dma_dst_addr)) {
+		dev_err(pthr->dma_chan->device->dev, "%d: Failed to map DMA addr\n",
+			pthr->tidx);
+		peer->dma_dst_addr = 0;
+		dma_release_channel(pthr->dma_chan);
+		goto err_free;
+	}
+	dev_dbg(pthr->dma_chan->device->dev, "%d: Map MMIO %pa to DMA addr %pad\n",
+			pthr->tidx,
+			&peer->out_phys_addr,
+			&peer->dma_dst_addr);
  
 	atomic_set(&pthr->dma_sync, 0);
-
 	return 0;
+
+err_free:
+	atomic_dec(&perf->tsync);
+	wake_up(&perf->twait);
+	kfree(pthr->src);
+	return -ENODEV;
 }
  
 static int perf_run_test(struct perf_thread *pthr)
  
@@ -978,9 +1005,14 @@
 	 * We call it anyway just to be sure of the transfers completion.
 	 */
 	(void)dmaengine_terminate_sync(pthr->dma_chan);
+	if (pthr->perf->test_peer->dma_dst_addr)
+		dma_unmap_resource(pthr->dma_chan->device->dev,
+				   pthr->perf->test_peer->dma_dst_addr,
+				   pthr->perf->test_peer->outbuf_size,
+				   DMA_FROM_DEVICE, 0);
+	if (pthr->dma_chan)
+		dma_release_channel(pthr->dma_chan);
  
-	dma_release_channel(pthr->dma_chan);
-
 no_dma_notify:
 	atomic_dec(&perf->tsync);
 	wake_up(&perf->twait);
@@ -1195,6 +1227,9 @@
 			"\tOut buffer addr 0x%pK\n", peer->outbuf);
  
 		pos += scnprintf(buf + pos, buf_size - pos,
+			"\tOut buff phys addr %pa[p]\n", &peer->out_phys_addr);
+
+		pos += scnprintf(buf + pos, buf_size - pos,
 			"\tOut buffer size %pa\n", &peer->outbuf_size);
  
 		pos += scnprintf(buf + pos, buf_size - pos,
@@ -1387,6 +1422,8 @@
 					peer->outbuf_size);
 	if (!peer->outbuf)
 		return -ENOMEM;
+
+	peer->out_phys_addr = phys_addr;
  
 	if (max_mw_size && peer->outbuf_size > max_mw_size) {
 		peer->outbuf_size = max_mw_size;
@@ -678,19 +678,19 @@
 			 &inmw->dma_base);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Window Size    \t%pa[p]\n",
+			 "Window Size    \t%pap\n",
 			 &inmw->size);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Alignment      \t%pa[p]\n",
+			 "Alignment      \t%pap\n",
 			 &addr_align);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Size Alignment \t%pa[p]\n",
+			 "Size Alignment \t%pap\n",
 			 &size_align);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Size Max       \t%pa[p]\n",
+			 "Size Max       \t%pap\n",
 			 &size_max);
  
 	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
  
  
@@ -907,16 +907,16 @@
 			 "Virtual address     \t0x%pK\n", outmw->io_base);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Phys Address        \t%pa[p]\n", &map_base);
+			 "Phys Address        \t%pap\n", &map_base);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Mapping Size        \t%pa[p]\n", &map_size);
+			 "Mapping Size        \t%pap\n", &map_size);
  
 	off += scnprintf(buf + off, buf_size - off,
 			 "Translation Address \t0x%016llx\n", outmw->tr_base);
  
 	off += scnprintf(buf + off, buf_size - off,
-			 "Window Size         \t%pa[p]\n", &outmw->size);
+			 "Window Size         \t%pap\n", &outmw->size);
  
 	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 	kfree(buf);
...	...	@@ -11896,6 +11896,7 @@
11896	11896	F: Documentation/core-api/symbol-namespaces.rst
11897	11897
11898	11898	NTB AMD DRIVER
	11899	+M: Sanjay R Mehta <sanju.mehta@amd.com>
11899	11900	M: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
11900	11901	L: linux-ntb@googlegroups.com
11901	11902	S: Supported
...	...	@@ -195,27 +195,118 @@
195	195	return 0;
196	196	}
197	197
198		-static int amd_link_is_up(struct amd_ntb_dev *ndev)
	198	+static int amd_ntb_get_link_status(struct amd_ntb_dev *ndev)
199	199	{
200		- if (!ndev->peer_sta)
201		- return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
	200	+ struct pci_dev *pdev = NULL;
	201	+ struct pci_dev *pci_swds = NULL;
	202	+ struct pci_dev *pci_swus = NULL;
	203	+ u32 stat;
	204	+ int rc;
202	205
203		- if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
204		- ndev->peer_sta = 0;
205		- return 1;
	206	+ if (ndev->ntb.topo == NTB_TOPO_SEC) {
	207	+ /* Locate the pointer to Downstream Switch for this device */
	208	+ pci_swds = pci_upstream_bridge(ndev->ntb.pdev);
	209	+ if (pci_swds) {
	210	+ /*
	211	+ * Locate the pointer to Upstream Switch for
	212	+ * the Downstream Switch.
	213	+ */
	214	+ pci_swus = pci_upstream_bridge(pci_swds);
	215	+ if (pci_swus) {
	216	+ rc = pcie_capability_read_dword(pci_swus,
	217	+ PCI_EXP_LNKCTL,
	218	+ &stat);
	219	+ if (rc)
	220	+ return 0;
	221	+ } else {
	222	+ return 0;
	223	+ }
	224	+ } else {
	225	+ return 0;
	226	+ }
	227	+ } else if (ndev->ntb.topo == NTB_TOPO_PRI) {
	228	+ /*
	229	+ * For NTB primary, we simply read the Link Status and control
	230	+ * register of the NTB device itself.
	231	+ */
	232	+ pdev = ndev->ntb.pdev;
	233	+ rc = pcie_capability_read_dword(pdev, PCI_EXP_LNKCTL, &stat);
	234	+ if (rc)
	235	+ return 0;
	236	+ } else {
	237	+ /* Catch all for everything else */
	238	+ return 0;
206	239	}
207	240
208		- /* If peer_sta is reset or D0 event, the ISR has
209		- * started a timer to check link status of hardware.
210		- * So here just clear status bit. And if peer_sta is
211		- * D3 or PME_TO, D0/reset event will be happened when
212		- * system wakeup/poweron, so do nothing here.
	241	+ ndev->lnk_sta = stat;
	242	+
	243	+ return 1;
	244	+}
	245	+
	246	+static int amd_link_is_up(struct amd_ntb_dev *ndev)
	247	+{
	248	+ int ret;
	249	+
	250	+ /*
	251	+ * We consider the link to be up under two conditions:
	252	+ *
	253	+ * - When a link-up event is received. This is indicated by
	254	+ * AMD_LINK_UP_EVENT set in peer_sta.
	255	+ * - When driver on both sides of the link have been loaded.
	256	+ * This is indicated by bit 1 being set in the peer
	257	+ * SIDEINFO register.
	258	+ *
	259	+ * This function should return 1 when the latter of the above
	260	+ * two conditions is true.
	261	+ *
	262	+ * Now consider the sequence of events - Link-Up event occurs,
	263	+ * then the peer side driver loads. In this case, we would have
	264	+ * received LINK_UP event and bit 1 of peer SIDEINFO is also
	265	+ * set. What happens now if the link goes down? Bit 1 of
	266	+ * peer SIDEINFO remains set, but LINK_DOWN bit is set in
	267	+ * peer_sta. So we should return 0 from this function. Not only
	268	+ * that, we clear bit 1 of peer SIDEINFO to 0, since the peer
	269	+ * side driver did not even get a chance to clear it before
	270	+ * the link went down. This can be the case of surprise link
	271	+ * removal.
	272	+ *
	273	+ * LINK_UP event will always occur before the peer side driver
	274	+ * gets loaded the very first time. So there can be a case when
	275	+ * the LINK_UP event has occurred, but the peer side driver hasn't
	276	+ * yet loaded. We return 0 in that case.
	277	+ *
	278	+ * There is also a special case when the primary side driver is
	279	+ * unloaded and then loaded again. Since there is no change in
	280	+ * the status of NTB secondary in this case, there is no Link-Up
	281	+ * or Link-Down notification received. We recognize this condition
	282	+ * with peer_sta being set to 0.
	283	+ *
	284	+ * If bit 1 of peer SIDEINFO register is not set, then we
	285	+ * simply return 0 irrespective of the link up or down status
	286	+ * set in peer_sta.
213	287	*/
214		- if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
215		- ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
216		- else if (ndev->peer_sta & (AMD_PEER_D0_EVENT \| AMD_LINK_DOWN_EVENT))
217		- ndev->peer_sta = 0;
	288	+ ret = amd_poll_link(ndev);
	289	+ if (ret) {
	290	+ /*
	291	+ * We need to check the below only for NTB primary. For NTB
	292	+ * secondary, simply checking the result of PSIDE_INFO
	293	+ * register will suffice.
	294	+ */
	295	+ if (ndev->ntb.topo == NTB_TOPO_PRI) {
	296	+ if ((ndev->peer_sta & AMD_LINK_UP_EVENT) \|\|
	297	+ (ndev->peer_sta == 0))
	298	+ return ret;
	299	+ else if (ndev->peer_sta & AMD_LINK_DOWN_EVENT) {
	300	+ /* Clear peer sideinfo register */
	301	+ amd_clear_side_info_reg(ndev, true);
218	302
	303	+ return 0;
	304	+ }
	305	+ } else { /* NTB_TOPO_SEC */
	306	+ return ret;
	307	+ }
	308	+ }
	309	+
219	310	return 0;
220	311	}
221	312
...	...	@@ -253,7 +344,6 @@
253	344	{
254	345	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
255	346	void __iomem *mmio = ndev->self_mmio;
256		- u32 ntb_ctl;
257	347
258	348	/* Enable event interrupt */
259	349	ndev->int_mask &= ~AMD_EVENT_INTMASK;
...	...	@@ -263,10 +353,6 @@
263	353	return -EINVAL;
264	354	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
265	355
266		- ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
267		- ntb_ctl \|= (PMM_REG_CTL \| SMM_REG_CTL);
268		- writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
269		-
270	356	return 0;
271	357	}
272	358
...	...	@@ -274,7 +360,6 @@
274	360	{
275	361	struct amd_ntb_dev *ndev = ntb_ndev(ntb);
276	362	void __iomem *mmio = ndev->self_mmio;
277		- u32 ntb_ctl;
278	363
279	364	/* Disable event interrupt */
280	365	ndev->int_mask \|= AMD_EVENT_INTMASK;
...	...	@@ -284,10 +369,6 @@
284	369	return -EINVAL;
285	370	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
286	371
287		- ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
288		- ntb_ctl &= ~(PMM_REG_CTL \| SMM_REG_CTL);
289		- writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
290		-
291	372	return 0;
292	373	}
293	374
...	...	@@ -493,8 +574,6 @@
493	574	reg = readl(mmio + AMD_SMUACK_OFFSET);
494	575	reg \|= bit;
495	576	writel(reg, mmio + AMD_SMUACK_OFFSET);
496		-
497		- ndev->peer_sta \|= bit;
498	577	}
499	578
500	579	static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
501	580
502	581
...	...	@@ -512,11 +591,17 @@
512	591	status &= AMD_EVENT_INTMASK;
513	592	switch (status) {
514	593	case AMD_PEER_FLUSH_EVENT:
	594	+ ndev->peer_sta \|= AMD_PEER_FLUSH_EVENT;
515	595	dev_info(dev, "Flush is done.\n");
516	596	break;
517	597	case AMD_PEER_RESET_EVENT:
518		- amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
	598	+ case AMD_LINK_DOWN_EVENT:
	599	+ ndev->peer_sta \|= status;
	600	+ if (status == AMD_LINK_DOWN_EVENT)
	601	+ ndev->peer_sta &= ~AMD_LINK_UP_EVENT;
519	602
	603	+ amd_ack_smu(ndev, status);
	604	+
520	605	/* link down first */
521	606	ntb_link_event(&ndev->ntb);
522	607	/* polling peer status */
...	...	@@ -526,7 +611,12 @@
526	611	case AMD_PEER_D3_EVENT:
527	612	case AMD_PEER_PMETO_EVENT:
528	613	case AMD_LINK_UP_EVENT:
529		- case AMD_LINK_DOWN_EVENT:
	614	+ ndev->peer_sta \|= status;
	615	+ if (status == AMD_LINK_UP_EVENT)
	616	+ ndev->peer_sta &= ~AMD_LINK_DOWN_EVENT;
	617	+ else if (status == AMD_PEER_D3_EVENT)
	618	+ ndev->peer_sta &= ~AMD_PEER_D0_EVENT;
	619	+
530	620	amd_ack_smu(ndev, status);
531	621
532	622	/* link down */
...	...	@@ -540,6 +630,8 @@
540	630	if (status & 0x1)
541	631	dev_info(dev, "Wakeup is done.\n");
542	632
	633	+ ndev->peer_sta \|= AMD_PEER_D0_EVENT;
	634	+ ndev->peer_sta &= ~AMD_PEER_D3_EVENT;
543	635	amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
544	636
545	637	/* start a timer to poll link status */
546	638
...	...	@@ -550,8 +642,41 @@
550	642	dev_info(dev, "event status = 0x%x.\n", status);
551	643	break;
552	644	}
	645	+
	646	+ /* Clear the interrupt status */
	647	+ writel(status, mmio + AMD_INTSTAT_OFFSET);
553	648	}
554	649
	650	+static void amd_handle_db_event(struct amd_ntb_dev *ndev, int vec)
	651	+{
	652	+ struct device *dev = &ndev->ntb.pdev->dev;
	653	+ u64 status;
	654	+
	655	+ status = amd_ntb_db_read(&ndev->ntb);
	656	+
	657	+ dev_dbg(dev, "status = 0x%llx and vec = %d\n", status, vec);
	658	+
	659	+ /*
	660	+ * Since we had reserved highest order bit of DB for signaling peer of
	661	+ * a special event, this is the only status bit we should be concerned
	662	+ * here now.
	663	+ */
	664	+ if (status & BIT(ndev->db_last_bit)) {
	665	+ ntb_db_clear(&ndev->ntb, BIT(ndev->db_last_bit));
	666	+ /* send link down event notification */
	667	+ ntb_link_event(&ndev->ntb);
	668	+
	669	+ /*
	670	+ * If we are here, that means the peer has signalled a special
	671	+ * event which notifies that the peer driver has been
	672	+ * un-loaded for some reason. Since there is a chance that the
	673	+ * peer will load its driver again sometime, we schedule link
	674	+ * polling routine.
	675	+ */
	676	+ schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
	677	+ }
	678	+}
	679	+
555	680	static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
556	681	{
557	682	dev_dbg(&ndev->ntb.pdev->dev, "vec %d\n", vec);
558	683
...	...	@@ -559,8 +684,10 @@
559	684	if (vec > (AMD_DB_CNT - 1) \|\| (ndev->msix_vec_count == 1))
560	685	amd_handle_event(ndev, vec);
561	686
562		- if (vec < AMD_DB_CNT)
	687	+ if (vec < AMD_DB_CNT) {
	688	+ amd_handle_db_event(ndev, vec);
563	689	ntb_db_event(&ndev->ntb, vec);
	690	+ }
564	691
565	692	return IRQ_HANDLED;
566	693	}
567	694
568	695
569	696
570	697
...	...	@@ -842,26 +969,18 @@
842	969	static int amd_poll_link(struct amd_ntb_dev *ndev)
843	970	{
844	971	void __iomem *mmio = ndev->peer_mmio;
845		- u32 reg, stat;
846		- int rc;
	972	+ u32 reg;
847	973
848	974	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
849		- reg &= NTB_LIN_STA_ACTIVE_BIT;
	975	+ reg &= AMD_SIDE_READY;
850	976
851	977	dev_dbg(&ndev->ntb.pdev->dev, "%s: reg_val = 0x%x.\n", __func__, reg);
852	978
853		- if (reg == ndev->cntl_sta)
854		- return 0;
855		-
856	979	ndev->cntl_sta = reg;
857	980
858		- rc = pci_read_config_dword(ndev->ntb.pdev,
859		- AMD_LINK_STATUS_OFFSET, &stat);
860		- if (rc)
861		- return 0;
862		- ndev->lnk_sta = stat;
	981	+ amd_ntb_get_link_status(ndev);
863	982
864		- return 1;
	983	+ return ndev->cntl_sta;
865	984	}
866	985
867	986	static void amd_link_hb(struct work_struct *work)
868	987
869	988
...	...	@@ -880,11 +999,16 @@
880	999	return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
881	1000	}
882	1001
883		-static void amd_init_side_info(struct amd_ntb_dev *ndev)
	1002	+static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
884	1003	{
885		- void __iomem *mmio = ndev->self_mmio;
	1004	+ void __iomem *mmio = NULL;
886	1005	unsigned int reg;
887	1006
	1007	+ if (peer)
	1008	+ mmio = ndev->peer_mmio;
	1009	+ else
	1010	+ mmio = ndev->self_mmio;
	1011	+
888	1012	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
889	1013	if (!(reg & AMD_SIDE_READY)) {
890	1014	reg \|= AMD_SIDE_READY;
891	1015
892	1016
...	...	@@ -892,11 +1016,16 @@
892	1016	}
893	1017	}
894	1018
895		-static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
	1019	+static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer)
896	1020	{
897		- void __iomem *mmio = ndev->self_mmio;
	1021	+ void __iomem *mmio = NULL;
898	1022	unsigned int reg;
899	1023
	1024	+ if (peer)
	1025	+ mmio = ndev->peer_mmio;
	1026	+ else
	1027	+ mmio = ndev->self_mmio;
	1028	+
900	1029	reg = readl(mmio + AMD_SIDEINFO_OFFSET);
901	1030	if (reg & AMD_SIDE_READY) {
902	1031	reg &= ~AMD_SIDE_READY;
...	...	@@ -905,6 +1034,30 @@
905	1034	}
906	1035	}
907	1036
	1037	+static void amd_init_side_info(struct amd_ntb_dev *ndev)
	1038	+{
	1039	+ void __iomem *mmio = ndev->self_mmio;
	1040	+ u32 ntb_ctl;
	1041	+
	1042	+ amd_set_side_info_reg(ndev, false);
	1043	+
	1044	+ ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
	1045	+ ntb_ctl \|= (PMM_REG_CTL \| SMM_REG_CTL);
	1046	+ writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
	1047	+}
	1048	+
	1049	+static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
	1050	+{
	1051	+ void __iomem *mmio = ndev->self_mmio;
	1052	+ u32 ntb_ctl;
	1053	+
	1054	+ amd_clear_side_info_reg(ndev, false);
	1055	+
	1056	+ ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
	1057	+ ntb_ctl &= ~(PMM_REG_CTL \| SMM_REG_CTL);
	1058	+ writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
	1059	+}
	1060	+
908	1061	static int amd_init_ntb(struct amd_ntb_dev *ndev)
909	1062	{
910	1063	void __iomem *mmio = ndev->self_mmio;
...	...	@@ -935,8 +1088,6 @@
935	1088	return -EINVAL;
936	1089	}
937	1090
938		- ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
939		-
940	1091	/* Mask event interrupts */
941	1092	writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
942	1093
...	...	@@ -957,6 +1108,7 @@
957	1108
958	1109	static int amd_init_dev(struct amd_ntb_dev *ndev)
959	1110	{
	1111	+ void __iomem *mmio = ndev->self_mmio;
960	1112	struct pci_dev *pdev;
961	1113	int rc = 0;
962	1114
963	1115
...	...	@@ -977,7 +1129,26 @@
977	1129	}
978	1130
979	1131	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
	1132	+ /*
	1133	+ * We reserve the highest order bit of the DB register which will
	1134	+ * be used to notify peer when the driver on this side is being
	1135	+ * un-loaded.
	1136	+ */
	1137	+ ndev->db_last_bit =
	1138	+ find_last_bit((unsigned long *)&ndev->db_valid_mask,
	1139	+ hweight64(ndev->db_valid_mask));
	1140	+ writew((u16)~BIT(ndev->db_last_bit), mmio + AMD_DBMASK_OFFSET);
	1141	+ /*
	1142	+ * Since now there is one less bit to account for, the DB count
	1143	+ * and DB mask should be adjusted accordingly.
	1144	+ */
	1145	+ ndev->db_count -= 1;
	1146	+ ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
980	1147
	1148	+ /* Enable Link-Up and Link-Down event interrupts */
	1149	+ ndev->int_mask &= ~(AMD_LINK_UP_EVENT \| AMD_LINK_DOWN_EVENT);
	1150	+ writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
	1151	+
981	1152	return 0;
982	1153	}
983	1154
984	1155
985	1156
...	...	@@ -1111,9 +1282,31 @@
1111	1282	{
1112	1283	struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1113	1284
	1285	+ /*
	1286	+ * Clear the READY bit in SIDEINFO register before sending DB event
	1287	+ * to the peer. This will make sure that when the peer handles the
	1288	+ * DB event, it correctly reads this bit as being 0.
	1289	+ */
	1290	+ amd_deinit_side_info(ndev);
	1291	+ ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
1114	1292	ntb_unregister_device(&ndev->ntb);
1115	1293	ndev_deinit_debugfs(ndev);
	1294	+ amd_deinit_dev(ndev);
	1295	+ amd_ntb_deinit_pci(ndev);
	1296	+ kfree(ndev);
	1297	+}
	1298	+
	1299	+static void amd_ntb_pci_shutdown(struct pci_dev *pdev)
	1300	+{
	1301	+ struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
	1302	+
	1303	+ /* Send link down notification */
	1304	+ ntb_link_event(&ndev->ntb);
	1305	+
1116	1306	amd_deinit_side_info(ndev);
	1307	+ ntb_peer_db_set(&ndev->ntb, BIT_ULL(ndev->db_last_bit));
	1308	+ ntb_unregister_device(&ndev->ntb);
	1309	+ ndev_deinit_debugfs(ndev);
1117	1310	amd_deinit_dev(ndev);
1118	1311	amd_ntb_deinit_pci(ndev);
1119	1312	kfree(ndev);
...	...	@@ -1149,6 +1342,7 @@
1149	1342	.id_table = amd_ntb_pci_tbl,
1150	1343	.probe = amd_ntb_pci_probe,
1151	1344	.remove = amd_ntb_pci_remove,
	1345	+ .shutdown = amd_ntb_pci_shutdown,
1152	1346	};
1153	1347
1154	1348	static int __init amd_ntb_pci_driver_init(void)
...	...	@@ -53,11 +53,8 @@
53	53	#include <linux/pci.h>
54	54
55	55	#define AMD_LINK_HB_TIMEOUT msecs_to_jiffies(1000)
56		-#define AMD_LINK_STATUS_OFFSET 0x68
57		-#define NTB_LIN_STA_ACTIVE_BIT 0x00000002
58	56	#define NTB_LNK_STA_SPEED_MASK 0x000F0000
59	57	#define NTB_LNK_STA_WIDTH_MASK 0x03F00000
60		-#define NTB_LNK_STA_ACTIVE(x) (!!((x) & NTB_LIN_STA_ACTIVE_BIT))
61	58	#define NTB_LNK_STA_SPEED(x) (((x) & NTB_LNK_STA_SPEED_MASK) >> 16)
62	59	#define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 20)
63	60
...	...	@@ -196,6 +193,7 @@
196	193
197	194	u64 db_valid_mask;
198	195	u64 db_mask;
	196	+ u64 db_last_bit;
199	197	u32 int_mask;
200	198
201	199	struct msix_entry *msix;
...	...	@@ -217,6 +215,10 @@
217	215
218	216	#define ntb_ndev(__ntb) container_of(__ntb, struct amd_ntb_dev, ntb)
219	217	#define hb_ndev(__work) container_of(__work, struct amd_ntb_dev, hb_timer.work)
	218	+
	219	+static void amd_set_side_info_reg(struct amd_ntb_dev *ndev, bool peer);
	220	+static void amd_clear_side_info_reg(struct amd_ntb_dev *ndev, bool peer);
	221	+static int amd_poll_link(struct amd_ntb_dev *ndev);
220	222
221	223	#endif
...	...	@@ -285,7 +285,7 @@
285	285	if (widx >= switchtec_ntb_mw_count(ntb, pidx))
286	286	return -EINVAL;
287	287
288		- if (xlate_pos < 12)
	288	+ if (size != 0 && xlate_pos < 12)
289	289	return -EINVAL;
290	290
291	291	if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
...	...	@@ -481,70 +481,70 @@
481	481	return -ENOMEM;
482	482
483	483	out_offset = 0;
484		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	484	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
485	485	"\nNTB QP stats:\n\n");
486		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	486	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
487	487	"rx_bytes - \t%llu\n", qp->rx_bytes);
488		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	488	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
489	489	"rx_pkts - \t%llu\n", qp->rx_pkts);
490		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	490	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
491	491	"rx_memcpy - \t%llu\n", qp->rx_memcpy);
492		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	492	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
493	493	"rx_async - \t%llu\n", qp->rx_async);
494		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	494	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
495	495	"rx_ring_empty - %llu\n", qp->rx_ring_empty);
496		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	496	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
497	497	"rx_err_no_buf - %llu\n", qp->rx_err_no_buf);
498		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	498	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
499	499	"rx_err_oflow - \t%llu\n", qp->rx_err_oflow);
500		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	500	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
501	501	"rx_err_ver - \t%llu\n", qp->rx_err_ver);
502		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	502	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
503	503	"rx_buff - \t0x%p\n", qp->rx_buff);
504		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	504	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
505	505	"rx_index - \t%u\n", qp->rx_index);
506		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	506	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
507	507	"rx_max_entry - \t%u\n", qp->rx_max_entry);
508		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	508	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
509	509	"rx_alloc_entry - \t%u\n\n", qp->rx_alloc_entry);
510	510
511		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	511	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
512	512	"tx_bytes - \t%llu\n", qp->tx_bytes);
513		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	513	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
514	514	"tx_pkts - \t%llu\n", qp->tx_pkts);
515		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	515	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
516	516	"tx_memcpy - \t%llu\n", qp->tx_memcpy);
517		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	517	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
518	518	"tx_async - \t%llu\n", qp->tx_async);
519		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	519	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
520	520	"tx_ring_full - \t%llu\n", qp->tx_ring_full);
521		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	521	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
522	522	"tx_err_no_buf - %llu\n", qp->tx_err_no_buf);
523		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	523	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
524	524	"tx_mw - \t0x%p\n", qp->tx_mw);
525		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	525	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
526	526	"tx_index (H) - \t%u\n", qp->tx_index);
527		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	527	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
528	528	"RRI (T) - \t%u\n",
529	529	qp->remote_rx_info->entry);
530		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	530	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
531	531	"tx_max_entry - \t%u\n", qp->tx_max_entry);
532		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	532	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
533	533	"free tx - \t%u\n",
534	534	ntb_transport_tx_free_entry(qp));
535	535
536		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	536	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
537	537	"\n");
538		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	538	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
539	539	"Using TX DMA - \t%s\n",
540	540	qp->tx_dma_chan ? "Yes" : "No");
541		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	541	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
542	542	"Using RX DMA - \t%s\n",
543	543	qp->rx_dma_chan ? "Yes" : "No");
544		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	544	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
545	545	"QP Link - \t%s\n",
546	546	qp->link_is_up ? "Up" : "Down");
547		- out_offset += snprintf(buf + out_offset, out_count - out_offset,
	547	+ out_offset += scnprintf(buf + out_offset, out_count - out_offset,
548	548	"\n");
549	549
550	550	if (out_offset > out_count)
...	...	@@ -149,7 +149,8 @@
149	149	u64 outbuf_xlat;
150	150	resource_size_t outbuf_size;
151	151	void __iomem *outbuf;
152		-
	152	+ phys_addr_t out_phys_addr;
	153	+ dma_addr_t dma_dst_addr;
153	154	/* Inbound MW params */
154	155	dma_addr_t inbuf_xlat;
155	156	resource_size_t inbuf_size;
...	...	@@ -782,6 +783,10 @@
782	783	struct dmaengine_unmap_data *unmap;
783	784	struct device *dma_dev;
784	785	int try = 0, ret = 0;
	786	+ struct perf_peer *peer = pthr->perf->test_peer;
	787	+ void __iomem *vbase;
	788	+ void __iomem *dst_vaddr;
	789	+ dma_addr_t dst_dma_addr;
785	790
786	791	if (!use_dma) {
787	792	memcpy_toio(dst, src, len);
...	...	@@ -794,6 +799,10 @@
794	799	offset_in_page(dst), len))
795	800	return -EIO;
796	801
	802	+ vbase = peer->outbuf;
	803	+ dst_vaddr = dst;
	804	+ dst_dma_addr = peer->dma_dst_addr + (dst_vaddr - vbase);
	805	+
797	806	unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
798	807	if (!unmap)
799	808	return -ENOMEM;
...	...	@@ -807,8 +816,7 @@
807	816	}
808	817	unmap->to_cnt = 1;
809	818
810		- unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
811		- offset_in_page(dst), len, DMA_FROM_DEVICE);
	819	+ unmap->addr[1] = dst_dma_addr;
812	820	if (dma_mapping_error(dma_dev, unmap->addr[1])) {
813	821	ret = -EIO;
814	822	goto err_free_resource;
...	...	@@ -865,6 +873,7 @@
865	873	{
866	874	struct perf_ctx *perf = pthr->perf;
867	875	dma_cap_mask_t dma_mask;
	876	+ struct perf_peer *peer = pthr->perf->test_peer;
868	877
869	878	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
870	879	dev_to_node(&perf->ntb->dev));
871	880
872	881
873	882
...	...	@@ -882,15 +891,33 @@
882	891	if (!pthr->dma_chan) {
883	892	dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
884	893	pthr->tidx);
885		- atomic_dec(&perf->tsync);
886		- wake_up(&perf->twait);
887		- kfree(pthr->src);
888		- return -ENODEV;
	894	+ goto err_free;
889	895	}
	896	+ peer->dma_dst_addr =
	897	+ dma_map_resource(pthr->dma_chan->device->dev,
	898	+ peer->out_phys_addr, peer->outbuf_size,
	899	+ DMA_FROM_DEVICE, 0);
	900	+ if (dma_mapping_error(pthr->dma_chan->device->dev,
	901	+ peer->dma_dst_addr)) {
	902	+ dev_err(pthr->dma_chan->device->dev, "%d: Failed to map DMA addr\n",
	903	+ pthr->tidx);
	904	+ peer->dma_dst_addr = 0;
	905	+ dma_release_channel(pthr->dma_chan);
	906	+ goto err_free;
	907	+ }
	908	+ dev_dbg(pthr->dma_chan->device->dev, "%d: Map MMIO %pa to DMA addr %pad\n",
	909	+ pthr->tidx,
	910	+ &peer->out_phys_addr,
	911	+ &peer->dma_dst_addr);
890	912
891	913	atomic_set(&pthr->dma_sync, 0);
892		-
893	914	return 0;
	915	+
	916	+err_free:
	917	+ atomic_dec(&perf->tsync);
	918	+ wake_up(&perf->twait);
	919	+ kfree(pthr->src);
	920	+ return -ENODEV;
894	921	}
895	922
896	923	static int perf_run_test(struct perf_thread *pthr)
897	924
...	...	@@ -978,9 +1005,14 @@
978	1005	* We call it anyway just to be sure of the transfers completion.
979	1006	*/
980	1007	(void)dmaengine_terminate_sync(pthr->dma_chan);
	1008	+ if (pthr->perf->test_peer->dma_dst_addr)
	1009	+ dma_unmap_resource(pthr->dma_chan->device->dev,
	1010	+ pthr->perf->test_peer->dma_dst_addr,
	1011	+ pthr->perf->test_peer->outbuf_size,
	1012	+ DMA_FROM_DEVICE, 0);
	1013	+ if (pthr->dma_chan)
	1014	+ dma_release_channel(pthr->dma_chan);
981	1015
982		- dma_release_channel(pthr->dma_chan);
983		-
984	1016	no_dma_notify:
985	1017	atomic_dec(&perf->tsync);
986	1018	wake_up(&perf->twait);
...	...	@@ -1195,6 +1227,9 @@
1195	1227	"\tOut buffer addr 0x%pK\n", peer->outbuf);
1196	1228
1197	1229	pos += scnprintf(buf + pos, buf_size - pos,
	1230	+ "\tOut buff phys addr %pa[p]\n", &peer->out_phys_addr);
	1231	+
	1232	+ pos += scnprintf(buf + pos, buf_size - pos,
1198	1233	"\tOut buffer size %pa\n", &peer->outbuf_size);
1199	1234
1200	1235	pos += scnprintf(buf + pos, buf_size - pos,
...	...	@@ -1387,6 +1422,8 @@
1387	1422	peer->outbuf_size);
1388	1423	if (!peer->outbuf)
1389	1424	return -ENOMEM;
	1425	+
	1426	+ peer->out_phys_addr = phys_addr;
1390	1427
1391	1428	if (max_mw_size && peer->outbuf_size > max_mw_size) {
1392	1429	peer->outbuf_size = max_mw_size;
...	...	@@ -678,19 +678,19 @@
678	678	&inmw->dma_base);
679	679
680	680	off += scnprintf(buf + off, buf_size - off,
681		- "Window Size \t%pa[p]\n",
	681	+ "Window Size \t%pap\n",
682	682	&inmw->size);
683	683
684	684	off += scnprintf(buf + off, buf_size - off,
685		- "Alignment \t%pa[p]\n",
	685	+ "Alignment \t%pap\n",
686	686	&addr_align);
687	687
688	688	off += scnprintf(buf + off, buf_size - off,
689		- "Size Alignment \t%pa[p]\n",
	689	+ "Size Alignment \t%pap\n",
690	690	&size_align);
691	691
692	692	off += scnprintf(buf + off, buf_size - off,
693		- "Size Max \t%pa[p]\n",
	693	+ "Size Max \t%pap\n",
694	694	&size_max);
695	695
696	696	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
697	697
698	698
...	...	@@ -907,16 +907,16 @@
907	907	"Virtual address \t0x%pK\n", outmw->io_base);
908	908
909	909	off += scnprintf(buf + off, buf_size - off,
910		- "Phys Address \t%pa[p]\n", &map_base);
	910	+ "Phys Address \t%pap\n", &map_base);
911	911
912	912	off += scnprintf(buf + off, buf_size - off,
913		- "Mapping Size \t%pa[p]\n", &map_size);
	913	+ "Mapping Size \t%pap\n", &map_size);
914	914
915	915	off += scnprintf(buf + off, buf_size - off,
916	916	"Translation Address \t0x%016llx\n", outmw->tr_base);
917	917
918	918	off += scnprintf(buf + off, buf_size - off,
919		- "Window Size \t%pa[p]\n", &outmw->size);
	919	+ "Window Size \t%pap\n", &outmw->size);
920	920
921	921	ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
922	922	kfree(buf);