iommu/arm-smmu: Don't use VMIDs for stage-1 translations

Although permitted by the architecture, using VMIDs for stage-1 translations causes a complete nightmare for hypervisors, who end up having to virtualise the VMID space across VMs, which may be using multiple VMIDs each. To make life easier for hypervisors (which might just decide not to support this VMID virtualisation), this patch reworks the stage-1 context-bank TLB invalidation so that: - Stage-1 mappings are marked non-global in the ptes - Each Stage-1 context-bank is assigned an ASID in TTBR0 - VMID 0 is reserved for Stage-1 context-banks This allows the hypervisor to overwrite the Stage-1 VMID in the CBAR when trapping the write from the guest. Signed-off-by: Will Deacon <will.deacon@arm.com> Signed-off-by: Joerg Roedel <joro@8bytes.org>

iommu/arm-smmu: Don't use VMIDs for stage-1 translations
Although permitted by the architecture, using VMIDs for stage-1 translations causes a complete nightmare for hypervisors, who end up having to virtualise the VMID space across VMs, which may be using multiple VMIDs each. To make life easier for hypervisors (which might just decide not to support this VMID virtualisation), this patch reworks the stage-1 context-bank TLB invalidation so that: - Stage-1 mappings are marked non-global in the ptes - Each Stage-1 context-bank is assigned an ASID in TTBR0 - VMID 0 is reserved for Stage-1 context-banks This allows the hypervisor to overwrite the Stage-1 VMID in the CBAR when trapping the write from the guest. Signed-off-by: Will Deacon <will.deacon@arm.com> Signed-off-by: Joerg Roedel <joro@8bytes.org>
Will Deacon · Joerg Roedel
1 parent adaba32091
Showing 1 changed file with 46 additions and 13 deletions Side-by-side Diff
drivers/iommu/arm-smmu.c
@@ -87,6 +87,7 @@
 #define ARM_SMMU_PTE_AP_UNPRIV		(((pteval_t)1) << 6)
 #define ARM_SMMU_PTE_AP_RDONLY		(((pteval_t)2) << 6)
 #define ARM_SMMU_PTE_ATTRINDX_SHIFT	2
+#define ARM_SMMU_PTE_nG			(((pteval_t)1) << 11)
  
 /* Stage-2 PTE */
 #define ARM_SMMU_PTE_HAP_FAULT		(((pteval_t)0) << 6)
@@ -223,6 +224,7 @@
 #define ARM_SMMU_CB_FAR_LO		0x60
 #define ARM_SMMU_CB_FAR_HI		0x64
 #define ARM_SMMU_CB_FSYNR0		0x68
+#define ARM_SMMU_CB_S1_TLBIASID		0x610
  
 #define SCTLR_S1_ASIDPNE		(1 << 12)
 #define SCTLR_CFCFG			(1 << 7)
@@ -282,6 +284,8 @@
 #define TTBCR2_ADDR_44			4
 #define TTBCR2_ADDR_48			5
  
+#define TTBRn_HI_ASID_SHIFT		16
+
 #define MAIR_ATTR_SHIFT(n)		((n) << 3)
 #define MAIR_ATTR_MASK			0xff
 #define MAIR_ATTR_DEVICE		0x04
@@ -533,6 +537,23 @@
 	}
 }
  
+static void arm_smmu_tlb_inv_context(struct arm_smmu_cfg *cfg)
+{
+	struct arm_smmu_device *smmu = cfg->smmu;
+	void __iomem *base = ARM_SMMU_GR0(smmu);
+	bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
+
+	if (stage1) {
+		base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
+		writel_relaxed(cfg->vmid, base + ARM_SMMU_CB_S1_TLBIASID);
+	} else {
+		base = ARM_SMMU_GR0(smmu);
+		writel_relaxed(cfg->vmid, base + ARM_SMMU_GR0_TLBIVMID);
+	}
+
+	arm_smmu_tlb_sync(smmu);
+}
+
 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
 {
 	int flags, ret;
  
@@ -621,14 +642,15 @@
 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
  
 	/* CBAR */
-	reg = root_cfg->cbar |
-	      (root_cfg->vmid << CBAR_VMID_SHIFT);
+	reg = root_cfg->cbar;
 	if (smmu->version == 1)
 	      reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
  
 	/* Use the weakest memory type, so it is overridden by the pte */
 	if (stage1)
 		reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
+	else
+		reg |= root_cfg->vmid << CBAR_VMID_SHIFT;
 	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
  
 	if (smmu->version > 1) {
@@ -692,6 +714,8 @@
 	reg = __pa(root_cfg->pgd);
 	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
 	reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
+	if (stage1)
+		reg |= root_cfg->vmid << TTBRn_HI_ASID_SHIFT;
 	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI);
  
 	/*
@@ -747,10 +771,6 @@
 		writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
 	}
  
-	/* Nuke the TLB */
-	writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
-	arm_smmu_tlb_sync(smmu);
-
 	/* SCTLR */
 	reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP;
 	if (stage1)
@@ -787,7 +807,8 @@
 		return -ENODEV;
 	}
  
-	ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 0, ARM_SMMU_NUM_VMIDS);
+	/* VMID zero is reserved for stage-1 mappings */
+	ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 1, ARM_SMMU_NUM_VMIDS);
 	if (IS_ERR_VALUE(ret))
 		return ret;
  
  
@@ -847,11 +868,17 @@
 	struct arm_smmu_domain *smmu_domain = domain->priv;
 	struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
 	struct arm_smmu_device *smmu = root_cfg->smmu;
+	void __iomem *cb_base;
 	int irq;
  
 	if (!smmu)
 		return;
  
+	/* Disable the context bank and nuke the TLB before freeing it. */
+	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
+	writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
+	arm_smmu_tlb_inv_context(root_cfg);
+
 	if (root_cfg->irptndx != -1) {
 		irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx];
 		free_irq(irq, domain);
@@ -956,6 +983,11 @@
 static void arm_smmu_domain_destroy(struct iommu_domain *domain)
 {
 	struct arm_smmu_domain *smmu_domain = domain->priv;
+
+	/*
+	 * Free the domain resources. We assume that all devices have
+	 * already been detached.
+	 */
 	arm_smmu_destroy_domain_context(domain);
 	arm_smmu_free_pgtables(smmu_domain);
 	kfree(smmu_domain);
@@ -1196,7 +1228,7 @@
 	}
  
 	if (stage == 1) {
-		pteval |= ARM_SMMU_PTE_AP_UNPRIV;
+		pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG;
 		if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
 			pteval |= ARM_SMMU_PTE_AP_RDONLY;
  
  
@@ -1412,13 +1444,9 @@
 {
 	int ret;
 	struct arm_smmu_domain *smmu_domain = domain->priv;
-	struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
-	struct arm_smmu_device *smmu = root_cfg->smmu;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
  
 	ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0);
-	writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
-	arm_smmu_tlb_sync(smmu);
+	arm_smmu_tlb_inv_context(&smmu_domain->root_cfg);
 	return ret ? ret : size;
 }
  
@@ -1541,6 +1569,7 @@
 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 {
 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+	void __iomem *sctlr_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB_SCTLR;
 	int i = 0;
 	u32 scr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
  
@@ -1549,6 +1578,10 @@
 		writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i));
 		writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i));
 	}
+
+	/* Make sure all context banks are disabled */
+	for (i = 0; i < smmu->num_context_banks; ++i)
+		writel_relaxed(0, sctlr_base + ARM_SMMU_CB(smmu, i));
  
 	/* Invalidate the TLB, just in case */
 	writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL);
...	...	@@ -87,6 +87,7 @@
87	87	#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6)
88	88	#define ARM_SMMU_PTE_AP_RDONLY (((pteval_t)2) << 6)
89	89	#define ARM_SMMU_PTE_ATTRINDX_SHIFT 2
	90	+#define ARM_SMMU_PTE_nG (((pteval_t)1) << 11)
90	91
91	92	/* Stage-2 PTE */
92	93	#define ARM_SMMU_PTE_HAP_FAULT (((pteval_t)0) << 6)
...	...	@@ -223,6 +224,7 @@
223	224	#define ARM_SMMU_CB_FAR_LO 0x60
224	225	#define ARM_SMMU_CB_FAR_HI 0x64
225	226	#define ARM_SMMU_CB_FSYNR0 0x68
	227	+#define ARM_SMMU_CB_S1_TLBIASID 0x610
226	228
227	229	#define SCTLR_S1_ASIDPNE (1 << 12)
228	230	#define SCTLR_CFCFG (1 << 7)
...	...	@@ -282,6 +284,8 @@
282	284	#define TTBCR2_ADDR_44 4
283	285	#define TTBCR2_ADDR_48 5
284	286
	287	+#define TTBRn_HI_ASID_SHIFT 16
	288	+
285	289	#define MAIR_ATTR_SHIFT(n) ((n) << 3)
286	290	#define MAIR_ATTR_MASK 0xff
287	291	#define MAIR_ATTR_DEVICE 0x04
...	...	@@ -533,6 +537,23 @@
533	537	}
534	538	}
535	539
	540	+static void arm_smmu_tlb_inv_context(struct arm_smmu_cfg *cfg)
	541	+{
	542	+ struct arm_smmu_device *smmu = cfg->smmu;
	543	+ void __iomem *base = ARM_SMMU_GR0(smmu);
	544	+ bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
	545	+
	546	+ if (stage1) {
	547	+ base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);
	548	+ writel_relaxed(cfg->vmid, base + ARM_SMMU_CB_S1_TLBIASID);
	549	+ } else {
	550	+ base = ARM_SMMU_GR0(smmu);
	551	+ writel_relaxed(cfg->vmid, base + ARM_SMMU_GR0_TLBIVMID);
	552	+ }
	553	+
	554	+ arm_smmu_tlb_sync(smmu);
	555	+}
	556	+
536	557	static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
537	558	{
538	559	int flags, ret;
539	560
...	...	@@ -621,14 +642,15 @@
621	642	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
622	643
623	644	/* CBAR */
624		- reg = root_cfg->cbar \|
625		- (root_cfg->vmid << CBAR_VMID_SHIFT);
	645	+ reg = root_cfg->cbar;
626	646	if (smmu->version == 1)
627	647	reg \|= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT;
628	648
629	649	/* Use the weakest memory type, so it is overridden by the pte */
630	650	if (stage1)
631	651	reg \|= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT);
	652	+ else
	653	+ reg \|= root_cfg->vmid << CBAR_VMID_SHIFT;
632	654	writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx));
633	655
634	656	if (smmu->version > 1) {
...	...	@@ -692,6 +714,8 @@
692	714	reg = __pa(root_cfg->pgd);
693	715	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO);
694	716	reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32;
	717	+ if (stage1)
	718	+ reg \|= root_cfg->vmid << TTBRn_HI_ASID_SHIFT;
695	719	writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI);
696	720
697	721	/*
...	...	@@ -747,10 +771,6 @@
747	771	writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0);
748	772	}
749	773
750		- /* Nuke the TLB */
751		- writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
752		- arm_smmu_tlb_sync(smmu);
753		-
754	774	/* SCTLR */
755	775	reg = SCTLR_CFCFG \| SCTLR_CFIE \| SCTLR_CFRE \| SCTLR_M \| SCTLR_EAE_SBOP;
756	776	if (stage1)
...	...	@@ -787,7 +807,8 @@
787	807	return -ENODEV;
788	808	}
789	809
790		- ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 0, ARM_SMMU_NUM_VMIDS);
	810	+ /* VMID zero is reserved for stage-1 mappings */
	811	+ ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 1, ARM_SMMU_NUM_VMIDS);
791	812	if (IS_ERR_VALUE(ret))
792	813	return ret;
793	814
794	815
...	...	@@ -847,11 +868,17 @@
847	868	struct arm_smmu_domain *smmu_domain = domain->priv;
848	869	struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
849	870	struct arm_smmu_device *smmu = root_cfg->smmu;
	871	+ void __iomem *cb_base;
850	872	int irq;
851	873
852	874	if (!smmu)
853	875	return;
854	876
	877	+ /* Disable the context bank and nuke the TLB before freeing it. */
	878	+ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx);
	879	+ writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR);
	880	+ arm_smmu_tlb_inv_context(root_cfg);
	881	+
855	882	if (root_cfg->irptndx != -1) {
856	883	irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx];
857	884	free_irq(irq, domain);
...	...	@@ -956,6 +983,11 @@
956	983	static void arm_smmu_domain_destroy(struct iommu_domain *domain)
957	984	{
958	985	struct arm_smmu_domain *smmu_domain = domain->priv;
	986	+
	987	+ /*
	988	+ * Free the domain resources. We assume that all devices have
	989	+ * already been detached.
	990	+ */
959	991	arm_smmu_destroy_domain_context(domain);
960	992	arm_smmu_free_pgtables(smmu_domain);
961	993	kfree(smmu_domain);
...	...	@@ -1196,7 +1228,7 @@
1196	1228	}
1197	1229
1198	1230	if (stage == 1) {
1199		- pteval \|= ARM_SMMU_PTE_AP_UNPRIV;
	1231	+ pteval \|= ARM_SMMU_PTE_AP_UNPRIV \| ARM_SMMU_PTE_nG;
1200	1232	if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ))
1201	1233	pteval \|= ARM_SMMU_PTE_AP_RDONLY;
1202	1234
1203	1235
...	...	@@ -1412,13 +1444,9 @@
1412	1444	{
1413	1445	int ret;
1414	1446	struct arm_smmu_domain *smmu_domain = domain->priv;
1415		- struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg;
1416		- struct arm_smmu_device *smmu = root_cfg->smmu;
1417		- void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
1418	1447
1419	1448	ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0);
1420		- writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID);
1421		- arm_smmu_tlb_sync(smmu);
	1449	+ arm_smmu_tlb_inv_context(&smmu_domain->root_cfg);
1422	1450	return ret ? ret : size;
1423	1451	}
1424	1452
...	...	@@ -1541,6 +1569,7 @@
1541	1569	static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1542	1570	{
1543	1571	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
	1572	+ void __iomem *sctlr_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB_SCTLR;
1544	1573	int i = 0;
1545	1574	u32 scr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
1546	1575
...	...	@@ -1549,6 +1578,10 @@
1549	1578	writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i));
1550	1579	writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i));
1551	1580	}
	1581	+
	1582	+ /* Make sure all context banks are disabled */
	1583	+ for (i = 0; i < smmu->num_context_banks; ++i)
	1584	+ writel_relaxed(0, sctlr_base + ARM_SMMU_CB(smmu, i));
1552	1585
1553	1586	/* Invalidate the TLB, just in case */
1554	1587	writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL);