Blame view
drivers/iommu/arm-smmu.c
57.2 KB
45ae7cff3 iommu/arm: Add su... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
/* * IOMMU API for ARM architected SMMU implementations. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) 2013 ARM Limited * * Author: Will Deacon <will.deacon@arm.com> * * This driver currently supports: * - SMMUv1 and v2 implementations * - Stream-matching and stream-indexing * - v7/v8 long-descriptor format * - Non-secure access to the SMMU |
45ae7cff3 iommu/arm: Add su... |
26 27 28 29 |
* - Context fault reporting */ #define pr_fmt(fmt) "arm-smmu: " fmt |
1f3d5ca43 iommu/arm-smmu: C... |
30 |
#include <linux/atomic.h> |
45ae7cff3 iommu/arm: Add su... |
31 |
#include <linux/delay.h> |
9adb95949 iommu/arm-smmu: S... |
32 |
#include <linux/dma-iommu.h> |
45ae7cff3 iommu/arm: Add su... |
33 34 35 36 |
#include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> |
f9a05f05b iommu/arm-smmu: T... |
37 |
#include <linux/io-64-nonatomic-hi-lo.h> |
45ae7cff3 iommu/arm: Add su... |
38 |
#include <linux/iommu.h> |
859a732e4 iommu/arm-smmu: a... |
39 |
#include <linux/iopoll.h> |
45ae7cff3 iommu/arm: Add su... |
40 41 |
#include <linux/module.h> #include <linux/of.h> |
bae2c2d42 iommu/arm-smmu: S... |
42 |
#include <linux/of_address.h> |
d6fc5d977 iommu/arm-smmu: S... |
43 |
#include <linux/of_device.h> |
adfec2e70 iommu/arm-smmu: C... |
44 |
#include <linux/of_iommu.h> |
a9a1b0b53 iommu/arm-smmu: a... |
45 |
#include <linux/pci.h> |
45ae7cff3 iommu/arm: Add su... |
46 47 48 49 50 |
#include <linux/platform_device.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/amba/bus.h> |
518f71362 iommu/arm-smmu: m... |
51 |
#include "io-pgtable.h" |
45ae7cff3 iommu/arm: Add su... |
52 |
|
45ae7cff3 iommu/arm: Add su... |
53 54 |
/* Maximum number of context banks per SMMU */ #define ARM_SMMU_MAX_CBS 128 |
45ae7cff3 iommu/arm: Add su... |
55 56 |
/* SMMU global address space */ #define ARM_SMMU_GR0(smmu) ((smmu)->base) |
c757e8528 iommu/arm-smmu: u... |
57 |
#define ARM_SMMU_GR1(smmu) ((smmu)->base + (1 << (smmu)->pgshift)) |
45ae7cff3 iommu/arm: Add su... |
58 |
|
3a5df8ff3 iommu/arm-smmu: s... |
59 60 61 62 63 64 65 66 67 |
/* * SMMU global address space with conditional offset to access secure * aliases of non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448, * nsGFSYNR0: 0x450) */ #define ARM_SMMU_GR0_NS(smmu) \ ((smmu)->base + \ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ ? 0x400 : 0)) |
f9a05f05b iommu/arm-smmu: T... |
68 69 70 71 72 |
/* * Some 64-bit registers only make sense to write atomically, but in such * cases all the data relevant to AArch32 formats lies within the lower word, * therefore this actually makes more sense than it might first appear. */ |
668b4ada1 iommu/arm-smmu: T... |
73 |
#ifdef CONFIG_64BIT |
f9a05f05b iommu/arm-smmu: T... |
74 |
#define smmu_write_atomic_lq writeq_relaxed |
668b4ada1 iommu/arm-smmu: T... |
75 |
#else |
f9a05f05b iommu/arm-smmu: T... |
76 |
#define smmu_write_atomic_lq writel_relaxed |
668b4ada1 iommu/arm-smmu: T... |
77 |
#endif |
45ae7cff3 iommu/arm: Add su... |
78 79 80 81 82 83 84 85 86 87 88 |
/* Configuration registers */ #define ARM_SMMU_GR0_sCR0 0x0 #define sCR0_CLIENTPD (1 << 0) #define sCR0_GFRE (1 << 1) #define sCR0_GFIE (1 << 2) #define sCR0_GCFGFRE (1 << 4) #define sCR0_GCFGFIE (1 << 5) #define sCR0_USFCFG (1 << 10) #define sCR0_VMIDPNE (1 << 11) #define sCR0_PTM (1 << 12) #define sCR0_FB (1 << 13) |
4e3e9b699 iommu/arm-smmu: A... |
89 |
#define sCR0_VMID16EN (1 << 31) |
45ae7cff3 iommu/arm: Add su... |
90 91 |
#define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 |
3ca3712a4 iommu/arm-smmu: C... |
92 93 |
/* Auxiliary Configuration register */ #define ARM_SMMU_GR0_sACR 0x10 |
45ae7cff3 iommu/arm: Add su... |
94 95 96 97 98 99 100 101 102 103 104 105 106 |
/* Identification registers */ #define ARM_SMMU_GR0_ID0 0x20 #define ARM_SMMU_GR0_ID1 0x24 #define ARM_SMMU_GR0_ID2 0x28 #define ARM_SMMU_GR0_ID3 0x2c #define ARM_SMMU_GR0_ID4 0x30 #define ARM_SMMU_GR0_ID5 0x34 #define ARM_SMMU_GR0_ID6 0x38 #define ARM_SMMU_GR0_ID7 0x3c #define ARM_SMMU_GR0_sGFSR 0x48 #define ARM_SMMU_GR0_sGFSYNR0 0x50 #define ARM_SMMU_GR0_sGFSYNR1 0x54 #define ARM_SMMU_GR0_sGFSYNR2 0x58 |
45ae7cff3 iommu/arm: Add su... |
107 108 109 110 111 |
#define ID0_S1TS (1 << 30) #define ID0_S2TS (1 << 29) #define ID0_NTS (1 << 28) #define ID0_SMS (1 << 27) |
859a732e4 iommu/arm-smmu: a... |
112 |
#define ID0_ATOSNS (1 << 26) |
7602b8710 iommu/arm-smmu: D... |
113 114 |
#define ID0_PTFS_NO_AARCH32 (1 << 25) #define ID0_PTFS_NO_AARCH32S (1 << 24) |
45ae7cff3 iommu/arm: Add su... |
115 116 117 |
#define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff |
3c8766d0c iommu/arm-smmu: D... |
118 119 |
#define ID0_NUMSIDB_SHIFT 9 #define ID0_NUMSIDB_MASK 0xf |
45ae7cff3 iommu/arm: Add su... |
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 |
#define ID0_NUMSMRG_SHIFT 0 #define ID0_NUMSMRG_MASK 0xff #define ID1_PAGESIZE (1 << 31) #define ID1_NUMPAGENDXB_SHIFT 28 #define ID1_NUMPAGENDXB_MASK 7 #define ID1_NUMS2CB_SHIFT 16 #define ID1_NUMS2CB_MASK 0xff #define ID1_NUMCB_SHIFT 0 #define ID1_NUMCB_MASK 0xff #define ID2_OAS_SHIFT 4 #define ID2_OAS_MASK 0xf #define ID2_IAS_SHIFT 0 #define ID2_IAS_MASK 0xf #define ID2_UBS_SHIFT 8 #define ID2_UBS_MASK 0xf #define ID2_PTFS_4K (1 << 12) #define ID2_PTFS_16K (1 << 13) #define ID2_PTFS_64K (1 << 14) |
4e3e9b699 iommu/arm-smmu: A... |
140 |
#define ID2_VMID16 (1 << 15) |
45ae7cff3 iommu/arm: Add su... |
141 |
|
3ca3712a4 iommu/arm-smmu: C... |
142 143 |
#define ID7_MAJOR_SHIFT 4 #define ID7_MAJOR_MASK 0xf |
45ae7cff3 iommu/arm: Add su... |
144 |
|
45ae7cff3 iommu/arm: Add su... |
145 |
/* Global TLB invalidation */ |
45ae7cff3 iommu/arm: Add su... |
146 147 148 149 150 151 152 153 154 155 156 157 |
#define ARM_SMMU_GR0_TLBIVMID 0x64 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 #define ARM_SMMU_GR0_TLBIALLH 0x6c #define ARM_SMMU_GR0_sTLBGSYNC 0x70 #define ARM_SMMU_GR0_sTLBGSTATUS 0x74 #define sTLBGSTATUS_GSACTIVE (1 << 0) #define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ /* Stream mapping registers */ #define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) #define SMR_VALID (1 << 31) #define SMR_MASK_SHIFT 16 |
45ae7cff3 iommu/arm: Add su... |
158 |
#define SMR_ID_SHIFT 0 |
45ae7cff3 iommu/arm: Add su... |
159 160 161 162 163 164 |
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) #define S2CR_CBNDX_SHIFT 0 #define S2CR_CBNDX_MASK 0xff #define S2CR_TYPE_SHIFT 16 #define S2CR_TYPE_MASK 0x3 |
8e8b203ea iommu/arm-smmu: K... |
165 166 167 168 169 |
enum arm_smmu_s2cr_type { S2CR_TYPE_TRANS, S2CR_TYPE_BYPASS, S2CR_TYPE_FAULT, }; |
45ae7cff3 iommu/arm: Add su... |
170 |
|
d346180e7 iommu/arm-smmu: T... |
171 |
#define S2CR_PRIVCFG_SHIFT 24 |
8e8b203ea iommu/arm-smmu: K... |
172 173 174 175 176 177 178 |
#define S2CR_PRIVCFG_MASK 0x3 enum arm_smmu_s2cr_privcfg { S2CR_PRIVCFG_DEFAULT, S2CR_PRIVCFG_DIPAN, S2CR_PRIVCFG_UNPRIV, S2CR_PRIVCFG_PRIV, }; |
d346180e7 iommu/arm-smmu: T... |
179 |
|
45ae7cff3 iommu/arm: Add su... |
180 181 182 183 |
/* Context bank attribute registers */ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) #define CBAR_VMID_SHIFT 0 #define CBAR_VMID_MASK 0xff |
57ca90f68 iommu/arm-smmu: s... |
184 185 186 |
#define CBAR_S1_BPSHCFG_SHIFT 8 #define CBAR_S1_BPSHCFG_MASK 3 #define CBAR_S1_BPSHCFG_NSH 3 |
45ae7cff3 iommu/arm: Add su... |
187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 |
#define CBAR_S1_MEMATTR_SHIFT 12 #define CBAR_S1_MEMATTR_MASK 0xf #define CBAR_S1_MEMATTR_WB 0xf #define CBAR_TYPE_SHIFT 16 #define CBAR_TYPE_MASK 0x3 #define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT) #define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT) #define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT) #define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT) #define CBAR_IRPTNDX_SHIFT 24 #define CBAR_IRPTNDX_MASK 0xff #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define CBA2R_RW64_32BIT (0 << 0) #define CBA2R_RW64_64BIT (1 << 0) |
4e3e9b699 iommu/arm-smmu: A... |
202 203 |
#define CBA2R_VMID_SHIFT 16 #define CBA2R_VMID_MASK 0xffff |
45ae7cff3 iommu/arm: Add su... |
204 205 206 |
/* Translation context bank */ #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) |
c757e8528 iommu/arm-smmu: u... |
207 |
#define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift)) |
45ae7cff3 iommu/arm: Add su... |
208 209 |
#define ARM_SMMU_CB_SCTLR 0x0 |
f0cfffc48 iommu/arm-smmu: W... |
210 |
#define ARM_SMMU_CB_ACTLR 0x4 |
45ae7cff3 iommu/arm: Add su... |
211 212 |
#define ARM_SMMU_CB_RESUME 0x8 #define ARM_SMMU_CB_TTBCR2 0x10 |
668b4ada1 iommu/arm-smmu: T... |
213 214 |
#define ARM_SMMU_CB_TTBR0 0x20 #define ARM_SMMU_CB_TTBR1 0x28 |
45ae7cff3 iommu/arm: Add su... |
215 |
#define ARM_SMMU_CB_TTBCR 0x30 |
6070529be iommu/arm-smmu: S... |
216 |
#define ARM_SMMU_CB_CONTEXTIDR 0x34 |
45ae7cff3 iommu/arm: Add su... |
217 |
#define ARM_SMMU_CB_S1_MAIR0 0x38 |
518f71362 iommu/arm-smmu: m... |
218 |
#define ARM_SMMU_CB_S1_MAIR1 0x3c |
f9a05f05b iommu/arm-smmu: T... |
219 |
#define ARM_SMMU_CB_PAR 0x50 |
45ae7cff3 iommu/arm: Add su... |
220 |
#define ARM_SMMU_CB_FSR 0x58 |
f9a05f05b iommu/arm-smmu: T... |
221 |
#define ARM_SMMU_CB_FAR 0x60 |
45ae7cff3 iommu/arm: Add su... |
222 |
#define ARM_SMMU_CB_FSYNR0 0x68 |
518f71362 iommu/arm-smmu: m... |
223 |
#define ARM_SMMU_CB_S1_TLBIVA 0x600 |
1463fe44f iommu/arm-smmu: D... |
224 |
#define ARM_SMMU_CB_S1_TLBIASID 0x610 |
518f71362 iommu/arm-smmu: m... |
225 226 227 |
#define ARM_SMMU_CB_S1_TLBIVAL 0x620 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 |
661d962f1 iommu/arm-smmu: F... |
228 |
#define ARM_SMMU_CB_ATS1PR 0x800 |
859a732e4 iommu/arm-smmu: a... |
229 |
#define ARM_SMMU_CB_ATSR 0x8f0 |
45ae7cff3 iommu/arm: Add su... |
230 231 232 233 234 235 236 237 238 |
#define SCTLR_S1_ASIDPNE (1 << 12) #define SCTLR_CFCFG (1 << 7) #define SCTLR_CFIE (1 << 6) #define SCTLR_CFRE (1 << 5) #define SCTLR_E (1 << 4) #define SCTLR_AFE (1 << 2) #define SCTLR_TRE (1 << 1) #define SCTLR_M (1 << 0) |
45ae7cff3 iommu/arm: Add su... |
239 |
|
f0cfffc48 iommu/arm-smmu: W... |
240 |
#define ARM_MMU500_ACTLR_CPRE (1 << 1) |
3ca3712a4 iommu/arm-smmu: C... |
241 |
#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) |
859a732e4 iommu/arm-smmu: a... |
242 243 244 |
#define CB_PAR_F (1 << 0) #define ATSR_ACTIVE (1 << 0) |
45ae7cff3 iommu/arm: Add su... |
245 246 |
#define RESUME_RETRY (0 << 0) #define RESUME_TERMINATE (1 << 0) |
45ae7cff3 iommu/arm: Add su... |
247 |
#define TTBCR2_SEP_SHIFT 15 |
5dc5616ee iommu/arm-smmu: F... |
248 |
#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT) |
45ae7cff3 iommu/arm: Add su... |
249 |
|
668b4ada1 iommu/arm-smmu: T... |
250 |
#define TTBRn_ASID_SHIFT 48 |
45ae7cff3 iommu/arm: Add su... |
251 252 253 254 255 256 257 258 259 260 261 |
#define FSR_MULTI (1 << 31) #define FSR_SS (1 << 30) #define FSR_UUT (1 << 8) #define FSR_ASF (1 << 7) #define FSR_TLBLKF (1 << 6) #define FSR_TLBMCF (1 << 5) #define FSR_EF (1 << 4) #define FSR_PF (1 << 3) #define FSR_AFF (1 << 2) #define FSR_TF (1 << 1) |
2907320df iommu/arm-smmu: f... |
262 263 264 |
#define FSR_IGN (FSR_AFF | FSR_ASF | \ FSR_TLBMCF | FSR_TLBLKF) #define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \ |
adaba3209 iommu/arm-smmu: T... |
265 |
FSR_EF | FSR_PF | FSR_TF | FSR_IGN) |
45ae7cff3 iommu/arm: Add su... |
266 267 |
#define FSYNR0_WNR (1 << 4) |
4cf740b0b iommu/arm-smmu: a... |
268 |
static int force_stage; |
25a1c96cd iommu/arm-smmu: A... |
269 |
module_param(force_stage, int, S_IRUGO); |
4cf740b0b iommu/arm-smmu: a... |
270 271 |
MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); |
25a1c96cd iommu/arm-smmu: A... |
272 273 274 275 |
static bool disable_bypass; module_param(disable_bypass, bool, S_IRUGO); MODULE_PARM_DESC(disable_bypass, "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); |
4cf740b0b iommu/arm-smmu: a... |
276 |
|
093604033 iommu/arm-smmu: f... |
277 |
enum arm_smmu_arch_version { |
b7862e355 iommu/arm-smmu: S... |
278 279 |
ARM_SMMU_V1, ARM_SMMU_V1_64K, |
093604033 iommu/arm-smmu: f... |
280 281 |
ARM_SMMU_V2, }; |
67b65a3fb iommu/arm-smmu: D... |
282 283 |
enum arm_smmu_implementation { GENERIC_SMMU, |
f0cfffc48 iommu/arm-smmu: W... |
284 |
ARM_MMU500, |
e086d912d iommu/arm-smmu: C... |
285 |
CAVIUM_SMMUV2, |
67b65a3fb iommu/arm-smmu: D... |
286 |
}; |
8e8b203ea iommu/arm-smmu: K... |
287 |
struct arm_smmu_s2cr { |
588888a73 iommu/arm-smmu: I... |
288 289 |
struct iommu_group *group; int count; |
8e8b203ea iommu/arm-smmu: K... |
290 291 292 293 294 295 296 297 |
enum arm_smmu_s2cr_type type; enum arm_smmu_s2cr_privcfg privcfg; u8 cbndx; }; #define s2cr_init_val (struct arm_smmu_s2cr){ \ .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ } |
45ae7cff3 iommu/arm: Add su... |
298 |
struct arm_smmu_smr { |
45ae7cff3 iommu/arm: Add su... |
299 300 |
u16 mask; u16 id; |
1f3d5ca43 iommu/arm-smmu: C... |
301 |
bool valid; |
45ae7cff3 iommu/arm: Add su... |
302 |
}; |
a9a1b0b53 iommu/arm-smmu: a... |
303 |
struct arm_smmu_master_cfg { |
f80cd885f iommu/arm-smmu: R... |
304 |
struct arm_smmu_device *smmu; |
adfec2e70 iommu/arm-smmu: C... |
305 |
s16 smendx[]; |
45ae7cff3 iommu/arm: Add su... |
306 |
}; |
1f3d5ca43 iommu/arm-smmu: C... |
307 |
#define INVALID_SMENDX -1 |
adfec2e70 iommu/arm-smmu: C... |
308 309 |
#define __fwspec_cfg(fw) ((struct arm_smmu_master_cfg *)fw->iommu_priv) #define fwspec_smmu(fw) (__fwspec_cfg(fw)->smmu) |
8c82d6ec5 iommu/arm-smmu: F... |
310 311 |
#define fwspec_smendx(fw, i) \ (i >= fw->num_ids ? INVALID_SMENDX : __fwspec_cfg(fw)->smendx[i]) |
adfec2e70 iommu/arm-smmu: C... |
312 |
#define for_each_cfg_sme(fw, i, idx) \ |
8c82d6ec5 iommu/arm-smmu: F... |
313 |
for (i = 0; idx = fwspec_smendx(fw, i), i < fw->num_ids; ++i) |
45ae7cff3 iommu/arm: Add su... |
314 315 316 |
struct arm_smmu_device { struct device *dev; |
45ae7cff3 iommu/arm: Add su... |
317 318 319 |
void __iomem *base; unsigned long size; |
c757e8528 iommu/arm-smmu: u... |
320 |
unsigned long pgshift; |
45ae7cff3 iommu/arm: Add su... |
321 322 323 324 325 326 |
#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) #define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) |
859a732e4 iommu/arm-smmu: a... |
327 |
#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) |
4e3e9b699 iommu/arm-smmu: A... |
328 |
#define ARM_SMMU_FEAT_VMID16 (1 << 6) |
7602b8710 iommu/arm-smmu: D... |
329 330 331 332 333 |
#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) #define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) #define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) #define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) #define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) |
45ae7cff3 iommu/arm: Add su... |
334 |
u32 features; |
3a5df8ff3 iommu/arm-smmu: s... |
335 336 337 |
#define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) u32 options; |
093604033 iommu/arm-smmu: f... |
338 |
enum arm_smmu_arch_version version; |
67b65a3fb iommu/arm-smmu: D... |
339 |
enum arm_smmu_implementation model; |
45ae7cff3 iommu/arm: Add su... |
340 341 342 343 344 345 346 |
u32 num_context_banks; u32 num_s2_context_banks; DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); atomic_t irptndx; u32 num_mapping_groups; |
21174240e iommu/arm-smmu: H... |
347 348 |
u16 streamid_mask; u16 smr_mask_mask; |
1f3d5ca43 iommu/arm-smmu: C... |
349 |
struct arm_smmu_smr *smrs; |
8e8b203ea iommu/arm-smmu: K... |
350 |
struct arm_smmu_s2cr *s2crs; |
588888a73 iommu/arm-smmu: I... |
351 |
struct mutex stream_map_mutex; |
45ae7cff3 iommu/arm: Add su... |
352 |
|
518f71362 iommu/arm-smmu: m... |
353 354 355 |
unsigned long va_size; unsigned long ipa_size; unsigned long pa_size; |
d54663573 iommu/arm-smmu: U... |
356 |
unsigned long pgsize_bitmap; |
45ae7cff3 iommu/arm: Add su... |
357 358 359 360 |
u32 num_global_irqs; u32 num_context_irqs; unsigned int *irqs; |
1bd37a683 iommu/arm-smmu: W... |
361 |
u32 cavium_id_base; /* Specific to Cavium */ |
45ae7cff3 iommu/arm: Add su... |
362 |
}; |
7602b8710 iommu/arm-smmu: D... |
363 364 365 366 367 |
enum arm_smmu_context_fmt { ARM_SMMU_CTX_FMT_NONE, ARM_SMMU_CTX_FMT_AARCH64, ARM_SMMU_CTX_FMT_AARCH32_L, ARM_SMMU_CTX_FMT_AARCH32_S, |
45ae7cff3 iommu/arm: Add su... |
368 369 370 |
}; struct arm_smmu_cfg { |
45ae7cff3 iommu/arm: Add su... |
371 372 373 |
u8 cbndx; u8 irptndx; u32 cbar; |
7602b8710 iommu/arm-smmu: D... |
374 |
enum arm_smmu_context_fmt fmt; |
45ae7cff3 iommu/arm: Add su... |
375 |
}; |
faea13b72 iommu/arm-smmu: f... |
376 |
#define INVALID_IRPTNDX 0xff |
45ae7cff3 iommu/arm: Add su... |
377 |
|
1bd37a683 iommu/arm-smmu: W... |
378 379 |
#define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx) #define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1) |
ecfadb6e5 iommu/arm-smmu: S... |
380 |
|
c752ce45b iommu/arm-smmu: a... |
381 382 383 384 385 |
enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, ARM_SMMU_DOMAIN_S2, ARM_SMMU_DOMAIN_NESTED, }; |
45ae7cff3 iommu/arm: Add su... |
386 |
struct arm_smmu_domain { |
44680eedf iommu/arm-smmu: r... |
387 |
struct arm_smmu_device *smmu; |
518f71362 iommu/arm-smmu: m... |
388 389 |
struct io_pgtable_ops *pgtbl_ops; spinlock_t pgtbl_lock; |
44680eedf iommu/arm-smmu: r... |
390 |
struct arm_smmu_cfg cfg; |
c752ce45b iommu/arm-smmu: a... |
391 |
enum arm_smmu_domain_stage stage; |
518f71362 iommu/arm-smmu: m... |
392 |
struct mutex init_mutex; /* Protects smmu pointer */ |
1d672638f iommu/arm-smmu: M... |
393 |
struct iommu_domain domain; |
45ae7cff3 iommu/arm: Add su... |
394 |
}; |
3a5df8ff3 iommu/arm-smmu: s... |
395 396 397 398 |
struct arm_smmu_option_prop { u32 opt; const char *prop; }; |
1bd37a683 iommu/arm-smmu: W... |
399 |
static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0); |
021bb8420 iommu/arm-smmu: W... |
400 |
static bool using_legacy_binding, using_generic_binding; |
2907320df iommu/arm-smmu: f... |
401 |
static struct arm_smmu_option_prop arm_smmu_options[] = { |
3a5df8ff3 iommu/arm-smmu: s... |
402 403 404 |
{ ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, { 0, NULL}, }; |
1d672638f iommu/arm-smmu: M... |
405 406 407 408 |
static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) { return container_of(dom, struct arm_smmu_domain, domain); } |
3a5df8ff3 iommu/arm-smmu: s... |
409 410 411 |
static void parse_driver_options(struct arm_smmu_device *smmu) { int i = 0; |
2907320df iommu/arm-smmu: f... |
412 |
|
3a5df8ff3 iommu/arm-smmu: s... |
413 414 415 416 417 418 419 420 421 422 |
do { if (of_property_read_bool(smmu->dev->of_node, arm_smmu_options[i].prop)) { smmu->options |= arm_smmu_options[i].opt; dev_notice(smmu->dev, "option %s ", arm_smmu_options[i].prop); } } while (arm_smmu_options[++i].opt); } |
8f68f8e28 iommu/arm-smmu: a... |
423 |
static struct device_node *dev_get_dev_node(struct device *dev) |
a9a1b0b53 iommu/arm-smmu: a... |
424 425 426 |
{ if (dev_is_pci(dev)) { struct pci_bus *bus = to_pci_dev(dev)->bus; |
2907320df iommu/arm-smmu: f... |
427 |
|
a9a1b0b53 iommu/arm-smmu: a... |
428 429 |
while (!pci_is_root_bus(bus)) bus = bus->parent; |
f80cd885f iommu/arm-smmu: R... |
430 |
return of_node_get(bus->bridge->parent->of_node); |
a9a1b0b53 iommu/arm-smmu: a... |
431 |
} |
f80cd885f iommu/arm-smmu: R... |
432 |
return of_node_get(dev->of_node); |
a9a1b0b53 iommu/arm-smmu: a... |
433 |
} |
f80cd885f iommu/arm-smmu: R... |
434 |
static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) |
45ae7cff3 iommu/arm: Add su... |
435 |
{ |
f80cd885f iommu/arm-smmu: R... |
436 437 |
*((__be32 *)data) = cpu_to_be32(alias); return 0; /* Continue walking */ |
45ae7cff3 iommu/arm: Add su... |
438 |
} |
f80cd885f iommu/arm-smmu: R... |
439 |
static int __find_legacy_master_phandle(struct device *dev, void *data) |
a9a1b0b53 iommu/arm-smmu: a... |
440 |
{ |
f80cd885f iommu/arm-smmu: R... |
441 442 443 444 445 446 447 448 449 450 451 452 |
struct of_phandle_iterator *it = *(void **)data; struct device_node *np = it->node; int err; of_for_each_phandle(it, err, dev->of_node, "mmu-masters", "#stream-id-cells", 0) if (it->node == np) { *(void **)data = dev; return 1; } it->node = np; return err == -ENOENT ? 0 : err; |
a9a1b0b53 iommu/arm-smmu: a... |
453 |
} |
d6fc5d977 iommu/arm-smmu: S... |
454 |
static struct platform_driver arm_smmu_driver; |
adfec2e70 iommu/arm-smmu: C... |
455 |
static struct iommu_ops arm_smmu_ops; |
d6fc5d977 iommu/arm-smmu: S... |
456 |
|
adfec2e70 iommu/arm-smmu: C... |
457 458 |
static int arm_smmu_register_legacy_master(struct device *dev, struct arm_smmu_device **smmu) |
45ae7cff3 iommu/arm: Add su... |
459 |
{ |
adfec2e70 iommu/arm-smmu: C... |
460 |
struct device *smmu_dev; |
f80cd885f iommu/arm-smmu: R... |
461 462 463 |
struct device_node *np; struct of_phandle_iterator it; void *data = ⁢ |
adfec2e70 iommu/arm-smmu: C... |
464 |
u32 *sids; |
f80cd885f iommu/arm-smmu: R... |
465 466 |
__be32 pci_sid; int err; |
45ae7cff3 iommu/arm: Add su... |
467 |
|
f80cd885f iommu/arm-smmu: R... |
468 469 470 471 472 |
np = dev_get_dev_node(dev); if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { of_node_put(np); return -ENODEV; } |
45ae7cff3 iommu/arm: Add su... |
473 |
|
f80cd885f iommu/arm-smmu: R... |
474 |
it.node = np; |
d6fc5d977 iommu/arm-smmu: S... |
475 476 |
err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, __find_legacy_master_phandle); |
adfec2e70 iommu/arm-smmu: C... |
477 |
smmu_dev = data; |
f80cd885f iommu/arm-smmu: R... |
478 479 480 481 482 |
of_node_put(np); if (err == 0) return -ENODEV; if (err < 0) return err; |
45ae7cff3 iommu/arm: Add su... |
483 |
|
f80cd885f iommu/arm-smmu: R... |
484 485 486 487 488 489 490 |
if (dev_is_pci(dev)) { /* "mmu-masters" assumes Stream ID == Requester ID */ pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, &pci_sid); it.cur = &pci_sid; it.cur_count = 1; } |
45ae7cff3 iommu/arm: Add su... |
491 |
|
adfec2e70 iommu/arm-smmu: C... |
492 493 494 495 |
err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, &arm_smmu_ops); if (err) return err; |
45ae7cff3 iommu/arm: Add su... |
496 |
|
adfec2e70 iommu/arm-smmu: C... |
497 498 499 |
sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); if (!sids) return -ENOMEM; |
44680eedf iommu/arm-smmu: r... |
500 |
|
adfec2e70 iommu/arm-smmu: C... |
501 502 503 504 505 |
*smmu = dev_get_drvdata(smmu_dev); of_phandle_iterator_args(&it, sids, it.cur_count); err = iommu_fwspec_add_ids(dev, sids, it.cur_count); kfree(sids); return err; |
45ae7cff3 iommu/arm: Add su... |
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 |
} static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) { int idx; do { idx = find_next_zero_bit(map, end, start); if (idx == end) return -ENOSPC; } while (test_and_set_bit(idx, map)); return idx; } static void __arm_smmu_free_bitmap(unsigned long *map, int idx) { clear_bit(idx, map); } /* Wait for any pending TLB invalidations to complete */ |
518f71362 iommu/arm-smmu: m... |
527 |
static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu) |
45ae7cff3 iommu/arm: Add su... |
528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 |
{ int count = 0; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC); while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS) & sTLBGSTATUS_GSACTIVE) { cpu_relax(); if (++count == TLB_LOOP_TIMEOUT) { dev_err_ratelimited(smmu->dev, "TLB sync timed out -- SMMU may be deadlocked "); return; } udelay(1); } } |
518f71362 iommu/arm-smmu: m... |
545 546 547 548 549 550 551 |
static void arm_smmu_tlb_sync(void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; __arm_smmu_tlb_sync(smmu_domain->smmu); } static void arm_smmu_tlb_inv_context(void *cookie) |
1463fe44f iommu/arm-smmu: D... |
552 |
{ |
518f71362 iommu/arm-smmu: m... |
553 |
struct arm_smmu_domain *smmu_domain = cookie; |
44680eedf iommu/arm-smmu: r... |
554 555 |
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; |
1463fe44f iommu/arm-smmu: D... |
556 |
bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; |
518f71362 iommu/arm-smmu: m... |
557 |
void __iomem *base; |
1463fe44f iommu/arm-smmu: D... |
558 559 560 |
if (stage1) { base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); |
1bd37a683 iommu/arm-smmu: W... |
561 |
writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg), |
ecfadb6e5 iommu/arm-smmu: S... |
562 |
base + ARM_SMMU_CB_S1_TLBIASID); |
1463fe44f iommu/arm-smmu: D... |
563 564 |
} else { base = ARM_SMMU_GR0(smmu); |
1bd37a683 iommu/arm-smmu: W... |
565 |
writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), |
ecfadb6e5 iommu/arm-smmu: S... |
566 |
base + ARM_SMMU_GR0_TLBIVMID); |
1463fe44f iommu/arm-smmu: D... |
567 |
} |
518f71362 iommu/arm-smmu: m... |
568 569 570 571 |
__arm_smmu_tlb_sync(smmu); } static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, |
06c610e8f iommu/io-pgtable:... |
572 |
size_t granule, bool leaf, void *cookie) |
518f71362 iommu/arm-smmu: m... |
573 574 575 576 577 578 579 580 581 582 |
{ struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; void __iomem *reg; if (stage1) { reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; |
7602b8710 iommu/arm-smmu: D... |
583 |
if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { |
518f71362 iommu/arm-smmu: m... |
584 |
iova &= ~12UL; |
1bd37a683 iommu/arm-smmu: W... |
585 |
iova |= ARM_SMMU_CB_ASID(smmu, cfg); |
75df13865 iommu/arm-smmu: I... |
586 587 588 589 |
do { writel_relaxed(iova, reg); iova += granule; } while (size -= granule); |
518f71362 iommu/arm-smmu: m... |
590 591 |
} else { iova >>= 12; |
1bd37a683 iommu/arm-smmu: W... |
592 |
iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48; |
75df13865 iommu/arm-smmu: I... |
593 594 595 596 |
do { writeq_relaxed(iova, reg); iova += granule >> 12; } while (size -= granule); |
518f71362 iommu/arm-smmu: m... |
597 |
} |
518f71362 iommu/arm-smmu: m... |
598 599 600 601 |
} else if (smmu->version == ARM_SMMU_V2) { reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2; |
75df13865 iommu/arm-smmu: I... |
602 603 |
iova >>= 12; do { |
f9a05f05b iommu/arm-smmu: T... |
604 |
smmu_write_atomic_lq(iova, reg); |
75df13865 iommu/arm-smmu: I... |
605 606 |
iova += granule >> 12; } while (size -= granule); |
518f71362 iommu/arm-smmu: m... |
607 608 |
} else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; |
1bd37a683 iommu/arm-smmu: W... |
609 |
writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg); |
518f71362 iommu/arm-smmu: m... |
610 611 |
} } |
518f71362 iommu/arm-smmu: m... |
612 613 614 615 |
static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, |
518f71362 iommu/arm-smmu: m... |
616 |
}; |
45ae7cff3 iommu/arm: Add su... |
617 618 |
static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { |
3714ce1d6 iommu/arm-smmu: D... |
619 |
u32 fsr, fsynr; |
45ae7cff3 iommu/arm: Add su... |
620 621 |
unsigned long iova; struct iommu_domain *domain = dev; |
1d672638f iommu/arm-smmu: M... |
622 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eedf iommu/arm-smmu: r... |
623 624 |
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; |
45ae7cff3 iommu/arm: Add su... |
625 |
void __iomem *cb_base; |
44680eedf iommu/arm-smmu: r... |
626 |
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); |
45ae7cff3 iommu/arm: Add su... |
627 628 629 630 |
fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); if (!(fsr & FSR_FAULT)) return IRQ_NONE; |
45ae7cff3 iommu/arm: Add su... |
631 |
fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); |
f9a05f05b iommu/arm-smmu: T... |
632 |
iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); |
45ae7cff3 iommu/arm: Add su... |
633 |
|
3714ce1d6 iommu/arm-smmu: D... |
634 635 636 637 |
dev_err_ratelimited(smmu->dev, "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cb=%d ", fsr, iova, fsynr, cfg->cbndx); |
45ae7cff3 iommu/arm: Add su... |
638 |
|
3714ce1d6 iommu/arm-smmu: D... |
639 640 |
writel(fsr, cb_base + ARM_SMMU_CB_FSR); return IRQ_HANDLED; |
45ae7cff3 iommu/arm: Add su... |
641 642 643 644 645 646 |
} static irqreturn_t arm_smmu_global_fault(int irq, void *dev) { u32 gfsr, gfsynr0, gfsynr1, gfsynr2; struct arm_smmu_device *smmu = dev; |
3a5df8ff3 iommu/arm-smmu: s... |
647 |
void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu); |
45ae7cff3 iommu/arm: Add su... |
648 649 650 651 652 |
gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); |
3a5df8ff3 iommu/arm-smmu: s... |
653 654 |
if (!gfsr) return IRQ_NONE; |
45ae7cff3 iommu/arm: Add su... |
655 656 657 658 659 660 661 662 663 |
dev_err_ratelimited(smmu->dev, "Unexpected global fault, this could be serious "); dev_err_ratelimited(smmu->dev, "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x ", gfsr, gfsynr0, gfsynr1, gfsynr2); writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); |
adaba3209 iommu/arm-smmu: T... |
664 |
return IRQ_HANDLED; |
45ae7cff3 iommu/arm: Add su... |
665 |
} |
518f71362 iommu/arm-smmu: m... |
666 667 |
static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, struct io_pgtable_cfg *pgtbl_cfg) |
45ae7cff3 iommu/arm: Add su... |
668 |
{ |
6070529be iommu/arm-smmu: S... |
669 |
u32 reg, reg2; |
668b4ada1 iommu/arm-smmu: T... |
670 |
u64 reg64; |
45ae7cff3 iommu/arm: Add su... |
671 |
bool stage1; |
44680eedf iommu/arm-smmu: r... |
672 673 |
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; |
c88ae5de7 iommu/arm-smmu: R... |
674 |
void __iomem *cb_base, *gr1_base; |
45ae7cff3 iommu/arm: Add su... |
675 |
|
45ae7cff3 iommu/arm: Add su... |
676 |
gr1_base = ARM_SMMU_GR1(smmu); |
44680eedf iommu/arm-smmu: r... |
677 678 |
stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); |
45ae7cff3 iommu/arm: Add su... |
679 |
|
4a1c93cbe iommu/arm-smmu: e... |
680 |
if (smmu->version > ARM_SMMU_V1) { |
7602b8710 iommu/arm-smmu: D... |
681 682 683 684 |
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) reg = CBA2R_RW64_64BIT; else reg = CBA2R_RW64_32BIT; |
4e3e9b699 iommu/arm-smmu: A... |
685 686 |
/* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) |
1bd37a683 iommu/arm-smmu: W... |
687 |
reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT; |
4e3e9b699 iommu/arm-smmu: A... |
688 |
|
4a1c93cbe iommu/arm-smmu: e... |
689 690 |
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } |
45ae7cff3 iommu/arm: Add su... |
691 |
/* CBAR */ |
44680eedf iommu/arm-smmu: r... |
692 |
reg = cfg->cbar; |
b7862e355 iommu/arm-smmu: S... |
693 |
if (smmu->version < ARM_SMMU_V2) |
2907320df iommu/arm-smmu: f... |
694 |
reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; |
45ae7cff3 iommu/arm: Add su... |
695 |
|
57ca90f68 iommu/arm-smmu: s... |
696 697 698 699 700 701 702 |
/* * Use the weakest shareability/memory types, so they are * overridden by the ttbcr/pte. */ if (stage1) { reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); |
4e3e9b699 iommu/arm-smmu: A... |
703 704 |
} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { /* 8-bit VMIDs live in CBAR */ |
1bd37a683 iommu/arm-smmu: W... |
705 |
reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT; |
57ca90f68 iommu/arm-smmu: s... |
706 |
} |
44680eedf iommu/arm-smmu: r... |
707 |
writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); |
45ae7cff3 iommu/arm: Add su... |
708 |
|
518f71362 iommu/arm-smmu: m... |
709 710 |
/* TTBRs */ if (stage1) { |
6070529be iommu/arm-smmu: S... |
711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 |
u16 asid = ARM_SMMU_CB_ASID(smmu, cfg); if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { reg = pgtbl_cfg->arm_v7s_cfg.ttbr[0]; writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0); reg = pgtbl_cfg->arm_v7s_cfg.ttbr[1]; writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1); writel_relaxed(asid, cb_base + ARM_SMMU_CB_CONTEXTIDR); } else { reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; reg64 |= (u64)asid << TTBRn_ASID_SHIFT; writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; reg64 |= (u64)asid << TTBRn_ASID_SHIFT; writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1); } |
518f71362 iommu/arm-smmu: m... |
727 |
} else { |
668b4ada1 iommu/arm-smmu: T... |
728 |
reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; |
f9a05f05b iommu/arm-smmu: T... |
729 |
writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); |
518f71362 iommu/arm-smmu: m... |
730 |
} |
a65217a4b iommu/arm-smmu: f... |
731 |
|
518f71362 iommu/arm-smmu: m... |
732 733 |
/* TTBCR */ if (stage1) { |
6070529be iommu/arm-smmu: S... |
734 735 736 737 738 739 740 |
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { reg = pgtbl_cfg->arm_v7s_cfg.tcr; reg2 = 0; } else { reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; reg2 = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; reg2 |= TTBCR2_SEP_UPSTREAM; |
45ae7cff3 iommu/arm: Add su... |
741 |
} |
6070529be iommu/arm-smmu: S... |
742 743 |
if (smmu->version > ARM_SMMU_V1) writel_relaxed(reg2, cb_base + ARM_SMMU_CB_TTBCR2); |
45ae7cff3 iommu/arm: Add su... |
744 |
} else { |
518f71362 iommu/arm-smmu: m... |
745 |
reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; |
45ae7cff3 iommu/arm: Add su... |
746 |
} |
6070529be iommu/arm-smmu: S... |
747 |
writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); |
45ae7cff3 iommu/arm: Add su... |
748 |
|
518f71362 iommu/arm-smmu: m... |
749 |
/* MAIRs (stage-1 only) */ |
45ae7cff3 iommu/arm: Add su... |
750 |
if (stage1) { |
6070529be iommu/arm-smmu: S... |
751 752 753 754 755 756 757 |
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { reg = pgtbl_cfg->arm_v7s_cfg.prrr; reg2 = pgtbl_cfg->arm_v7s_cfg.nmrr; } else { reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; reg2 = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; } |
45ae7cff3 iommu/arm: Add su... |
758 |
writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0); |
6070529be iommu/arm-smmu: S... |
759 |
writel_relaxed(reg2, cb_base + ARM_SMMU_CB_S1_MAIR1); |
45ae7cff3 iommu/arm: Add su... |
760 |
} |
45ae7cff3 iommu/arm: Add su... |
761 |
/* SCTLR */ |
6070529be iommu/arm-smmu: S... |
762 |
reg = SCTLR_CFIE | SCTLR_CFRE | SCTLR_AFE | SCTLR_TRE | SCTLR_M; |
45ae7cff3 iommu/arm: Add su... |
763 764 765 766 767 |
if (stage1) reg |= SCTLR_S1_ASIDPNE; #ifdef __BIG_ENDIAN reg |= SCTLR_E; #endif |
25724841d iommu/arm-smmu: u... |
768 |
writel_relaxed(reg, cb_base + ARM_SMMU_CB_SCTLR); |
45ae7cff3 iommu/arm: Add su... |
769 770 771 |
} static int arm_smmu_init_domain_context(struct iommu_domain *domain, |
44680eedf iommu/arm-smmu: r... |
772 |
struct arm_smmu_device *smmu) |
45ae7cff3 iommu/arm: Add su... |
773 |
{ |
a18037b27 iommu/arm-smmu: a... |
774 |
int irq, start, ret = 0; |
518f71362 iommu/arm-smmu: m... |
775 776 777 778 |
unsigned long ias, oas; struct io_pgtable_ops *pgtbl_ops; struct io_pgtable_cfg pgtbl_cfg; enum io_pgtable_fmt fmt; |
1d672638f iommu/arm-smmu: M... |
779 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eedf iommu/arm-smmu: r... |
780 |
struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
45ae7cff3 iommu/arm: Add su... |
781 |
|
518f71362 iommu/arm-smmu: m... |
782 |
mutex_lock(&smmu_domain->init_mutex); |
a18037b27 iommu/arm-smmu: a... |
783 784 |
if (smmu_domain->smmu) goto out_unlock; |
c752ce45b iommu/arm-smmu: a... |
785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 |
/* * Mapping the requested stage onto what we support is surprisingly * complicated, mainly because the spec allows S1+S2 SMMUs without * support for nested translation. That means we end up with the * following table: * * Requested Supported Actual * S1 N S1 * S1 S1+S2 S1 * S1 S2 S2 * S1 S1 S1 * N N N * N S1+S2 S2 * N S2 S2 * N S1 S1 * * Note that you can't actually request stage-2 mappings. */ if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) smmu_domain->stage = ARM_SMMU_DOMAIN_S2; if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; |
7602b8710 iommu/arm-smmu: D... |
807 808 809 810 811 812 813 814 815 816 |
/* * Choosing a suitable context format is even more fiddly. Until we * grow some way for the caller to express a preference, and/or move * the decision into the io-pgtable code where it arguably belongs, * just aim for the closest thing to the rest of the system, and hope * that the hardware isn't esoteric enough that we can't assume AArch64 * support to be a superset of AArch32 support... */ if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; |
6070529be iommu/arm-smmu: S... |
817 818 819 820 821 |
if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; |
7602b8710 iommu/arm-smmu: D... |
822 823 824 825 826 827 828 829 830 831 |
if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | ARM_SMMU_FEAT_FMT_AARCH64_16K | ARM_SMMU_FEAT_FMT_AARCH64_4K))) cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { ret = -EINVAL; goto out_unlock; } |
c752ce45b iommu/arm-smmu: a... |
832 833 834 835 |
switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; start = smmu->num_s2_context_banks; |
518f71362 iommu/arm-smmu: m... |
836 837 |
ias = smmu->va_size; oas = smmu->ipa_size; |
7602b8710 iommu/arm-smmu: D... |
838 |
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { |
518f71362 iommu/arm-smmu: m... |
839 |
fmt = ARM_64_LPAE_S1; |
6070529be iommu/arm-smmu: S... |
840 |
} else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { |
518f71362 iommu/arm-smmu: m... |
841 |
fmt = ARM_32_LPAE_S1; |
7602b8710 iommu/arm-smmu: D... |
842 843 |
ias = min(ias, 32UL); oas = min(oas, 40UL); |
6070529be iommu/arm-smmu: S... |
844 845 846 847 |
} else { fmt = ARM_V7S; ias = min(ias, 32UL); oas = min(oas, 32UL); |
7602b8710 iommu/arm-smmu: D... |
848 |
} |
c752ce45b iommu/arm-smmu: a... |
849 850 |
break; case ARM_SMMU_DOMAIN_NESTED: |
45ae7cff3 iommu/arm: Add su... |
851 852 853 854 |
/* * We will likely want to change this if/when KVM gets * involved. */ |
c752ce45b iommu/arm-smmu: a... |
855 |
case ARM_SMMU_DOMAIN_S2: |
9c5c92e35 iommu/arm-smmu: p... |
856 857 |
cfg->cbar = CBAR_TYPE_S2_TRANS; start = 0; |
518f71362 iommu/arm-smmu: m... |
858 859 |
ias = smmu->ipa_size; oas = smmu->pa_size; |
7602b8710 iommu/arm-smmu: D... |
860 |
if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { |
518f71362 iommu/arm-smmu: m... |
861 |
fmt = ARM_64_LPAE_S2; |
7602b8710 iommu/arm-smmu: D... |
862 |
} else { |
518f71362 iommu/arm-smmu: m... |
863 |
fmt = ARM_32_LPAE_S2; |
7602b8710 iommu/arm-smmu: D... |
864 865 866 |
ias = min(ias, 40UL); oas = min(oas, 40UL); } |
c752ce45b iommu/arm-smmu: a... |
867 868 869 870 |
break; default: ret = -EINVAL; goto out_unlock; |
45ae7cff3 iommu/arm: Add su... |
871 872 873 874 |
} ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); |
287980e49 remove lots of IS... |
875 |
if (ret < 0) |
a18037b27 iommu/arm-smmu: a... |
876 |
goto out_unlock; |
45ae7cff3 iommu/arm: Add su... |
877 |
|
44680eedf iommu/arm-smmu: r... |
878 |
cfg->cbndx = ret; |
b7862e355 iommu/arm-smmu: S... |
879 |
if (smmu->version < ARM_SMMU_V2) { |
44680eedf iommu/arm-smmu: r... |
880 881 |
cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; |
45ae7cff3 iommu/arm: Add su... |
882 |
} else { |
44680eedf iommu/arm-smmu: r... |
883 |
cfg->irptndx = cfg->cbndx; |
45ae7cff3 iommu/arm: Add su... |
884 |
} |
518f71362 iommu/arm-smmu: m... |
885 |
pgtbl_cfg = (struct io_pgtable_cfg) { |
d54663573 iommu/arm-smmu: U... |
886 |
.pgsize_bitmap = smmu->pgsize_bitmap, |
518f71362 iommu/arm-smmu: m... |
887 888 889 |
.ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, |
2df7a25ce iommu/arm-smmu: C... |
890 |
.iommu_dev = smmu->dev, |
518f71362 iommu/arm-smmu: m... |
891 892 893 894 895 896 897 898 |
}; smmu_domain->smmu = smmu; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); if (!pgtbl_ops) { ret = -ENOMEM; goto out_clear_smmu; } |
d54663573 iommu/arm-smmu: U... |
899 900 |
/* Update the domain's page sizes to reflect the page table format */ domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; |
455eb7d34 iommu/arm-smmu: S... |
901 902 |
domain->geometry.aperture_end = (1UL << ias) - 1; domain->geometry.force_aperture = true; |
a18037b27 iommu/arm-smmu: a... |
903 |
|
518f71362 iommu/arm-smmu: m... |
904 905 906 907 908 909 910 |
/* Initialise the context bank with our page table cfg */ arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); /* * Request context fault interrupt. Do this last to avoid the * handler seeing a half-initialised domain state. */ |
44680eedf iommu/arm-smmu: r... |
911 |
irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; |
bee140044 iommu/arm-smmu: U... |
912 913 |
ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); |
287980e49 remove lots of IS... |
914 |
if (ret < 0) { |
45ae7cff3 iommu/arm: Add su... |
915 916 |
dev_err(smmu->dev, "failed to request context IRQ %d (%u) ", |
44680eedf iommu/arm-smmu: r... |
917 918 |
cfg->irptndx, irq); cfg->irptndx = INVALID_IRPTNDX; |
45ae7cff3 iommu/arm: Add su... |
919 |
} |
518f71362 iommu/arm-smmu: m... |
920 921 922 923 |
mutex_unlock(&smmu_domain->init_mutex); /* Publish page table ops for map/unmap */ smmu_domain->pgtbl_ops = pgtbl_ops; |
a9a1b0b53 iommu/arm-smmu: a... |
924 |
return 0; |
45ae7cff3 iommu/arm: Add su... |
925 |
|
518f71362 iommu/arm-smmu: m... |
926 927 |
out_clear_smmu: smmu_domain->smmu = NULL; |
a18037b27 iommu/arm-smmu: a... |
928 |
out_unlock: |
518f71362 iommu/arm-smmu: m... |
929 |
mutex_unlock(&smmu_domain->init_mutex); |
45ae7cff3 iommu/arm: Add su... |
930 931 932 933 934 |
return ret; } static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) { |
1d672638f iommu/arm-smmu: M... |
935 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
44680eedf iommu/arm-smmu: r... |
936 937 |
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; |
1463fe44f iommu/arm-smmu: D... |
938 |
void __iomem *cb_base; |
45ae7cff3 iommu/arm: Add su... |
939 |
int irq; |
021bb8420 iommu/arm-smmu: W... |
940 |
if (!smmu) |
45ae7cff3 iommu/arm: Add su... |
941 |
return; |
518f71362 iommu/arm-smmu: m... |
942 943 944 945 |
/* * Disable the context bank and free the page tables before freeing * it. */ |
44680eedf iommu/arm-smmu: r... |
946 |
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); |
1463fe44f iommu/arm-smmu: D... |
947 |
writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); |
1463fe44f iommu/arm-smmu: D... |
948 |
|
44680eedf iommu/arm-smmu: r... |
949 950 |
if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; |
bee140044 iommu/arm-smmu: U... |
951 |
devm_free_irq(smmu->dev, irq, domain); |
45ae7cff3 iommu/arm: Add su... |
952 |
} |
44830b0cb iommu/arm-smmu: D... |
953 |
free_io_pgtable_ops(smmu_domain->pgtbl_ops); |
44680eedf iommu/arm-smmu: r... |
954 |
__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); |
45ae7cff3 iommu/arm: Add su... |
955 |
} |
1d672638f iommu/arm-smmu: M... |
956 |
static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) |
45ae7cff3 iommu/arm: Add su... |
957 958 |
{ struct arm_smmu_domain *smmu_domain; |
45ae7cff3 iommu/arm: Add su... |
959 |
|
9adb95949 iommu/arm-smmu: S... |
960 |
if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) |
1d672638f iommu/arm-smmu: M... |
961 |
return NULL; |
45ae7cff3 iommu/arm: Add su... |
962 963 964 965 966 967 968 |
/* * Allocate the domain and initialise some of its data structures. * We can't really do anything meaningful until we've added a * master. */ smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); if (!smmu_domain) |
1d672638f iommu/arm-smmu: M... |
969 |
return NULL; |
45ae7cff3 iommu/arm: Add su... |
970 |
|
021bb8420 iommu/arm-smmu: W... |
971 972 |
if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || iommu_get_dma_cookie(&smmu_domain->domain))) { |
9adb95949 iommu/arm-smmu: S... |
973 974 975 |
kfree(smmu_domain); return NULL; } |
518f71362 iommu/arm-smmu: m... |
976 977 |
mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); |
1d672638f iommu/arm-smmu: M... |
978 979 |
return &smmu_domain->domain; |
45ae7cff3 iommu/arm: Add su... |
980 |
} |
1d672638f iommu/arm-smmu: M... |
981 |
static void arm_smmu_domain_free(struct iommu_domain *domain) |
45ae7cff3 iommu/arm: Add su... |
982 |
{ |
1d672638f iommu/arm-smmu: M... |
983 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
1463fe44f iommu/arm-smmu: D... |
984 985 986 987 988 |
/* * Free the domain resources. We assume that all devices have * already been detached. */ |
9adb95949 iommu/arm-smmu: S... |
989 |
iommu_put_dma_cookie(domain); |
45ae7cff3 iommu/arm: Add su... |
990 |
arm_smmu_destroy_domain_context(domain); |
45ae7cff3 iommu/arm: Add su... |
991 992 |
kfree(smmu_domain); } |
1f3d5ca43 iommu/arm-smmu: C... |
993 994 995 |
static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_smr *smr = smmu->smrs + idx; |
f80cd885f iommu/arm-smmu: R... |
996 |
u32 reg = smr->id << SMR_ID_SHIFT | smr->mask << SMR_MASK_SHIFT; |
1f3d5ca43 iommu/arm-smmu: C... |
997 998 999 1000 1001 |
if (smr->valid) reg |= SMR_VALID; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_SMR(idx)); } |
8e8b203ea iommu/arm-smmu: K... |
1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 |
static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) { struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; u32 reg = (s2cr->type & S2CR_TYPE_MASK) << S2CR_TYPE_SHIFT | (s2cr->cbndx & S2CR_CBNDX_MASK) << S2CR_CBNDX_SHIFT | (s2cr->privcfg & S2CR_PRIVCFG_MASK) << S2CR_PRIVCFG_SHIFT; writel_relaxed(reg, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_S2CR(idx)); } static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) { arm_smmu_write_s2cr(smmu, idx); if (smmu->smrs) arm_smmu_write_smr(smmu, idx); } |
588888a73 iommu/arm-smmu: I... |
1018 |
static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) |
1f3d5ca43 iommu/arm-smmu: C... |
1019 1020 |
{ struct arm_smmu_smr *smrs = smmu->smrs; |
588888a73 iommu/arm-smmu: I... |
1021 |
int i, free_idx = -ENOSPC; |
1f3d5ca43 iommu/arm-smmu: C... |
1022 |
|
588888a73 iommu/arm-smmu: I... |
1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 |
/* Stream indexing is blissfully easy */ if (!smrs) return id; /* Validating SMRs is... less so */ for (i = 0; i < smmu->num_mapping_groups; ++i) { if (!smrs[i].valid) { /* * Note the first free entry we come across, which * we'll claim in the end if nothing else matches. */ if (free_idx < 0) free_idx = i; |
1f3d5ca43 iommu/arm-smmu: C... |
1036 1037 |
continue; } |
588888a73 iommu/arm-smmu: I... |
1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 |
/* * If the new entry is _entirely_ matched by an existing entry, * then reuse that, with the guarantee that there also cannot * be any subsequent conflicting entries. In normal use we'd * expect simply identical entries for this case, but there's * no harm in accommodating the generalisation. */ if ((mask & smrs[i].mask) == mask && !((id ^ smrs[i].id) & ~smrs[i].mask)) return i; /* * If the new entry has any other overlap with an existing one, * though, then there always exists at least one stream ID * which would cause a conflict, and we can't allow that risk. */ if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) return -EINVAL; } |
1f3d5ca43 iommu/arm-smmu: C... |
1056 |
|
588888a73 iommu/arm-smmu: I... |
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 |
return free_idx; } static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) { if (--smmu->s2crs[idx].count) return false; smmu->s2crs[idx] = s2cr_init_val; if (smmu->smrs) smmu->smrs[idx].valid = false; return true; } static int arm_smmu_master_alloc_smes(struct device *dev) { |
adfec2e70 iommu/arm-smmu: C... |
1074 1075 |
struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; |
588888a73 iommu/arm-smmu: I... |
1076 1077 1078 1079 1080 1081 1082 |
struct arm_smmu_device *smmu = cfg->smmu; struct arm_smmu_smr *smrs = smmu->smrs; struct iommu_group *group; int i, idx, ret; mutex_lock(&smmu->stream_map_mutex); /* Figure out a viable stream map entry allocation */ |
adfec2e70 iommu/arm-smmu: C... |
1083 |
for_each_cfg_sme(fwspec, i, idx) { |
021bb8420 iommu/arm-smmu: W... |
1084 1085 |
u16 sid = fwspec->ids[i]; u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; |
588888a73 iommu/arm-smmu: I... |
1086 1087 1088 |
if (idx != INVALID_SMENDX) { ret = -EEXIST; goto out_err; |
45ae7cff3 iommu/arm: Add su... |
1089 |
} |
021bb8420 iommu/arm-smmu: W... |
1090 |
ret = arm_smmu_find_sme(smmu, sid, mask); |
588888a73 iommu/arm-smmu: I... |
1091 1092 1093 1094 1095 |
if (ret < 0) goto out_err; idx = ret; if (smrs && smmu->s2crs[idx].count == 0) { |
021bb8420 iommu/arm-smmu: W... |
1096 1097 |
smrs[idx].id = sid; smrs[idx].mask = mask; |
588888a73 iommu/arm-smmu: I... |
1098 1099 1100 1101 |
smrs[idx].valid = true; } smmu->s2crs[idx].count++; cfg->smendx[i] = (s16)idx; |
45ae7cff3 iommu/arm: Add su... |
1102 |
} |
588888a73 iommu/arm-smmu: I... |
1103 1104 1105 1106 1107 1108 1109 1110 |
group = iommu_group_get_for_dev(dev); if (!group) group = ERR_PTR(-ENOMEM); if (IS_ERR(group)) { ret = PTR_ERR(group); goto out_err; } iommu_group_put(group); |
1f3d5ca43 iommu/arm-smmu: C... |
1111 |
|
45ae7cff3 iommu/arm: Add su... |
1112 |
/* It worked! Now, poke the actual hardware */ |
adfec2e70 iommu/arm-smmu: C... |
1113 |
for_each_cfg_sme(fwspec, i, idx) { |
588888a73 iommu/arm-smmu: I... |
1114 1115 1116 |
arm_smmu_write_sme(smmu, idx); smmu->s2crs[idx].group = group; } |
45ae7cff3 iommu/arm: Add su... |
1117 |
|
588888a73 iommu/arm-smmu: I... |
1118 |
mutex_unlock(&smmu->stream_map_mutex); |
45ae7cff3 iommu/arm: Add su... |
1119 |
return 0; |
588888a73 iommu/arm-smmu: I... |
1120 |
out_err: |
1f3d5ca43 iommu/arm-smmu: C... |
1121 |
while (i--) { |
588888a73 iommu/arm-smmu: I... |
1122 |
arm_smmu_free_sme(smmu, cfg->smendx[i]); |
1f3d5ca43 iommu/arm-smmu: C... |
1123 1124 |
cfg->smendx[i] = INVALID_SMENDX; } |
588888a73 iommu/arm-smmu: I... |
1125 1126 |
mutex_unlock(&smmu->stream_map_mutex); return ret; |
45ae7cff3 iommu/arm: Add su... |
1127 |
} |
adfec2e70 iommu/arm-smmu: C... |
1128 |
static void arm_smmu_master_free_smes(struct iommu_fwspec *fwspec) |
45ae7cff3 iommu/arm: Add su... |
1129 |
{ |
adfec2e70 iommu/arm-smmu: C... |
1130 1131 |
struct arm_smmu_device *smmu = fwspec_smmu(fwspec); struct arm_smmu_master_cfg *cfg = fwspec->iommu_priv; |
d3097e393 iommu/arm-smmu: A... |
1132 |
int i, idx; |
43b412bed iommu/arm-smmu: f... |
1133 |
|
588888a73 iommu/arm-smmu: I... |
1134 |
mutex_lock(&smmu->stream_map_mutex); |
adfec2e70 iommu/arm-smmu: C... |
1135 |
for_each_cfg_sme(fwspec, i, idx) { |
588888a73 iommu/arm-smmu: I... |
1136 1137 |
if (arm_smmu_free_sme(smmu, idx)) arm_smmu_write_sme(smmu, idx); |
1f3d5ca43 iommu/arm-smmu: C... |
1138 |
cfg->smendx[i] = INVALID_SMENDX; |
45ae7cff3 iommu/arm: Add su... |
1139 |
} |
588888a73 iommu/arm-smmu: I... |
1140 |
mutex_unlock(&smmu->stream_map_mutex); |
45ae7cff3 iommu/arm: Add su... |
1141 |
} |
45ae7cff3 iommu/arm: Add su... |
1142 |
static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, |
adfec2e70 iommu/arm-smmu: C... |
1143 |
struct iommu_fwspec *fwspec) |
45ae7cff3 iommu/arm: Add su... |
1144 |
{ |
44680eedf iommu/arm-smmu: r... |
1145 |
struct arm_smmu_device *smmu = smmu_domain->smmu; |
8e8b203ea iommu/arm-smmu: K... |
1146 1147 1148 |
struct arm_smmu_s2cr *s2cr = smmu->s2crs; enum arm_smmu_s2cr_type type = S2CR_TYPE_TRANS; u8 cbndx = smmu_domain->cfg.cbndx; |
588888a73 iommu/arm-smmu: I... |
1149 |
int i, idx; |
45ae7cff3 iommu/arm: Add su... |
1150 |
|
adfec2e70 iommu/arm-smmu: C... |
1151 |
for_each_cfg_sme(fwspec, i, idx) { |
8e8b203ea iommu/arm-smmu: K... |
1152 |
if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) |
588888a73 iommu/arm-smmu: I... |
1153 |
continue; |
1f3d5ca43 iommu/arm-smmu: C... |
1154 |
|
8e8b203ea iommu/arm-smmu: K... |
1155 1156 1157 1158 |
s2cr[idx].type = type; s2cr[idx].privcfg = S2CR_PRIVCFG_UNPRIV; s2cr[idx].cbndx = cbndx; arm_smmu_write_s2cr(smmu, idx); |
43b412bed iommu/arm-smmu: f... |
1159 |
} |
8e8b203ea iommu/arm-smmu: K... |
1160 |
return 0; |
bc7f2ce0a iommu/arm-smmu: D... |
1161 |
} |
45ae7cff3 iommu/arm: Add su... |
1162 1163 |
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { |
a18037b27 iommu/arm-smmu: a... |
1164 |
int ret; |
adfec2e70 iommu/arm-smmu: C... |
1165 1166 |
struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu; |
1d672638f iommu/arm-smmu: M... |
1167 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
45ae7cff3 iommu/arm: Add su... |
1168 |
|
adfec2e70 iommu/arm-smmu: C... |
1169 |
if (!fwspec || fwspec->ops != &arm_smmu_ops) { |
45ae7cff3 iommu/arm: Add su... |
1170 1171 1172 1173 |
dev_err(dev, "cannot attach to SMMU, is it on the same bus? "); return -ENXIO; } |
fba4f8e5c iommu/arm-smmu: W... |
1174 1175 1176 1177 1178 1179 1180 1181 1182 |
/* * FIXME: The arch/arm DMA API code tries to attach devices to its own * domains between of_xlate() and add_device() - we have no way to cope * with that, so until ARM gets converted to rely on groups and default * domains, just say no (but more politely than by dereferencing NULL). * This should be at least a WARN_ON once that's sorted. */ if (!fwspec->iommu_priv) return -ENODEV; |
adfec2e70 iommu/arm-smmu: C... |
1183 |
smmu = fwspec_smmu(fwspec); |
518f71362 iommu/arm-smmu: m... |
1184 |
/* Ensure that the domain is finalised */ |
adfec2e70 iommu/arm-smmu: C... |
1185 |
ret = arm_smmu_init_domain_context(domain, smmu); |
287980e49 remove lots of IS... |
1186 |
if (ret < 0) |
518f71362 iommu/arm-smmu: m... |
1187 |
return ret; |
45ae7cff3 iommu/arm: Add su... |
1188 |
/* |
44680eedf iommu/arm-smmu: r... |
1189 1190 |
* Sanity check the domain. We don't support domains across * different SMMUs. |
45ae7cff3 iommu/arm: Add su... |
1191 |
*/ |
adfec2e70 iommu/arm-smmu: C... |
1192 |
if (smmu_domain->smmu != smmu) { |
45ae7cff3 iommu/arm: Add su... |
1193 1194 1195 |
dev_err(dev, "cannot attach to SMMU %s whilst already attached to domain on SMMU %s ", |
adfec2e70 iommu/arm-smmu: C... |
1196 |
dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); |
a18037b27 iommu/arm-smmu: a... |
1197 |
return -EINVAL; |
45ae7cff3 iommu/arm: Add su... |
1198 |
} |
45ae7cff3 iommu/arm: Add su... |
1199 1200 |
/* Looks ok, so add the device to the domain */ |
adfec2e70 iommu/arm-smmu: C... |
1201 |
return arm_smmu_domain_add_master(smmu_domain, fwspec); |
45ae7cff3 iommu/arm: Add su... |
1202 |
} |
45ae7cff3 iommu/arm: Add su... |
1203 |
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, |
b410aed93 iommu/arm-smmu: c... |
1204 |
phys_addr_t paddr, size_t size, int prot) |
45ae7cff3 iommu/arm: Add su... |
1205 |
{ |
518f71362 iommu/arm-smmu: m... |
1206 1207 |
int ret; unsigned long flags; |
1d672638f iommu/arm-smmu: M... |
1208 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
518f71362 iommu/arm-smmu: m... |
1209 |
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; |
45ae7cff3 iommu/arm: Add su... |
1210 |
|
518f71362 iommu/arm-smmu: m... |
1211 |
if (!ops) |
45ae7cff3 iommu/arm: Add su... |
1212 |
return -ENODEV; |
518f71362 iommu/arm-smmu: m... |
1213 1214 1215 1216 |
spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); ret = ops->map(ops, iova, paddr, size, prot); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); return ret; |
45ae7cff3 iommu/arm: Add su... |
1217 1218 1219 1220 1221 |
} static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { |
518f71362 iommu/arm-smmu: m... |
1222 1223 |
size_t ret; unsigned long flags; |
1d672638f iommu/arm-smmu: M... |
1224 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
518f71362 iommu/arm-smmu: m... |
1225 |
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; |
45ae7cff3 iommu/arm: Add su... |
1226 |
|
518f71362 iommu/arm-smmu: m... |
1227 1228 1229 1230 1231 1232 1233 |
if (!ops) return 0; spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); ret = ops->unmap(ops, iova, size); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); return ret; |
45ae7cff3 iommu/arm: Add su... |
1234 |
} |
859a732e4 iommu/arm-smmu: a... |
1235 1236 1237 |
static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, dma_addr_t iova) { |
1d672638f iommu/arm-smmu: M... |
1238 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
859a732e4 iommu/arm-smmu: a... |
1239 1240 1241 1242 1243 1244 1245 |
struct arm_smmu_device *smmu = smmu_domain->smmu; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; struct device *dev = smmu->dev; void __iomem *cb_base; u32 tmp; u64 phys; |
661d962f1 iommu/arm-smmu: F... |
1246 |
unsigned long va; |
859a732e4 iommu/arm-smmu: a... |
1247 1248 |
cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); |
661d962f1 iommu/arm-smmu: F... |
1249 1250 |
/* ATS1 registers can only be written atomically */ va = iova & ~0xfffUL; |
661d962f1 iommu/arm-smmu: F... |
1251 |
if (smmu->version == ARM_SMMU_V2) |
f9a05f05b iommu/arm-smmu: T... |
1252 1253 |
smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR); else /* Register is only 32-bit in v1 */ |
661d962f1 iommu/arm-smmu: F... |
1254 |
writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); |
859a732e4 iommu/arm-smmu: a... |
1255 1256 1257 1258 |
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) { dev_err(dev, |
077124c98 iommu/arm-smmu: R... |
1259 1260 |
"iova to phys timed out on %pad. Falling back to software table walk. ", |
859a732e4 iommu/arm-smmu: a... |
1261 1262 1263 |
&iova); return ops->iova_to_phys(ops, iova); } |
f9a05f05b iommu/arm-smmu: T... |
1264 |
phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); |
859a732e4 iommu/arm-smmu: a... |
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 |
if (phys & CB_PAR_F) { dev_err(dev, "translation fault! "); dev_err(dev, "PAR = 0x%llx ", phys); return 0; } return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); } |
45ae7cff3 iommu/arm: Add su... |
1275 |
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, |
859a732e4 iommu/arm-smmu: a... |
1276 |
dma_addr_t iova) |
45ae7cff3 iommu/arm: Add su... |
1277 |
{ |
518f71362 iommu/arm-smmu: m... |
1278 1279 |
phys_addr_t ret; unsigned long flags; |
1d672638f iommu/arm-smmu: M... |
1280 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
518f71362 iommu/arm-smmu: m... |
1281 |
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; |
45ae7cff3 iommu/arm: Add su... |
1282 |
|
518f71362 iommu/arm-smmu: m... |
1283 |
if (!ops) |
a44a9791e iommu/arm-smmu: u... |
1284 |
return 0; |
45ae7cff3 iommu/arm: Add su... |
1285 |
|
518f71362 iommu/arm-smmu: m... |
1286 |
spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); |
83a60ed8f iommu/arm-smmu: f... |
1287 1288 |
if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { |
859a732e4 iommu/arm-smmu: a... |
1289 |
ret = arm_smmu_iova_to_phys_hard(domain, iova); |
83a60ed8f iommu/arm-smmu: f... |
1290 |
} else { |
859a732e4 iommu/arm-smmu: a... |
1291 |
ret = ops->iova_to_phys(ops, iova); |
83a60ed8f iommu/arm-smmu: f... |
1292 |
} |
518f71362 iommu/arm-smmu: m... |
1293 |
spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); |
859a732e4 iommu/arm-smmu: a... |
1294 |
|
518f71362 iommu/arm-smmu: m... |
1295 |
return ret; |
45ae7cff3 iommu/arm: Add su... |
1296 |
} |
1fd0c775a iommu/arm-smmu: C... |
1297 |
static bool arm_smmu_capable(enum iommu_cap cap) |
45ae7cff3 iommu/arm: Add su... |
1298 |
{ |
d09489456 iommu/arm-smmu: c... |
1299 1300 |
switch (cap) { case IOMMU_CAP_CACHE_COHERENCY: |
1fd0c775a iommu/arm-smmu: C... |
1301 1302 1303 1304 1305 |
/* * Return true here as the SMMU can always send out coherent * requests. */ return true; |
d09489456 iommu/arm-smmu: c... |
1306 |
case IOMMU_CAP_INTR_REMAP: |
1fd0c775a iommu/arm-smmu: C... |
1307 |
return true; /* MSIs are just memory writes */ |
0029a8dd6 iommu/arm-smmu: a... |
1308 1309 |
case IOMMU_CAP_NOEXEC: return true; |
d09489456 iommu/arm-smmu: c... |
1310 |
default: |
1fd0c775a iommu/arm-smmu: C... |
1311 |
return false; |
d09489456 iommu/arm-smmu: c... |
1312 |
} |
45ae7cff3 iommu/arm: Add su... |
1313 |
} |
45ae7cff3 iommu/arm: Add su... |
1314 |
|
021bb8420 iommu/arm-smmu: W... |
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 |
static int arm_smmu_match_node(struct device *dev, void *data) { return dev->of_node == data; } static struct arm_smmu_device *arm_smmu_get_by_node(struct device_node *np) { struct device *dev = driver_find_device(&arm_smmu_driver.driver, NULL, np, arm_smmu_match_node); put_device(dev); return dev ? dev_get_drvdata(dev) : NULL; } |
f80cd885f iommu/arm-smmu: R... |
1327 |
static int arm_smmu_add_device(struct device *dev) |
45ae7cff3 iommu/arm: Add su... |
1328 |
{ |
adfec2e70 iommu/arm-smmu: C... |
1329 |
struct arm_smmu_device *smmu; |
03edb2264 iommu/arm-smmu: h... |
1330 |
struct arm_smmu_master_cfg *cfg; |
021bb8420 iommu/arm-smmu: W... |
1331 |
struct iommu_fwspec *fwspec = dev->iommu_fwspec; |
f80cd885f iommu/arm-smmu: R... |
1332 |
int i, ret; |
8f68f8e28 iommu/arm-smmu: a... |
1333 |
|
021bb8420 iommu/arm-smmu: W... |
1334 1335 1336 1337 1338 |
if (using_legacy_binding) { ret = arm_smmu_register_legacy_master(dev, &smmu); fwspec = dev->iommu_fwspec; if (ret) goto out_free; |
3c117b543 iommu/arm-smmu: C... |
1339 |
} else if (fwspec && fwspec->ops == &arm_smmu_ops) { |
021bb8420 iommu/arm-smmu: W... |
1340 1341 1342 1343 |
smmu = arm_smmu_get_by_node(to_of_node(fwspec->iommu_fwnode)); } else { return -ENODEV; } |
a9a1b0b53 iommu/arm-smmu: a... |
1344 |
|
f80cd885f iommu/arm-smmu: R... |
1345 |
ret = -EINVAL; |
adfec2e70 iommu/arm-smmu: C... |
1346 1347 |
for (i = 0; i < fwspec->num_ids; i++) { u16 sid = fwspec->ids[i]; |
021bb8420 iommu/arm-smmu: W... |
1348 |
u16 mask = fwspec->ids[i] >> SMR_MASK_SHIFT; |
03edb2264 iommu/arm-smmu: h... |
1349 |
|
adfec2e70 iommu/arm-smmu: C... |
1350 |
if (sid & ~smmu->streamid_mask) { |
f80cd885f iommu/arm-smmu: R... |
1351 1352 |
dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x) ", |
021bb8420 iommu/arm-smmu: W... |
1353 1354 1355 1356 1357 1358 1359 |
sid, smmu->streamid_mask); goto out_free; } if (mask & ~smmu->smr_mask_mask) { dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x) ", sid, smmu->smr_mask_mask); |
f80cd885f iommu/arm-smmu: R... |
1360 1361 |
goto out_free; } |
1f3d5ca43 iommu/arm-smmu: C... |
1362 |
} |
5fc63a7c4 iommu/arm-smmu: a... |
1363 |
|
adfec2e70 iommu/arm-smmu: C... |
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 |
ret = -ENOMEM; cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), GFP_KERNEL); if (!cfg) goto out_free; cfg->smmu = smmu; fwspec->iommu_priv = cfg; while (i--) cfg->smendx[i] = INVALID_SMENDX; |
588888a73 iommu/arm-smmu: I... |
1374 |
ret = arm_smmu_master_alloc_smes(dev); |
adfec2e70 iommu/arm-smmu: C... |
1375 1376 1377 1378 |
if (ret) goto out_free; return 0; |
f80cd885f iommu/arm-smmu: R... |
1379 1380 |
out_free: |
adfec2e70 iommu/arm-smmu: C... |
1381 1382 1383 |
if (fwspec) kfree(fwspec->iommu_priv); iommu_fwspec_free(dev); |
f80cd885f iommu/arm-smmu: R... |
1384 |
return ret; |
03edb2264 iommu/arm-smmu: h... |
1385 |
} |
45ae7cff3 iommu/arm: Add su... |
1386 1387 |
static void arm_smmu_remove_device(struct device *dev) { |
adfec2e70 iommu/arm-smmu: C... |
1388 |
struct iommu_fwspec *fwspec = dev->iommu_fwspec; |
8e8b203ea iommu/arm-smmu: K... |
1389 |
|
adfec2e70 iommu/arm-smmu: C... |
1390 |
if (!fwspec || fwspec->ops != &arm_smmu_ops) |
f80cd885f iommu/arm-smmu: R... |
1391 |
return; |
8e8b203ea iommu/arm-smmu: K... |
1392 |
|
adfec2e70 iommu/arm-smmu: C... |
1393 |
arm_smmu_master_free_smes(fwspec); |
5fc63a7c4 iommu/arm-smmu: a... |
1394 |
iommu_group_remove_device(dev); |
adfec2e70 iommu/arm-smmu: C... |
1395 1396 |
kfree(fwspec->iommu_priv); iommu_fwspec_free(dev); |
45ae7cff3 iommu/arm: Add su... |
1397 |
} |
af6599322 iommu/arm-smmu: S... |
1398 1399 |
static struct iommu_group *arm_smmu_device_group(struct device *dev) { |
adfec2e70 iommu/arm-smmu: C... |
1400 1401 |
struct iommu_fwspec *fwspec = dev->iommu_fwspec; struct arm_smmu_device *smmu = fwspec_smmu(fwspec); |
588888a73 iommu/arm-smmu: I... |
1402 1403 |
struct iommu_group *group = NULL; int i, idx; |
adfec2e70 iommu/arm-smmu: C... |
1404 |
for_each_cfg_sme(fwspec, i, idx) { |
588888a73 iommu/arm-smmu: I... |
1405 1406 1407 1408 1409 1410 1411 1412 1413 |
if (group && smmu->s2crs[idx].group && group != smmu->s2crs[idx].group) return ERR_PTR(-EINVAL); group = smmu->s2crs[idx].group; } if (group) return group; |
af6599322 iommu/arm-smmu: S... |
1414 1415 1416 1417 1418 |
if (dev_is_pci(dev)) group = pci_device_group(dev); else group = generic_device_group(dev); |
af6599322 iommu/arm-smmu: S... |
1419 1420 |
return group; } |
c752ce45b iommu/arm-smmu: a... |
1421 1422 1423 |
static int arm_smmu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { |
1d672638f iommu/arm-smmu: M... |
1424 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
c752ce45b iommu/arm-smmu: a... |
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 |
switch (attr) { case DOMAIN_ATTR_NESTING: *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); return 0; default: return -ENODEV; } } static int arm_smmu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { |
518f71362 iommu/arm-smmu: m... |
1438 |
int ret = 0; |
1d672638f iommu/arm-smmu: M... |
1439 |
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); |
c752ce45b iommu/arm-smmu: a... |
1440 |
|
518f71362 iommu/arm-smmu: m... |
1441 |
mutex_lock(&smmu_domain->init_mutex); |
c752ce45b iommu/arm-smmu: a... |
1442 1443 |
switch (attr) { case DOMAIN_ATTR_NESTING: |
518f71362 iommu/arm-smmu: m... |
1444 1445 1446 1447 |
if (smmu_domain->smmu) { ret = -EPERM; goto out_unlock; } |
c752ce45b iommu/arm-smmu: a... |
1448 1449 1450 1451 |
if (*(int *)data) smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; else smmu_domain->stage = ARM_SMMU_DOMAIN_S1; |
518f71362 iommu/arm-smmu: m... |
1452 |
break; |
c752ce45b iommu/arm-smmu: a... |
1453 |
default: |
518f71362 iommu/arm-smmu: m... |
1454 |
ret = -ENODEV; |
c752ce45b iommu/arm-smmu: a... |
1455 |
} |
518f71362 iommu/arm-smmu: m... |
1456 1457 1458 1459 |
out_unlock: mutex_unlock(&smmu_domain->init_mutex); return ret; |
c752ce45b iommu/arm-smmu: a... |
1460 |
} |
021bb8420 iommu/arm-smmu: W... |
1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 |
static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) { u32 fwid = 0; if (args->args_count > 0) fwid |= (u16)args->args[0]; if (args->args_count > 1) fwid |= (u16)args->args[1] << SMR_MASK_SHIFT; return iommu_fwspec_add_ids(dev, &fwid, 1); } |
518f71362 iommu/arm-smmu: m... |
1473 |
static struct iommu_ops arm_smmu_ops = { |
c752ce45b iommu/arm-smmu: a... |
1474 |
.capable = arm_smmu_capable, |
1d672638f iommu/arm-smmu: M... |
1475 1476 |
.domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, |
c752ce45b iommu/arm-smmu: a... |
1477 |
.attach_dev = arm_smmu_attach_dev, |
c752ce45b iommu/arm-smmu: a... |
1478 1479 |
.map = arm_smmu_map, .unmap = arm_smmu_unmap, |
76771c938 Merge branches 'a... |
1480 |
.map_sg = default_iommu_map_sg, |
c752ce45b iommu/arm-smmu: a... |
1481 1482 1483 |
.iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, |
af6599322 iommu/arm-smmu: S... |
1484 |
.device_group = arm_smmu_device_group, |
c752ce45b iommu/arm-smmu: a... |
1485 1486 |
.domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, |
021bb8420 iommu/arm-smmu: W... |
1487 |
.of_xlate = arm_smmu_of_xlate, |
518f71362 iommu/arm-smmu: m... |
1488 |
.pgsize_bitmap = -1UL, /* Restricted during device attach */ |
45ae7cff3 iommu/arm: Add su... |
1489 1490 1491 1492 1493 |
}; static void arm_smmu_device_reset(struct arm_smmu_device *smmu) { void __iomem *gr0_base = ARM_SMMU_GR0(smmu); |
659db6f6b iommu/arm-smmu: C... |
1494 |
void __iomem *cb_base; |
1f3d5ca43 iommu/arm-smmu: C... |
1495 |
int i; |
3ca3712a4 iommu/arm-smmu: C... |
1496 |
u32 reg, major; |
659db6f6b iommu/arm-smmu: C... |
1497 |
|
3a5df8ff3 iommu/arm-smmu: s... |
1498 1499 1500 |
/* clear global FSR */ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); |
45ae7cff3 iommu/arm: Add su... |
1501 |
|
1f3d5ca43 iommu/arm-smmu: C... |
1502 1503 1504 1505 |
/* * Reset stream mapping groups: Initial values mark all SMRn as * invalid and all S2CRn as bypass unless overridden. */ |
8e8b203ea iommu/arm-smmu: K... |
1506 1507 |
for (i = 0; i < smmu->num_mapping_groups; ++i) arm_smmu_write_sme(smmu, i); |
45ae7cff3 iommu/arm: Add su... |
1508 |
|
3ca3712a4 iommu/arm-smmu: C... |
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 |
/* * Before clearing ARM_MMU500_ACTLR_CPRE, need to * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK * bit is only present in MMU-500r2 onwards. */ reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; if ((smmu->model == ARM_MMU500) && (major >= 2)) { reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); reg &= ~ARM_MMU500_ACR_CACHE_LOCK; writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); } |
659db6f6b iommu/arm-smmu: C... |
1521 1522 1523 1524 1525 |
/* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); |
f0cfffc48 iommu/arm-smmu: W... |
1526 1527 1528 1529 1530 1531 1532 1533 1534 |
/* * Disable MMU-500's not-particularly-beneficial next-page * prefetcher for the sake of errata #841119 and #826419. */ if (smmu->model == ARM_MMU500) { reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); reg &= ~ARM_MMU500_ACTLR_CPRE; writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); } |
659db6f6b iommu/arm-smmu: C... |
1535 |
} |
1463fe44f iommu/arm-smmu: D... |
1536 |
|
45ae7cff3 iommu/arm: Add su... |
1537 |
/* Invalidate the TLB, just in case */ |
45ae7cff3 iommu/arm: Add su... |
1538 1539 |
writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH); writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); |
3a5df8ff3 iommu/arm-smmu: s... |
1540 |
reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
659db6f6b iommu/arm-smmu: C... |
1541 |
|
45ae7cff3 iommu/arm: Add su... |
1542 |
/* Enable fault reporting */ |
659db6f6b iommu/arm-smmu: C... |
1543 |
reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); |
45ae7cff3 iommu/arm: Add su... |
1544 1545 |
/* Disable TLB broadcasting. */ |
659db6f6b iommu/arm-smmu: C... |
1546 |
reg |= (sCR0_VMIDPNE | sCR0_PTM); |
45ae7cff3 iommu/arm: Add su... |
1547 |
|
25a1c96cd iommu/arm-smmu: A... |
1548 1549 1550 1551 1552 1553 |
/* Enable client access, handling unmatched streams as appropriate */ reg &= ~sCR0_CLIENTPD; if (disable_bypass) reg |= sCR0_USFCFG; else reg &= ~sCR0_USFCFG; |
45ae7cff3 iommu/arm: Add su... |
1554 1555 |
/* Disable forced broadcasting */ |
659db6f6b iommu/arm-smmu: C... |
1556 |
reg &= ~sCR0_FB; |
45ae7cff3 iommu/arm: Add su... |
1557 1558 |
/* Don't upgrade barriers */ |
659db6f6b iommu/arm-smmu: C... |
1559 |
reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); |
45ae7cff3 iommu/arm: Add su... |
1560 |
|
4e3e9b699 iommu/arm-smmu: A... |
1561 1562 |
if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= sCR0_VMID16EN; |
45ae7cff3 iommu/arm: Add su... |
1563 |
/* Push the button */ |
518f71362 iommu/arm-smmu: m... |
1564 |
__arm_smmu_tlb_sync(smmu); |
3a5df8ff3 iommu/arm-smmu: s... |
1565 |
writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
45ae7cff3 iommu/arm: Add su... |
1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 |
} static int arm_smmu_id_size_to_bits(int size) { switch (size) { case 0: return 32; case 1: return 36; case 2: return 40; case 3: return 42; case 4: return 44; case 5: default: return 48; } } static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) { unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; |
bae2c2d42 iommu/arm-smmu: S... |
1592 |
bool cttw_dt, cttw_reg; |
8e8b203ea iommu/arm-smmu: K... |
1593 |
int i; |
45ae7cff3 iommu/arm: Add su... |
1594 1595 1596 |
dev_notice(smmu->dev, "probing hardware configuration... "); |
b7862e355 iommu/arm-smmu: S... |
1597 1598 1599 |
dev_notice(smmu->dev, "SMMUv%d with: ", smmu->version == ARM_SMMU_V2 ? 2 : 1); |
45ae7cff3 iommu/arm: Add su... |
1600 1601 1602 |
/* ID0 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); |
4cf740b0b iommu/arm-smmu: a... |
1603 1604 1605 1606 1607 1608 |
/* Restrict available stages based on module parameter */ if (force_stage == 1) id &= ~(ID0_S2TS | ID0_NTS); else if (force_stage == 2) id &= ~(ID0_S1TS | ID0_NTS); |
45ae7cff3 iommu/arm: Add su... |
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 |
if (id & ID0_S1TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S1; dev_notice(smmu->dev, "\tstage 1 translation "); } if (id & ID0_S2TS) { smmu->features |= ARM_SMMU_FEAT_TRANS_S2; dev_notice(smmu->dev, "\tstage 2 translation "); } if (id & ID0_NTS) { smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; dev_notice(smmu->dev, "\tnested translation "); } if (!(smmu->features & |
4cf740b0b iommu/arm-smmu: a... |
1628 |
(ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { |
45ae7cff3 iommu/arm: Add su... |
1629 1630 1631 1632 |
dev_err(smmu->dev, "\tno translation support! "); return -ENODEV; } |
b7862e355 iommu/arm-smmu: S... |
1633 1634 |
if ((id & ID0_S1TS) && ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { |
859a732e4 iommu/arm-smmu: a... |
1635 1636 1637 1638 |
smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; dev_notice(smmu->dev, "\taddress translation ops "); } |
bae2c2d42 iommu/arm-smmu: S... |
1639 1640 1641 1642 1643 1644 1645 1646 1647 |
/* * In order for DMA API calls to work properly, we must defer to what * the DT says about coherency, regardless of what the hardware claims. * Fortunately, this also opens up a workaround for systems where the * ID register value has ended up configured incorrectly. */ cttw_dt = of_dma_is_coherent(smmu->dev->of_node); cttw_reg = !!(id & ID0_CTTW); if (cttw_dt) |
45ae7cff3 iommu/arm: Add su... |
1648 |
smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; |
bae2c2d42 iommu/arm-smmu: S... |
1649 1650 1651 1652 1653 1654 1655 1656 |
if (cttw_dt || cttw_reg) dev_notice(smmu->dev, "\t%scoherent table walk ", cttw_dt ? "" : "non-"); if (cttw_dt != cttw_reg) dev_notice(smmu->dev, "\t(IDR0.CTTW overridden by dma-coherent property) "); |
45ae7cff3 iommu/arm: Add su... |
1657 |
|
21174240e iommu/arm-smmu: H... |
1658 1659 1660 |
/* Max. number of entries we have for stream matching/indexing */ size = 1 << ((id >> ID0_NUMSIDB_SHIFT) & ID0_NUMSIDB_MASK); smmu->streamid_mask = size - 1; |
45ae7cff3 iommu/arm: Add su... |
1661 |
if (id & ID0_SMS) { |
21174240e iommu/arm-smmu: H... |
1662 |
u32 smr; |
45ae7cff3 iommu/arm: Add su... |
1663 1664 |
smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; |
21174240e iommu/arm-smmu: H... |
1665 1666 |
size = (id >> ID0_NUMSMRG_SHIFT) & ID0_NUMSMRG_MASK; if (size == 0) { |
45ae7cff3 iommu/arm: Add su... |
1667 1668 1669 1670 1671 |
dev_err(smmu->dev, "stream-matching supported, but no SMRs present! "); return -ENODEV; } |
21174240e iommu/arm-smmu: H... |
1672 1673 1674 1675 1676 1677 |
/* * SMR.ID bits may not be preserved if the corresponding MASK * bits are set, so check each one separately. We can reject * masters later if they try to claim IDs outside these masks. */ smr = smmu->streamid_mask << SMR_ID_SHIFT; |
45ae7cff3 iommu/arm: Add su... |
1678 1679 |
writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); |
21174240e iommu/arm-smmu: H... |
1680 |
smmu->streamid_mask = smr >> SMR_ID_SHIFT; |
45ae7cff3 iommu/arm: Add su... |
1681 |
|
21174240e iommu/arm-smmu: H... |
1682 1683 1684 1685 |
smr = smmu->streamid_mask << SMR_MASK_SHIFT; writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); smmu->smr_mask_mask = smr >> SMR_MASK_SHIFT; |
45ae7cff3 iommu/arm: Add su... |
1686 |
|
1f3d5ca43 iommu/arm-smmu: C... |
1687 1688 1689 1690 1691 |
/* Zero-initialised to mark as invalid */ smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), GFP_KERNEL); if (!smmu->smrs) return -ENOMEM; |
45ae7cff3 iommu/arm: Add su... |
1692 |
dev_notice(smmu->dev, |
21174240e iommu/arm-smmu: H... |
1693 1694 |
"\tstream matching with %lu register groups, mask 0x%x", size, smmu->smr_mask_mask); |
45ae7cff3 iommu/arm: Add su... |
1695 |
} |
8e8b203ea iommu/arm-smmu: K... |
1696 1697 1698 1699 1700 1701 1702 |
/* s2cr->type == 0 means translation, so initialise explicitly */ smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), GFP_KERNEL); if (!smmu->s2crs) return -ENOMEM; for (i = 0; i < size; i++) smmu->s2crs[i] = s2cr_init_val; |
21174240e iommu/arm-smmu: H... |
1703 |
smmu->num_mapping_groups = size; |
588888a73 iommu/arm-smmu: I... |
1704 |
mutex_init(&smmu->stream_map_mutex); |
45ae7cff3 iommu/arm: Add su... |
1705 |
|
7602b8710 iommu/arm-smmu: D... |
1706 1707 1708 1709 1710 |
if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; if (!(id & ID0_PTFS_NO_AARCH32S)) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; } |
45ae7cff3 iommu/arm: Add su... |
1711 1712 |
/* ID1 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); |
c757e8528 iommu/arm-smmu: u... |
1713 |
smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; |
45ae7cff3 iommu/arm: Add su... |
1714 |
|
c55af7f71 iommu/arm-smmu: R... |
1715 |
/* Check for size mismatch of SMMU address space from mapped region */ |
518f71362 iommu/arm-smmu: m... |
1716 |
size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); |
c757e8528 iommu/arm-smmu: u... |
1717 |
size *= 2 << smmu->pgshift; |
c55af7f71 iommu/arm-smmu: R... |
1718 |
if (smmu->size != size) |
2907320df iommu/arm-smmu: f... |
1719 1720 1721 1722 |
dev_warn(smmu->dev, "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)! ", size, smmu->size); |
45ae7cff3 iommu/arm: Add su... |
1723 |
|
518f71362 iommu/arm-smmu: m... |
1724 |
smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; |
45ae7cff3 iommu/arm: Add su... |
1725 1726 1727 1728 1729 1730 1731 1732 1733 |
smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; if (smmu->num_s2_context_banks > smmu->num_context_banks) { dev_err(smmu->dev, "impossible number of S2 context banks! "); return -ENODEV; } dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only) ", smmu->num_context_banks, smmu->num_s2_context_banks); |
e086d912d iommu/arm-smmu: C... |
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 |
/* * Cavium CN88xx erratum #27704. * Ensure ASID and VMID allocation is unique across all SMMUs in * the system. */ if (smmu->model == CAVIUM_SMMUV2) { smmu->cavium_id_base = atomic_add_return(smmu->num_context_banks, &cavium_smmu_context_count); smmu->cavium_id_base -= smmu->num_context_banks; } |
45ae7cff3 iommu/arm: Add su... |
1745 1746 1747 1748 |
/* ID2 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); |
518f71362 iommu/arm-smmu: m... |
1749 |
smmu->ipa_size = size; |
45ae7cff3 iommu/arm: Add su... |
1750 |
|
518f71362 iommu/arm-smmu: m... |
1751 |
/* The output mask is also applied for bypass */ |
45ae7cff3 iommu/arm: Add su... |
1752 |
size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); |
518f71362 iommu/arm-smmu: m... |
1753 |
smmu->pa_size = size; |
45ae7cff3 iommu/arm: Add su... |
1754 |
|
4e3e9b699 iommu/arm-smmu: A... |
1755 1756 |
if (id & ID2_VMID16) smmu->features |= ARM_SMMU_FEAT_VMID16; |
f1d845486 iommu/arm-smmu: s... |
1757 1758 1759 1760 1761 1762 1763 1764 1765 |
/* * What the page table walker can address actually depends on which * descriptor format is in use, but since a) we don't know that yet, * and b) it can vary per context bank, this will have to do... */ if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) dev_warn(smmu->dev, "failed to set DMA mask for table walker "); |
b7862e355 iommu/arm-smmu: S... |
1766 |
if (smmu->version < ARM_SMMU_V2) { |
518f71362 iommu/arm-smmu: m... |
1767 |
smmu->va_size = smmu->ipa_size; |
b7862e355 iommu/arm-smmu: S... |
1768 1769 |
if (smmu->version == ARM_SMMU_V1_64K) smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; |
45ae7cff3 iommu/arm: Add su... |
1770 |
} else { |
45ae7cff3 iommu/arm: Add su... |
1771 |
size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; |
518f71362 iommu/arm-smmu: m... |
1772 |
smmu->va_size = arm_smmu_id_size_to_bits(size); |
518f71362 iommu/arm-smmu: m... |
1773 |
if (id & ID2_PTFS_4K) |
7602b8710 iommu/arm-smmu: D... |
1774 |
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; |
518f71362 iommu/arm-smmu: m... |
1775 |
if (id & ID2_PTFS_16K) |
7602b8710 iommu/arm-smmu: D... |
1776 |
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; |
518f71362 iommu/arm-smmu: m... |
1777 |
if (id & ID2_PTFS_64K) |
7602b8710 iommu/arm-smmu: D... |
1778 |
smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; |
45ae7cff3 iommu/arm: Add su... |
1779 |
} |
7602b8710 iommu/arm-smmu: D... |
1780 |
/* Now we've corralled the various formats, what'll it do? */ |
7602b8710 iommu/arm-smmu: D... |
1781 |
if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) |
d54663573 iommu/arm-smmu: U... |
1782 |
smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; |
7602b8710 iommu/arm-smmu: D... |
1783 1784 |
if (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) |
d54663573 iommu/arm-smmu: U... |
1785 |
smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; |
7602b8710 iommu/arm-smmu: D... |
1786 |
if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) |
d54663573 iommu/arm-smmu: U... |
1787 |
smmu->pgsize_bitmap |= SZ_16K | SZ_32M; |
7602b8710 iommu/arm-smmu: D... |
1788 |
if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) |
d54663573 iommu/arm-smmu: U... |
1789 1790 1791 1792 1793 1794 1795 1796 1797 |
smmu->pgsize_bitmap |= SZ_64K | SZ_512M; if (arm_smmu_ops.pgsize_bitmap == -1UL) arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; else arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx ", smmu->pgsize_bitmap); |
7602b8710 iommu/arm-smmu: D... |
1798 |
|
518f71362 iommu/arm-smmu: m... |
1799 |
|
28d6007ba iommu/arm-smmu: d... |
1800 1801 1802 |
if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA ", |
518f71362 iommu/arm-smmu: m... |
1803 |
smmu->va_size, smmu->ipa_size); |
28d6007ba iommu/arm-smmu: d... |
1804 1805 1806 1807 |
if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA ", |
518f71362 iommu/arm-smmu: m... |
1808 |
smmu->ipa_size, smmu->pa_size); |
28d6007ba iommu/arm-smmu: d... |
1809 |
|
45ae7cff3 iommu/arm: Add su... |
1810 1811 |
return 0; } |
67b65a3fb iommu/arm-smmu: D... |
1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 |
struct arm_smmu_match_data { enum arm_smmu_arch_version version; enum arm_smmu_implementation model; }; #define ARM_SMMU_MATCH_DATA(name, ver, imp) \ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); |
b7862e355 iommu/arm-smmu: S... |
1822 |
ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); |
f0cfffc48 iommu/arm-smmu: W... |
1823 |
ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); |
e086d912d iommu/arm-smmu: C... |
1824 |
ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); |
67b65a3fb iommu/arm-smmu: D... |
1825 |
|
09b5269a1 Merge branches 'a... |
1826 |
static const struct of_device_id arm_smmu_of_match[] = { |
67b65a3fb iommu/arm-smmu: D... |
1827 1828 1829 |
{ .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, |
b7862e355 iommu/arm-smmu: S... |
1830 |
{ .compatible = "arm,mmu-401", .data = &arm_mmu401 }, |
f0cfffc48 iommu/arm-smmu: W... |
1831 |
{ .compatible = "arm,mmu-500", .data = &arm_mmu500 }, |
e086d912d iommu/arm-smmu: C... |
1832 |
{ .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, |
093604033 iommu/arm-smmu: f... |
1833 1834 1835 |
{ }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); |
45ae7cff3 iommu/arm: Add su... |
1836 1837 |
static int arm_smmu_device_dt_probe(struct platform_device *pdev) { |
67b65a3fb iommu/arm-smmu: D... |
1838 |
const struct arm_smmu_match_data *data; |
45ae7cff3 iommu/arm: Add su... |
1839 1840 |
struct resource *res; struct arm_smmu_device *smmu; |
45ae7cff3 iommu/arm: Add su... |
1841 |
struct device *dev = &pdev->dev; |
45ae7cff3 iommu/arm: Add su... |
1842 |
int num_irqs, i, err; |
021bb8420 iommu/arm-smmu: W... |
1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 |
bool legacy_binding; legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); if (legacy_binding && !using_generic_binding) { if (!using_legacy_binding) pr_notice("deprecated \"mmu-masters\" DT property in use; DMA API support unavailable "); using_legacy_binding = true; } else if (!legacy_binding && !using_legacy_binding) { using_generic_binding = true; } else { dev_err(dev, "not probing due to mismatched DT properties "); return -ENODEV; } |
45ae7cff3 iommu/arm: Add su... |
1858 1859 1860 1861 1862 1863 1864 1865 |
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); if (!smmu) { dev_err(dev, "failed to allocate arm_smmu_device "); return -ENOMEM; } smmu->dev = dev; |
d6fc5d977 iommu/arm-smmu: S... |
1866 |
data = of_device_get_match_data(dev); |
67b65a3fb iommu/arm-smmu: D... |
1867 1868 |
smmu->version = data->version; smmu->model = data->model; |
093604033 iommu/arm-smmu: f... |
1869 |
|
45ae7cff3 iommu/arm: Add su... |
1870 |
res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
8a7f43122 iommu/arm-smmu: r... |
1871 1872 1873 |
smmu->base = devm_ioremap_resource(dev, res); if (IS_ERR(smmu->base)) return PTR_ERR(smmu->base); |
45ae7cff3 iommu/arm: Add su... |
1874 |
smmu->size = resource_size(res); |
45ae7cff3 iommu/arm: Add su... |
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 |
if (of_property_read_u32(dev->of_node, "#global-interrupts", &smmu->num_global_irqs)) { dev_err(dev, "missing #global-interrupts property "); return -ENODEV; } num_irqs = 0; while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { num_irqs++; if (num_irqs > smmu->num_global_irqs) smmu->num_context_irqs++; } |
44a08de2a iommu/arm-smmu: C... |
1889 1890 1891 1892 1893 |
if (!smmu->num_context_irqs) { dev_err(dev, "found %d interrupts but expected at least %d ", num_irqs, smmu->num_global_irqs + 1); return -ENODEV; |
45ae7cff3 iommu/arm: Add su... |
1894 |
} |
45ae7cff3 iommu/arm: Add su... |
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 |
smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs, GFP_KERNEL); if (!smmu->irqs) { dev_err(dev, "failed to allocate %d irqs ", num_irqs); return -ENOMEM; } for (i = 0; i < num_irqs; ++i) { int irq = platform_get_irq(pdev, i); |
2907320df iommu/arm-smmu: f... |
1906 |
|
45ae7cff3 iommu/arm: Add su... |
1907 1908 1909 1910 1911 1912 1913 |
if (irq < 0) { dev_err(dev, "failed to get irq index %d ", i); return -ENODEV; } smmu->irqs[i] = irq; } |
3c8766d0c iommu/arm-smmu: D... |
1914 1915 1916 |
err = arm_smmu_device_cfg_probe(smmu); if (err) return err; |
3a5df8ff3 iommu/arm-smmu: s... |
1917 |
parse_driver_options(smmu); |
b7862e355 iommu/arm-smmu: S... |
1918 |
if (smmu->version == ARM_SMMU_V2 && |
45ae7cff3 iommu/arm: Add su... |
1919 1920 1921 1922 1923 |
smmu->num_context_banks != smmu->num_context_irqs) { dev_err(dev, "found only %d context interrupt(s) but %d required ", smmu->num_context_irqs, smmu->num_context_banks); |
f80cd885f iommu/arm-smmu: R... |
1924 |
return -ENODEV; |
45ae7cff3 iommu/arm: Add su... |
1925 |
} |
45ae7cff3 iommu/arm: Add su... |
1926 |
for (i = 0; i < smmu->num_global_irqs; ++i) { |
bee140044 iommu/arm-smmu: U... |
1927 1928 1929 1930 1931 |
err = devm_request_irq(smmu->dev, smmu->irqs[i], arm_smmu_global_fault, IRQF_SHARED, "arm-smmu global fault", smmu); |
45ae7cff3 iommu/arm: Add su... |
1932 1933 1934 1935 |
if (err) { dev_err(dev, "failed to request global IRQ %d (%u) ", i, smmu->irqs[i]); |
f80cd885f iommu/arm-smmu: R... |
1936 |
return err; |
45ae7cff3 iommu/arm: Add su... |
1937 1938 |
} } |
adfec2e70 iommu/arm-smmu: C... |
1939 |
of_iommu_set_ops(dev->of_node, &arm_smmu_ops); |
d6fc5d977 iommu/arm-smmu: S... |
1940 |
platform_set_drvdata(pdev, smmu); |
fd90cecbd iommu/arm-smmu: d... |
1941 |
arm_smmu_device_reset(smmu); |
021bb8420 iommu/arm-smmu: W... |
1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 |
/* Oh, for a proper bus abstraction */ if (!iommu_present(&platform_bus_type)) bus_set_iommu(&platform_bus_type, &arm_smmu_ops); #ifdef CONFIG_ARM_AMBA if (!iommu_present(&amba_bustype)) bus_set_iommu(&amba_bustype, &arm_smmu_ops); #endif #ifdef CONFIG_PCI if (!iommu_present(&pci_bus_type)) { pci_request_acs(); bus_set_iommu(&pci_bus_type, &arm_smmu_ops); } #endif |
45ae7cff3 iommu/arm: Add su... |
1956 |
return 0; |
45ae7cff3 iommu/arm: Add su... |
1957 1958 1959 1960 |
} static int arm_smmu_device_remove(struct platform_device *pdev) { |
d6fc5d977 iommu/arm-smmu: S... |
1961 |
struct arm_smmu_device *smmu = platform_get_drvdata(pdev); |
45ae7cff3 iommu/arm: Add su... |
1962 1963 1964 |
if (!smmu) return -ENODEV; |
ecfadb6e5 iommu/arm-smmu: S... |
1965 |
if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) |
d6fc5d977 iommu/arm-smmu: S... |
1966 1967 |
dev_err(&pdev->dev, "removing device with active domains! "); |
45ae7cff3 iommu/arm: Add su... |
1968 |
|
45ae7cff3 iommu/arm: Add su... |
1969 |
/* Turn the thing off */ |
2907320df iommu/arm-smmu: f... |
1970 |
writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); |
45ae7cff3 iommu/arm: Add su... |
1971 1972 |
return 0; } |
45ae7cff3 iommu/arm: Add su... |
1973 1974 |
static struct platform_driver arm_smmu_driver = { .driver = { |
45ae7cff3 iommu/arm: Add su... |
1975 1976 1977 1978 1979 1980 1981 1982 1983 |
.name = "arm-smmu", .of_match_table = of_match_ptr(arm_smmu_of_match), }, .probe = arm_smmu_device_dt_probe, .remove = arm_smmu_device_remove, }; static int __init arm_smmu_init(void) { |
021bb8420 iommu/arm-smmu: W... |
1984 1985 |
static bool registered; int ret = 0; |
45ae7cff3 iommu/arm: Add su... |
1986 |
|
021bb8420 iommu/arm-smmu: W... |
1987 1988 1989 |
if (!registered) { ret = platform_driver_register(&arm_smmu_driver); registered = !ret; |
112c898b5 iommu/arm-smmu: r... |
1990 |
} |
021bb8420 iommu/arm-smmu: W... |
1991 |
return ret; |
45ae7cff3 iommu/arm: Add su... |
1992 1993 1994 1995 1996 1997 |
} static void __exit arm_smmu_exit(void) { return platform_driver_unregister(&arm_smmu_driver); } |
b1950b279 iommu/arm-smmu: S... |
1998 |
subsys_initcall(arm_smmu_init); |
45ae7cff3 iommu/arm: Add su... |
1999 |
module_exit(arm_smmu_exit); |
021bb8420 iommu/arm-smmu: W... |
2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 |
static int __init arm_smmu_of_init(struct device_node *np) { int ret = arm_smmu_init(); if (ret) return ret; if (!of_platform_device_create(np, NULL, platform_bus_type.dev_root)) return -ENODEV; return 0; } IOMMU_OF_DECLARE(arm_smmuv1, "arm,smmu-v1", arm_smmu_of_init); IOMMU_OF_DECLARE(arm_smmuv2, "arm,smmu-v2", arm_smmu_of_init); IOMMU_OF_DECLARE(arm_mmu400, "arm,mmu-400", arm_smmu_of_init); IOMMU_OF_DECLARE(arm_mmu401, "arm,mmu-401", arm_smmu_of_init); IOMMU_OF_DECLARE(arm_mmu500, "arm,mmu-500", arm_smmu_of_init); IOMMU_OF_DECLARE(cavium_smmuv2, "cavium,smmu-v2", arm_smmu_of_init); |
45ae7cff3 iommu/arm: Add su... |
2018 2019 2020 |
MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); MODULE_LICENSE("GPL v2"); |