Blame view
drivers/iommu/exynos-iommu.c
36.6 KB
740a01eee iommu/exynos: Add... |
1 2 |
/* * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. |
2a96536e7 iommu/exynos: Add... |
3 4 5 6 7 8 9 10 11 12 |
* http://www.samsung.com * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ #ifdef CONFIG_EXYNOS_IOMMU_DEBUG #define DEBUG #endif |
2a96536e7 iommu/exynos: Add... |
13 |
#include <linux/clk.h> |
8ed55c812 iommu/exynos: Ini... |
14 |
#include <linux/dma-mapping.h> |
2a96536e7 iommu/exynos: Add... |
15 |
#include <linux/err.h> |
312900c60 iommu/exynos: Rem... |
16 |
#include <linux/io.h> |
2a96536e7 iommu/exynos: Add... |
17 |
#include <linux/iommu.h> |
312900c60 iommu/exynos: Rem... |
18 |
#include <linux/interrupt.h> |
2a96536e7 iommu/exynos: Add... |
19 |
#include <linux/list.h> |
8ed55c812 iommu/exynos: Ini... |
20 21 22 |
#include <linux/of.h> #include <linux/of_iommu.h> #include <linux/of_platform.h> |
312900c60 iommu/exynos: Rem... |
23 24 25 |
#include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> |
58c6f6a3d iommu/exynos: Add... |
26 |
#include <linux/dma-iommu.h> |
2a96536e7 iommu/exynos: Add... |
27 |
|
d09d78fc9 iommu/exynos: Use... |
28 29 |
typedef u32 sysmmu_iova_t; typedef u32 sysmmu_pte_t; |
f171abab8 iommu/exynos: Fix... |
30 |
/* We do not consider super section mapping (16MB) */ |
2a96536e7 iommu/exynos: Add... |
31 32 33 34 35 36 37 38 39 40 41 |
#define SECT_ORDER 20 #define LPAGE_ORDER 16 #define SPAGE_ORDER 12 #define SECT_SIZE (1 << SECT_ORDER) #define LPAGE_SIZE (1 << LPAGE_ORDER) #define SPAGE_SIZE (1 << SPAGE_ORDER) #define SECT_MASK (~(SECT_SIZE - 1)) #define LPAGE_MASK (~(LPAGE_SIZE - 1)) #define SPAGE_MASK (~(SPAGE_SIZE - 1)) |
66a7ed84b iommu/exynos: App... |
42 43 44 45 46 47 |
#define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \ ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) #define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK) #define lv1ent_page_zero(sent) ((*(sent) & 3) == 1) #define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \ ((*(sent) & 3) == 1)) |
2a96536e7 iommu/exynos: Add... |
48 49 50 51 52 |
#define lv1ent_section(sent) ((*(sent) & 3) == 2) #define lv2ent_fault(pent) ((*(pent) & 3) == 0) #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) |
6ae5343c2 iommu/exynos: upd... |
53 54 55 |
#ifdef CONFIG_BIG_ENDIAN #warning "revisit driver if we can enable big-endian ptes" #endif |
740a01eee iommu/exynos: Add... |
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
/* * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces * v5.0 introduced support for 36bit physical address space by shifting * all page entry values by 4 bits. * All SYSMMU controllers in the system support the address spaces of the same * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper * value (0 or 4). */ static short PG_ENT_SHIFT = -1; #define SYSMMU_PG_ENT_SHIFT 0 #define SYSMMU_V5_PG_ENT_SHIFT 4 #define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) #define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) #define section_offs(iova) (iova & (SECT_SIZE - 1)) #define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) #define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) #define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) #define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) |
2a96536e7 iommu/exynos: Add... |
75 76 |
#define NUM_LV1ENTRIES 4096 |
d09d78fc9 iommu/exynos: Use... |
77 |
#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) |
2a96536e7 iommu/exynos: Add... |
78 |
|
d09d78fc9 iommu/exynos: Use... |
79 80 81 82 83 84 85 86 87 |
static u32 lv1ent_offset(sysmmu_iova_t iova) { return iova >> SECT_ORDER; } static u32 lv2ent_offset(sysmmu_iova_t iova) { return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); } |
5e3435eb7 iommu/exynos: Rem... |
88 |
#define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t)) |
d09d78fc9 iommu/exynos: Use... |
89 |
#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) |
2a96536e7 iommu/exynos: Add... |
90 91 |
#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) |
740a01eee iommu/exynos: Add... |
92 |
#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) |
2a96536e7 iommu/exynos: Add... |
93 |
|
740a01eee iommu/exynos: Add... |
94 95 96 97 |
#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2) #define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) #define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1) #define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2) |
2a96536e7 iommu/exynos: Add... |
98 99 100 101 |
#define CTRL_ENABLE 0x5 #define CTRL_BLOCK 0x7 #define CTRL_DISABLE 0x0 |
eeb5184bb iommu/exynos: Tur... |
102 103 |
#define CFG_LRU 0x1 #define CFG_QOS(n) ((n & 0xF) << 7) |
eeb5184bb iommu/exynos: Tur... |
104 105 106 |
#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ |
740a01eee iommu/exynos: Add... |
107 |
/* common registers */ |
2a96536e7 iommu/exynos: Add... |
108 109 110 |
#define REG_MMU_CTRL 0x000 #define REG_MMU_CFG 0x004 #define REG_MMU_STATUS 0x008 |
740a01eee iommu/exynos: Add... |
111 112 113 114 115 116 117 118 119 |
#define REG_MMU_VERSION 0x034 #define MMU_MAJ_VER(val) ((val) >> 7) #define MMU_MIN_VER(val) ((val) & 0x7F) #define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ #define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) /* v1.x - v3.x registers */ |
2a96536e7 iommu/exynos: Add... |
120 121 122 123 124 125 126 127 128 129 |
#define REG_MMU_FLUSH 0x00C #define REG_MMU_FLUSH_ENTRY 0x010 #define REG_PT_BASE_ADDR 0x014 #define REG_INT_STATUS 0x018 #define REG_INT_CLEAR 0x01C #define REG_PAGE_FAULT_ADDR 0x024 #define REG_AW_FAULT_ADDR 0x028 #define REG_AR_FAULT_ADDR 0x02C #define REG_DEFAULT_SLAVE_ADDR 0x030 |
740a01eee iommu/exynos: Add... |
130 131 132 133 134 135 136 137 |
/* v5.x registers */ #define REG_V5_PT_BASE_PFN 0x00C #define REG_V5_MMU_FLUSH_ALL 0x010 #define REG_V5_MMU_FLUSH_ENTRY 0x014 #define REG_V5_INT_STATUS 0x060 #define REG_V5_INT_CLEAR 0x064 #define REG_V5_FAULT_AR_VA 0x070 #define REG_V5_FAULT_AW_VA 0x080 |
2a96536e7 iommu/exynos: Add... |
138 |
|
6b21a5db3 iommu/exynos: Sup... |
139 |
#define has_sysmmu(dev) (dev->archdata.iommu != NULL) |
5e3435eb7 iommu/exynos: Rem... |
140 |
static struct device *dma_dev; |
734c3c732 iommu/exynos: All... |
141 |
static struct kmem_cache *lv2table_kmem_cache; |
66a7ed84b iommu/exynos: App... |
142 143 |
static sysmmu_pte_t *zero_lv2_table; #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) |
734c3c732 iommu/exynos: All... |
144 |
|
d09d78fc9 iommu/exynos: Use... |
145 |
static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) |
2a96536e7 iommu/exynos: Add... |
146 147 148 |
{ return pgtable + lv1ent_offset(iova); } |
d09d78fc9 iommu/exynos: Use... |
149 |
static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) |
2a96536e7 iommu/exynos: Add... |
150 |
{ |
d09d78fc9 iommu/exynos: Use... |
151 |
return (sysmmu_pte_t *)phys_to_virt( |
7222e8db2 iommu/exynos: Fix... |
152 |
lv2table_base(sent)) + lv2ent_offset(iova); |
2a96536e7 iommu/exynos: Add... |
153 |
} |
d093fc7e8 iommu/exynos: Ref... |
154 155 156 157 158 159 160 161 |
/* * IOMMU fault information register */ struct sysmmu_fault_info { unsigned int bit; /* bit number in STATUS register */ unsigned short addr_reg; /* register to read VA fault address */ const char *name; /* human readable fault name */ unsigned int type; /* fault type for report_iommu_fault */ |
2a96536e7 iommu/exynos: Add... |
162 |
}; |
d093fc7e8 iommu/exynos: Ref... |
163 164 165 166 167 168 169 170 171 |
static const struct sysmmu_fault_info sysmmu_faults[] = { { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, |
2a96536e7 iommu/exynos: Add... |
172 |
}; |
740a01eee iommu/exynos: Add... |
173 174 175 176 177 178 179 180 181 182 183 184 |
static const struct sysmmu_fault_info sysmmu_v5_faults[] = { { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, }; |
2860af3c8 iommu/exynos: Doc... |
185 186 187 188 189 190 |
/* * This structure is attached to dev.archdata.iommu of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, * which are bound to given master device. It is usually referenced by 'owner' * pointer. */ |
6b21a5db3 iommu/exynos: Sup... |
191 |
struct exynos_iommu_owner { |
1b0920543 iommu/exynos: Add... |
192 |
struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ |
5fa61cbff iommu/exynos: Sup... |
193 |
struct iommu_domain *domain; /* domain this device is attached */ |
6b21a5db3 iommu/exynos: Sup... |
194 |
}; |
2860af3c8 iommu/exynos: Doc... |
195 196 197 198 199 200 |
/* * This structure exynos specific generalization of struct iommu_domain. * It contains list of SYSMMU controllers from all master devices, which has * been attached to this domain and page tables of IO address space defined by * it. It is usually referenced by 'domain' pointer. */ |
2a96536e7 iommu/exynos: Add... |
201 |
struct exynos_iommu_domain { |
2860af3c8 iommu/exynos: Doc... |
202 203 204 205 206 |
struct list_head clients; /* list of sysmmu_drvdata.domain_node */ sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ short *lv2entcnt; /* free lv2 entry counter for each section */ spinlock_t lock; /* lock for modyfying list of clients */ spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ |
e1fd1eaa3 iommu/exynos: Mak... |
207 |
struct iommu_domain domain; /* generic domain data structure */ |
2a96536e7 iommu/exynos: Add... |
208 |
}; |
2860af3c8 iommu/exynos: Doc... |
209 210 211 212 213 214 |
/* * This structure hold all data of a single SYSMMU controller, this includes * hw resources like registers and clocks, pointers and list nodes to connect * it to all other structures, internal state and parameters read from device * tree. It is usually referenced by 'data' pointer. */ |
2a96536e7 iommu/exynos: Add... |
215 |
struct sysmmu_drvdata { |
2860af3c8 iommu/exynos: Doc... |
216 217 218 219 |
struct device *sysmmu; /* SYSMMU controller device */ struct device *master; /* master device (owner) */ void __iomem *sfrbase; /* our registers */ struct clk *clk; /* SYSMMU's clock */ |
740a01eee iommu/exynos: Add... |
220 221 |
struct clk *aclk; /* SYSMMU's aclk clock */ struct clk *pclk; /* SYSMMU's pclk clock */ |
2860af3c8 iommu/exynos: Doc... |
222 223 224 225 226 |
struct clk *clk_master; /* master's device clock */ int activations; /* number of calls to sysmmu_enable */ spinlock_t lock; /* lock for modyfying state */ struct exynos_iommu_domain *domain; /* domain we belong to */ struct list_head domain_node; /* node for domain clients list */ |
1b0920543 iommu/exynos: Add... |
227 |
struct list_head owner_node; /* node for owner controllers list */ |
2860af3c8 iommu/exynos: Doc... |
228 229 |
phys_addr_t pgtable; /* assigned page table structure */ unsigned int version; /* our version */ |
2a96536e7 iommu/exynos: Add... |
230 |
}; |
e1fd1eaa3 iommu/exynos: Mak... |
231 232 233 234 |
static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) { return container_of(dom, struct exynos_iommu_domain, domain); } |
2a96536e7 iommu/exynos: Add... |
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 |
static bool set_sysmmu_active(struct sysmmu_drvdata *data) { /* return true if the System MMU was not active previously and it needs to be initialized */ return ++data->activations == 1; } static bool set_sysmmu_inactive(struct sysmmu_drvdata *data) { /* return true if the System MMU is needed to be disabled */ BUG_ON(data->activations < 1); return --data->activations == 0; } static bool is_sysmmu_active(struct sysmmu_drvdata *data) { return data->activations > 0; } |
02cdc365c iommu/exynos: Ref... |
253 |
static void sysmmu_unblock(struct sysmmu_drvdata *data) |
2a96536e7 iommu/exynos: Add... |
254 |
{ |
84bd04286 iommu/exynos: Use... |
255 |
writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); |
2a96536e7 iommu/exynos: Add... |
256 |
} |
02cdc365c iommu/exynos: Ref... |
257 |
static bool sysmmu_block(struct sysmmu_drvdata *data) |
2a96536e7 iommu/exynos: Add... |
258 259 |
{ int i = 120; |
84bd04286 iommu/exynos: Use... |
260 261 |
writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) |
2a96536e7 iommu/exynos: Add... |
262 |
--i; |
84bd04286 iommu/exynos: Use... |
263 |
if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { |
02cdc365c iommu/exynos: Ref... |
264 |
sysmmu_unblock(data); |
2a96536e7 iommu/exynos: Add... |
265 266 267 268 269 |
return false; } return true; } |
02cdc365c iommu/exynos: Ref... |
270 |
static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) |
2a96536e7 iommu/exynos: Add... |
271 |
{ |
740a01eee iommu/exynos: Add... |
272 |
if (MMU_MAJ_VER(data->version) < 5) |
84bd04286 iommu/exynos: Use... |
273 |
writel(0x1, data->sfrbase + REG_MMU_FLUSH); |
740a01eee iommu/exynos: Add... |
274 |
else |
84bd04286 iommu/exynos: Use... |
275 |
writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); |
2a96536e7 iommu/exynos: Add... |
276 |
} |
02cdc365c iommu/exynos: Ref... |
277 |
static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, |
d09d78fc9 iommu/exynos: Use... |
278 |
sysmmu_iova_t iova, unsigned int num_inv) |
2a96536e7 iommu/exynos: Add... |
279 |
{ |
3ad6b7f3a iommu/exynos: Fix... |
280 |
unsigned int i; |
365409db5 iommu/exynos: Fix... |
281 |
|
3ad6b7f3a iommu/exynos: Fix... |
282 |
for (i = 0; i < num_inv; i++) { |
740a01eee iommu/exynos: Add... |
283 |
if (MMU_MAJ_VER(data->version) < 5) |
84bd04286 iommu/exynos: Use... |
284 |
writel((iova & SPAGE_MASK) | 1, |
740a01eee iommu/exynos: Add... |
285 286 |
data->sfrbase + REG_MMU_FLUSH_ENTRY); else |
84bd04286 iommu/exynos: Use... |
287 |
writel((iova & SPAGE_MASK) | 1, |
740a01eee iommu/exynos: Add... |
288 |
data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); |
3ad6b7f3a iommu/exynos: Fix... |
289 290 |
iova += SPAGE_SIZE; } |
2a96536e7 iommu/exynos: Add... |
291 |
} |
02cdc365c iommu/exynos: Ref... |
292 |
static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) |
2a96536e7 iommu/exynos: Add... |
293 |
{ |
740a01eee iommu/exynos: Add... |
294 |
if (MMU_MAJ_VER(data->version) < 5) |
84bd04286 iommu/exynos: Use... |
295 |
writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); |
740a01eee iommu/exynos: Add... |
296 |
else |
84bd04286 iommu/exynos: Use... |
297 |
writel(pgd >> PAGE_SHIFT, |
740a01eee iommu/exynos: Add... |
298 |
data->sfrbase + REG_V5_PT_BASE_PFN); |
2a96536e7 iommu/exynos: Add... |
299 |
|
02cdc365c iommu/exynos: Ref... |
300 |
__sysmmu_tlb_invalidate(data); |
2a96536e7 iommu/exynos: Add... |
301 |
} |
fecc49db8 iommu/exynos: Pre... |
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 |
static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) { BUG_ON(clk_prepare_enable(data->clk_master)); BUG_ON(clk_prepare_enable(data->clk)); BUG_ON(clk_prepare_enable(data->pclk)); BUG_ON(clk_prepare_enable(data->aclk)); } static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) { clk_disable_unprepare(data->aclk); clk_disable_unprepare(data->pclk); clk_disable_unprepare(data->clk); clk_disable_unprepare(data->clk_master); } |
850d313e2 iommu/exynos: Add... |
317 318 319 |
static void __sysmmu_get_version(struct sysmmu_drvdata *data) { u32 ver; |
fecc49db8 iommu/exynos: Pre... |
320 |
__sysmmu_enable_clocks(data); |
850d313e2 iommu/exynos: Add... |
321 |
|
84bd04286 iommu/exynos: Use... |
322 |
ver = readl(data->sfrbase + REG_MMU_VERSION); |
850d313e2 iommu/exynos: Add... |
323 324 325 326 327 328 329 330 331 332 |
/* controllers on some SoCs don't report proper version */ if (ver == 0x80000001u) data->version = MAKE_MMU_VER(1, 0); else data->version = MMU_RAW_VER(ver); dev_dbg(data->sysmmu, "hardware version: %d.%d ", MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); |
fecc49db8 iommu/exynos: Pre... |
333 |
__sysmmu_disable_clocks(data); |
850d313e2 iommu/exynos: Add... |
334 |
} |
d093fc7e8 iommu/exynos: Ref... |
335 336 337 |
static void show_fault_information(struct sysmmu_drvdata *data, const struct sysmmu_fault_info *finfo, sysmmu_iova_t fault_addr) |
2a96536e7 iommu/exynos: Add... |
338 |
{ |
d09d78fc9 iommu/exynos: Use... |
339 |
sysmmu_pte_t *ent; |
2a96536e7 iommu/exynos: Add... |
340 |
|
d093fc7e8 iommu/exynos: Ref... |
341 342 343 344 345 346 |
dev_err(data->sysmmu, "%s FAULT occurred at %#x (page table base: %pa) ", finfo->name, fault_addr, &data->pgtable); ent = section_entry(phys_to_virt(data->pgtable), fault_addr); dev_err(data->sysmmu, "\tLv1 entry: %#x ", *ent); |
2a96536e7 iommu/exynos: Add... |
347 348 |
if (lv1ent_page(ent)) { ent = page_entry(ent, fault_addr); |
d093fc7e8 iommu/exynos: Ref... |
349 350 |
dev_err(data->sysmmu, "\t Lv2 entry: %#x ", *ent); |
2a96536e7 iommu/exynos: Add... |
351 |
} |
2a96536e7 iommu/exynos: Add... |
352 353 354 355 |
} static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { |
f171abab8 iommu/exynos: Fix... |
356 |
/* SYSMMU is in blocked state when interrupt occurred. */ |
2a96536e7 iommu/exynos: Add... |
357 |
struct sysmmu_drvdata *data = dev_id; |
740a01eee iommu/exynos: Add... |
358 359 |
const struct sysmmu_fault_info *finfo; unsigned int i, n, itype; |
d093fc7e8 iommu/exynos: Ref... |
360 |
sysmmu_iova_t fault_addr = -1; |
740a01eee iommu/exynos: Add... |
361 |
unsigned short reg_status, reg_clear; |
7222e8db2 iommu/exynos: Fix... |
362 |
int ret = -ENOSYS; |
2a96536e7 iommu/exynos: Add... |
363 |
|
2a96536e7 iommu/exynos: Add... |
364 |
WARN_ON(!is_sysmmu_active(data)); |
740a01eee iommu/exynos: Add... |
365 366 367 368 369 370 371 372 373 374 375 |
if (MMU_MAJ_VER(data->version) < 5) { reg_status = REG_INT_STATUS; reg_clear = REG_INT_CLEAR; finfo = sysmmu_faults; n = ARRAY_SIZE(sysmmu_faults); } else { reg_status = REG_V5_INT_STATUS; reg_clear = REG_V5_INT_CLEAR; finfo = sysmmu_v5_faults; n = ARRAY_SIZE(sysmmu_v5_faults); } |
9d4e7a24d iommu/exynos: Cha... |
376 |
spin_lock(&data->lock); |
b398af216 iommu/exynos: Sim... |
377 |
clk_enable(data->clk_master); |
9d4e7a24d iommu/exynos: Cha... |
378 |
|
84bd04286 iommu/exynos: Use... |
379 |
itype = __ffs(readl(data->sfrbase + reg_status)); |
d093fc7e8 iommu/exynos: Ref... |
380 381 382 383 384 385 386 |
for (i = 0; i < n; i++, finfo++) if (finfo->bit == itype) break; /* unknown/unsupported fault */ BUG_ON(i == n); /* print debug message */ |
84bd04286 iommu/exynos: Use... |
387 |
fault_addr = readl(data->sfrbase + finfo->addr_reg); |
d093fc7e8 iommu/exynos: Ref... |
388 |
show_fault_information(data, finfo, fault_addr); |
2a96536e7 iommu/exynos: Add... |
389 |
|
d093fc7e8 iommu/exynos: Ref... |
390 391 392 |
if (data->domain) ret = report_iommu_fault(&data->domain->domain, data->master, fault_addr, finfo->type); |
1fab7fa72 iommu/exynos: Rem... |
393 394 |
/* fault is not recovered by fault handler */ BUG_ON(ret != 0); |
2a96536e7 iommu/exynos: Add... |
395 |
|
84bd04286 iommu/exynos: Use... |
396 |
writel(1 << itype, data->sfrbase + reg_clear); |
1fab7fa72 iommu/exynos: Rem... |
397 |
|
02cdc365c iommu/exynos: Ref... |
398 |
sysmmu_unblock(data); |
2a96536e7 iommu/exynos: Add... |
399 |
|
b398af216 iommu/exynos: Sim... |
400 |
clk_disable(data->clk_master); |
706058705 iommu/exynos: Gat... |
401 |
|
9d4e7a24d iommu/exynos: Cha... |
402 |
spin_unlock(&data->lock); |
2a96536e7 iommu/exynos: Add... |
403 404 405 |
return IRQ_HANDLED; } |
6b21a5db3 iommu/exynos: Sup... |
406 |
static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) |
2a96536e7 iommu/exynos: Add... |
407 |
{ |
b398af216 iommu/exynos: Sim... |
408 |
clk_enable(data->clk_master); |
706058705 iommu/exynos: Gat... |
409 |
|
84bd04286 iommu/exynos: Use... |
410 411 |
writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); writel(0, data->sfrbase + REG_MMU_CFG); |
2a96536e7 iommu/exynos: Add... |
412 |
|
fecc49db8 iommu/exynos: Pre... |
413 |
__sysmmu_disable_clocks(data); |
2a96536e7 iommu/exynos: Add... |
414 |
} |
6b21a5db3 iommu/exynos: Sup... |
415 |
static bool __sysmmu_disable(struct sysmmu_drvdata *data) |
2a96536e7 iommu/exynos: Add... |
416 |
{ |
6b21a5db3 iommu/exynos: Sup... |
417 |
bool disabled; |
2a96536e7 iommu/exynos: Add... |
418 |
unsigned long flags; |
9d4e7a24d iommu/exynos: Cha... |
419 |
spin_lock_irqsave(&data->lock, flags); |
2a96536e7 iommu/exynos: Add... |
420 |
|
6b21a5db3 iommu/exynos: Sup... |
421 422 423 424 425 426 427 |
disabled = set_sysmmu_inactive(data); if (disabled) { data->pgtable = 0; data->domain = NULL; __sysmmu_disable_nocount(data); |
2a96536e7 iommu/exynos: Add... |
428 |
|
6b21a5db3 iommu/exynos: Sup... |
429 430 431 432 433 434 |
dev_dbg(data->sysmmu, "Disabled "); } else { dev_dbg(data->sysmmu, "%d times left to disable ", data->activations); |
2a96536e7 iommu/exynos: Add... |
435 |
} |
6b21a5db3 iommu/exynos: Sup... |
436 437 438 439 |
spin_unlock_irqrestore(&data->lock, flags); return disabled; } |
2a96536e7 iommu/exynos: Add... |
440 |
|
6b21a5db3 iommu/exynos: Sup... |
441 442 |
static void __sysmmu_init_config(struct sysmmu_drvdata *data) { |
83addecdb iommu/exynos: Ref... |
443 |
unsigned int cfg; |
83addecdb iommu/exynos: Ref... |
444 445 446 447 448 449 |
if (data->version <= MAKE_MMU_VER(3, 1)) cfg = CFG_LRU | CFG_QOS(15); else if (data->version <= MAKE_MMU_VER(3, 2)) cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL; else cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; |
6b21a5db3 iommu/exynos: Sup... |
450 |
|
84bd04286 iommu/exynos: Use... |
451 |
writel(cfg, data->sfrbase + REG_MMU_CFG); |
6b21a5db3 iommu/exynos: Sup... |
452 453 454 455 |
} static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { |
fecc49db8 iommu/exynos: Pre... |
456 |
__sysmmu_enable_clocks(data); |
706058705 iommu/exynos: Gat... |
457 |
|
84bd04286 iommu/exynos: Use... |
458 |
writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); |
6b21a5db3 iommu/exynos: Sup... |
459 460 |
__sysmmu_init_config(data); |
02cdc365c iommu/exynos: Ref... |
461 |
__sysmmu_set_ptbase(data, data->pgtable); |
2a96536e7 iommu/exynos: Add... |
462 |
|
84bd04286 iommu/exynos: Use... |
463 |
writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); |
7222e8db2 iommu/exynos: Fix... |
464 |
|
fecc49db8 iommu/exynos: Pre... |
465 466 467 468 469 470 |
/* * SYSMMU driver keeps master's clock enabled only for the short * time, while accessing the registers. For performing address * translation during DMA transaction it relies on the client * driver to enable it. */ |
b398af216 iommu/exynos: Sim... |
471 |
clk_disable(data->clk_master); |
6b21a5db3 iommu/exynos: Sup... |
472 |
} |
706058705 iommu/exynos: Gat... |
473 |
|
bfa004893 iommu/exynos: Ren... |
474 |
static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, |
a9133b993 iommu/exynos: Use... |
475 |
struct exynos_iommu_domain *domain) |
6b21a5db3 iommu/exynos: Sup... |
476 477 478 479 480 481 482 |
{ int ret = 0; unsigned long flags; spin_lock_irqsave(&data->lock, flags); if (set_sysmmu_active(data)) { data->pgtable = pgtable; |
a9133b993 iommu/exynos: Use... |
483 |
data->domain = domain; |
6b21a5db3 iommu/exynos: Sup... |
484 485 486 487 488 489 490 491 492 493 494 495 496 497 |
__sysmmu_enable_nocount(data); dev_dbg(data->sysmmu, "Enabled "); } else { ret = (pgtable == data->pgtable) ? 1 : -EBUSY; dev_dbg(data->sysmmu, "already enabled "); } if (WARN_ON(ret < 0)) set_sysmmu_inactive(data); /* decrement count */ |
2a96536e7 iommu/exynos: Add... |
498 |
|
9d4e7a24d iommu/exynos: Cha... |
499 |
spin_unlock_irqrestore(&data->lock, flags); |
2a96536e7 iommu/exynos: Add... |
500 501 502 |
return ret; } |
469acebe4 iommu/exynos: Ref... |
503 |
static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, |
66a7ed84b iommu/exynos: App... |
504 505 506 |
sysmmu_iova_t iova) { unsigned long flags; |
66a7ed84b iommu/exynos: App... |
507 |
|
66a7ed84b iommu/exynos: App... |
508 509 |
spin_lock_irqsave(&data->lock, flags); |
01324ab2c iommu/exynos: Fix... |
510 511 512 513 |
if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) { clk_enable(data->clk_master); __sysmmu_tlb_invalidate_entry(data, iova, 1); clk_disable(data->clk_master); |
d631ea980 iommu/exynos: Uni... |
514 |
} |
66a7ed84b iommu/exynos: App... |
515 |
spin_unlock_irqrestore(&data->lock, flags); |
66a7ed84b iommu/exynos: App... |
516 |
} |
469acebe4 iommu/exynos: Ref... |
517 518 |
static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_iova_t iova, size_t size) |
2a96536e7 iommu/exynos: Add... |
519 520 |
{ unsigned long flags; |
2a96536e7 iommu/exynos: Add... |
521 |
|
6b21a5db3 iommu/exynos: Sup... |
522 |
spin_lock_irqsave(&data->lock, flags); |
2a96536e7 iommu/exynos: Add... |
523 |
if (is_sysmmu_active(data)) { |
3ad6b7f3a iommu/exynos: Fix... |
524 |
unsigned int num_inv = 1; |
706058705 iommu/exynos: Gat... |
525 |
|
b398af216 iommu/exynos: Sim... |
526 |
clk_enable(data->clk_master); |
706058705 iommu/exynos: Gat... |
527 |
|
3ad6b7f3a iommu/exynos: Fix... |
528 529 530 |
/* * L2TLB invalidation required * 4KB page: 1 invalidation |
f171abab8 iommu/exynos: Fix... |
531 532 |
* 64KB page: 16 invalidations * 1MB page: 64 invalidations |
3ad6b7f3a iommu/exynos: Fix... |
533 534 535 536 537 |
* because it is set-associative TLB * with 8-way and 64 sets. * 1MB page can be cached in one of all sets. * 64KB page can be one of 16 consecutive sets. */ |
512bd0c6c iommu/exynos: Don... |
538 |
if (MMU_MAJ_VER(data->version) == 2) |
3ad6b7f3a iommu/exynos: Fix... |
539 |
num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); |
02cdc365c iommu/exynos: Ref... |
540 541 542 |
if (sysmmu_block(data)) { __sysmmu_tlb_invalidate_entry(data, iova, num_inv); sysmmu_unblock(data); |
2a96536e7 iommu/exynos: Add... |
543 |
} |
b398af216 iommu/exynos: Sim... |
544 |
clk_disable(data->clk_master); |
2a96536e7 iommu/exynos: Add... |
545 |
} else { |
469acebe4 iommu/exynos: Ref... |
546 547 548 |
dev_dbg(data->master, "disabled. Skipping TLB invalidation @ %#x ", iova); |
2a96536e7 iommu/exynos: Add... |
549 |
} |
9d4e7a24d iommu/exynos: Cha... |
550 |
spin_unlock_irqrestore(&data->lock, flags); |
2a96536e7 iommu/exynos: Add... |
551 |
} |
96f665570 iommu/exynos: Pre... |
552 |
static struct iommu_ops exynos_iommu_ops; |
6b21a5db3 iommu/exynos: Sup... |
553 |
static int __init exynos_sysmmu_probe(struct platform_device *pdev) |
2a96536e7 iommu/exynos: Add... |
554 |
{ |
46c16d1e4 iommu/exynos: Use... |
555 |
int irq, ret; |
7222e8db2 iommu/exynos: Fix... |
556 |
struct device *dev = &pdev->dev; |
2a96536e7 iommu/exynos: Add... |
557 |
struct sysmmu_drvdata *data; |
7222e8db2 iommu/exynos: Fix... |
558 |
struct resource *res; |
2a96536e7 iommu/exynos: Add... |
559 |
|
46c16d1e4 iommu/exynos: Use... |
560 561 562 |
data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; |
2a96536e7 iommu/exynos: Add... |
563 |
|
7222e8db2 iommu/exynos: Fix... |
564 |
res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
46c16d1e4 iommu/exynos: Use... |
565 566 567 |
data->sfrbase = devm_ioremap_resource(dev, res); if (IS_ERR(data->sfrbase)) return PTR_ERR(data->sfrbase); |
2a96536e7 iommu/exynos: Add... |
568 |
|
46c16d1e4 iommu/exynos: Use... |
569 570 |
irq = platform_get_irq(pdev, 0); if (irq <= 0) { |
0bf4e54db iommu/exynos: Enh... |
571 572 |
dev_err(dev, "Unable to find IRQ resource "); |
46c16d1e4 iommu/exynos: Use... |
573 |
return irq; |
2a96536e7 iommu/exynos: Add... |
574 |
} |
46c16d1e4 iommu/exynos: Use... |
575 |
ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0, |
7222e8db2 iommu/exynos: Fix... |
576 577 |
dev_name(dev), data); if (ret) { |
46c16d1e4 iommu/exynos: Use... |
578 579 580 |
dev_err(dev, "Unabled to register handler of irq %d ", irq); return ret; |
2a96536e7 iommu/exynos: Add... |
581 |
} |
46c16d1e4 iommu/exynos: Use... |
582 |
data->clk = devm_clk_get(dev, "sysmmu"); |
0c2b063f1 iommu/exynos: Ret... |
583 |
if (PTR_ERR(data->clk) == -ENOENT) |
740a01eee iommu/exynos: Add... |
584 |
data->clk = NULL; |
0c2b063f1 iommu/exynos: Ret... |
585 586 |
else if (IS_ERR(data->clk)) return PTR_ERR(data->clk); |
740a01eee iommu/exynos: Add... |
587 588 |
data->aclk = devm_clk_get(dev, "aclk"); |
0c2b063f1 iommu/exynos: Ret... |
589 |
if (PTR_ERR(data->aclk) == -ENOENT) |
740a01eee iommu/exynos: Add... |
590 |
data->aclk = NULL; |
0c2b063f1 iommu/exynos: Ret... |
591 592 |
else if (IS_ERR(data->aclk)) return PTR_ERR(data->aclk); |
740a01eee iommu/exynos: Add... |
593 594 |
data->pclk = devm_clk_get(dev, "pclk"); |
0c2b063f1 iommu/exynos: Ret... |
595 |
if (PTR_ERR(data->pclk) == -ENOENT) |
740a01eee iommu/exynos: Add... |
596 |
data->pclk = NULL; |
0c2b063f1 iommu/exynos: Ret... |
597 598 |
else if (IS_ERR(data->pclk)) return PTR_ERR(data->pclk); |
740a01eee iommu/exynos: Add... |
599 600 601 602 603 |
if (!data->clk && (!data->aclk || !data->pclk)) { dev_err(dev, "Failed to get device clock(s)! "); return -ENOSYS; |
2a96536e7 iommu/exynos: Add... |
604 |
} |
706058705 iommu/exynos: Gat... |
605 |
data->clk_master = devm_clk_get(dev, "master"); |
0c2b063f1 iommu/exynos: Ret... |
606 |
if (PTR_ERR(data->clk_master) == -ENOENT) |
b398af216 iommu/exynos: Sim... |
607 |
data->clk_master = NULL; |
0c2b063f1 iommu/exynos: Ret... |
608 609 |
else if (IS_ERR(data->clk_master)) return PTR_ERR(data->clk_master); |
706058705 iommu/exynos: Gat... |
610 |
|
2a96536e7 iommu/exynos: Add... |
611 |
data->sysmmu = dev; |
9d4e7a24d iommu/exynos: Cha... |
612 |
spin_lock_init(&data->lock); |
2a96536e7 iommu/exynos: Add... |
613 |
|
7222e8db2 iommu/exynos: Fix... |
614 |
platform_set_drvdata(pdev, data); |
850d313e2 iommu/exynos: Add... |
615 |
__sysmmu_get_version(data); |
740a01eee iommu/exynos: Add... |
616 617 618 619 620 621 |
if (PG_ENT_SHIFT < 0) { if (MMU_MAJ_VER(data->version) < 5) PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; else PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; } |
f4723ec17 iommu/exynos: Alw... |
622 |
pm_runtime_enable(dev); |
2a96536e7 iommu/exynos: Add... |
623 |
|
96f665570 iommu/exynos: Pre... |
624 |
of_iommu_set_ops(dev->of_node, &exynos_iommu_ops); |
2a96536e7 iommu/exynos: Add... |
625 |
return 0; |
2a96536e7 iommu/exynos: Add... |
626 |
} |
622015e40 iommu/exynos: Add... |
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 |
#ifdef CONFIG_PM_SLEEP static int exynos_sysmmu_suspend(struct device *dev) { struct sysmmu_drvdata *data = dev_get_drvdata(dev); dev_dbg(dev, "suspend "); if (is_sysmmu_active(data)) { __sysmmu_disable_nocount(data); pm_runtime_put(dev); } return 0; } static int exynos_sysmmu_resume(struct device *dev) { struct sysmmu_drvdata *data = dev_get_drvdata(dev); dev_dbg(dev, "resume "); if (is_sysmmu_active(data)) { pm_runtime_get_sync(dev); __sysmmu_enable_nocount(data); } return 0; } #endif static const struct dev_pm_ops sysmmu_pm_ops = { SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume) }; |
6b21a5db3 iommu/exynos: Sup... |
658 659 660 661 662 663 664 665 |
static const struct of_device_id sysmmu_of_match[] __initconst = { { .compatible = "samsung,exynos-sysmmu", }, { }, }; static struct platform_driver exynos_sysmmu_driver __refdata = { .probe = exynos_sysmmu_probe, .driver = { |
2a96536e7 iommu/exynos: Add... |
666 |
.name = "exynos-sysmmu", |
6b21a5db3 iommu/exynos: Sup... |
667 |
.of_match_table = sysmmu_of_match, |
622015e40 iommu/exynos: Add... |
668 |
.pm = &sysmmu_pm_ops, |
b54b874fb iommu/exynos: Sup... |
669 |
.suppress_bind_attrs = true, |
2a96536e7 iommu/exynos: Add... |
670 671 |
} }; |
5e3435eb7 iommu/exynos: Rem... |
672 |
static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) |
2a96536e7 iommu/exynos: Add... |
673 |
{ |
5e3435eb7 iommu/exynos: Rem... |
674 675 |
dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); |
6ae5343c2 iommu/exynos: upd... |
676 |
*ent = cpu_to_le32(val); |
5e3435eb7 iommu/exynos: Rem... |
677 678 |
dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
679 |
} |
e1fd1eaa3 iommu/exynos: Mak... |
680 |
static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) |
2a96536e7 iommu/exynos: Add... |
681 |
{ |
bfa004893 iommu/exynos: Ren... |
682 |
struct exynos_iommu_domain *domain; |
5e3435eb7 iommu/exynos: Rem... |
683 |
dma_addr_t handle; |
66a7ed84b iommu/exynos: App... |
684 |
int i; |
2a96536e7 iommu/exynos: Add... |
685 |
|
740a01eee iommu/exynos: Add... |
686 687 |
/* Check if correct PTE offsets are initialized */ BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); |
e1fd1eaa3 iommu/exynos: Mak... |
688 |
|
bfa004893 iommu/exynos: Ren... |
689 690 |
domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) |
e1fd1eaa3 iommu/exynos: Mak... |
691 |
return NULL; |
2a96536e7 iommu/exynos: Add... |
692 |
|
58c6f6a3d iommu/exynos: Add... |
693 694 695 696 697 698 |
if (type == IOMMU_DOMAIN_DMA) { if (iommu_get_dma_cookie(&domain->domain) != 0) goto err_pgtable; } else if (type != IOMMU_DOMAIN_UNMANAGED) { goto err_pgtable; } |
bfa004893 iommu/exynos: Ren... |
699 700 |
domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); if (!domain->pgtable) |
58c6f6a3d iommu/exynos: Add... |
701 |
goto err_dma_cookie; |
2a96536e7 iommu/exynos: Add... |
702 |
|
bfa004893 iommu/exynos: Ren... |
703 704 |
domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); if (!domain->lv2entcnt) |
2a96536e7 iommu/exynos: Add... |
705 |
goto err_counter; |
f171abab8 iommu/exynos: Fix... |
706 |
/* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ |
66a7ed84b iommu/exynos: App... |
707 |
for (i = 0; i < NUM_LV1ENTRIES; i += 8) { |
bfa004893 iommu/exynos: Ren... |
708 709 710 711 712 713 714 715 |
domain->pgtable[i + 0] = ZERO_LV2LINK; domain->pgtable[i + 1] = ZERO_LV2LINK; domain->pgtable[i + 2] = ZERO_LV2LINK; domain->pgtable[i + 3] = ZERO_LV2LINK; domain->pgtable[i + 4] = ZERO_LV2LINK; domain->pgtable[i + 5] = ZERO_LV2LINK; domain->pgtable[i + 6] = ZERO_LV2LINK; domain->pgtable[i + 7] = ZERO_LV2LINK; |
66a7ed84b iommu/exynos: App... |
716 |
} |
5e3435eb7 iommu/exynos: Rem... |
717 718 719 720 |
handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, DMA_TO_DEVICE); /* For mapping page table entries we rely on dma == phys */ BUG_ON(handle != virt_to_phys(domain->pgtable)); |
2a96536e7 iommu/exynos: Add... |
721 |
|
bfa004893 iommu/exynos: Ren... |
722 723 724 |
spin_lock_init(&domain->lock); spin_lock_init(&domain->pgtablelock); INIT_LIST_HEAD(&domain->clients); |
2a96536e7 iommu/exynos: Add... |
725 |
|
bfa004893 iommu/exynos: Ren... |
726 727 728 |
domain->domain.geometry.aperture_start = 0; domain->domain.geometry.aperture_end = ~0UL; domain->domain.geometry.force_aperture = true; |
3177bb76a iommu/exynos: Imp... |
729 |
|
bfa004893 iommu/exynos: Ren... |
730 |
return &domain->domain; |
2a96536e7 iommu/exynos: Add... |
731 732 |
err_counter: |
bfa004893 iommu/exynos: Ren... |
733 |
free_pages((unsigned long)domain->pgtable, 2); |
58c6f6a3d iommu/exynos: Add... |
734 735 736 |
err_dma_cookie: if (type == IOMMU_DOMAIN_DMA) iommu_put_dma_cookie(&domain->domain); |
2a96536e7 iommu/exynos: Add... |
737 |
err_pgtable: |
bfa004893 iommu/exynos: Ren... |
738 |
kfree(domain); |
e1fd1eaa3 iommu/exynos: Mak... |
739 |
return NULL; |
2a96536e7 iommu/exynos: Add... |
740 |
} |
bfa004893 iommu/exynos: Ren... |
741 |
static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) |
2a96536e7 iommu/exynos: Add... |
742 |
{ |
bfa004893 iommu/exynos: Ren... |
743 |
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); |
469acebe4 iommu/exynos: Ref... |
744 |
struct sysmmu_drvdata *data, *next; |
2a96536e7 iommu/exynos: Add... |
745 746 |
unsigned long flags; int i; |
bfa004893 iommu/exynos: Ren... |
747 |
WARN_ON(!list_empty(&domain->clients)); |
2a96536e7 iommu/exynos: Add... |
748 |
|
bfa004893 iommu/exynos: Ren... |
749 |
spin_lock_irqsave(&domain->lock, flags); |
2a96536e7 iommu/exynos: Add... |
750 |
|
bfa004893 iommu/exynos: Ren... |
751 |
list_for_each_entry_safe(data, next, &domain->clients, domain_node) { |
469acebe4 iommu/exynos: Ref... |
752 753 754 |
if (__sysmmu_disable(data)) data->master = NULL; list_del_init(&data->domain_node); |
2a96536e7 iommu/exynos: Add... |
755 |
} |
bfa004893 iommu/exynos: Ren... |
756 |
spin_unlock_irqrestore(&domain->lock, flags); |
2a96536e7 iommu/exynos: Add... |
757 |
|
58c6f6a3d iommu/exynos: Add... |
758 759 |
if (iommu_domain->type == IOMMU_DOMAIN_DMA) iommu_put_dma_cookie(iommu_domain); |
5e3435eb7 iommu/exynos: Rem... |
760 761 |
dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
762 |
for (i = 0; i < NUM_LV1ENTRIES; i++) |
5e3435eb7 iommu/exynos: Rem... |
763 764 765 766 767 |
if (lv1ent_page(domain->pgtable + i)) { phys_addr_t base = lv2table_base(domain->pgtable + i); dma_unmap_single(dma_dev, base, LV2TABLE_SIZE, DMA_TO_DEVICE); |
734c3c732 iommu/exynos: All... |
768 |
kmem_cache_free(lv2table_kmem_cache, |
5e3435eb7 iommu/exynos: Rem... |
769 770 |
phys_to_virt(base)); } |
2a96536e7 iommu/exynos: Add... |
771 |
|
bfa004893 iommu/exynos: Ren... |
772 773 774 |
free_pages((unsigned long)domain->pgtable, 2); free_pages((unsigned long)domain->lv2entcnt, 1); kfree(domain); |
2a96536e7 iommu/exynos: Add... |
775 |
} |
5fa61cbff iommu/exynos: Sup... |
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 |
static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, struct device *dev) { struct exynos_iommu_owner *owner = dev->archdata.iommu; struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); phys_addr_t pagetable = virt_to_phys(domain->pgtable); struct sysmmu_drvdata *data, *next; unsigned long flags; bool found = false; if (!has_sysmmu(dev) || owner->domain != iommu_domain) return; spin_lock_irqsave(&domain->lock, flags); list_for_each_entry_safe(data, next, &domain->clients, domain_node) { if (data->master == dev) { if (__sysmmu_disable(data)) { data->master = NULL; list_del_init(&data->domain_node); } pm_runtime_put(data->sysmmu); found = true; } } spin_unlock_irqrestore(&domain->lock, flags); owner->domain = NULL; if (found) dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa ", __func__, &pagetable); else dev_err(dev, "%s: No IOMMU is attached ", __func__); } |
bfa004893 iommu/exynos: Ren... |
812 |
static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, |
2a96536e7 iommu/exynos: Add... |
813 814 |
struct device *dev) { |
6b21a5db3 iommu/exynos: Sup... |
815 |
struct exynos_iommu_owner *owner = dev->archdata.iommu; |
bfa004893 iommu/exynos: Ren... |
816 |
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); |
469acebe4 iommu/exynos: Ref... |
817 |
struct sysmmu_drvdata *data; |
bfa004893 iommu/exynos: Ren... |
818 |
phys_addr_t pagetable = virt_to_phys(domain->pgtable); |
2a96536e7 iommu/exynos: Add... |
819 |
unsigned long flags; |
469acebe4 iommu/exynos: Ref... |
820 |
int ret = -ENODEV; |
2a96536e7 iommu/exynos: Add... |
821 |
|
469acebe4 iommu/exynos: Ref... |
822 823 |
if (!has_sysmmu(dev)) return -ENODEV; |
2a96536e7 iommu/exynos: Add... |
824 |
|
5fa61cbff iommu/exynos: Sup... |
825 826 |
if (owner->domain) exynos_iommu_detach_device(owner->domain, dev); |
1b0920543 iommu/exynos: Add... |
827 |
list_for_each_entry(data, &owner->controllers, owner_node) { |
ce70ca562 iommu/exynos: Add... |
828 |
pm_runtime_get_sync(data->sysmmu); |
a9133b993 iommu/exynos: Use... |
829 |
ret = __sysmmu_enable(data, pagetable, domain); |
469acebe4 iommu/exynos: Ref... |
830 831 |
if (ret >= 0) { data->master = dev; |
bfa004893 iommu/exynos: Ren... |
832 833 834 |
spin_lock_irqsave(&domain->lock, flags); list_add_tail(&data->domain_node, &domain->clients); spin_unlock_irqrestore(&domain->lock, flags); |
469acebe4 iommu/exynos: Ref... |
835 836 |
} } |
2a96536e7 iommu/exynos: Add... |
837 838 |
if (ret < 0) { |
7222e8db2 iommu/exynos: Fix... |
839 840 841 |
dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa ", __func__, &pagetable); |
7222e8db2 iommu/exynos: Fix... |
842 |
return ret; |
2a96536e7 iommu/exynos: Add... |
843 |
} |
5fa61cbff iommu/exynos: Sup... |
844 |
owner->domain = iommu_domain; |
7222e8db2 iommu/exynos: Fix... |
845 846 847 |
dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s ", __func__, &pagetable, (ret == 0) ? "" : ", again"); |
2a96536e7 iommu/exynos: Add... |
848 849 |
return ret; } |
bfa004893 iommu/exynos: Ren... |
850 |
static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, |
66a7ed84b iommu/exynos: App... |
851 |
sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) |
2a96536e7 iommu/exynos: Add... |
852 |
{ |
61128f08f iommu/exynos: Cha... |
853 |
if (lv1ent_section(sent)) { |
d09d78fc9 iommu/exynos: Use... |
854 |
WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova); |
61128f08f iommu/exynos: Cha... |
855 856 |
return ERR_PTR(-EADDRINUSE); } |
2a96536e7 iommu/exynos: Add... |
857 |
if (lv1ent_fault(sent)) { |
d09d78fc9 iommu/exynos: Use... |
858 |
sysmmu_pte_t *pent; |
66a7ed84b iommu/exynos: App... |
859 |
bool need_flush_flpd_cache = lv1ent_zero(sent); |
2a96536e7 iommu/exynos: Add... |
860 |
|
734c3c732 iommu/exynos: All... |
861 |
pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); |
dbf6c6efa iommu/exynos: Poi... |
862 |
BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); |
2a96536e7 iommu/exynos: Add... |
863 |
if (!pent) |
61128f08f iommu/exynos: Cha... |
864 |
return ERR_PTR(-ENOMEM); |
2a96536e7 iommu/exynos: Add... |
865 |
|
5e3435eb7 iommu/exynos: Rem... |
866 |
update_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); |
dc3814f40 iommu/exynos: Tel... |
867 |
kmemleak_ignore(pent); |
2a96536e7 iommu/exynos: Add... |
868 |
*pgcounter = NUM_LV2ENTRIES; |
5e3435eb7 iommu/exynos: Rem... |
869 |
dma_map_single(dma_dev, pent, LV2TABLE_SIZE, DMA_TO_DEVICE); |
66a7ed84b iommu/exynos: App... |
870 871 |
/* |
f171abab8 iommu/exynos: Fix... |
872 873 874 875 |
* If pre-fetched SLPD is a faulty SLPD in zero_l2_table, * FLPD cache may cache the address of zero_l2_table. This * function replaces the zero_l2_table with new L2 page table * to write valid mappings. |
66a7ed84b iommu/exynos: App... |
876 |
* Accessing the valid area may cause page fault since FLPD |
f171abab8 iommu/exynos: Fix... |
877 878 879 |
* cache may still cache zero_l2_table for the valid area * instead of new L2 page table that has the mapping * information of the valid area. |
66a7ed84b iommu/exynos: App... |
880 881 882 883 884 885 886 887 888 |
* Thus any replacement of zero_l2_table with other valid L2 * page table must involve FLPD cache invalidation for System * MMU v3.3. * FLPD cache invalidation is performed with TLB invalidation * by VPN without blocking. It is safe to invalidate TLB without * blocking because the target address of TLB invalidation is * not currently mapped. */ if (need_flush_flpd_cache) { |
469acebe4 iommu/exynos: Ref... |
889 |
struct sysmmu_drvdata *data; |
365409db5 iommu/exynos: Fix... |
890 |
|
bfa004893 iommu/exynos: Ren... |
891 892 |
spin_lock(&domain->lock); list_for_each_entry(data, &domain->clients, domain_node) |
469acebe4 iommu/exynos: Ref... |
893 |
sysmmu_tlb_invalidate_flpdcache(data, iova); |
bfa004893 iommu/exynos: Ren... |
894 |
spin_unlock(&domain->lock); |
66a7ed84b iommu/exynos: App... |
895 |
} |
2a96536e7 iommu/exynos: Add... |
896 897 898 899 |
} return page_entry(sent, iova); } |
bfa004893 iommu/exynos: Ren... |
900 |
static int lv1set_section(struct exynos_iommu_domain *domain, |
66a7ed84b iommu/exynos: App... |
901 |
sysmmu_pte_t *sent, sysmmu_iova_t iova, |
61128f08f iommu/exynos: Cha... |
902 |
phys_addr_t paddr, short *pgcnt) |
2a96536e7 iommu/exynos: Add... |
903 |
{ |
61128f08f iommu/exynos: Cha... |
904 |
if (lv1ent_section(sent)) { |
d09d78fc9 iommu/exynos: Use... |
905 |
WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", |
61128f08f iommu/exynos: Cha... |
906 |
iova); |
2a96536e7 iommu/exynos: Add... |
907 |
return -EADDRINUSE; |
61128f08f iommu/exynos: Cha... |
908 |
} |
2a96536e7 iommu/exynos: Add... |
909 910 |
if (lv1ent_page(sent)) { |
61128f08f iommu/exynos: Cha... |
911 |
if (*pgcnt != NUM_LV2ENTRIES) { |
d09d78fc9 iommu/exynos: Use... |
912 |
WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", |
61128f08f iommu/exynos: Cha... |
913 |
iova); |
2a96536e7 iommu/exynos: Add... |
914 |
return -EADDRINUSE; |
61128f08f iommu/exynos: Cha... |
915 |
} |
2a96536e7 iommu/exynos: Add... |
916 |
|
734c3c732 iommu/exynos: All... |
917 |
kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0)); |
2a96536e7 iommu/exynos: Add... |
918 919 |
*pgcnt = 0; } |
5e3435eb7 iommu/exynos: Rem... |
920 |
update_pte(sent, mk_lv1ent_sect(paddr)); |
2a96536e7 iommu/exynos: Add... |
921 |
|
bfa004893 iommu/exynos: Ren... |
922 |
spin_lock(&domain->lock); |
66a7ed84b iommu/exynos: App... |
923 |
if (lv1ent_page_zero(sent)) { |
469acebe4 iommu/exynos: Ref... |
924 |
struct sysmmu_drvdata *data; |
66a7ed84b iommu/exynos: App... |
925 926 927 928 |
/* * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD * entry by speculative prefetch of SLPD which has no mapping. */ |
bfa004893 iommu/exynos: Ren... |
929 |
list_for_each_entry(data, &domain->clients, domain_node) |
469acebe4 iommu/exynos: Ref... |
930 |
sysmmu_tlb_invalidate_flpdcache(data, iova); |
66a7ed84b iommu/exynos: App... |
931 |
} |
bfa004893 iommu/exynos: Ren... |
932 |
spin_unlock(&domain->lock); |
66a7ed84b iommu/exynos: App... |
933 |
|
2a96536e7 iommu/exynos: Add... |
934 935 |
return 0; } |
d09d78fc9 iommu/exynos: Use... |
936 |
static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, |
2a96536e7 iommu/exynos: Add... |
937 938 939 |
short *pgcnt) { if (size == SPAGE_SIZE) { |
0bf4e54db iommu/exynos: Enh... |
940 |
if (WARN_ON(!lv2ent_fault(pent))) |
2a96536e7 iommu/exynos: Add... |
941 |
return -EADDRINUSE; |
5e3435eb7 iommu/exynos: Rem... |
942 |
update_pte(pent, mk_lv2ent_spage(paddr)); |
2a96536e7 iommu/exynos: Add... |
943 944 945 |
*pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; |
5e3435eb7 iommu/exynos: Rem... |
946 |
dma_addr_t pent_base = virt_to_phys(pent); |
365409db5 iommu/exynos: Fix... |
947 |
|
5e3435eb7 iommu/exynos: Rem... |
948 949 950 |
dma_sync_single_for_cpu(dma_dev, pent_base, sizeof(*pent) * SPAGES_PER_LPAGE, DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
951 |
for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { |
0bf4e54db iommu/exynos: Enh... |
952 |
if (WARN_ON(!lv2ent_fault(pent))) { |
61128f08f iommu/exynos: Cha... |
953 954 |
if (i > 0) memset(pent - i, 0, sizeof(*pent) * i); |
2a96536e7 iommu/exynos: Add... |
955 956 957 958 959 |
return -EADDRINUSE; } *pent = mk_lv2ent_lpage(paddr); } |
5e3435eb7 iommu/exynos: Rem... |
960 961 962 |
dma_sync_single_for_device(dma_dev, pent_base, sizeof(*pent) * SPAGES_PER_LPAGE, DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
963 964 965 966 967 |
*pgcnt -= SPAGES_PER_LPAGE; } return 0; } |
66a7ed84b iommu/exynos: App... |
968 969 970 |
/* * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: * |
f171abab8 iommu/exynos: Fix... |
971 |
* System MMU v3.x has advanced logic to improve address translation |
66a7ed84b iommu/exynos: App... |
972 |
* performance with caching more page table entries by a page table walk. |
f171abab8 iommu/exynos: Fix... |
973 974 975 976 977 978 |
* However, the logic has a bug that while caching faulty page table entries, * System MMU reports page fault if the cached fault entry is hit even though * the fault entry is updated to a valid entry after the entry is cached. * To prevent caching faulty page table entries which may be updated to valid * entries later, the virtual memory manager should care about the workaround * for the problem. The following describes the workaround. |
66a7ed84b iommu/exynos: App... |
979 980 |
* * Any two consecutive I/O virtual address regions must have a hole of 128KiB |
f171abab8 iommu/exynos: Fix... |
981 |
* at maximum to prevent misbehavior of System MMU 3.x (workaround for h/w bug). |
66a7ed84b iommu/exynos: App... |
982 |
* |
f171abab8 iommu/exynos: Fix... |
983 |
* Precisely, any start address of I/O virtual region must be aligned with |
66a7ed84b iommu/exynos: App... |
984 985 986 987 988 |
* the following sizes for System MMU v3.1 and v3.2. * System MMU v3.1: 128KiB * System MMU v3.2: 256KiB * * Because System MMU v3.3 caches page table entries more aggressively, it needs |
f171abab8 iommu/exynos: Fix... |
989 990 991 |
* more workarounds. * - Any two consecutive I/O virtual regions must have a hole of size larger * than or equal to 128KiB. |
66a7ed84b iommu/exynos: App... |
992 993 |
* - Start address of an I/O virtual region must be aligned by 128KiB. */ |
bfa004893 iommu/exynos: Ren... |
994 995 996 |
static int exynos_iommu_map(struct iommu_domain *iommu_domain, unsigned long l_iova, phys_addr_t paddr, size_t size, int prot) |
2a96536e7 iommu/exynos: Add... |
997 |
{ |
bfa004893 iommu/exynos: Ren... |
998 |
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); |
d09d78fc9 iommu/exynos: Use... |
999 1000 |
sysmmu_pte_t *entry; sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; |
2a96536e7 iommu/exynos: Add... |
1001 1002 |
unsigned long flags; int ret = -ENOMEM; |
bfa004893 iommu/exynos: Ren... |
1003 |
BUG_ON(domain->pgtable == NULL); |
2a96536e7 iommu/exynos: Add... |
1004 |
|
bfa004893 iommu/exynos: Ren... |
1005 |
spin_lock_irqsave(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1006 |
|
bfa004893 iommu/exynos: Ren... |
1007 |
entry = section_entry(domain->pgtable, iova); |
2a96536e7 iommu/exynos: Add... |
1008 1009 |
if (size == SECT_SIZE) { |
bfa004893 iommu/exynos: Ren... |
1010 1011 |
ret = lv1set_section(domain, entry, iova, paddr, &domain->lv2entcnt[lv1ent_offset(iova)]); |
2a96536e7 iommu/exynos: Add... |
1012 |
} else { |
d09d78fc9 iommu/exynos: Use... |
1013 |
sysmmu_pte_t *pent; |
2a96536e7 iommu/exynos: Add... |
1014 |
|
bfa004893 iommu/exynos: Ren... |
1015 1016 |
pent = alloc_lv2entry(domain, entry, iova, &domain->lv2entcnt[lv1ent_offset(iova)]); |
2a96536e7 iommu/exynos: Add... |
1017 |
|
61128f08f iommu/exynos: Cha... |
1018 1019 |
if (IS_ERR(pent)) ret = PTR_ERR(pent); |
2a96536e7 iommu/exynos: Add... |
1020 1021 |
else ret = lv2set_page(pent, paddr, size, |
bfa004893 iommu/exynos: Ren... |
1022 |
&domain->lv2entcnt[lv1ent_offset(iova)]); |
2a96536e7 iommu/exynos: Add... |
1023 |
} |
61128f08f iommu/exynos: Cha... |
1024 |
if (ret) |
0bf4e54db iommu/exynos: Enh... |
1025 1026 1027 |
pr_err("%s: Failed(%d) to map %#zx bytes @ %#x ", __func__, ret, size, iova); |
2a96536e7 iommu/exynos: Add... |
1028 |
|
bfa004893 iommu/exynos: Ren... |
1029 |
spin_unlock_irqrestore(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1030 1031 1032 |
return ret; } |
bfa004893 iommu/exynos: Ren... |
1033 1034 |
static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, sysmmu_iova_t iova, size_t size) |
66a7ed84b iommu/exynos: App... |
1035 |
{ |
469acebe4 iommu/exynos: Ref... |
1036 |
struct sysmmu_drvdata *data; |
66a7ed84b iommu/exynos: App... |
1037 |
unsigned long flags; |
bfa004893 iommu/exynos: Ren... |
1038 |
spin_lock_irqsave(&domain->lock, flags); |
66a7ed84b iommu/exynos: App... |
1039 |
|
bfa004893 iommu/exynos: Ren... |
1040 |
list_for_each_entry(data, &domain->clients, domain_node) |
469acebe4 iommu/exynos: Ref... |
1041 |
sysmmu_tlb_invalidate_entry(data, iova, size); |
66a7ed84b iommu/exynos: App... |
1042 |
|
bfa004893 iommu/exynos: Ren... |
1043 |
spin_unlock_irqrestore(&domain->lock, flags); |
66a7ed84b iommu/exynos: App... |
1044 |
} |
bfa004893 iommu/exynos: Ren... |
1045 1046 |
static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, unsigned long l_iova, size_t size) |
2a96536e7 iommu/exynos: Add... |
1047 |
{ |
bfa004893 iommu/exynos: Ren... |
1048 |
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); |
d09d78fc9 iommu/exynos: Use... |
1049 1050 |
sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; sysmmu_pte_t *ent; |
61128f08f iommu/exynos: Cha... |
1051 |
size_t err_pgsize; |
d09d78fc9 iommu/exynos: Use... |
1052 |
unsigned long flags; |
2a96536e7 iommu/exynos: Add... |
1053 |
|
bfa004893 iommu/exynos: Ren... |
1054 |
BUG_ON(domain->pgtable == NULL); |
2a96536e7 iommu/exynos: Add... |
1055 |
|
bfa004893 iommu/exynos: Ren... |
1056 |
spin_lock_irqsave(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1057 |
|
bfa004893 iommu/exynos: Ren... |
1058 |
ent = section_entry(domain->pgtable, iova); |
2a96536e7 iommu/exynos: Add... |
1059 1060 |
if (lv1ent_section(ent)) { |
0bf4e54db iommu/exynos: Enh... |
1061 |
if (WARN_ON(size < SECT_SIZE)) { |
61128f08f iommu/exynos: Cha... |
1062 1063 1064 |
err_pgsize = SECT_SIZE; goto err; } |
2a96536e7 iommu/exynos: Add... |
1065 |
|
f171abab8 iommu/exynos: Fix... |
1066 |
/* workaround for h/w bug in System MMU v3.3 */ |
5e3435eb7 iommu/exynos: Rem... |
1067 |
update_pte(ent, ZERO_LV2LINK); |
2a96536e7 iommu/exynos: Add... |
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 |
size = SECT_SIZE; goto done; } if (unlikely(lv1ent_fault(ent))) { if (size > SECT_SIZE) size = SECT_SIZE; goto done; } /* lv1ent_page(sent) == true here */ ent = page_entry(ent, iova); if (unlikely(lv2ent_fault(ent))) { size = SPAGE_SIZE; goto done; } if (lv2ent_small(ent)) { |
5e3435eb7 iommu/exynos: Rem... |
1088 |
update_pte(ent, 0); |
2a96536e7 iommu/exynos: Add... |
1089 |
size = SPAGE_SIZE; |
bfa004893 iommu/exynos: Ren... |
1090 |
domain->lv2entcnt[lv1ent_offset(iova)] += 1; |
2a96536e7 iommu/exynos: Add... |
1091 1092 1093 1094 |
goto done; } /* lv1ent_large(ent) == true here */ |
0bf4e54db iommu/exynos: Enh... |
1095 |
if (WARN_ON(size < LPAGE_SIZE)) { |
61128f08f iommu/exynos: Cha... |
1096 1097 1098 |
err_pgsize = LPAGE_SIZE; goto err; } |
2a96536e7 iommu/exynos: Add... |
1099 |
|
5e3435eb7 iommu/exynos: Rem... |
1100 1101 1102 |
dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent) * SPAGES_PER_LPAGE, DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
1103 |
memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); |
5e3435eb7 iommu/exynos: Rem... |
1104 1105 1106 |
dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent) * SPAGES_PER_LPAGE, DMA_TO_DEVICE); |
2a96536e7 iommu/exynos: Add... |
1107 |
size = LPAGE_SIZE; |
bfa004893 iommu/exynos: Ren... |
1108 |
domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; |
2a96536e7 iommu/exynos: Add... |
1109 |
done: |
bfa004893 iommu/exynos: Ren... |
1110 |
spin_unlock_irqrestore(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1111 |
|
bfa004893 iommu/exynos: Ren... |
1112 |
exynos_iommu_tlb_invalidate_entry(domain, iova, size); |
2a96536e7 iommu/exynos: Add... |
1113 |
|
2a96536e7 iommu/exynos: Add... |
1114 |
return size; |
61128f08f iommu/exynos: Cha... |
1115 |
err: |
bfa004893 iommu/exynos: Ren... |
1116 |
spin_unlock_irqrestore(&domain->pgtablelock, flags); |
61128f08f iommu/exynos: Cha... |
1117 |
|
0bf4e54db iommu/exynos: Enh... |
1118 1119 1120 |
pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx ", __func__, size, iova, err_pgsize); |
61128f08f iommu/exynos: Cha... |
1121 1122 |
return 0; |
2a96536e7 iommu/exynos: Add... |
1123 |
} |
bfa004893 iommu/exynos: Ren... |
1124 |
static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, |
bb5547acf iommu/fsl: Make i... |
1125 |
dma_addr_t iova) |
2a96536e7 iommu/exynos: Add... |
1126 |
{ |
bfa004893 iommu/exynos: Ren... |
1127 |
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); |
d09d78fc9 iommu/exynos: Use... |
1128 |
sysmmu_pte_t *entry; |
2a96536e7 iommu/exynos: Add... |
1129 1130 |
unsigned long flags; phys_addr_t phys = 0; |
bfa004893 iommu/exynos: Ren... |
1131 |
spin_lock_irqsave(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1132 |
|
bfa004893 iommu/exynos: Ren... |
1133 |
entry = section_entry(domain->pgtable, iova); |
2a96536e7 iommu/exynos: Add... |
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 |
if (lv1ent_section(entry)) { phys = section_phys(entry) + section_offs(iova); } else if (lv1ent_page(entry)) { entry = page_entry(entry, iova); if (lv2ent_large(entry)) phys = lpage_phys(entry) + lpage_offs(iova); else if (lv2ent_small(entry)) phys = spage_phys(entry) + spage_offs(iova); } |
bfa004893 iommu/exynos: Ren... |
1145 |
spin_unlock_irqrestore(&domain->pgtablelock, flags); |
2a96536e7 iommu/exynos: Add... |
1146 1147 1148 |
return phys; } |
6c2ae7e29 iommu/exynos: Rew... |
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 |
static struct iommu_group *get_device_iommu_group(struct device *dev) { struct iommu_group *group; group = iommu_group_get(dev); if (!group) group = iommu_group_alloc(); return group; } |
bf4a1c920 iommu/exynos: Add... |
1159 1160 1161 |
static int exynos_iommu_add_device(struct device *dev) { struct iommu_group *group; |
bf4a1c920 iommu/exynos: Add... |
1162 |
|
06801db0d iommu/exynos: Add... |
1163 1164 |
if (!has_sysmmu(dev)) return -ENODEV; |
6c2ae7e29 iommu/exynos: Rew... |
1165 |
group = iommu_group_get_for_dev(dev); |
bf4a1c920 iommu/exynos: Add... |
1166 |
|
6c2ae7e29 iommu/exynos: Rew... |
1167 1168 |
if (IS_ERR(group)) return PTR_ERR(group); |
bf4a1c920 iommu/exynos: Add... |
1169 |
|
bf4a1c920 iommu/exynos: Add... |
1170 |
iommu_group_put(group); |
6c2ae7e29 iommu/exynos: Rew... |
1171 |
return 0; |
bf4a1c920 iommu/exynos: Add... |
1172 1173 1174 1175 |
} static void exynos_iommu_remove_device(struct device *dev) { |
06801db0d iommu/exynos: Add... |
1176 1177 |
if (!has_sysmmu(dev)) return; |
bf4a1c920 iommu/exynos: Add... |
1178 1179 |
iommu_group_remove_device(dev); } |
aa759fd37 iommu/exynos: Add... |
1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 |
static int exynos_iommu_of_xlate(struct device *dev, struct of_phandle_args *spec) { struct exynos_iommu_owner *owner = dev->archdata.iommu; struct platform_device *sysmmu = of_find_device_by_node(spec->np); struct sysmmu_drvdata *data; if (!sysmmu) return -ENODEV; data = platform_get_drvdata(sysmmu); if (!data) return -ENODEV; if (!owner) { owner = kzalloc(sizeof(*owner), GFP_KERNEL); if (!owner) return -ENOMEM; INIT_LIST_HEAD(&owner->controllers); dev->archdata.iommu = owner; } list_add_tail(&data->owner_node, &owner->controllers); return 0; } |
8ed55c812 iommu/exynos: Ini... |
1206 |
static struct iommu_ops exynos_iommu_ops = { |
e1fd1eaa3 iommu/exynos: Mak... |
1207 1208 |
.domain_alloc = exynos_iommu_domain_alloc, .domain_free = exynos_iommu_domain_free, |
ba5fa6f65 iommu/exynos: Rem... |
1209 1210 1211 1212 |
.attach_dev = exynos_iommu_attach_device, .detach_dev = exynos_iommu_detach_device, .map = exynos_iommu_map, .unmap = exynos_iommu_unmap, |
315786ebb iommu: Add iommu_... |
1213 |
.map_sg = default_iommu_map_sg, |
ba5fa6f65 iommu/exynos: Rem... |
1214 |
.iova_to_phys = exynos_iommu_iova_to_phys, |
6c2ae7e29 iommu/exynos: Rew... |
1215 |
.device_group = get_device_iommu_group, |
ba5fa6f65 iommu/exynos: Rem... |
1216 1217 |
.add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, |
2a96536e7 iommu/exynos: Add... |
1218 |
.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, |
aa759fd37 iommu/exynos: Add... |
1219 |
.of_xlate = exynos_iommu_of_xlate, |
2a96536e7 iommu/exynos: Add... |
1220 |
}; |
8ed55c812 iommu/exynos: Ini... |
1221 |
static bool init_done; |
2a96536e7 iommu/exynos: Add... |
1222 1223 1224 |
static int __init exynos_iommu_init(void) { int ret; |
734c3c732 iommu/exynos: All... |
1225 1226 1227 1228 1229 1230 1231 |
lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); if (!lv2table_kmem_cache) { pr_err("%s: Failed to create kmem cache ", __func__); return -ENOMEM; } |
2a96536e7 iommu/exynos: Add... |
1232 |
ret = platform_driver_register(&exynos_sysmmu_driver); |
734c3c732 iommu/exynos: All... |
1233 1234 1235 1236 1237 |
if (ret) { pr_err("%s: Failed to register driver ", __func__); goto err_reg_driver; } |
2a96536e7 iommu/exynos: Add... |
1238 |
|
66a7ed84b iommu/exynos: App... |
1239 1240 1241 1242 1243 1244 1245 1246 |
zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL); if (zero_lv2_table == NULL) { pr_err("%s: Failed to allocate zero level2 page table ", __func__); ret = -ENOMEM; goto err_zero_lv2; } |
734c3c732 iommu/exynos: All... |
1247 1248 1249 1250 1251 1252 1253 |
ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); if (ret) { pr_err("%s: Failed to register exynos-iommu driver. ", __func__); goto err_set_iommu; } |
2a96536e7 iommu/exynos: Add... |
1254 |
|
8ed55c812 iommu/exynos: Ini... |
1255 |
init_done = true; |
734c3c732 iommu/exynos: All... |
1256 1257 |
return 0; err_set_iommu: |
66a7ed84b iommu/exynos: App... |
1258 1259 |
kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); err_zero_lv2: |
734c3c732 iommu/exynos: All... |
1260 1261 1262 |
platform_driver_unregister(&exynos_sysmmu_driver); err_reg_driver: kmem_cache_destroy(lv2table_kmem_cache); |
2a96536e7 iommu/exynos: Add... |
1263 1264 |
return ret; } |
8ed55c812 iommu/exynos: Ini... |
1265 1266 1267 1268 1269 1270 1271 1272 1273 |
static int __init exynos_iommu_of_setup(struct device_node *np) { struct platform_device *pdev; if (!init_done) exynos_iommu_init(); pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); |
423595e89 iommu/exynos: Fix... |
1274 1275 |
if (!pdev) return -ENODEV; |
8ed55c812 iommu/exynos: Ini... |
1276 |
|
5e3435eb7 iommu/exynos: Rem... |
1277 1278 1279 1280 1281 1282 |
/* * use the first registered sysmmu device for performing * dma mapping operations on iommu page tables (cpu cache flush) */ if (!dma_dev) dma_dev = &pdev->dev; |
8ed55c812 iommu/exynos: Ini... |
1283 1284 1285 1286 1287 |
return 0; } IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", exynos_iommu_of_setup); |