Eric Lee / smarc-ti-linux-kernel

Download as
Browse Code ยป

Commit 29ef7a3e26283606f181000b11e5fd37d73ab98f

Authored by Aneesh Kumar K.V
Committed by Benjamin Herrenschmidt
1 parent 2d6b63bbdd
Exists in master and in 13 other branches dlt-processor-sdk-linux-03.00.00.04, processor-sdk-linux-02.00.01, smarc-ti-linux-3.15.y, smarc-ti-lsk-linux-4.1.y, smarct3x-processor-sdk-04.01.00.06, smarct3x-processor-sdk-linux-02.00.01, smarct3x-processor-sdk-linux-03.00.00.04, smarct4x-800-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-04.01.00.06, smarct4x-processor-sdk-linux-02.00.01, smarct4x-processor-sdk-linux-03.00.00.04, ti-linux-3.15.y, ti-lsk-linux-4.1.y

powerpc/mm: Fix tlbie to add AVAL fields for 64K pages 

The if condition check was based on a draft ISA doc. Remove the same.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

Showing 1 changed file with 16 additions and 22 deletions Inline Diff

arch/powerpc/mm/hash_native_64.c
Diff comments   View file @ 29ef7a3
1 /* 1 /*
2 * native hashtable management. 2 * native hashtable management.
3 * 3 *
4 * SMP scalability work: 4 * SMP scalability work:
5 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM 5 * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
6 * 6 *
7 * This program is free software; you can redistribute it and/or 7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License 8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; either version 9 * as published by the Free Software Foundation; either version
10 * 2 of the License, or (at your option) any later version. 10 * 2 of the License, or (at your option) any later version.
11 */ 11 */
12 12
13 #undef DEBUG_LOW 13 #undef DEBUG_LOW
14 14
15 #include <linux/spinlock.h> 15 #include <linux/spinlock.h>
16 #include <linux/bitops.h> 16 #include <linux/bitops.h>
17 #include <linux/of.h> 17 #include <linux/of.h>
18 #include <linux/threads.h> 18 #include <linux/threads.h>
19 #include <linux/smp.h> 19 #include <linux/smp.h>
20 20
21 #include <asm/machdep.h> 21 #include <asm/machdep.h>
22 #include <asm/mmu.h> 22 #include <asm/mmu.h>
23 #include <asm/mmu_context.h> 23 #include <asm/mmu_context.h>
24 #include <asm/pgtable.h> 24 #include <asm/pgtable.h>
25 #include <asm/tlbflush.h> 25 #include <asm/tlbflush.h>
26 #include <asm/tlb.h> 26 #include <asm/tlb.h>
27 #include <asm/cputable.h> 27 #include <asm/cputable.h>
28 #include <asm/udbg.h> 28 #include <asm/udbg.h>
29 #include <asm/kexec.h> 29 #include <asm/kexec.h>
30 #include <asm/ppc-opcode.h> 30 #include <asm/ppc-opcode.h>
31 31
32 #ifdef DEBUG_LOW 32 #ifdef DEBUG_LOW
33 #define DBG_LOW(fmt...) udbg_printf(fmt) 33 #define DBG_LOW(fmt...) udbg_printf(fmt)
34 #else 34 #else
35 #define DBG_LOW(fmt...) 35 #define DBG_LOW(fmt...)
36 #endif 36 #endif
37 37
38 #ifdef __BIG_ENDIAN__ 38 #ifdef __BIG_ENDIAN__
39 #define HPTE_LOCK_BIT 3 39 #define HPTE_LOCK_BIT 3
40 #else 40 #else
41 #define HPTE_LOCK_BIT (56+3) 41 #define HPTE_LOCK_BIT (56+3)
42 #endif 42 #endif
43 43
44 DEFINE_RAW_SPINLOCK(native_tlbie_lock); 44 DEFINE_RAW_SPINLOCK(native_tlbie_lock);
45 45
46 static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) 46 static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
47 { 47 {
48 unsigned long va; 48 unsigned long va;
49 unsigned int penc; 49 unsigned int penc;
50 unsigned long sllp; 50 unsigned long sllp;
51 51
52 /* 52 /*
53 * We need 14 to 65 bits of va for a tlibe of 4K page 53 * We need 14 to 65 bits of va for a tlibe of 4K page
54 * With vpn we ignore the lower VPN_SHIFT bits already. 54 * With vpn we ignore the lower VPN_SHIFT bits already.
55 * And top two bits are already ignored because we can 55 * And top two bits are already ignored because we can
56 * only accomadate 76 bits in a 64 bit vpn with a VPN_SHIFT 56 * only accomadate 76 bits in a 64 bit vpn with a VPN_SHIFT
57 * of 12. 57 * of 12.
58 */ 58 */
59 va = vpn << VPN_SHIFT; 59 va = vpn << VPN_SHIFT;
60 /* 60 /*
61 * clear top 16 bits of 64bit va, non SLS segment 61 * clear top 16 bits of 64bit va, non SLS segment
62 * Older versions of the architecture (2.02 and earler) require the 62 * Older versions of the architecture (2.02 and earler) require the
63 * masking of the top 16 bits. 63 * masking of the top 16 bits.
64 */ 64 */
65 va &= ~(0xffffULL << 48); 65 va &= ~(0xffffULL << 48);
66 66
67 switch (psize) { 67 switch (psize) {
68 case MMU_PAGE_4K: 68 case MMU_PAGE_4K:
69 /* clear out bits after (52) [0....52.....63] */ 69 /* clear out bits after (52) [0....52.....63] */
70 va &= ~((1ul << (64 - 52)) - 1); 70 va &= ~((1ul << (64 - 52)) - 1);
71 va |= ssize << 8; 71 va |= ssize << 8;
72 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) | 72 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
73 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4); 73 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
74 va |= sllp << 5; 74 va |= sllp << 5;
75 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) 75 asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
76 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) 76 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
77 : "memory"); 77 : "memory");
78 break; 78 break;
79 default: 79 default:
80 /* We need 14 to 14 + i bits of va */ 80 /* We need 14 to 14 + i bits of va */
81 penc = mmu_psize_defs[psize].penc[apsize]; 81 penc = mmu_psize_defs[psize].penc[apsize];
82 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); 82 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
83 va |= penc << 12; 83 va |= penc << 12;
84 va |= ssize << 8; 84 va |= ssize << 8;
85 /* Add AVAL part */ 85 /*
86 if (psize != apsize) { 86 * AVAL bits:
87 /* 87 * We don't need all the bits, but rest of the bits
88 * MPSS, 64K base page size and 16MB parge page size 88 * must be ignored by the processor.
89 * We don't need all the bits, but rest of the bits 89 * vpn cover upto 65 bits of va. (0...65) and we need
90 * must be ignored by the processor. 90 * 58..64 bits of va.
91 * vpn cover upto 65 bits of va. (0...65) and we need 91 */
92 * 58..64 bits of va. 92 va |= (vpn & 0xfe); /* AVAL */
93 */
94 va |= (vpn & 0xfe);
95 }
96 va |= 1; /* L */ 93 va |= 1; /* L */
97 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) 94 asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
98 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) 95 : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
99 : "memory"); 96 : "memory");
100 break; 97 break;
101 } 98 }
102 } 99 }
103 100
104 static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) 101 static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
105 { 102 {
106 unsigned long va; 103 unsigned long va;
107 unsigned int penc; 104 unsigned int penc;
108 unsigned long sllp; 105 unsigned long sllp;
109 106
110 /* VPN_SHIFT can be atmost 12 */ 107 /* VPN_SHIFT can be atmost 12 */
111 va = vpn << VPN_SHIFT; 108 va = vpn << VPN_SHIFT;
112 /* 109 /*
113 * clear top 16 bits of 64 bit va, non SLS segment 110 * clear top 16 bits of 64 bit va, non SLS segment
114 * Older versions of the architecture (2.02 and earler) require the 111 * Older versions of the architecture (2.02 and earler) require the
115 * masking of the top 16 bits. 112 * masking of the top 16 bits.
116 */ 113 */
117 va &= ~(0xffffULL << 48); 114 va &= ~(0xffffULL << 48);
118 115
119 switch (psize) { 116 switch (psize) {
120 case MMU_PAGE_4K: 117 case MMU_PAGE_4K:
121 /* clear out bits after(52) [0....52.....63] */ 118 /* clear out bits after(52) [0....52.....63] */
122 va &= ~((1ul << (64 - 52)) - 1); 119 va &= ~((1ul << (64 - 52)) - 1);
123 va |= ssize << 8; 120 va |= ssize << 8;
124 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) | 121 sllp = ((mmu_psize_defs[apsize].sllp & SLB_VSID_L) >> 6) |
125 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4); 122 ((mmu_psize_defs[apsize].sllp & SLB_VSID_LP) >> 4);
126 va |= sllp << 5; 123 va |= sllp << 5;
127 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" 124 asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)"
128 : : "r"(va) : "memory"); 125 : : "r"(va) : "memory");
129 break; 126 break;
130 default: 127 default:
131 /* We need 14 to 14 + i bits of va */ 128 /* We need 14 to 14 + i bits of va */
132 penc = mmu_psize_defs[psize].penc[apsize]; 129 penc = mmu_psize_defs[psize].penc[apsize];
133 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1); 130 va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
134 va |= penc << 12; 131 va |= penc << 12;
135 va |= ssize << 8; 132 va |= ssize << 8;
136 /* Add AVAL part */ 133 /*
137 if (psize != apsize) { 134 * AVAL bits:
138 /* 135 * We don't need all the bits, but rest of the bits
139 * MPSS, 64K base page size and 16MB parge page size 136 * must be ignored by the processor.
140 * We don't need all the bits, but rest of the bits 137 * vpn cover upto 65 bits of va. (0...65) and we need
141 * must be ignored by the processor. 138 * 58..64 bits of va.
142 * vpn cover upto 65 bits of va. (0...65) and we need 139 */
143 * 58..64 bits of va. 140 va |= (vpn & 0xfe);
144 */
145 va |= (vpn & 0xfe);
146 }
147 va |= 1; /* L */ 141 va |= 1; /* L */
148 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" 142 asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)"
149 : : "r"(va) : "memory"); 143 : : "r"(va) : "memory");
150 break; 144 break;
151 } 145 }
152 146
153 } 147 }
154 148
155 static inline void tlbie(unsigned long vpn, int psize, int apsize, 149 static inline void tlbie(unsigned long vpn, int psize, int apsize,
156 int ssize, int local) 150 int ssize, int local)
157 { 151 {
158 unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL); 152 unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL);
159 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 153 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
160 154
161 if (use_local) 155 if (use_local)
162 use_local = mmu_psize_defs[psize].tlbiel; 156 use_local = mmu_psize_defs[psize].tlbiel;
163 if (lock_tlbie && !use_local) 157 if (lock_tlbie && !use_local)
164 raw_spin_lock(&native_tlbie_lock); 158 raw_spin_lock(&native_tlbie_lock);
165 asm volatile("ptesync": : :"memory"); 159 asm volatile("ptesync": : :"memory");
166 if (use_local) { 160 if (use_local) {
167 __tlbiel(vpn, psize, apsize, ssize); 161 __tlbiel(vpn, psize, apsize, ssize);
168 asm volatile("ptesync": : :"memory"); 162 asm volatile("ptesync": : :"memory");
169 } else { 163 } else {
170 __tlbie(vpn, psize, apsize, ssize); 164 __tlbie(vpn, psize, apsize, ssize);
171 asm volatile("eieio; tlbsync; ptesync": : :"memory"); 165 asm volatile("eieio; tlbsync; ptesync": : :"memory");
172 } 166 }
173 if (lock_tlbie && !use_local) 167 if (lock_tlbie && !use_local)
174 raw_spin_unlock(&native_tlbie_lock); 168 raw_spin_unlock(&native_tlbie_lock);
175 } 169 }
176 170
177 static inline void native_lock_hpte(struct hash_pte *hptep) 171 static inline void native_lock_hpte(struct hash_pte *hptep)
178 { 172 {
179 unsigned long *word = (unsigned long *)&hptep->v; 173 unsigned long *word = (unsigned long *)&hptep->v;
180 174
181 while (1) { 175 while (1) {
182 if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) 176 if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
183 break; 177 break;
184 while(test_bit(HPTE_LOCK_BIT, word)) 178 while(test_bit(HPTE_LOCK_BIT, word))
185 cpu_relax(); 179 cpu_relax();
186 } 180 }
187 } 181 }
188 182
189 static inline void native_unlock_hpte(struct hash_pte *hptep) 183 static inline void native_unlock_hpte(struct hash_pte *hptep)
190 { 184 {
191 unsigned long *word = (unsigned long *)&hptep->v; 185 unsigned long *word = (unsigned long *)&hptep->v;
192 186
193 clear_bit_unlock(HPTE_LOCK_BIT, word); 187 clear_bit_unlock(HPTE_LOCK_BIT, word);
194 } 188 }
195 189
196 static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, 190 static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
197 unsigned long pa, unsigned long rflags, 191 unsigned long pa, unsigned long rflags,
198 unsigned long vflags, int psize, int apsize, int ssize) 192 unsigned long vflags, int psize, int apsize, int ssize)
199 { 193 {
200 struct hash_pte *hptep = htab_address + hpte_group; 194 struct hash_pte *hptep = htab_address + hpte_group;
201 unsigned long hpte_v, hpte_r; 195 unsigned long hpte_v, hpte_r;
202 int i; 196 int i;
203 197
204 if (!(vflags & HPTE_V_BOLTED)) { 198 if (!(vflags & HPTE_V_BOLTED)) {
205 DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx," 199 DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
206 " rflags=%lx, vflags=%lx, psize=%d)\n", 200 " rflags=%lx, vflags=%lx, psize=%d)\n",
207 hpte_group, vpn, pa, rflags, vflags, psize); 201 hpte_group, vpn, pa, rflags, vflags, psize);
208 } 202 }
209 203
210 for (i = 0; i < HPTES_PER_GROUP; i++) { 204 for (i = 0; i < HPTES_PER_GROUP; i++) {
211 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) { 205 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
212 /* retry with lock held */ 206 /* retry with lock held */
213 native_lock_hpte(hptep); 207 native_lock_hpte(hptep);
214 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) 208 if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
215 break; 209 break;
216 native_unlock_hpte(hptep); 210 native_unlock_hpte(hptep);
217 } 211 }
218 212
219 hptep++; 213 hptep++;
220 } 214 }
221 215
222 if (i == HPTES_PER_GROUP) 216 if (i == HPTES_PER_GROUP)
223 return -1; 217 return -1;
224 218
225 hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; 219 hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
226 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; 220 hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
227 221
228 if (!(vflags & HPTE_V_BOLTED)) { 222 if (!(vflags & HPTE_V_BOLTED)) {
229 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", 223 DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
230 i, hpte_v, hpte_r); 224 i, hpte_v, hpte_r);
231 } 225 }
232 226
233 hptep->r = cpu_to_be64(hpte_r); 227 hptep->r = cpu_to_be64(hpte_r);
234 /* Guarantee the second dword is visible before the valid bit */ 228 /* Guarantee the second dword is visible before the valid bit */
235 eieio(); 229 eieio();
236 /* 230 /*
237 * Now set the first dword including the valid bit 231 * Now set the first dword including the valid bit
238 * NOTE: this also unlocks the hpte 232 * NOTE: this also unlocks the hpte
239 */ 233 */
240 hptep->v = cpu_to_be64(hpte_v); 234 hptep->v = cpu_to_be64(hpte_v);
241 235
242 __asm__ __volatile__ ("ptesync" : : : "memory"); 236 __asm__ __volatile__ ("ptesync" : : : "memory");
243 237
244 return i | (!!(vflags & HPTE_V_SECONDARY) << 3); 238 return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
245 } 239 }
246 240
247 static long native_hpte_remove(unsigned long hpte_group) 241 static long native_hpte_remove(unsigned long hpte_group)
248 { 242 {
249 struct hash_pte *hptep; 243 struct hash_pte *hptep;
250 int i; 244 int i;
251 int slot_offset; 245 int slot_offset;
252 unsigned long hpte_v; 246 unsigned long hpte_v;
253 247
254 DBG_LOW(" remove(group=%lx)\n", hpte_group); 248 DBG_LOW(" remove(group=%lx)\n", hpte_group);
255 249
256 /* pick a random entry to start at */ 250 /* pick a random entry to start at */
257 slot_offset = mftb() & 0x7; 251 slot_offset = mftb() & 0x7;
258 252
259 for (i = 0; i < HPTES_PER_GROUP; i++) { 253 for (i = 0; i < HPTES_PER_GROUP; i++) {
260 hptep = htab_address + hpte_group + slot_offset; 254 hptep = htab_address + hpte_group + slot_offset;
261 hpte_v = be64_to_cpu(hptep->v); 255 hpte_v = be64_to_cpu(hptep->v);
262 256
263 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { 257 if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
264 /* retry with lock held */ 258 /* retry with lock held */
265 native_lock_hpte(hptep); 259 native_lock_hpte(hptep);
266 hpte_v = be64_to_cpu(hptep->v); 260 hpte_v = be64_to_cpu(hptep->v);
267 if ((hpte_v & HPTE_V_VALID) 261 if ((hpte_v & HPTE_V_VALID)
268 && !(hpte_v & HPTE_V_BOLTED)) 262 && !(hpte_v & HPTE_V_BOLTED))
269 break; 263 break;
270 native_unlock_hpte(hptep); 264 native_unlock_hpte(hptep);
271 } 265 }
272 266
273 slot_offset++; 267 slot_offset++;
274 slot_offset &= 0x7; 268 slot_offset &= 0x7;
275 } 269 }
276 270
277 if (i == HPTES_PER_GROUP) 271 if (i == HPTES_PER_GROUP)
278 return -1; 272 return -1;
279 273
280 /* Invalidate the hpte. NOTE: this also unlocks it */ 274 /* Invalidate the hpte. NOTE: this also unlocks it */
281 hptep->v = 0; 275 hptep->v = 0;
282 276
283 return i; 277 return i;
284 } 278 }
285 279
286 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, 280 static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
287 unsigned long vpn, int bpsize, 281 unsigned long vpn, int bpsize,
288 int apsize, int ssize, int local) 282 int apsize, int ssize, int local)
289 { 283 {
290 struct hash_pte *hptep = htab_address + slot; 284 struct hash_pte *hptep = htab_address + slot;
291 unsigned long hpte_v, want_v; 285 unsigned long hpte_v, want_v;
292 int ret = 0; 286 int ret = 0;
293 287
294 want_v = hpte_encode_avpn(vpn, bpsize, ssize); 288 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
295 289
296 DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)", 290 DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
297 vpn, want_v & HPTE_V_AVPN, slot, newpp); 291 vpn, want_v & HPTE_V_AVPN, slot, newpp);
298 292
299 native_lock_hpte(hptep); 293 native_lock_hpte(hptep);
300 294
301 hpte_v = be64_to_cpu(hptep->v); 295 hpte_v = be64_to_cpu(hptep->v);
302 /* 296 /*
303 * We need to invalidate the TLB always because hpte_remove doesn't do 297 * We need to invalidate the TLB always because hpte_remove doesn't do
304 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less 298 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
305 * random entry from it. When we do that we don't invalidate the TLB 299 * random entry from it. When we do that we don't invalidate the TLB
306 * (hpte_remove) because we assume the old translation is still 300 * (hpte_remove) because we assume the old translation is still
307 * technically "valid". 301 * technically "valid".
308 */ 302 */
309 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) { 303 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
310 DBG_LOW(" -> miss\n"); 304 DBG_LOW(" -> miss\n");
311 ret = -1; 305 ret = -1;
312 } else { 306 } else {
313 DBG_LOW(" -> hit\n"); 307 DBG_LOW(" -> hit\n");
314 /* Update the HPTE */ 308 /* Update the HPTE */
315 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & ~(HPTE_R_PP | HPTE_R_N)) | 309 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & ~(HPTE_R_PP | HPTE_R_N)) |
316 (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C))); 310 (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C)));
317 } 311 }
318 native_unlock_hpte(hptep); 312 native_unlock_hpte(hptep);
319 313
320 /* Ensure it is out of the tlb too. */ 314 /* Ensure it is out of the tlb too. */
321 tlbie(vpn, bpsize, apsize, ssize, local); 315 tlbie(vpn, bpsize, apsize, ssize, local);
322 316
323 return ret; 317 return ret;
324 } 318 }
325 319
326 static long native_hpte_find(unsigned long vpn, int psize, int ssize) 320 static long native_hpte_find(unsigned long vpn, int psize, int ssize)
327 { 321 {
328 struct hash_pte *hptep; 322 struct hash_pte *hptep;
329 unsigned long hash; 323 unsigned long hash;
330 unsigned long i; 324 unsigned long i;
331 long slot; 325 long slot;
332 unsigned long want_v, hpte_v; 326 unsigned long want_v, hpte_v;
333 327
334 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize); 328 hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
335 want_v = hpte_encode_avpn(vpn, psize, ssize); 329 want_v = hpte_encode_avpn(vpn, psize, ssize);
336 330
337 /* Bolted mappings are only ever in the primary group */ 331 /* Bolted mappings are only ever in the primary group */
338 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 332 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
339 for (i = 0; i < HPTES_PER_GROUP; i++) { 333 for (i = 0; i < HPTES_PER_GROUP; i++) {
340 hptep = htab_address + slot; 334 hptep = htab_address + slot;
341 hpte_v = be64_to_cpu(hptep->v); 335 hpte_v = be64_to_cpu(hptep->v);
342 336
343 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) 337 if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
344 /* HPTE matches */ 338 /* HPTE matches */
345 return slot; 339 return slot;
346 ++slot; 340 ++slot;
347 } 341 }
348 342
349 return -1; 343 return -1;
350 } 344 }
351 345
352 /* 346 /*
353 * Update the page protection bits. Intended to be used to create 347 * Update the page protection bits. Intended to be used to create
354 * guard pages for kernel data structures on pages which are bolted 348 * guard pages for kernel data structures on pages which are bolted
355 * in the HPT. Assumes pages being operated on will not be stolen. 349 * in the HPT. Assumes pages being operated on will not be stolen.
356 * 350 *
357 * No need to lock here because we should be the only user. 351 * No need to lock here because we should be the only user.
358 */ 352 */
359 static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, 353 static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
360 int psize, int ssize) 354 int psize, int ssize)
361 { 355 {
362 unsigned long vpn; 356 unsigned long vpn;
363 unsigned long vsid; 357 unsigned long vsid;
364 long slot; 358 long slot;
365 struct hash_pte *hptep; 359 struct hash_pte *hptep;
366 360
367 vsid = get_kernel_vsid(ea, ssize); 361 vsid = get_kernel_vsid(ea, ssize);
368 vpn = hpt_vpn(ea, vsid, ssize); 362 vpn = hpt_vpn(ea, vsid, ssize);
369 363
370 slot = native_hpte_find(vpn, psize, ssize); 364 slot = native_hpte_find(vpn, psize, ssize);
371 if (slot == -1) 365 if (slot == -1)
372 panic("could not find page to bolt\n"); 366 panic("could not find page to bolt\n");
373 hptep = htab_address + slot; 367 hptep = htab_address + slot;
374 368
375 /* Update the HPTE */ 369 /* Update the HPTE */
376 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) & 370 hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
377 ~(HPTE_R_PP | HPTE_R_N)) | 371 ~(HPTE_R_PP | HPTE_R_N)) |
378 (newpp & (HPTE_R_PP | HPTE_R_N))); 372 (newpp & (HPTE_R_PP | HPTE_R_N)));
379 /* 373 /*
380 * Ensure it is out of the tlb too. Bolted entries base and 374 * Ensure it is out of the tlb too. Bolted entries base and
381 * actual page size will be same. 375 * actual page size will be same.
382 */ 376 */
383 tlbie(vpn, psize, psize, ssize, 0); 377 tlbie(vpn, psize, psize, ssize, 0);
384 } 378 }
385 379
386 static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, 380 static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
387 int bpsize, int apsize, int ssize, int local) 381 int bpsize, int apsize, int ssize, int local)
388 { 382 {
389 struct hash_pte *hptep = htab_address + slot; 383 struct hash_pte *hptep = htab_address + slot;
390 unsigned long hpte_v; 384 unsigned long hpte_v;
391 unsigned long want_v; 385 unsigned long want_v;
392 unsigned long flags; 386 unsigned long flags;
393 387
394 local_irq_save(flags); 388 local_irq_save(flags);
395 389
396 DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot); 390 DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
397 391
398 want_v = hpte_encode_avpn(vpn, bpsize, ssize); 392 want_v = hpte_encode_avpn(vpn, bpsize, ssize);
399 native_lock_hpte(hptep); 393 native_lock_hpte(hptep);
400 hpte_v = be64_to_cpu(hptep->v); 394 hpte_v = be64_to_cpu(hptep->v);
401 395
402 /* 396 /*
403 * We need to invalidate the TLB always because hpte_remove doesn't do 397 * We need to invalidate the TLB always because hpte_remove doesn't do
404 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less 398 * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
405 * random entry from it. When we do that we don't invalidate the TLB 399 * random entry from it. When we do that we don't invalidate the TLB
406 * (hpte_remove) because we assume the old translation is still 400 * (hpte_remove) because we assume the old translation is still
407 * technically "valid". 401 * technically "valid".
408 */ 402 */
409 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) 403 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
410 native_unlock_hpte(hptep); 404 native_unlock_hpte(hptep);
411 else 405 else
412 /* Invalidate the hpte. NOTE: this also unlocks it */ 406 /* Invalidate the hpte. NOTE: this also unlocks it */
413 hptep->v = 0; 407 hptep->v = 0;
414 408
415 /* Invalidate the TLB */ 409 /* Invalidate the TLB */
416 tlbie(vpn, bpsize, apsize, ssize, local); 410 tlbie(vpn, bpsize, apsize, ssize, local);
417 411
418 local_irq_restore(flags); 412 local_irq_restore(flags);
419 } 413 }
420 414
421 static void native_hugepage_invalidate(struct mm_struct *mm, 415 static void native_hugepage_invalidate(struct mm_struct *mm,
422 unsigned char *hpte_slot_array, 416 unsigned char *hpte_slot_array,
423 unsigned long addr, int psize) 417 unsigned long addr, int psize)
424 { 418 {
425 int ssize = 0, i; 419 int ssize = 0, i;
426 int lock_tlbie; 420 int lock_tlbie;
427 struct hash_pte *hptep; 421 struct hash_pte *hptep;
428 int actual_psize = MMU_PAGE_16M; 422 int actual_psize = MMU_PAGE_16M;
429 unsigned int max_hpte_count, valid; 423 unsigned int max_hpte_count, valid;
430 unsigned long flags, s_addr = addr; 424 unsigned long flags, s_addr = addr;
431 unsigned long hpte_v, want_v, shift; 425 unsigned long hpte_v, want_v, shift;
432 unsigned long hidx, vpn = 0, vsid, hash, slot; 426 unsigned long hidx, vpn = 0, vsid, hash, slot;
433 427
434 shift = mmu_psize_defs[psize].shift; 428 shift = mmu_psize_defs[psize].shift;
435 max_hpte_count = 1U << (PMD_SHIFT - shift); 429 max_hpte_count = 1U << (PMD_SHIFT - shift);
436 430
437 local_irq_save(flags); 431 local_irq_save(flags);
438 for (i = 0; i < max_hpte_count; i++) { 432 for (i = 0; i < max_hpte_count; i++) {
439 valid = hpte_valid(hpte_slot_array, i); 433 valid = hpte_valid(hpte_slot_array, i);
440 if (!valid) 434 if (!valid)
441 continue; 435 continue;
442 hidx = hpte_hash_index(hpte_slot_array, i); 436 hidx = hpte_hash_index(hpte_slot_array, i);
443 437
444 /* get the vpn */ 438 /* get the vpn */
445 addr = s_addr + (i * (1ul << shift)); 439 addr = s_addr + (i * (1ul << shift));
446 if (!is_kernel_addr(addr)) { 440 if (!is_kernel_addr(addr)) {
447 ssize = user_segment_size(addr); 441 ssize = user_segment_size(addr);
448 vsid = get_vsid(mm->context.id, addr, ssize); 442 vsid = get_vsid(mm->context.id, addr, ssize);
449 WARN_ON(vsid == 0); 443 WARN_ON(vsid == 0);
450 } else { 444 } else {
451 vsid = get_kernel_vsid(addr, mmu_kernel_ssize); 445 vsid = get_kernel_vsid(addr, mmu_kernel_ssize);
452 ssize = mmu_kernel_ssize; 446 ssize = mmu_kernel_ssize;
453 } 447 }
454 448
455 vpn = hpt_vpn(addr, vsid, ssize); 449 vpn = hpt_vpn(addr, vsid, ssize);
456 hash = hpt_hash(vpn, shift, ssize); 450 hash = hpt_hash(vpn, shift, ssize);
457 if (hidx & _PTEIDX_SECONDARY) 451 if (hidx & _PTEIDX_SECONDARY)
458 hash = ~hash; 452 hash = ~hash;
459 453
460 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 454 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
461 slot += hidx & _PTEIDX_GROUP_IX; 455 slot += hidx & _PTEIDX_GROUP_IX;
462 456
463 hptep = htab_address + slot; 457 hptep = htab_address + slot;
464 want_v = hpte_encode_avpn(vpn, psize, ssize); 458 want_v = hpte_encode_avpn(vpn, psize, ssize);
465 native_lock_hpte(hptep); 459 native_lock_hpte(hptep);
466 hpte_v = be64_to_cpu(hptep->v); 460 hpte_v = be64_to_cpu(hptep->v);
467 461
468 /* Even if we miss, we need to invalidate the TLB */ 462 /* Even if we miss, we need to invalidate the TLB */
469 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) 463 if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
470 native_unlock_hpte(hptep); 464 native_unlock_hpte(hptep);
471 else 465 else
472 /* Invalidate the hpte. NOTE: this also unlocks it */ 466 /* Invalidate the hpte. NOTE: this also unlocks it */
473 hptep->v = 0; 467 hptep->v = 0;
474 } 468 }
475 /* 469 /*
476 * Since this is a hugepage, we just need a single tlbie. 470 * Since this is a hugepage, we just need a single tlbie.
477 * use the last vpn. 471 * use the last vpn.
478 */ 472 */
479 lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 473 lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
480 if (lock_tlbie) 474 if (lock_tlbie)
481 raw_spin_lock(&native_tlbie_lock); 475 raw_spin_lock(&native_tlbie_lock);
482 476
483 asm volatile("ptesync":::"memory"); 477 asm volatile("ptesync":::"memory");
484 __tlbie(vpn, psize, actual_psize, ssize); 478 __tlbie(vpn, psize, actual_psize, ssize);
485 asm volatile("eieio; tlbsync; ptesync":::"memory"); 479 asm volatile("eieio; tlbsync; ptesync":::"memory");
486 480
487 if (lock_tlbie) 481 if (lock_tlbie)
488 raw_spin_unlock(&native_tlbie_lock); 482 raw_spin_unlock(&native_tlbie_lock);
489 483
490 local_irq_restore(flags); 484 local_irq_restore(flags);
491 } 485 }
492 486
493 static inline int __hpte_actual_psize(unsigned int lp, int psize) 487 static inline int __hpte_actual_psize(unsigned int lp, int psize)
494 { 488 {
495 int i, shift; 489 int i, shift;
496 unsigned int mask; 490 unsigned int mask;
497 491
498 /* start from 1 ignoring MMU_PAGE_4K */ 492 /* start from 1 ignoring MMU_PAGE_4K */
499 for (i = 1; i < MMU_PAGE_COUNT; i++) { 493 for (i = 1; i < MMU_PAGE_COUNT; i++) {
500 494
501 /* invalid penc */ 495 /* invalid penc */
502 if (mmu_psize_defs[psize].penc[i] == -1) 496 if (mmu_psize_defs[psize].penc[i] == -1)
503 continue; 497 continue;
504 /* 498 /*
505 * encoding bits per actual page size 499 * encoding bits per actual page size
506 * PTE LP actual page size 500 * PTE LP actual page size
507 * rrrr rrrz >=8KB 501 * rrrr rrrz >=8KB
508 * rrrr rrzz >=16KB 502 * rrrr rrzz >=16KB
509 * rrrr rzzz >=32KB 503 * rrrr rzzz >=32KB
510 * rrrr zzzz >=64KB 504 * rrrr zzzz >=64KB
511 * ....... 505 * .......
512 */ 506 */
513 shift = mmu_psize_defs[i].shift - LP_SHIFT; 507 shift = mmu_psize_defs[i].shift - LP_SHIFT;
514 if (shift > LP_BITS) 508 if (shift > LP_BITS)
515 shift = LP_BITS; 509 shift = LP_BITS;
516 mask = (1 << shift) - 1; 510 mask = (1 << shift) - 1;
517 if ((lp & mask) == mmu_psize_defs[psize].penc[i]) 511 if ((lp & mask) == mmu_psize_defs[psize].penc[i])
518 return i; 512 return i;
519 } 513 }
520 return -1; 514 return -1;
521 } 515 }
522 516
523 static void hpte_decode(struct hash_pte *hpte, unsigned long slot, 517 static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
524 int *psize, int *apsize, int *ssize, unsigned long *vpn) 518 int *psize, int *apsize, int *ssize, unsigned long *vpn)
525 { 519 {
526 unsigned long avpn, pteg, vpi; 520 unsigned long avpn, pteg, vpi;
527 unsigned long hpte_v = be64_to_cpu(hpte->v); 521 unsigned long hpte_v = be64_to_cpu(hpte->v);
528 unsigned long hpte_r = be64_to_cpu(hpte->r); 522 unsigned long hpte_r = be64_to_cpu(hpte->r);
529 unsigned long vsid, seg_off; 523 unsigned long vsid, seg_off;
530 int size, a_size, shift; 524 int size, a_size, shift;
531 /* Look at the 8 bit LP value */ 525 /* Look at the 8 bit LP value */
532 unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1); 526 unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
533 527
534 if (!(hpte_v & HPTE_V_LARGE)) { 528 if (!(hpte_v & HPTE_V_LARGE)) {
535 size = MMU_PAGE_4K; 529 size = MMU_PAGE_4K;
536 a_size = MMU_PAGE_4K; 530 a_size = MMU_PAGE_4K;
537 } else { 531 } else {
538 for (size = 0; size < MMU_PAGE_COUNT; size++) { 532 for (size = 0; size < MMU_PAGE_COUNT; size++) {
539 533
540 /* valid entries have a shift value */ 534 /* valid entries have a shift value */
541 if (!mmu_psize_defs[size].shift) 535 if (!mmu_psize_defs[size].shift)
542 continue; 536 continue;
543 537
544 a_size = __hpte_actual_psize(lp, size); 538 a_size = __hpte_actual_psize(lp, size);
545 if (a_size != -1) 539 if (a_size != -1)
546 break; 540 break;
547 } 541 }
548 } 542 }
549 /* This works for all page sizes, and for 256M and 1T segments */ 543 /* This works for all page sizes, and for 256M and 1T segments */
550 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; 544 *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
551 shift = mmu_psize_defs[size].shift; 545 shift = mmu_psize_defs[size].shift;
552 546
553 avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); 547 avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
554 pteg = slot / HPTES_PER_GROUP; 548 pteg = slot / HPTES_PER_GROUP;
555 if (hpte_v & HPTE_V_SECONDARY) 549 if (hpte_v & HPTE_V_SECONDARY)
556 pteg = ~pteg; 550 pteg = ~pteg;
557 551
558 switch (*ssize) { 552 switch (*ssize) {
559 case MMU_SEGSIZE_256M: 553 case MMU_SEGSIZE_256M:
560 /* We only have 28 - 23 bits of seg_off in avpn */ 554 /* We only have 28 - 23 bits of seg_off in avpn */
561 seg_off = (avpn & 0x1f) << 23; 555 seg_off = (avpn & 0x1f) << 23;
562 vsid = avpn >> 5; 556 vsid = avpn >> 5;
563 /* We can find more bits from the pteg value */ 557 /* We can find more bits from the pteg value */
564 if (shift < 23) { 558 if (shift < 23) {
565 vpi = (vsid ^ pteg) & htab_hash_mask; 559 vpi = (vsid ^ pteg) & htab_hash_mask;
566 seg_off |= vpi << shift; 560 seg_off |= vpi << shift;
567 } 561 }
568 *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT; 562 *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
569 break; 563 break;
570 case MMU_SEGSIZE_1T: 564 case MMU_SEGSIZE_1T:
571 /* We only have 40 - 23 bits of seg_off in avpn */ 565 /* We only have 40 - 23 bits of seg_off in avpn */
572 seg_off = (avpn & 0x1ffff) << 23; 566 seg_off = (avpn & 0x1ffff) << 23;
573 vsid = avpn >> 17; 567 vsid = avpn >> 17;
574 if (shift < 23) { 568 if (shift < 23) {
575 vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask; 569 vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
576 seg_off |= vpi << shift; 570 seg_off |= vpi << shift;
577 } 571 }
578 *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT; 572 *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
579 break; 573 break;
580 default: 574 default:
581 *vpn = size = 0; 575 *vpn = size = 0;
582 } 576 }
583 *psize = size; 577 *psize = size;
584 *apsize = a_size; 578 *apsize = a_size;
585 } 579 }
586 580
587 /* 581 /*
588 * clear all mappings on kexec. All cpus are in real mode (or they will 582 * clear all mappings on kexec. All cpus are in real mode (or they will
589 * be when they isi), and we are the only one left. We rely on our kernel 583 * be when they isi), and we are the only one left. We rely on our kernel
590 * mapping being 0xC0's and the hardware ignoring those two real bits. 584 * mapping being 0xC0's and the hardware ignoring those two real bits.
591 * 585 *
592 * TODO: add batching support when enabled. remember, no dynamic memory here, 586 * TODO: add batching support when enabled. remember, no dynamic memory here,
593 * athough there is the control page available... 587 * athough there is the control page available...
594 */ 588 */
595 static void native_hpte_clear(void) 589 static void native_hpte_clear(void)
596 { 590 {
597 unsigned long vpn = 0; 591 unsigned long vpn = 0;
598 unsigned long slot, slots, flags; 592 unsigned long slot, slots, flags;
599 struct hash_pte *hptep = htab_address; 593 struct hash_pte *hptep = htab_address;
600 unsigned long hpte_v; 594 unsigned long hpte_v;
601 unsigned long pteg_count; 595 unsigned long pteg_count;
602 int psize, apsize, ssize; 596 int psize, apsize, ssize;
603 597
604 pteg_count = htab_hash_mask + 1; 598 pteg_count = htab_hash_mask + 1;
605 599
606 local_irq_save(flags); 600 local_irq_save(flags);
607 601
608 /* we take the tlbie lock and hold it. Some hardware will 602 /* we take the tlbie lock and hold it. Some hardware will
609 * deadlock if we try to tlbie from two processors at once. 603 * deadlock if we try to tlbie from two processors at once.
610 */ 604 */
611 raw_spin_lock(&native_tlbie_lock); 605 raw_spin_lock(&native_tlbie_lock);
612 606
613 slots = pteg_count * HPTES_PER_GROUP; 607 slots = pteg_count * HPTES_PER_GROUP;
614 608
615 for (slot = 0; slot < slots; slot++, hptep++) { 609 for (slot = 0; slot < slots; slot++, hptep++) {
616 /* 610 /*
617 * we could lock the pte here, but we are the only cpu 611 * we could lock the pte here, but we are the only cpu
618 * running, right? and for crash dump, we probably 612 * running, right? and for crash dump, we probably
619 * don't want to wait for a maybe bad cpu. 613 * don't want to wait for a maybe bad cpu.
620 */ 614 */
621 hpte_v = be64_to_cpu(hptep->v); 615 hpte_v = be64_to_cpu(hptep->v);
622 616
623 /* 617 /*
624 * Call __tlbie() here rather than tlbie() since we 618 * Call __tlbie() here rather than tlbie() since we
625 * already hold the native_tlbie_lock. 619 * already hold the native_tlbie_lock.
626 */ 620 */
627 if (hpte_v & HPTE_V_VALID) { 621 if (hpte_v & HPTE_V_VALID) {
628 hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn); 622 hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
629 hptep->v = 0; 623 hptep->v = 0;
630 __tlbie(vpn, psize, apsize, ssize); 624 __tlbie(vpn, psize, apsize, ssize);
631 } 625 }
632 } 626 }
633 627
634 asm volatile("eieio; tlbsync; ptesync":::"memory"); 628 asm volatile("eieio; tlbsync; ptesync":::"memory");
635 raw_spin_unlock(&native_tlbie_lock); 629 raw_spin_unlock(&native_tlbie_lock);
636 local_irq_restore(flags); 630 local_irq_restore(flags);
637 } 631 }
638 632
639 /* 633 /*
640 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing 634 * Batched hash table flush, we batch the tlbie's to avoid taking/releasing
641 * the lock all the time 635 * the lock all the time
642 */ 636 */
643 static void native_flush_hash_range(unsigned long number, int local) 637 static void native_flush_hash_range(unsigned long number, int local)
644 { 638 {
645 unsigned long vpn; 639 unsigned long vpn;
646 unsigned long hash, index, hidx, shift, slot; 640 unsigned long hash, index, hidx, shift, slot;
647 struct hash_pte *hptep; 641 struct hash_pte *hptep;
648 unsigned long hpte_v; 642 unsigned long hpte_v;
649 unsigned long want_v; 643 unsigned long want_v;
650 unsigned long flags; 644 unsigned long flags;
651 real_pte_t pte; 645 real_pte_t pte;
652 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); 646 struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
653 unsigned long psize = batch->psize; 647 unsigned long psize = batch->psize;
654 int ssize = batch->ssize; 648 int ssize = batch->ssize;
655 int i; 649 int i;
656 650
657 local_irq_save(flags); 651 local_irq_save(flags);
658 652
659 for (i = 0; i < number; i++) { 653 for (i = 0; i < number; i++) {
660 vpn = batch->vpn[i]; 654 vpn = batch->vpn[i];
661 pte = batch->pte[i]; 655 pte = batch->pte[i];
662 656
663 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { 657 pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
664 hash = hpt_hash(vpn, shift, ssize); 658 hash = hpt_hash(vpn, shift, ssize);
665 hidx = __rpte_to_hidx(pte, index); 659 hidx = __rpte_to_hidx(pte, index);
666 if (hidx & _PTEIDX_SECONDARY) 660 if (hidx & _PTEIDX_SECONDARY)
667 hash = ~hash; 661 hash = ~hash;
668 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 662 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
669 slot += hidx & _PTEIDX_GROUP_IX; 663 slot += hidx & _PTEIDX_GROUP_IX;
670 hptep = htab_address + slot; 664 hptep = htab_address + slot;
671 want_v = hpte_encode_avpn(vpn, psize, ssize); 665 want_v = hpte_encode_avpn(vpn, psize, ssize);
672 native_lock_hpte(hptep); 666 native_lock_hpte(hptep);
673 hpte_v = be64_to_cpu(hptep->v); 667 hpte_v = be64_to_cpu(hptep->v);
674 if (!HPTE_V_COMPARE(hpte_v, want_v) || 668 if (!HPTE_V_COMPARE(hpte_v, want_v) ||
675 !(hpte_v & HPTE_V_VALID)) 669 !(hpte_v & HPTE_V_VALID))
676 native_unlock_hpte(hptep); 670 native_unlock_hpte(hptep);
677 else 671 else
678 hptep->v = 0; 672 hptep->v = 0;
679 } pte_iterate_hashed_end(); 673 } pte_iterate_hashed_end();
680 } 674 }
681 675
682 if (mmu_has_feature(MMU_FTR_TLBIEL) && 676 if (mmu_has_feature(MMU_FTR_TLBIEL) &&
683 mmu_psize_defs[psize].tlbiel && local) { 677 mmu_psize_defs[psize].tlbiel && local) {
684 asm volatile("ptesync":::"memory"); 678 asm volatile("ptesync":::"memory");
685 for (i = 0; i < number; i++) { 679 for (i = 0; i < number; i++) {
686 vpn = batch->vpn[i]; 680 vpn = batch->vpn[i];
687 pte = batch->pte[i]; 681 pte = batch->pte[i];
688 682
689 pte_iterate_hashed_subpages(pte, psize, 683 pte_iterate_hashed_subpages(pte, psize,
690 vpn, index, shift) { 684 vpn, index, shift) {
691 __tlbiel(vpn, psize, psize, ssize); 685 __tlbiel(vpn, psize, psize, ssize);
692 } pte_iterate_hashed_end(); 686 } pte_iterate_hashed_end();
693 } 687 }
694 asm volatile("ptesync":::"memory"); 688 asm volatile("ptesync":::"memory");
695 } else { 689 } else {
696 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); 690 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
697 691
698 if (lock_tlbie) 692 if (lock_tlbie)
699 raw_spin_lock(&native_tlbie_lock); 693 raw_spin_lock(&native_tlbie_lock);
700 694
701 asm volatile("ptesync":::"memory"); 695 asm volatile("ptesync":::"memory");
702 for (i = 0; i < number; i++) { 696 for (i = 0; i < number; i++) {
703 vpn = batch->vpn[i]; 697 vpn = batch->vpn[i];
704 pte = batch->pte[i]; 698 pte = batch->pte[i];
705 699
706 pte_iterate_hashed_subpages(pte, psize, 700 pte_iterate_hashed_subpages(pte, psize,
707 vpn, index, shift) { 701 vpn, index, shift) {
708 __tlbie(vpn, psize, psize, ssize); 702 __tlbie(vpn, psize, psize, ssize);
709 } pte_iterate_hashed_end(); 703 } pte_iterate_hashed_end();
710 } 704 }
711 asm volatile("eieio; tlbsync; ptesync":::"memory"); 705 asm volatile("eieio; tlbsync; ptesync":::"memory");
712 706
713 if (lock_tlbie) 707 if (lock_tlbie)
714 raw_spin_unlock(&native_tlbie_lock); 708 raw_spin_unlock(&native_tlbie_lock);
715 } 709 }
716 710
717 local_irq_restore(flags); 711 local_irq_restore(flags);
718 } 712 }
719 713
720 void __init hpte_init_native(void) 714 void __init hpte_init_native(void)
721 { 715 {
722 ppc_md.hpte_invalidate = native_hpte_invalidate; 716 ppc_md.hpte_invalidate = native_hpte_invalidate;
723 ppc_md.hpte_updatepp = native_hpte_updatepp; 717 ppc_md.hpte_updatepp = native_hpte_updatepp;
724 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; 718 ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp;
725 ppc_md.hpte_insert = native_hpte_insert; 719 ppc_md.hpte_insert = native_hpte_insert;
726 ppc_md.hpte_remove = native_hpte_remove; 720 ppc_md.hpte_remove = native_hpte_remove;
727 ppc_md.hpte_clear_all = native_hpte_clear; 721 ppc_md.hpte_clear_all = native_hpte_clear;
728 ppc_md.flush_hash_range = native_flush_hash_range; 722 ppc_md.flush_hash_range = native_flush_hash_range;
729 ppc_md.hugepage_invalidate = native_hugepage_invalidate; 723 ppc_md.hugepage_invalidate = native_hugepage_invalidate;
730 } 724 }
731 725