Eric Lee / smarc-ti-linux-kernel

1

/*

1

/*

2

* arch/sparc64/mm/init.c

2

* arch/sparc64/mm/init.c

3

*

3

*

4

5

6

*/

6

*/

7

8

#include <linux/module.h>

8

#include <linux/module.h>

9

#include <linux/kernel.h>

9

#include <linux/kernel.h>

10

#include <linux/sched.h>

10

#include <linux/sched.h>

11

#include <linux/string.h>

11

#include <linux/string.h>

12

#include <linux/init.h>

12

#include <linux/init.h>

13

#include <linux/bootmem.h>

13

#include <linux/bootmem.h>

14

#include <linux/mm.h>

14

#include <linux/mm.h>

15

#include <linux/hugetlb.h>

15

#include <linux/hugetlb.h>

16

#include <linux/initrd.h>

16

#include <linux/initrd.h>

17

#include <linux/swap.h>

17

#include <linux/swap.h>

18

#include <linux/pagemap.h>

18

#include <linux/pagemap.h>

19

#include <linux/poison.h>

19

#include <linux/poison.h>

20

#include <linux/fs.h>

20

#include <linux/fs.h>

21

#include <linux/seq_file.h>

21

#include <linux/seq_file.h>

22

#include <linux/kprobes.h>

22

#include <linux/kprobes.h>

23

#include <linux/cache.h>

23

#include <linux/cache.h>

24

#include <linux/sort.h>

24

#include <linux/sort.h>

25

#include <linux/percpu.h>

25

#include <linux/percpu.h>

26

#include <linux/memblock.h>

26

#include <linux/memblock.h>

27

#include <linux/mmzone.h>

27

#include <linux/mmzone.h>

28

#include <linux/gfp.h>

28

#include <linux/gfp.h>

29

30

#include <asm/head.h>

30

#include <asm/head.h>

31

#include <asm/page.h>

31

#include <asm/page.h>

32

#include <asm/pgalloc.h>

32

#include <asm/pgalloc.h>

33

#include <asm/pgtable.h>

33

#include <asm/pgtable.h>

34

#include <asm/oplib.h>

34

#include <asm/oplib.h>

35

#include <asm/iommu.h>

35

#include <asm/iommu.h>

36

#include <asm/io.h>

36

#include <asm/io.h>

37

#include <asm/uaccess.h>

37

#include <asm/uaccess.h>

38

#include <asm/mmu_context.h>

38

#include <asm/mmu_context.h>

39

#include <asm/tlbflush.h>

39

#include <asm/tlbflush.h>

40

#include <asm/dma.h>

40

#include <asm/dma.h>

41

#include <asm/starfire.h>

41

#include <asm/starfire.h>

42

#include <asm/tlb.h>

42

#include <asm/tlb.h>

43

#include <asm/spitfire.h>

43

#include <asm/spitfire.h>

44

#include <asm/sections.h>

44

#include <asm/sections.h>

45

#include <asm/tsb.h>

45

#include <asm/tsb.h>

46

#include <asm/hypervisor.h>

46

#include <asm/hypervisor.h>

47

#include <asm/prom.h>

47

#include <asm/prom.h>

48

#include <asm/mdesc.h>

48

#include <asm/mdesc.h>

49

#include <asm/cpudata.h>

49

#include <asm/cpudata.h>

50

#include <asm/irq.h>

50

#include <asm/irq.h>

51

52

#include "init_64.h"

52

#include "init_64.h"

53

54

unsigned long kern_linear_pte_xor[4] __read_mostly;

54

unsigned long kern_linear_pte_xor[4] __read_mostly;

55

56

/* A bitmap, two bits for every 256MB of physical memory. These two

56

/* A bitmap, two bits for every 256MB of physical memory. These two

57

* bits determine what page size we use for kernel linear

57

* bits determine what page size we use for kernel linear

58

* translations. They form an index into kern_linear_pte_xor[]. The

58

* translations. They form an index into kern_linear_pte_xor[]. The

59

* value in the indexed slot is XOR'd with the TLB miss virtual

59

* value in the indexed slot is XOR'd with the TLB miss virtual

60

* address to form the resulting TTE. The mapping is:

60

* address to form the resulting TTE. The mapping is:

61

*

61

*

62

* 0 ==> 4MB

62

* 0 ==> 4MB

63

* 1 ==> 256MB

63

* 1 ==> 256MB

64

* 2 ==> 2GB

64

* 2 ==> 2GB

65

* 3 ==> 16GB

65

* 3 ==> 16GB

66

*

66

*

67

* All sun4v chips support 256MB pages. Only SPARC-T4 and later

67

* All sun4v chips support 256MB pages. Only SPARC-T4 and later

68

* support 2GB pages, and hopefully future cpus will support the 16GB

68

* support 2GB pages, and hopefully future cpus will support the 16GB

69

* pages as well. For slots 2 and 3, we encode a 256MB TTE xor there

69

* pages as well. For slots 2 and 3, we encode a 256MB TTE xor there

70

* if these larger page sizes are not supported by the cpu.

70

* if these larger page sizes are not supported by the cpu.

71

*

71

*

72

* It would be nice to determine this from the machine description

72

* It would be nice to determine this from the machine description

73

* 'cpu' properties, but we need to have this table setup before the

73

* 'cpu' properties, but we need to have this table setup before the

74

* MDESC is initialized.

74

* MDESC is initialized.

75

*/

75

*/

76

unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];

76

unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];

77

78

#ifndef CONFIG_DEBUG_PAGEALLOC

78

#ifndef CONFIG_DEBUG_PAGEALLOC

79

/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.

79

/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.

80

* Space is allocated for this right after the trap table in

80

* Space is allocated for this right after the trap table in

81

* arch/sparc64/kernel/head.S

81

* arch/sparc64/kernel/head.S

82

*/

82

*/

83

extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];

83

extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];

84

#endif

84

#endif

85

86

static unsigned long cpu_pgsz_mask;

86

static unsigned long cpu_pgsz_mask;

87

88

#define MAX_BANKS 32

88

#define MAX_BANKS 32

89

90

static struct linux_prom64_registers pavail[MAX_BANKS];

90

static struct linux_prom64_registers pavail[MAX_BANKS];

91

static int pavail_ents;

91

static int pavail_ents;

92

93

static int cmp_p64(const void *a, const void *b)

93

static int cmp_p64(const void *a, const void *b)

94

{

94

{

95

const struct linux_prom64_registers *x = a, *y = b;

95

const struct linux_prom64_registers *x = a, *y = b;

96

97

if (x->phys_addr > y->phys_addr)

97

if (x->phys_addr > y->phys_addr)

98

return 1;

98

return 1;

99

if (x->phys_addr < y->phys_addr)

99

if (x->phys_addr < y->phys_addr)

100

return -1;

100

return -1;

101

return 0;

101

return 0;

102

}

102

}

103

104

static void __init read_obp_memory(const char *property,

104

static void __init read_obp_memory(const char *property,

105

struct linux_prom64_registers *regs,

105

struct linux_prom64_registers *regs,

106

int *num_ents)

106

int *num_ents)

107

{

107

{

108

phandle node = prom_finddevice("/memory");

108

phandle node = prom_finddevice("/memory");

109

int prop_size = prom_getproplen(node, property);

109

int prop_size = prom_getproplen(node, property);

110

int ents, ret, i;

110

int ents, ret, i;

111

112

ents = prop_size / sizeof(struct linux_prom64_registers);

112

ents = prop_size / sizeof(struct linux_prom64_registers);

113

if (ents > MAX_BANKS) {

113

if (ents > MAX_BANKS) {

114

prom_printf("The machine has more %s property entries than "

114

prom_printf("The machine has more %s property entries than "

115

"this kernel can support (%d).\n",

115

"this kernel can support (%d).\n",

116

property, MAX_BANKS);

116

property, MAX_BANKS);

117

prom_halt();

117

prom_halt();

118

}

118

}

119

120

ret = prom_getproperty(node, property, (char *) regs, prop_size);

120

ret = prom_getproperty(node, property, (char *) regs, prop_size);

121

if (ret == -1) {

121

if (ret == -1) {

122

prom_printf("Couldn't get %s property from /memory.\n",

122

prom_printf("Couldn't get %s property from /memory.\n",

123

property);

123

property);

124

prom_halt();

124

prom_halt();

125

}

125

}

126

127

/* Sanitize what we got from the firmware, by page aligning

127

/* Sanitize what we got from the firmware, by page aligning

128

* everything.

128

* everything.

129

*/

129

*/

130

for (i = 0; i < ents; i++) {

130

for (i = 0; i < ents; i++) {

131

unsigned long base, size;

131

unsigned long base, size;

132

133

base = regs[i].phys_addr;

133

base = regs[i].phys_addr;

134

size = regs[i].reg_size;

134

size = regs[i].reg_size;

135

136

size &= PAGE_MASK;

136

size &= PAGE_MASK;

137

if (base & ~PAGE_MASK) {

137

if (base & ~PAGE_MASK) {

138

unsigned long new_base = PAGE_ALIGN(base);

138

unsigned long new_base = PAGE_ALIGN(base);

139

140

size -= new_base - base;

140

size -= new_base - base;

141

if ((long) size < 0L)

141

if ((long) size < 0L)

142

size = 0UL;

142

size = 0UL;

143

base = new_base;

143

base = new_base;

144

}

144

}

145

if (size == 0UL) {

145

if (size == 0UL) {

146

/* If it is empty, simply get rid of it.

146

/* If it is empty, simply get rid of it.

147

* This simplifies the logic of the other

147

* This simplifies the logic of the other

148

* functions that process these arrays.

148

* functions that process these arrays.

149

*/

149

*/

150

memmove(&regs[i], &regs[i + 1],

150

memmove(&regs[i], &regs[i + 1],

151

(ents - i - 1) * sizeof(regs[0]));

151

(ents - i - 1) * sizeof(regs[0]));

152

i--;

152

i--;

153

ents--;

153

ents--;

154

continue;

154

continue;

155

}

155

}

156

regs[i].phys_addr = base;

156

regs[i].phys_addr = base;

157

regs[i].reg_size = size;

157

regs[i].reg_size = size;

158

}

158

}

159

160

*num_ents = ents;

160

*num_ents = ents;

161

162

sort(regs, ents, sizeof(struct linux_prom64_registers),

162

sort(regs, ents, sizeof(struct linux_prom64_registers),

163

cmp_p64, NULL);

163

cmp_p64, NULL);

164

}

164

}

165

166

unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /

166

unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /

167

sizeof(unsigned long)];

167

sizeof(unsigned long)];

168

EXPORT_SYMBOL(sparc64_valid_addr_bitmap);

168

EXPORT_SYMBOL(sparc64_valid_addr_bitmap);

169

170

/* Kernel physical address base and size in bytes. */

170

/* Kernel physical address base and size in bytes. */

171

unsigned long kern_base __read_mostly;

171

unsigned long kern_base __read_mostly;

172

unsigned long kern_size __read_mostly;

172

unsigned long kern_size __read_mostly;

173

174

/* Initial ramdisk setup */

174

/* Initial ramdisk setup */

175

extern unsigned long sparc_ramdisk_image64;

175

extern unsigned long sparc_ramdisk_image64;

176

extern unsigned int sparc_ramdisk_image;

176

extern unsigned int sparc_ramdisk_image;

177

extern unsigned int sparc_ramdisk_size;

177

extern unsigned int sparc_ramdisk_size;

178

179

struct page *mem_map_zero __read_mostly;

179

struct page *mem_map_zero __read_mostly;

180

EXPORT_SYMBOL(mem_map_zero);

180

EXPORT_SYMBOL(mem_map_zero);

181

182

unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;

182

unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;

183

184

unsigned long sparc64_kern_pri_context __read_mostly;

184

unsigned long sparc64_kern_pri_context __read_mostly;

185

unsigned long sparc64_kern_pri_nuc_bits __read_mostly;

185

unsigned long sparc64_kern_pri_nuc_bits __read_mostly;

186

unsigned long sparc64_kern_sec_context __read_mostly;

186

unsigned long sparc64_kern_sec_context __read_mostly;

187

188

int num_kernel_image_mappings;

188

int num_kernel_image_mappings;

189

190

#ifdef CONFIG_DEBUG_DCFLUSH

190

#ifdef CONFIG_DEBUG_DCFLUSH

191

atomic_t dcpage_flushes = ATOMIC_INIT(0);

191

atomic_t dcpage_flushes = ATOMIC_INIT(0);

192

#ifdef CONFIG_SMP

192

#ifdef CONFIG_SMP

193

atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);

193

atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);

194

#endif

194

#endif

195

#endif

195

#endif

196

197

inline void flush_dcache_page_impl(struct page *page)

197

inline void flush_dcache_page_impl(struct page *page)

198

{

198

{

199

BUG_ON(tlb_type == hypervisor);

199

BUG_ON(tlb_type == hypervisor);

200

#ifdef CONFIG_DEBUG_DCFLUSH

200

#ifdef CONFIG_DEBUG_DCFLUSH

201

atomic_inc(&dcpage_flushes);

201

atomic_inc(&dcpage_flushes);

202

#endif

202

#endif

203

204

#ifdef DCACHE_ALIASING_POSSIBLE

204

#ifdef DCACHE_ALIASING_POSSIBLE

205

__flush_dcache_page(page_address(page),

205

__flush_dcache_page(page_address(page),

206

((tlb_type == spitfire) &&

206

((tlb_type == spitfire) &&

207

page_mapping(page) != NULL));

207

page_mapping(page) != NULL));

208

#else

208

#else

209

if (page_mapping(page) != NULL &&

209

if (page_mapping(page) != NULL &&

210

tlb_type == spitfire)

210

tlb_type == spitfire)

211

__flush_icache_page(__pa(page_address(page)));

211

__flush_icache_page(__pa(page_address(page)));

212

#endif

212

#endif

213

}

213

}

214

215

#define PG_dcache_dirty PG_arch_1

215

#define PG_dcache_dirty PG_arch_1

216

#define PG_dcache_cpu_shift 32UL

216

#define PG_dcache_cpu_shift 32UL

217

#define PG_dcache_cpu_mask \

217

#define PG_dcache_cpu_mask \

218

((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)

218

((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)

219

220

#define dcache_dirty_cpu(page) \

220

#define dcache_dirty_cpu(page) \

221

(((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)

221

(((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)

222

223

static inline void set_dcache_dirty(struct page *page, int this_cpu)

223

static inline void set_dcache_dirty(struct page *page, int this_cpu)

224

{

224

{

225

unsigned long mask = this_cpu;

225

unsigned long mask = this_cpu;

226

unsigned long non_cpu_bits;

226

unsigned long non_cpu_bits;

227

228

non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);

228

non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);

229

mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);

229

mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);

230

231

__asm__ __volatile__("1:\n\t"

231

__asm__ __volatile__("1:\n\t"

232

"ldx [%2], %%g7\n\t"

232

"ldx [%2], %%g7\n\t"

233

"and %%g7, %1, %%g1\n\t"

233

"and %%g7, %1, %%g1\n\t"

234

"or %%g1, %0, %%g1\n\t"

234

"or %%g1, %0, %%g1\n\t"

235

"casx [%2], %%g7, %%g1\n\t"

235

"casx [%2], %%g7, %%g1\n\t"

236

"cmp %%g7, %%g1\n\t"

236

"cmp %%g7, %%g1\n\t"

237

"bne,pn %%xcc, 1b\n\t"

237

"bne,pn %%xcc, 1b\n\t"

238

" nop"

238

" nop"

239

: /* no outputs */

239

: /* no outputs */

240

: "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)

240

: "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)

241

: "g1", "g7");

241

: "g1", "g7");

242

}

242

}

243

244

static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)

244

static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)

245

{

245

{

246

unsigned long mask = (1UL << PG_dcache_dirty);

246

unsigned long mask = (1UL << PG_dcache_dirty);

247

248

__asm__ __volatile__("! test_and_clear_dcache_dirty\n"

248

__asm__ __volatile__("! test_and_clear_dcache_dirty\n"

249

"1:\n\t"

249

"1:\n\t"

250

"ldx [%2], %%g7\n\t"

250

"ldx [%2], %%g7\n\t"

251

"srlx %%g7, %4, %%g1\n\t"

251

"srlx %%g7, %4, %%g1\n\t"

252

"and %%g1, %3, %%g1\n\t"

252

"and %%g1, %3, %%g1\n\t"

253

"cmp %%g1, %0\n\t"

253

"cmp %%g1, %0\n\t"

254

"bne,pn %%icc, 2f\n\t"

254

"bne,pn %%icc, 2f\n\t"

255

" andn %%g7, %1, %%g1\n\t"

255

" andn %%g7, %1, %%g1\n\t"

256

"casx [%2], %%g7, %%g1\n\t"

256

"casx [%2], %%g7, %%g1\n\t"

257

"cmp %%g7, %%g1\n\t"

257

"cmp %%g7, %%g1\n\t"

258

"bne,pn %%xcc, 1b\n\t"

258

"bne,pn %%xcc, 1b\n\t"

259

" nop\n"

259

" nop\n"

260

"2:"

260

"2:"

261

: /* no outputs */

261

: /* no outputs */

262

: "r" (cpu), "r" (mask), "r" (&page->flags),

262

: "r" (cpu), "r" (mask), "r" (&page->flags),

263

"i" (PG_dcache_cpu_mask),

263

"i" (PG_dcache_cpu_mask),

264

"i" (PG_dcache_cpu_shift)

264

"i" (PG_dcache_cpu_shift)

265

: "g1", "g7");

265

: "g1", "g7");

266

}

266

}

267

268

static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)

268

static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)

269

{

269

{

270

unsigned long tsb_addr = (unsigned long) ent;

270

unsigned long tsb_addr = (unsigned long) ent;

271

272

if (tlb_type == cheetah_plus || tlb_type == hypervisor)

272

if (tlb_type == cheetah_plus || tlb_type == hypervisor)

273

tsb_addr = __pa(tsb_addr);

273

tsb_addr = __pa(tsb_addr);

274

275

__tsb_insert(tsb_addr, tag, pte);

275

__tsb_insert(tsb_addr, tag, pte);

276

}

276

}

277

278

unsigned long _PAGE_ALL_SZ_BITS __read_mostly;

278

unsigned long _PAGE_ALL_SZ_BITS __read_mostly;

279

280

static void flush_dcache(unsigned long pfn)

280

static void flush_dcache(unsigned long pfn)

281

{

281

{

282

struct page *page;

282

struct page *page;

283

284

page = pfn_to_page(pfn);

284

page = pfn_to_page(pfn);

285

if (page) {

285

if (page) {

286

unsigned long pg_flags;

286

unsigned long pg_flags;

287

288

pg_flags = page->flags;

288

pg_flags = page->flags;

289

if (pg_flags & (1UL << PG_dcache_dirty)) {

289

if (pg_flags & (1UL << PG_dcache_dirty)) {

290

int cpu = ((pg_flags >> PG_dcache_cpu_shift) &

290

int cpu = ((pg_flags >> PG_dcache_cpu_shift) &

291

PG_dcache_cpu_mask);

291

PG_dcache_cpu_mask);

292

int this_cpu = get_cpu();

292

int this_cpu = get_cpu();

293

294

/* This is just to optimize away some function calls

294

/* This is just to optimize away some function calls

295

* in the SMP case.

295

* in the SMP case.

296

*/

296

*/

297

if (cpu == this_cpu)

297

if (cpu == this_cpu)

298

flush_dcache_page_impl(page);

298

flush_dcache_page_impl(page);

299

else

299

else

300

smp_flush_dcache_page_impl(page, cpu);

300

smp_flush_dcache_page_impl(page, cpu);

301

302

clear_dcache_dirty_cpu(page, cpu);

302

clear_dcache_dirty_cpu(page, cpu);

303

304

put_cpu();

304

put_cpu();

305

}

305

}

306

}

306

}

307

}

307

}

308

309

/* mm->context.lock must be held */

309

/* mm->context.lock must be held */

310

static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,

310

static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,

311

unsigned long tsb_hash_shift, unsigned long address,

311

unsigned long tsb_hash_shift, unsigned long address,

312

unsigned long tte)

312

unsigned long tte)

313

{

313

{

314

struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;

314

struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;

315

unsigned long tag;

315

unsigned long tag;

316

317

if (unlikely(!tsb))

317

if (unlikely(!tsb))

318

return;

318

return;

319

320

tsb += ((address >> tsb_hash_shift) &

320

tsb += ((address >> tsb_hash_shift) &

321

(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));

321

(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));

322

tag = (address >> 22UL);

322

tag = (address >> 22UL);

323

tsb_insert(tsb, tag, tte);

323

tsb_insert(tsb, tag, tte);

324

}

324

}

325

326

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

326

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

327

static inline bool is_hugetlb_pte(pte_t pte)

327

static inline bool is_hugetlb_pte(pte_t pte)

328

{

328

{

329

if ((tlb_type == hypervisor &&

329

if ((tlb_type == hypervisor &&

330

(pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||

330

(pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||

331

(tlb_type != hypervisor &&

331

(tlb_type != hypervisor &&

332

(pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))

332

(pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))

333

return true;

333

return true;

334

return false;

334

return false;

335

}

335

}

336

#endif

336

#endif

337

338

void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)

338

void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)

339

{

339

{

340

struct mm_struct *mm;

340

struct mm_struct *mm;

341

unsigned long flags;

341

unsigned long flags;

342

pte_t pte = *ptep;

342

pte_t pte = *ptep;

343

344

if (tlb_type != hypervisor) {

344

if (tlb_type != hypervisor) {

345

unsigned long pfn = pte_pfn(pte);

345

unsigned long pfn = pte_pfn(pte);

346

347

if (pfn_valid(pfn))

347

if (pfn_valid(pfn))

348

flush_dcache(pfn);

348

flush_dcache(pfn);

349

}

349

}

350

351

mm = vma->vm_mm;

351

mm = vma->vm_mm;

352

353

/* Don't insert a non-valid PTE into the TSB, we'll deadlock. */

354

if (!pte_accessible(mm, pte))

355

return;

356

353

spin_lock_irqsave(&mm->context.lock, flags);

357

spin_lock_irqsave(&mm->context.lock, flags);

354

358

355

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

359

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

356

if (mm->context.huge_pte_count && is_hugetlb_pte(pte))

360

if (mm->context.huge_pte_count && is_hugetlb_pte(pte))

357

__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,

361

__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,

358

address, pte_val(pte));

362

address, pte_val(pte));

359

else

363

else

360

#endif

364

#endif

361

__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,

365

__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,

362

address, pte_val(pte));

366

address, pte_val(pte));

363

367

364

spin_unlock_irqrestore(&mm->context.lock, flags);

368

spin_unlock_irqrestore(&mm->context.lock, flags);

365

}

369

}

366

370

367

void flush_dcache_page(struct page *page)

371

void flush_dcache_page(struct page *page)

368

{

372

{

369

struct address_space *mapping;

373

struct address_space *mapping;

370

int this_cpu;

374

int this_cpu;

371

375

372

if (tlb_type == hypervisor)

376

if (tlb_type == hypervisor)

373

return;

377

return;

374

378

375

/* Do not bother with the expensive D-cache flush if it

379

/* Do not bother with the expensive D-cache flush if it

376

* is merely the zero page. The 'bigcore' testcase in GDB

380

* is merely the zero page. The 'bigcore' testcase in GDB

377

* causes this case to run millions of times.

381

* causes this case to run millions of times.

378

*/

382

*/

379

if (page == ZERO_PAGE(0))

383

if (page == ZERO_PAGE(0))

380

return;

384

return;

381

385

382

this_cpu = get_cpu();

386

this_cpu = get_cpu();

383

387

384

mapping = page_mapping(page);

388

mapping = page_mapping(page);

385

if (mapping && !mapping_mapped(mapping)) {

389

if (mapping && !mapping_mapped(mapping)) {

386

int dirty = test_bit(PG_dcache_dirty, &page->flags);

390

int dirty = test_bit(PG_dcache_dirty, &page->flags);

387

if (dirty) {

391

if (dirty) {

388

int dirty_cpu = dcache_dirty_cpu(page);

392

int dirty_cpu = dcache_dirty_cpu(page);

389

393

390

if (dirty_cpu == this_cpu)

394

if (dirty_cpu == this_cpu)

391

goto out;

395

goto out;

392

smp_flush_dcache_page_impl(page, dirty_cpu);

396

smp_flush_dcache_page_impl(page, dirty_cpu);

393

}

397

}

394

set_dcache_dirty(page, this_cpu);

398

set_dcache_dirty(page, this_cpu);

395

} else {

399

} else {

396

/* We could delay the flush for the !page_mapping

400

/* We could delay the flush for the !page_mapping

397

* case too. But that case is for exec env/arg

401

* case too. But that case is for exec env/arg

398

* pages and those are %99 certainly going to get

402

* pages and those are %99 certainly going to get

399

* faulted into the tlb (and thus flushed) anyways.

403

* faulted into the tlb (and thus flushed) anyways.

400

*/

404

*/

401

flush_dcache_page_impl(page);

405

flush_dcache_page_impl(page);

402

}

406

}

403

407

404

out:

408

out:

405

put_cpu();

409

put_cpu();

406

}

410

}

407

EXPORT_SYMBOL(flush_dcache_page);

411

EXPORT_SYMBOL(flush_dcache_page);

408

412

409

void __kprobes flush_icache_range(unsigned long start, unsigned long end)

413

void __kprobes flush_icache_range(unsigned long start, unsigned long end)

410

{

414

{

411

/* Cheetah and Hypervisor platform cpus have coherent I-cache. */

415

/* Cheetah and Hypervisor platform cpus have coherent I-cache. */

412

if (tlb_type == spitfire) {

416

if (tlb_type == spitfire) {

413

unsigned long kaddr;

417

unsigned long kaddr;

414

418

415

/* This code only runs on Spitfire cpus so this is

419

/* This code only runs on Spitfire cpus so this is

416

* why we can assume _PAGE_PADDR_4U.

420

* why we can assume _PAGE_PADDR_4U.

417

*/

421

*/

418

for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {

422

for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {

419

unsigned long paddr, mask = _PAGE_PADDR_4U;

423

unsigned long paddr, mask = _PAGE_PADDR_4U;

420

424

421

if (kaddr >= PAGE_OFFSET)

425

if (kaddr >= PAGE_OFFSET)

422

paddr = kaddr & mask;

426

paddr = kaddr & mask;

423

else {

427

else {

424

pgd_t *pgdp = pgd_offset_k(kaddr);

428

pgd_t *pgdp = pgd_offset_k(kaddr);

425

pud_t *pudp = pud_offset(pgdp, kaddr);

429

pud_t *pudp = pud_offset(pgdp, kaddr);

426

pmd_t *pmdp = pmd_offset(pudp, kaddr);

430

pmd_t *pmdp = pmd_offset(pudp, kaddr);

427

pte_t *ptep = pte_offset_kernel(pmdp, kaddr);

431

pte_t *ptep = pte_offset_kernel(pmdp, kaddr);

428

432

429

paddr = pte_val(*ptep) & mask;

433

paddr = pte_val(*ptep) & mask;

430

}

434

}

431

__flush_icache_page(paddr);

435

__flush_icache_page(paddr);

432

}

436

}

433

}

437

}

434

}

438

}

435

EXPORT_SYMBOL(flush_icache_range);

439

EXPORT_SYMBOL(flush_icache_range);

436

440

437

void mmu_info(struct seq_file *m)

441

void mmu_info(struct seq_file *m)

438

{

442

{

439

static const char *pgsz_strings[] = {

443

static const char *pgsz_strings[] = {

440

"8K", "64K", "512K", "4MB", "32MB",

444

"8K", "64K", "512K", "4MB", "32MB",

441

"256MB", "2GB", "16GB",

445

"256MB", "2GB", "16GB",

442

};

446

};

443

int i, printed;

447

int i, printed;

444

448

445

if (tlb_type == cheetah)

449

if (tlb_type == cheetah)

446

seq_printf(m, "MMU Type\t: Cheetah\n");

450

seq_printf(m, "MMU Type\t: Cheetah\n");

447

else if (tlb_type == cheetah_plus)

451

else if (tlb_type == cheetah_plus)

448

seq_printf(m, "MMU Type\t: Cheetah+\n");

452

seq_printf(m, "MMU Type\t: Cheetah+\n");

449

else if (tlb_type == spitfire)

453

else if (tlb_type == spitfire)

450

seq_printf(m, "MMU Type\t: Spitfire\n");

454

seq_printf(m, "MMU Type\t: Spitfire\n");

451

else if (tlb_type == hypervisor)

455

else if (tlb_type == hypervisor)

452

seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");

456

seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");

453

else

457

else

454

seq_printf(m, "MMU Type\t: ???\n");

458

seq_printf(m, "MMU Type\t: ???\n");

455

459

456

seq_printf(m, "MMU PGSZs\t: ");

460

seq_printf(m, "MMU PGSZs\t: ");

457

printed = 0;

461

printed = 0;

458

for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {

462

for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {

459

if (cpu_pgsz_mask & (1UL << i)) {

463

if (cpu_pgsz_mask & (1UL << i)) {

460

seq_printf(m, "%s%s",

464

seq_printf(m, "%s%s",

461

printed ? "," : "", pgsz_strings[i]);

465

printed ? "," : "", pgsz_strings[i]);

462

printed++;

466

printed++;

463

}

467

}

464

}

468

}

465

seq_putc(m, '\n');

469

seq_putc(m, '\n');

466

470

467

#ifdef CONFIG_DEBUG_DCFLUSH

471

#ifdef CONFIG_DEBUG_DCFLUSH

468

seq_printf(m, "DCPageFlushes\t: %d\n",

472

seq_printf(m, "DCPageFlushes\t: %d\n",

469

atomic_read(&dcpage_flushes));

473

atomic_read(&dcpage_flushes));

470

#ifdef CONFIG_SMP

474

#ifdef CONFIG_SMP

471

seq_printf(m, "DCPageFlushesXC\t: %d\n",

475

seq_printf(m, "DCPageFlushesXC\t: %d\n",

472

atomic_read(&dcpage_flushes_xcall));

476

atomic_read(&dcpage_flushes_xcall));

473

#endif /* CONFIG_SMP */

477

#endif /* CONFIG_SMP */

474

#endif /* CONFIG_DEBUG_DCFLUSH */

478

#endif /* CONFIG_DEBUG_DCFLUSH */

475

}

479

}

476

480

477

struct linux_prom_translation prom_trans[512] __read_mostly;

481

struct linux_prom_translation prom_trans[512] __read_mostly;

478

unsigned int prom_trans_ents __read_mostly;

482

unsigned int prom_trans_ents __read_mostly;

479

483

480

unsigned long kern_locked_tte_data;

484

unsigned long kern_locked_tte_data;

481

485

482

/* The obp translations are saved based on 8k pagesize, since obp can

486

/* The obp translations are saved based on 8k pagesize, since obp can

483

* use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->

487

* use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->

484

* HI_OBP_ADDRESS range are handled in ktlb.S.

488

* HI_OBP_ADDRESS range are handled in ktlb.S.

485

*/

489

*/

486

static inline int in_obp_range(unsigned long vaddr)

490

static inline int in_obp_range(unsigned long vaddr)

487

{

491

{

488

return (vaddr >= LOW_OBP_ADDRESS &&

492

return (vaddr >= LOW_OBP_ADDRESS &&

489

vaddr < HI_OBP_ADDRESS);

493

vaddr < HI_OBP_ADDRESS);

490

}

494

}

491

495

492

static int cmp_ptrans(const void *a, const void *b)

496

static int cmp_ptrans(const void *a, const void *b)

493

{

497

{

494

const struct linux_prom_translation *x = a, *y = b;

498

const struct linux_prom_translation *x = a, *y = b;

495

499

496

if (x->virt > y->virt)

500

if (x->virt > y->virt)

497

return 1;

501

return 1;

498

if (x->virt < y->virt)

502

if (x->virt < y->virt)

499

return -1;

503

return -1;

500

return 0;

504

return 0;

501

}

505

}

502

506

503

/* Read OBP translations property into 'prom_trans[]'. */

507

/* Read OBP translations property into 'prom_trans[]'. */

504

static void __init read_obp_translations(void)

508

static void __init read_obp_translations(void)

505

{

509

{

506

int n, node, ents, first, last, i;

510

int n, node, ents, first, last, i;

507

511

508

node = prom_finddevice("/virtual-memory");

512

node = prom_finddevice("/virtual-memory");

509

n = prom_getproplen(node, "translations");

513

n = prom_getproplen(node, "translations");

510

if (unlikely(n == 0 || n == -1)) {

514

if (unlikely(n == 0 || n == -1)) {

511

prom_printf("prom_mappings: Couldn't get size.\n");

515

prom_printf("prom_mappings: Couldn't get size.\n");

512

prom_halt();

516

prom_halt();

513

}

517

}

514

if (unlikely(n > sizeof(prom_trans))) {

518

if (unlikely(n > sizeof(prom_trans))) {

515

prom_printf("prom_mappings: Size %d is too big.\n", n);

519

prom_printf("prom_mappings: Size %d is too big.\n", n);

516

prom_halt();

520

prom_halt();

517

}

521

}

518

522

519

if ((n = prom_getproperty(node, "translations",

523

if ((n = prom_getproperty(node, "translations",

520

(char *)&prom_trans[0],

524

(char *)&prom_trans[0],

521

sizeof(prom_trans))) == -1) {

525

sizeof(prom_trans))) == -1) {

522

prom_printf("prom_mappings: Couldn't get property.\n");

526

prom_printf("prom_mappings: Couldn't get property.\n");

523

prom_halt();

527

prom_halt();

524

}

528

}

525

529

526

n = n / sizeof(struct linux_prom_translation);

530

n = n / sizeof(struct linux_prom_translation);

527

531

528

ents = n;

532

ents = n;

529

533

530

sort(prom_trans, ents, sizeof(struct linux_prom_translation),

534

sort(prom_trans, ents, sizeof(struct linux_prom_translation),

531

cmp_ptrans, NULL);

535

cmp_ptrans, NULL);

532

536

533

/* Now kick out all the non-OBP entries. */

537

/* Now kick out all the non-OBP entries. */

534

for (i = 0; i < ents; i++) {

538

for (i = 0; i < ents; i++) {

535

if (in_obp_range(prom_trans[i].virt))

539

if (in_obp_range(prom_trans[i].virt))

536

break;

540

break;

537

}

541

}

538

first = i;

542

first = i;

539

for (; i < ents; i++) {

543

for (; i < ents; i++) {

540

if (!in_obp_range(prom_trans[i].virt))

544

if (!in_obp_range(prom_trans[i].virt))

541

break;

545

break;

542

}

546

}

543

last = i;

547

last = i;

544

548

545

for (i = 0; i < (last - first); i++) {

549

for (i = 0; i < (last - first); i++) {

546

struct linux_prom_translation *src = &prom_trans[i + first];

550

struct linux_prom_translation *src = &prom_trans[i + first];

547

struct linux_prom_translation *dest = &prom_trans[i];

551

struct linux_prom_translation *dest = &prom_trans[i];

548

552

549

*dest = *src;

553

*dest = *src;

550

}

554

}

551

for (; i < ents; i++) {

555

for (; i < ents; i++) {

552

struct linux_prom_translation *dest = &prom_trans[i];

556

struct linux_prom_translation *dest = &prom_trans[i];

553

dest->virt = dest->size = dest->data = 0x0UL;

557

dest->virt = dest->size = dest->data = 0x0UL;

554

}

558

}

555

559

556

prom_trans_ents = last - first;

560

prom_trans_ents = last - first;

557

561

558

if (tlb_type == spitfire) {

562

if (tlb_type == spitfire) {

559

/* Clear diag TTE bits. */

563

/* Clear diag TTE bits. */

560

for (i = 0; i < prom_trans_ents; i++)

564

for (i = 0; i < prom_trans_ents; i++)

561

prom_trans[i].data &= ~0x0003fe0000000000UL;

565

prom_trans[i].data &= ~0x0003fe0000000000UL;

562

}

566

}

563

567

564

/* Force execute bit on. */

568

/* Force execute bit on. */

565

for (i = 0; i < prom_trans_ents; i++)

569

for (i = 0; i < prom_trans_ents; i++)

566

prom_trans[i].data |= (tlb_type == hypervisor ?

570

prom_trans[i].data |= (tlb_type == hypervisor ?

567

_PAGE_EXEC_4V : _PAGE_EXEC_4U);

571

_PAGE_EXEC_4V : _PAGE_EXEC_4U);

568

}

572

}

569

573

570

static void __init hypervisor_tlb_lock(unsigned long vaddr,

574

static void __init hypervisor_tlb_lock(unsigned long vaddr,

571

unsigned long pte,

575

unsigned long pte,

572

unsigned long mmu)

576

unsigned long mmu)

573

{

577

{

574

unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);

578

unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);

575

579

576

if (ret != 0) {

580

if (ret != 0) {

577

prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "

581

prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "

578

"errors with %lx\n", vaddr, 0, pte, mmu, ret);

582

"errors with %lx\n", vaddr, 0, pte, mmu, ret);

579

prom_halt();

583

prom_halt();

580

}

584

}

581

}

585

}

582

586

583

static unsigned long kern_large_tte(unsigned long paddr);

587

static unsigned long kern_large_tte(unsigned long paddr);

584

588

585

static void __init remap_kernel(void)

589

static void __init remap_kernel(void)

586

{

590

{

587

unsigned long phys_page, tte_vaddr, tte_data;

591

unsigned long phys_page, tte_vaddr, tte_data;

588

int i, tlb_ent = sparc64_highest_locked_tlbent();

592

int i, tlb_ent = sparc64_highest_locked_tlbent();

589

593

590

tte_vaddr = (unsigned long) KERNBASE;

594

tte_vaddr = (unsigned long) KERNBASE;

591

phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;

595

phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;

592

tte_data = kern_large_tte(phys_page);

596

tte_data = kern_large_tte(phys_page);

593

597

594

kern_locked_tte_data = tte_data;

598

kern_locked_tte_data = tte_data;

595

599

596

/* Now lock us into the TLBs via Hypervisor or OBP. */

600

/* Now lock us into the TLBs via Hypervisor or OBP. */

597

if (tlb_type == hypervisor) {

601

if (tlb_type == hypervisor) {

598

for (i = 0; i < num_kernel_image_mappings; i++) {

602

for (i = 0; i < num_kernel_image_mappings; i++) {

599

hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);

603

hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);

600

hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);

604

hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);

601

tte_vaddr += 0x400000;

605

tte_vaddr += 0x400000;

602

tte_data += 0x400000;

606

tte_data += 0x400000;

603

}

607

}

604

} else {

608

} else {

605

for (i = 0; i < num_kernel_image_mappings; i++) {

609

for (i = 0; i < num_kernel_image_mappings; i++) {

606

prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);

610

prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);

607

prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);

611

prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);

608

tte_vaddr += 0x400000;

612

tte_vaddr += 0x400000;

609

tte_data += 0x400000;

613

tte_data += 0x400000;

610

}

614

}

611

sparc64_highest_unlocked_tlb_ent = tlb_ent - i;

615

sparc64_highest_unlocked_tlb_ent = tlb_ent - i;

612

}

616

}

613

if (tlb_type == cheetah_plus) {

617

if (tlb_type == cheetah_plus) {

614

sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |

618

sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |

615

CTX_CHEETAH_PLUS_NUC);

619

CTX_CHEETAH_PLUS_NUC);

616

sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;

620

sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;

617

sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;

621

sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;

618

}

622

}

619

}

623

}

620

624

621

625

622

static void __init inherit_prom_mappings(void)

626

static void __init inherit_prom_mappings(void)

623

{

627

{

624

/* Now fixup OBP's idea about where we really are mapped. */

628

/* Now fixup OBP's idea about where we really are mapped. */

625

printk("Remapping the kernel... ");

629

printk("Remapping the kernel... ");

626

remap_kernel();

630

remap_kernel();

627

printk("done.\n");

631

printk("done.\n");

628

}

632

}

629

633

630

void prom_world(int enter)

634

void prom_world(int enter)

631

{

635

{

632

if (!enter)

636

if (!enter)

633

set_fs(get_fs());

637

set_fs(get_fs());

634

638

635

__asm__ __volatile__("flushw");

639

__asm__ __volatile__("flushw");

636

}

640

}

637

641

638

void __flush_dcache_range(unsigned long start, unsigned long end)

642

void __flush_dcache_range(unsigned long start, unsigned long end)

639

{

643

{

640

unsigned long va;

644

unsigned long va;

641

645

642

if (tlb_type == spitfire) {

646

if (tlb_type == spitfire) {

643

int n = 0;

647

int n = 0;

644

648

645

for (va = start; va < end; va += 32) {

649

for (va = start; va < end; va += 32) {

646

spitfire_put_dcache_tag(va & 0x3fe0, 0x0);

650

spitfire_put_dcache_tag(va & 0x3fe0, 0x0);

647

if (++n >= 512)

651

if (++n >= 512)

648

break;

652

break;

649

}

653

}

650

} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {

654

} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {

651

start = __pa(start);

655

start = __pa(start);

652

end = __pa(end);

656

end = __pa(end);

653

for (va = start; va < end; va += 32)

657

for (va = start; va < end; va += 32)

654

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

658

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

655

"membar #Sync"

659

"membar #Sync"

656

: /* no outputs */

660

: /* no outputs */

657

: "r" (va),

661

: "r" (va),

658

"i" (ASI_DCACHE_INVALIDATE));

662

"i" (ASI_DCACHE_INVALIDATE));

659

}

663

}

660

}

664

}

661

EXPORT_SYMBOL(__flush_dcache_range);

665

EXPORT_SYMBOL(__flush_dcache_range);

662

666

663

/* get_new_mmu_context() uses "cache + 1". */

667

/* get_new_mmu_context() uses "cache + 1". */

664

DEFINE_SPINLOCK(ctx_alloc_lock);

668

DEFINE_SPINLOCK(ctx_alloc_lock);

665

unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;

669

unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;

666

#define MAX_CTX_NR (1UL << CTX_NR_BITS)

670

#define MAX_CTX_NR (1UL << CTX_NR_BITS)

667

#define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR)

671

#define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR)

668

DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);

672

DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);

669

673

670

/* Caller does TLB context flushing on local CPU if necessary.

674

/* Caller does TLB context flushing on local CPU if necessary.

671

* The caller also ensures that CTX_VALID(mm->context) is false.

675

* The caller also ensures that CTX_VALID(mm->context) is false.

672

*

676

*

673

* We must be careful about boundary cases so that we never

677

* We must be careful about boundary cases so that we never

674

* let the user have CTX 0 (nucleus) or we ever use a CTX

678

* let the user have CTX 0 (nucleus) or we ever use a CTX

675

* version of zero (and thus NO_CONTEXT would not be caught

679

* version of zero (and thus NO_CONTEXT would not be caught

676

* by version mis-match tests in mmu_context.h).

680

* by version mis-match tests in mmu_context.h).

677

*

681

*

678

* Always invoked with interrupts disabled.

682

* Always invoked with interrupts disabled.

679

*/

683

*/

680

void get_new_mmu_context(struct mm_struct *mm)

684

void get_new_mmu_context(struct mm_struct *mm)

681

{

685

{

682

unsigned long ctx, new_ctx;

686

unsigned long ctx, new_ctx;

683

unsigned long orig_pgsz_bits;

687

unsigned long orig_pgsz_bits;

684

int new_version;

688

int new_version;

685

689

686

spin_lock(&ctx_alloc_lock);

690

spin_lock(&ctx_alloc_lock);

687

orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);

691

orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);

688

ctx = (tlb_context_cache + 1) & CTX_NR_MASK;

692

ctx = (tlb_context_cache + 1) & CTX_NR_MASK;

689

new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);

693

new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);

690

new_version = 0;

694

new_version = 0;

691

if (new_ctx >= (1 << CTX_NR_BITS)) {

695

if (new_ctx >= (1 << CTX_NR_BITS)) {

692

new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);

696

new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);

693

if (new_ctx >= ctx) {

697

if (new_ctx >= ctx) {

694

int i;

698

int i;

695

new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +

699

new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +

696

CTX_FIRST_VERSION;

700

CTX_FIRST_VERSION;

697

if (new_ctx == 1)

701

if (new_ctx == 1)

698

new_ctx = CTX_FIRST_VERSION;

702

new_ctx = CTX_FIRST_VERSION;

699

703

700

/* Don't call memset, for 16 entries that's just

704

/* Don't call memset, for 16 entries that's just

701

* plain silly...

705

* plain silly...

702

*/

706

*/

703

mmu_context_bmap[0] = 3;

707

mmu_context_bmap[0] = 3;

704

mmu_context_bmap[1] = 0;

708

mmu_context_bmap[1] = 0;

705

mmu_context_bmap[2] = 0;

709

mmu_context_bmap[2] = 0;

706

mmu_context_bmap[3] = 0;

710

mmu_context_bmap[3] = 0;

707

for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {

711

for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {

708

mmu_context_bmap[i + 0] = 0;

712

mmu_context_bmap[i + 0] = 0;

709

mmu_context_bmap[i + 1] = 0;

713

mmu_context_bmap[i + 1] = 0;

710

mmu_context_bmap[i + 2] = 0;

714

mmu_context_bmap[i + 2] = 0;

711

mmu_context_bmap[i + 3] = 0;

715

mmu_context_bmap[i + 3] = 0;

712

}

716

}

713

new_version = 1;

717

new_version = 1;

714

goto out;

718

goto out;

715

}

719

}

716

}

720

}

717

mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));

721

mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));

718

new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);

722

new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);

719

out:

723

out:

720

tlb_context_cache = new_ctx;

724

tlb_context_cache = new_ctx;

721

mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;

725

mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;

722

spin_unlock(&ctx_alloc_lock);

726

spin_unlock(&ctx_alloc_lock);

723

727

724

if (unlikely(new_version))

728

if (unlikely(new_version))

725

smp_new_mmu_context_version();

729

smp_new_mmu_context_version();

726

}

730

}

727

731

728

static int numa_enabled = 1;

732

static int numa_enabled = 1;

729

static int numa_debug;

733

static int numa_debug;

730

734

731

static int __init early_numa(char *p)

735

static int __init early_numa(char *p)

732

{

736

{

733

if (!p)

737

if (!p)

734

return 0;

738

return 0;

735

739

736

if (strstr(p, "off"))

740

if (strstr(p, "off"))

737

numa_enabled = 0;

741

numa_enabled = 0;

738

742

739

if (strstr(p, "debug"))

743

if (strstr(p, "debug"))

740

numa_debug = 1;

744

numa_debug = 1;

741

745

742

return 0;

746

return 0;

743

}

747

}

744

early_param("numa", early_numa);

748

early_param("numa", early_numa);

745

749

746

#define numadbg(f, a...) \

750

#define numadbg(f, a...) \

747

do { if (numa_debug) \

751

do { if (numa_debug) \

748

printk(KERN_INFO f, ## a); \

752

printk(KERN_INFO f, ## a); \

749

} while (0)

753

} while (0)

750

754

751

static void __init find_ramdisk(unsigned long phys_base)

755

static void __init find_ramdisk(unsigned long phys_base)

752

{

756

{

753

#ifdef CONFIG_BLK_DEV_INITRD

757

#ifdef CONFIG_BLK_DEV_INITRD

754

if (sparc_ramdisk_image || sparc_ramdisk_image64) {

758

if (sparc_ramdisk_image || sparc_ramdisk_image64) {

755

unsigned long ramdisk_image;

759

unsigned long ramdisk_image;

756

760

757

/* Older versions of the bootloader only supported a

761

/* Older versions of the bootloader only supported a

758

* 32-bit physical address for the ramdisk image

762

* 32-bit physical address for the ramdisk image

759

* location, stored at sparc_ramdisk_image. Newer

763

* location, stored at sparc_ramdisk_image. Newer

760

* SILO versions set sparc_ramdisk_image to zero and

764

* SILO versions set sparc_ramdisk_image to zero and

761

* provide a full 64-bit physical address at

765

* provide a full 64-bit physical address at

762

* sparc_ramdisk_image64.

766

* sparc_ramdisk_image64.

763

*/

767

*/

764

ramdisk_image = sparc_ramdisk_image;

768

ramdisk_image = sparc_ramdisk_image;

765

if (!ramdisk_image)

769

if (!ramdisk_image)

766

ramdisk_image = sparc_ramdisk_image64;

770

ramdisk_image = sparc_ramdisk_image64;

767

771

768

/* Another bootloader quirk. The bootloader normalizes

772

/* Another bootloader quirk. The bootloader normalizes

769

* the physical address to KERNBASE, so we have to

773

* the physical address to KERNBASE, so we have to

770

* factor that back out and add in the lowest valid

774

* factor that back out and add in the lowest valid

771

* physical page address to get the true physical address.

775

* physical page address to get the true physical address.

772

*/

776

*/

773

ramdisk_image -= KERNBASE;

777

ramdisk_image -= KERNBASE;

774

ramdisk_image += phys_base;

778

ramdisk_image += phys_base;

775

779

776

numadbg("Found ramdisk at physical address 0x%lx, size %u\n",

780

numadbg("Found ramdisk at physical address 0x%lx, size %u\n",

777

ramdisk_image, sparc_ramdisk_size);

781

ramdisk_image, sparc_ramdisk_size);

778

782

779

initrd_start = ramdisk_image;

783

initrd_start = ramdisk_image;

780

initrd_end = ramdisk_image + sparc_ramdisk_size;

784

initrd_end = ramdisk_image + sparc_ramdisk_size;

781

785

782

memblock_reserve(initrd_start, sparc_ramdisk_size);

786

memblock_reserve(initrd_start, sparc_ramdisk_size);

783

787

784

initrd_start += PAGE_OFFSET;

788

initrd_start += PAGE_OFFSET;

785

initrd_end += PAGE_OFFSET;

789

initrd_end += PAGE_OFFSET;

786

}

790

}

787

#endif

791

#endif

788

}

792

}

789

793

790

struct node_mem_mask {

794

struct node_mem_mask {

791

unsigned long mask;

795

unsigned long mask;

792

unsigned long val;

796

unsigned long val;

793

};

797

};

794

static struct node_mem_mask node_masks[MAX_NUMNODES];

798

static struct node_mem_mask node_masks[MAX_NUMNODES];

795

static int num_node_masks;

799

static int num_node_masks;

796

800

797

int numa_cpu_lookup_table[NR_CPUS];

801

int numa_cpu_lookup_table[NR_CPUS];

798

cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];

802

cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];

799

803

800

#ifdef CONFIG_NEED_MULTIPLE_NODES

804

#ifdef CONFIG_NEED_MULTIPLE_NODES

801

805

802

struct mdesc_mblock {

806

struct mdesc_mblock {

803

u64 base;

807

u64 base;

804

u64 size;

808

u64 size;

805

u64 offset; /* RA-to-PA */

809

u64 offset; /* RA-to-PA */

806

};

810

};

807

static struct mdesc_mblock *mblocks;

811

static struct mdesc_mblock *mblocks;

808

static int num_mblocks;

812

static int num_mblocks;

809

813

810

static unsigned long ra_to_pa(unsigned long addr)

814

static unsigned long ra_to_pa(unsigned long addr)

811

{

815

{

812

int i;

816

int i;

813

817

814

for (i = 0; i < num_mblocks; i++) {

818

for (i = 0; i < num_mblocks; i++) {

815

struct mdesc_mblock *m = &mblocks[i];

819

struct mdesc_mblock *m = &mblocks[i];

816

820

817

if (addr >= m->base &&

821

if (addr >= m->base &&

818

addr < (m->base + m->size)) {

822

addr < (m->base + m->size)) {

819

addr += m->offset;

823

addr += m->offset;

820

break;

824

break;

821

}

825

}

822

}

826

}

823

return addr;

827

return addr;

824

}

828

}

825

829

826

static int find_node(unsigned long addr)

830

static int find_node(unsigned long addr)

827

{

831

{

828

int i;

832

int i;

829

833

830

addr = ra_to_pa(addr);

834

addr = ra_to_pa(addr);

831

for (i = 0; i < num_node_masks; i++) {

835

for (i = 0; i < num_node_masks; i++) {

832

struct node_mem_mask *p = &node_masks[i];

836

struct node_mem_mask *p = &node_masks[i];

833

837

834

if ((addr & p->mask) == p->val)

838

if ((addr & p->mask) == p->val)

835

return i;

839

return i;

836

}

840

}

837

return -1;

841

return -1;

838

}

842

}

839

843

840

static u64 memblock_nid_range(u64 start, u64 end, int *nid)

844

static u64 memblock_nid_range(u64 start, u64 end, int *nid)

841

{

845

{

842

*nid = find_node(start);

846

*nid = find_node(start);

843

start += PAGE_SIZE;

847

start += PAGE_SIZE;

844

while (start < end) {

848

while (start < end) {

845

int n = find_node(start);

849

int n = find_node(start);

846

850

847

if (n != *nid)

851

if (n != *nid)

848

break;

852

break;

849

start += PAGE_SIZE;

853

start += PAGE_SIZE;

850

}

854

}

851

855

852

if (start > end)

856

if (start > end)

853

start = end;

857

start = end;

854

858

855

return start;

859

return start;

856

}

860

}

857

#endif

861

#endif

858

862

859

/* This must be invoked after performing all of the necessary

863

/* This must be invoked after performing all of the necessary

860

* memblock_set_node() calls for 'nid'. We need to be able to get

864

* memblock_set_node() calls for 'nid'. We need to be able to get

861

* correct data from get_pfn_range_for_nid().

865

* correct data from get_pfn_range_for_nid().

862

*/

866

*/

863

static void __init allocate_node_data(int nid)

867

static void __init allocate_node_data(int nid)

864

{

868

{

865

struct pglist_data *p;

869

struct pglist_data *p;

866

unsigned long start_pfn, end_pfn;

870

unsigned long start_pfn, end_pfn;

867

#ifdef CONFIG_NEED_MULTIPLE_NODES

871

#ifdef CONFIG_NEED_MULTIPLE_NODES

868

unsigned long paddr;

872

unsigned long paddr;

869

873

870

paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);

874

paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);

871

if (!paddr) {

875

if (!paddr) {

872

prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);

876

prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);

873

prom_halt();

877

prom_halt();

874

}

878

}

875

NODE_DATA(nid) = __va(paddr);

879

NODE_DATA(nid) = __va(paddr);

876

memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));

880

memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));

877

881

878

NODE_DATA(nid)->node_id = nid;

882

NODE_DATA(nid)->node_id = nid;

879

#endif

883

#endif

880

884

881

p = NODE_DATA(nid);

885

p = NODE_DATA(nid);

882

886

883

get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);

887

get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);

884

p->node_start_pfn = start_pfn;

888

p->node_start_pfn = start_pfn;

885

p->node_spanned_pages = end_pfn - start_pfn;

889

p->node_spanned_pages = end_pfn - start_pfn;

886

}

890

}

887

891

888

static void init_node_masks_nonnuma(void)

892

static void init_node_masks_nonnuma(void)

889

{

893

{

890

int i;

894

int i;

891

895

892

numadbg("Initializing tables for non-numa.\n");

896

numadbg("Initializing tables for non-numa.\n");

893

897

894

node_masks[0].mask = node_masks[0].val = 0;

898

node_masks[0].mask = node_masks[0].val = 0;

895

num_node_masks = 1;

899

num_node_masks = 1;

896

900

897

for (i = 0; i < NR_CPUS; i++)

901

for (i = 0; i < NR_CPUS; i++)

898

numa_cpu_lookup_table[i] = 0;

902

numa_cpu_lookup_table[i] = 0;

899

903

900

cpumask_setall(&numa_cpumask_lookup_table[0]);

904

cpumask_setall(&numa_cpumask_lookup_table[0]);

901

}

905

}

902

906

903

#ifdef CONFIG_NEED_MULTIPLE_NODES

907

#ifdef CONFIG_NEED_MULTIPLE_NODES

904

struct pglist_data *node_data[MAX_NUMNODES];

908

struct pglist_data *node_data[MAX_NUMNODES];

905

909

906

EXPORT_SYMBOL(numa_cpu_lookup_table);

910

EXPORT_SYMBOL(numa_cpu_lookup_table);

907

EXPORT_SYMBOL(numa_cpumask_lookup_table);

911

EXPORT_SYMBOL(numa_cpumask_lookup_table);

908

EXPORT_SYMBOL(node_data);

912

EXPORT_SYMBOL(node_data);

909

913

910

struct mdesc_mlgroup {

914

struct mdesc_mlgroup {

911

u64 node;

915

u64 node;

912

u64 latency;

916

u64 latency;

913

u64 match;

917

u64 match;

914

u64 mask;

918

u64 mask;

915

};

919

};

916

static struct mdesc_mlgroup *mlgroups;

920

static struct mdesc_mlgroup *mlgroups;

917

static int num_mlgroups;

921

static int num_mlgroups;

918

922

919

static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,

923

static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,

920

u32 cfg_handle)

924

u32 cfg_handle)

921

{

925

{

922

u64 arc;

926

u64 arc;

923

927

924

mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {

928

mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {

925

u64 target = mdesc_arc_target(md, arc);

929

u64 target = mdesc_arc_target(md, arc);

926

const u64 *val;

930

const u64 *val;

927

931

928

val = mdesc_get_property(md, target,

932

val = mdesc_get_property(md, target,

929

"cfg-handle", NULL);

933

"cfg-handle", NULL);

930

if (val && *val == cfg_handle)

934

if (val && *val == cfg_handle)

931

return 0;

935

return 0;

932

}

936

}

933

return -ENODEV;

937

return -ENODEV;

934

}

938

}

935

939

936

static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,

940

static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,

937

u32 cfg_handle)

941

u32 cfg_handle)

938

{

942

{

939

u64 arc, candidate, best_latency = ~(u64)0;

943

u64 arc, candidate, best_latency = ~(u64)0;

940

944

941

candidate = MDESC_NODE_NULL;

945

candidate = MDESC_NODE_NULL;

942

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {

946

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {

943

u64 target = mdesc_arc_target(md, arc);

947

u64 target = mdesc_arc_target(md, arc);

944

const char *name = mdesc_node_name(md, target);

948

const char *name = mdesc_node_name(md, target);

945

const u64 *val;

949

const u64 *val;

946

950

947

if (strcmp(name, "pio-latency-group"))

951

if (strcmp(name, "pio-latency-group"))

948

continue;

952

continue;

949

953

950

val = mdesc_get_property(md, target, "latency", NULL);

954

val = mdesc_get_property(md, target, "latency", NULL);

951

if (!val)

955

if (!val)

952

continue;

956

continue;

953

957

954

if (*val < best_latency) {

958

if (*val < best_latency) {

955

candidate = target;

959

candidate = target;

956

best_latency = *val;

960

best_latency = *val;

957

}

961

}

958

}

962

}

959

963

960

if (candidate == MDESC_NODE_NULL)

964

if (candidate == MDESC_NODE_NULL)

961

return -ENODEV;

965

return -ENODEV;

962

966

963

return scan_pio_for_cfg_handle(md, candidate, cfg_handle);

967

return scan_pio_for_cfg_handle(md, candidate, cfg_handle);

964

}

968

}

965

969

966

int of_node_to_nid(struct device_node *dp)

970

int of_node_to_nid(struct device_node *dp)

967

{

971

{

968

const struct linux_prom64_registers *regs;

972

const struct linux_prom64_registers *regs;

969

struct mdesc_handle *md;

973

struct mdesc_handle *md;

970

u32 cfg_handle;

974

u32 cfg_handle;

971

int count, nid;

975

int count, nid;

972

u64 grp;

976

u64 grp;

973

977

974

/* This is the right thing to do on currently supported

978

/* This is the right thing to do on currently supported

975

* SUN4U NUMA platforms as well, as the PCI controller does

979

* SUN4U NUMA platforms as well, as the PCI controller does

976

* not sit behind any particular memory controller.

980

* not sit behind any particular memory controller.

977

*/

981

*/

978

if (!mlgroups)

982

if (!mlgroups)

979

return -1;

983

return -1;

980

984

981

regs = of_get_property(dp, "reg", NULL);

985

regs = of_get_property(dp, "reg", NULL);

982

if (!regs)

986

if (!regs)

983

return -1;

987

return -1;

984

988

985

cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;

989

cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;

986

990

987

md = mdesc_grab();

991

md = mdesc_grab();

988

992

989

count = 0;

993

count = 0;

990

nid = -1;

994

nid = -1;

991

mdesc_for_each_node_by_name(md, grp, "group") {

995

mdesc_for_each_node_by_name(md, grp, "group") {

992

if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {

996

if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {

993

nid = count;

997

nid = count;

994

break;

998

break;

995

}

999

}

996

count++;

1000

count++;

997

}

1001

}

998

1002

999

mdesc_release(md);

1003

mdesc_release(md);

1000

1004

1001

return nid;

1005

return nid;

1002

}

1006

}

1003

1007

1004

static void __init add_node_ranges(void)

1008

static void __init add_node_ranges(void)

1005

{

1009

{

1006

struct memblock_region *reg;

1010

struct memblock_region *reg;

1007

1011

1008

for_each_memblock(memory, reg) {

1012

for_each_memblock(memory, reg) {

1009

unsigned long size = reg->size;

1013

unsigned long size = reg->size;

1010

unsigned long start, end;

1014

unsigned long start, end;

1011

1015

1012

start = reg->base;

1016

start = reg->base;

1013

end = start + size;

1017

end = start + size;

1014

while (start < end) {

1018

while (start < end) {

1015

unsigned long this_end;

1019

unsigned long this_end;

1016

int nid;

1020

int nid;

1017

1021

1018

this_end = memblock_nid_range(start, end, &nid);

1022

this_end = memblock_nid_range(start, end, &nid);

1019

1023

1020

numadbg("Setting memblock NUMA node nid[%d] "

1024

numadbg("Setting memblock NUMA node nid[%d] "

1021

"start[%lx] end[%lx]\n",

1025

"start[%lx] end[%lx]\n",

1022

nid, start, this_end);

1026

nid, start, this_end);

1023

1027

1024

memblock_set_node(start, this_end - start,

1028

memblock_set_node(start, this_end - start,

1025

&memblock.memory, nid);

1029

&memblock.memory, nid);

1026

start = this_end;

1030

start = this_end;

1027

}

1031

}

1028

}

1032

}

1029

}

1033

}

1030

1034

1031

static int __init grab_mlgroups(struct mdesc_handle *md)

1035

static int __init grab_mlgroups(struct mdesc_handle *md)

1032

{

1036

{

1033

unsigned long paddr;

1037

unsigned long paddr;

1034

int count = 0;

1038

int count = 0;

1035

u64 node;

1039

u64 node;

1036

1040

1037

mdesc_for_each_node_by_name(md, node, "memory-latency-group")

1041

mdesc_for_each_node_by_name(md, node, "memory-latency-group")

1038

count++;

1042

count++;

1039

if (!count)

1043

if (!count)

1040

return -ENOENT;

1044

return -ENOENT;

1041

1045

1042

paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),

1046

paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),

1043

SMP_CACHE_BYTES);

1047

SMP_CACHE_BYTES);

1044

if (!paddr)

1048

if (!paddr)

1045

return -ENOMEM;

1049

return -ENOMEM;

1046

1050

1047

mlgroups = __va(paddr);

1051

mlgroups = __va(paddr);

1048

num_mlgroups = count;

1052

num_mlgroups = count;

1049

1053

1050

count = 0;

1054

count = 0;

1051

mdesc_for_each_node_by_name(md, node, "memory-latency-group") {

1055

mdesc_for_each_node_by_name(md, node, "memory-latency-group") {

1052

struct mdesc_mlgroup *m = &mlgroups[count++];

1056

struct mdesc_mlgroup *m = &mlgroups[count++];

1053

const u64 *val;

1057

const u64 *val;

1054

1058

1055

m->node = node;

1059

m->node = node;

1056

1060

1057

val = mdesc_get_property(md, node, "latency", NULL);

1061

val = mdesc_get_property(md, node, "latency", NULL);

1058

m->latency = *val;

1062

m->latency = *val;

1059

val = mdesc_get_property(md, node, "address-match", NULL);

1063

val = mdesc_get_property(md, node, "address-match", NULL);

1060

m->match = *val;

1064

m->match = *val;

1061

val = mdesc_get_property(md, node, "address-mask", NULL);

1065

val = mdesc_get_property(md, node, "address-mask", NULL);

1062

m->mask = *val;

1066

m->mask = *val;

1063

1067

1064

numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "

1068

numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "

1065

"match[%llx] mask[%llx]\n",

1069

"match[%llx] mask[%llx]\n",

1066

count - 1, m->node, m->latency, m->match, m->mask);

1070

count - 1, m->node, m->latency, m->match, m->mask);

1067

}

1071

}

1068

1072

1069

return 0;

1073

return 0;

1070

}

1074

}

1071

1075

1072

static int __init grab_mblocks(struct mdesc_handle *md)

1076

static int __init grab_mblocks(struct mdesc_handle *md)

1073

{

1077

{

1074

unsigned long paddr;

1078

unsigned long paddr;

1075

int count = 0;

1079

int count = 0;

1076

u64 node;

1080

u64 node;

1077

1081

1078

mdesc_for_each_node_by_name(md, node, "mblock")

1082

mdesc_for_each_node_by_name(md, node, "mblock")

1079

count++;

1083

count++;

1080

if (!count)

1084

if (!count)

1081

return -ENOENT;

1085

return -ENOENT;

1082

1086

1083

paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),

1087

paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),

1084

SMP_CACHE_BYTES);

1088

SMP_CACHE_BYTES);

1085

if (!paddr)

1089

if (!paddr)

1086

return -ENOMEM;

1090

return -ENOMEM;

1087

1091

1088

mblocks = __va(paddr);

1092

mblocks = __va(paddr);

1089

num_mblocks = count;

1093

num_mblocks = count;

1090

1094

1091

count = 0;

1095

count = 0;

1092

mdesc_for_each_node_by_name(md, node, "mblock") {

1096

mdesc_for_each_node_by_name(md, node, "mblock") {

1093

struct mdesc_mblock *m = &mblocks[count++];

1097

struct mdesc_mblock *m = &mblocks[count++];

1094

const u64 *val;

1098

const u64 *val;

1095

1099

1096

val = mdesc_get_property(md, node, "base", NULL);

1100

val = mdesc_get_property(md, node, "base", NULL);

1097

m->base = *val;

1101

m->base = *val;

1098

val = mdesc_get_property(md, node, "size", NULL);

1102

val = mdesc_get_property(md, node, "size", NULL);

1099

m->size = *val;

1103

m->size = *val;

1100

val = mdesc_get_property(md, node,

1104

val = mdesc_get_property(md, node,

1101

"address-congruence-offset", NULL);

1105

"address-congruence-offset", NULL);

1102

1106

1103

/* The address-congruence-offset property is optional.

1107

/* The address-congruence-offset property is optional.

1104

* Explicity zero it be identifty this.

1108

* Explicity zero it be identifty this.

1105

*/

1109

*/

1106

if (val)

1110

if (val)

1107

m->offset = *val;

1111

m->offset = *val;

1108

else

1112

else

1109

m->offset = 0UL;

1113

m->offset = 0UL;

1110

1114

1111

numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",

1115

numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",

1112

count - 1, m->base, m->size, m->offset);

1116

count - 1, m->base, m->size, m->offset);

1113

}

1117

}

1114

1118

1115

return 0;

1119

return 0;

1116

}

1120

}

1117

1121

1118

static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,

1122

static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,

1119

u64 grp, cpumask_t *mask)

1123

u64 grp, cpumask_t *mask)

1120

{

1124

{

1121

u64 arc;

1125

u64 arc;

1122

1126

1123

cpumask_clear(mask);

1127

cpumask_clear(mask);

1124

1128

1125

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {

1129

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {

1126

u64 target = mdesc_arc_target(md, arc);

1130

u64 target = mdesc_arc_target(md, arc);

1127

const char *name = mdesc_node_name(md, target);

1131

const char *name = mdesc_node_name(md, target);

1128

const u64 *id;

1132

const u64 *id;

1129

1133

1130

if (strcmp(name, "cpu"))

1134

if (strcmp(name, "cpu"))

1131

continue;

1135

continue;

1132

id = mdesc_get_property(md, target, "id", NULL);

1136

id = mdesc_get_property(md, target, "id", NULL);

1133

if (*id < nr_cpu_ids)

1137

if (*id < nr_cpu_ids)

1134

cpumask_set_cpu(*id, mask);

1138

cpumask_set_cpu(*id, mask);

1135

}

1139

}

1136

}

1140

}

1137

1141

1138

static struct mdesc_mlgroup * __init find_mlgroup(u64 node)

1142

static struct mdesc_mlgroup * __init find_mlgroup(u64 node)

1139

{

1143

{

1140

int i;

1144

int i;

1141

1145

1142

for (i = 0; i < num_mlgroups; i++) {

1146

for (i = 0; i < num_mlgroups; i++) {

1143

struct mdesc_mlgroup *m = &mlgroups[i];

1147

struct mdesc_mlgroup *m = &mlgroups[i];

1144

if (m->node == node)

1148

if (m->node == node)

1145

return m;

1149

return m;

1146

}

1150

}

1147

return NULL;

1151

return NULL;

1148

}

1152

}

1149

1153

1150

static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,

1154

static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,

1151

int index)

1155

int index)

1152

{

1156

{

1153

struct mdesc_mlgroup *candidate = NULL;

1157

struct mdesc_mlgroup *candidate = NULL;

1154

u64 arc, best_latency = ~(u64)0;

1158

u64 arc, best_latency = ~(u64)0;

1155

struct node_mem_mask *n;

1159

struct node_mem_mask *n;

1156

1160

1157

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {

1161

mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {

1158

u64 target = mdesc_arc_target(md, arc);

1162

u64 target = mdesc_arc_target(md, arc);

1159

struct mdesc_mlgroup *m = find_mlgroup(target);

1163

struct mdesc_mlgroup *m = find_mlgroup(target);

1160

if (!m)

1164

if (!m)

1161

continue;

1165

continue;

1162

if (m->latency < best_latency) {

1166

if (m->latency < best_latency) {

1163

candidate = m;

1167

candidate = m;

1164

best_latency = m->latency;

1168

best_latency = m->latency;

1165

}

1169

}

1166

}

1170

}

1167

if (!candidate)

1171

if (!candidate)

1168

return -ENOENT;

1172

return -ENOENT;

1169

1173

1170

if (num_node_masks != index) {

1174

if (num_node_masks != index) {

1171

printk(KERN_ERR "Inconsistent NUMA state, "

1175

printk(KERN_ERR "Inconsistent NUMA state, "

1172

"index[%d] != num_node_masks[%d]\n",

1176

"index[%d] != num_node_masks[%d]\n",

1173

index, num_node_masks);

1177

index, num_node_masks);

1174

return -EINVAL;

1178

return -EINVAL;

1175

}

1179

}

1176

1180

1177

n = &node_masks[num_node_masks++];

1181

n = &node_masks[num_node_masks++];

1178

1182

1179

n->mask = candidate->mask;

1183

n->mask = candidate->mask;

1180

n->val = candidate->match;

1184

n->val = candidate->match;

1181

1185

1182

numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n",

1186

numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n",

1183

index, n->mask, n->val, candidate->latency);

1187

index, n->mask, n->val, candidate->latency);

1184

1188

1185

return 0;

1189

return 0;

1186

}

1190

}

1187

1191

1188

static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,

1192

static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,

1189

int index)

1193

int index)

1190

{

1194

{

1191

cpumask_t mask;

1195

cpumask_t mask;

1192

int cpu;

1196

int cpu;

1193

1197

1194

numa_parse_mdesc_group_cpus(md, grp, &mask);

1198

numa_parse_mdesc_group_cpus(md, grp, &mask);

1195

1199

1196

for_each_cpu(cpu, &mask)

1200

for_each_cpu(cpu, &mask)

1197

numa_cpu_lookup_table[cpu] = index;

1201

numa_cpu_lookup_table[cpu] = index;

1198

cpumask_copy(&numa_cpumask_lookup_table[index], &mask);

1202

cpumask_copy(&numa_cpumask_lookup_table[index], &mask);

1199

1203

1200

if (numa_debug) {

1204

if (numa_debug) {

1201

printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);

1205

printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);

1202

for_each_cpu(cpu, &mask)

1206

for_each_cpu(cpu, &mask)

1203

printk("%d ", cpu);

1207

printk("%d ", cpu);

1204

printk("]\n");

1208

printk("]\n");

1205

}

1209

}

1206

1210

1207

return numa_attach_mlgroup(md, grp, index);

1211

return numa_attach_mlgroup(md, grp, index);

1208

}

1212

}

1209

1213

1210

static int __init numa_parse_mdesc(void)

1214

static int __init numa_parse_mdesc(void)

1211

{

1215

{

1212

struct mdesc_handle *md = mdesc_grab();

1216

struct mdesc_handle *md = mdesc_grab();

1213

int i, err, count;

1217

int i, err, count;

1214

u64 node;

1218

u64 node;

1215

1219

1216

node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");

1220

node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");

1217

if (node == MDESC_NODE_NULL) {

1221

if (node == MDESC_NODE_NULL) {

1218

mdesc_release(md);

1222

mdesc_release(md);

1219

return -ENOENT;

1223

return -ENOENT;

1220

}

1224

}

1221

1225

1222

err = grab_mblocks(md);

1226

err = grab_mblocks(md);

1223

if (err < 0)

1227

if (err < 0)

1224

goto out;

1228

goto out;

1225

1229

1226

err = grab_mlgroups(md);

1230

err = grab_mlgroups(md);

1227

if (err < 0)

1231

if (err < 0)

1228

goto out;

1232

goto out;

1229

1233

1230

count = 0;

1234

count = 0;

1231

mdesc_for_each_node_by_name(md, node, "group") {

1235

mdesc_for_each_node_by_name(md, node, "group") {

1232

err = numa_parse_mdesc_group(md, node, count);

1236

err = numa_parse_mdesc_group(md, node, count);

1233

if (err < 0)

1237

if (err < 0)

1234

break;

1238

break;

1235

count++;

1239

count++;

1236

}

1240

}

1237

1241

1238

add_node_ranges();

1242

add_node_ranges();

1239

1243

1240

for (i = 0; i < num_node_masks; i++) {

1244

for (i = 0; i < num_node_masks; i++) {

1241

allocate_node_data(i);

1245

allocate_node_data(i);

1242

node_set_online(i);

1246

node_set_online(i);

1243

}

1247

}

1244

1248

1245

err = 0;

1249

err = 0;

1246

out:

1250

out:

1247

mdesc_release(md);

1251

mdesc_release(md);

1248

return err;

1252

return err;

1249

}

1253

}

1250

1254

1251

static int __init numa_parse_jbus(void)

1255

static int __init numa_parse_jbus(void)

1252

{

1256

{

1253

unsigned long cpu, index;

1257

unsigned long cpu, index;

1254

1258

1255

/* NUMA node id is encoded in bits 36 and higher, and there is

1259

/* NUMA node id is encoded in bits 36 and higher, and there is

1256

* a 1-to-1 mapping from CPU ID to NUMA node ID.

1260

* a 1-to-1 mapping from CPU ID to NUMA node ID.

1257

*/

1261

*/

1258

index = 0;

1262

index = 0;

1259

for_each_present_cpu(cpu) {

1263

for_each_present_cpu(cpu) {

1260

numa_cpu_lookup_table[cpu] = index;

1264

numa_cpu_lookup_table[cpu] = index;

1261

cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));

1265

cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));

1262

node_masks[index].mask = ~((1UL << 36UL) - 1UL);

1266

node_masks[index].mask = ~((1UL << 36UL) - 1UL);

1263

node_masks[index].val = cpu << 36UL;

1267

node_masks[index].val = cpu << 36UL;

1264

1268

1265

index++;

1269

index++;

1266

}

1270

}

1267

num_node_masks = index;

1271

num_node_masks = index;

1268

1272

1269

add_node_ranges();

1273

add_node_ranges();

1270

1274

1271

for (index = 0; index < num_node_masks; index++) {

1275

for (index = 0; index < num_node_masks; index++) {

1272

allocate_node_data(index);

1276

allocate_node_data(index);

1273

node_set_online(index);

1277

node_set_online(index);

1274

}

1278

}

1275

1279

1276

return 0;

1280

return 0;

1277

}

1281

}

1278

1282

1279

static int __init numa_parse_sun4u(void)

1283

static int __init numa_parse_sun4u(void)

1280

{

1284

{

1281

if (tlb_type == cheetah || tlb_type == cheetah_plus) {

1285

if (tlb_type == cheetah || tlb_type == cheetah_plus) {

1282

unsigned long ver;

1286

unsigned long ver;

1283

1287

1284

__asm__ ("rdpr %%ver, %0" : "=r" (ver));

1288

__asm__ ("rdpr %%ver, %0" : "=r" (ver));

1285

if ((ver >> 32UL) == __JALAPENO_ID ||

1289

if ((ver >> 32UL) == __JALAPENO_ID ||

1286

(ver >> 32UL) == __SERRANO_ID)

1290

(ver >> 32UL) == __SERRANO_ID)

1287

return numa_parse_jbus();

1291

return numa_parse_jbus();

1288

}

1292

}

1289

return -1;

1293

return -1;

1290

}

1294

}

1291

1295

1292

static int __init bootmem_init_numa(void)

1296

static int __init bootmem_init_numa(void)

1293

{

1297

{

1294

int err = -1;

1298

int err = -1;

1295

1299

1296

numadbg("bootmem_init_numa()\n");

1300

numadbg("bootmem_init_numa()\n");

1297

1301

1298

if (numa_enabled) {

1302

if (numa_enabled) {

1299

if (tlb_type == hypervisor)

1303

if (tlb_type == hypervisor)

1300

err = numa_parse_mdesc();

1304

err = numa_parse_mdesc();

1301

else

1305

else

1302

err = numa_parse_sun4u();

1306

err = numa_parse_sun4u();

1303

}

1307

}

1304

return err;

1308

return err;

1305

}

1309

}

1306

1310

1307

#else

1311

#else

1308

1312

1309

static int bootmem_init_numa(void)

1313

static int bootmem_init_numa(void)

1310

{

1314

{

1311

return -1;

1315

return -1;

1312

}

1316

}

1313

1317

1314

#endif

1318

#endif

1315

1319

1316

static void __init bootmem_init_nonnuma(void)

1320

static void __init bootmem_init_nonnuma(void)

1317

{

1321

{

1318

unsigned long top_of_ram = memblock_end_of_DRAM();

1322

unsigned long top_of_ram = memblock_end_of_DRAM();

1319

unsigned long total_ram = memblock_phys_mem_size();

1323

unsigned long total_ram = memblock_phys_mem_size();

1320

1324

1321

numadbg("bootmem_init_nonnuma()\n");

1325

numadbg("bootmem_init_nonnuma()\n");

1322

1326

1323

printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",

1327

printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",

1324

top_of_ram, total_ram);

1328

top_of_ram, total_ram);

1325

printk(KERN_INFO "Memory hole size: %ldMB\n",

1329

printk(KERN_INFO "Memory hole size: %ldMB\n",

1326

(top_of_ram - total_ram) >> 20);

1330

(top_of_ram - total_ram) >> 20);

1327

1331

1328

init_node_masks_nonnuma();

1332

init_node_masks_nonnuma();

1329

memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);

1333

memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);

1330

allocate_node_data(0);

1334

allocate_node_data(0);

1331

node_set_online(0);

1335

node_set_online(0);

1332

}

1336

}

1333

1337

1334

static unsigned long __init bootmem_init(unsigned long phys_base)

1338

static unsigned long __init bootmem_init(unsigned long phys_base)

1335

{

1339

{

1336

unsigned long end_pfn;

1340

unsigned long end_pfn;

1337

1341

1338

end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;

1342

end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;

1339

max_pfn = max_low_pfn = end_pfn;

1343

max_pfn = max_low_pfn = end_pfn;

1340

min_low_pfn = (phys_base >> PAGE_SHIFT);

1344

min_low_pfn = (phys_base >> PAGE_SHIFT);

1341

1345

1342

if (bootmem_init_numa() < 0)

1346

if (bootmem_init_numa() < 0)

1343

bootmem_init_nonnuma();

1347

bootmem_init_nonnuma();

1344

1348

1345

/* Dump memblock with node info. */

1349

/* Dump memblock with node info. */

1346

memblock_dump_all();

1350

memblock_dump_all();

1347

1351

1348

/* XXX cpu notifier XXX */

1352

/* XXX cpu notifier XXX */

1349

1353

1350

sparse_memory_present_with_active_regions(MAX_NUMNODES);

1354

sparse_memory_present_with_active_regions(MAX_NUMNODES);

1351

sparse_init();

1355

sparse_init();

1352

1356

1353

return end_pfn;

1357

return end_pfn;

1354

}

1358

}

1355

1359

1356

static struct linux_prom64_registers pall[MAX_BANKS] __initdata;

1360

static struct linux_prom64_registers pall[MAX_BANKS] __initdata;

1357

static int pall_ents __initdata;

1361

static int pall_ents __initdata;

1358

1362

1359

#ifdef CONFIG_DEBUG_PAGEALLOC

1363

#ifdef CONFIG_DEBUG_PAGEALLOC

1360

static unsigned long __ref kernel_map_range(unsigned long pstart,

1364

static unsigned long __ref kernel_map_range(unsigned long pstart,

1361

unsigned long pend, pgprot_t prot)

1365

unsigned long pend, pgprot_t prot)

1362

{

1366

{

1363

unsigned long vstart = PAGE_OFFSET + pstart;

1367

unsigned long vstart = PAGE_OFFSET + pstart;

1364

unsigned long vend = PAGE_OFFSET + pend;

1368

unsigned long vend = PAGE_OFFSET + pend;

1365

unsigned long alloc_bytes = 0UL;

1369

unsigned long alloc_bytes = 0UL;

1366

1370

1367

if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {

1371

if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {

1368

prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",

1372

prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",

1369

vstart, vend);

1373

vstart, vend);

1370

prom_halt();

1374

prom_halt();

1371

}

1375

}

1372

1376

1373

while (vstart < vend) {

1377

while (vstart < vend) {

1374

unsigned long this_end, paddr = __pa(vstart);

1378

unsigned long this_end, paddr = __pa(vstart);

1375

pgd_t *pgd = pgd_offset_k(vstart);

1379

pgd_t *pgd = pgd_offset_k(vstart);

1376

pud_t *pud;

1380

pud_t *pud;

1377

pmd_t *pmd;

1381

pmd_t *pmd;

1378

pte_t *pte;

1382

pte_t *pte;

1379

1383

1380

pud = pud_offset(pgd, vstart);

1384

pud = pud_offset(pgd, vstart);

1381

if (pud_none(*pud)) {

1385

if (pud_none(*pud)) {

1382

pmd_t *new;

1386

pmd_t *new;

1383

1387

1384

new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

1388

new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

1385

alloc_bytes += PAGE_SIZE;

1389

alloc_bytes += PAGE_SIZE;

1386

pud_populate(&init_mm, pud, new);

1390

pud_populate(&init_mm, pud, new);

1387

}

1391

}

1388

1392

1389

pmd = pmd_offset(pud, vstart);

1393

pmd = pmd_offset(pud, vstart);

1390

if (!pmd_present(*pmd)) {

1394

if (!pmd_present(*pmd)) {

1391

pte_t *new;

1395

pte_t *new;

1392

1396

1393

new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

1397

new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);

1394

alloc_bytes += PAGE_SIZE;

1398

alloc_bytes += PAGE_SIZE;

1395

pmd_populate_kernel(&init_mm, pmd, new);

1399

pmd_populate_kernel(&init_mm, pmd, new);

1396

}

1400

}

1397

1401

1398

pte = pte_offset_kernel(pmd, vstart);

1402

pte = pte_offset_kernel(pmd, vstart);

1399

this_end = (vstart + PMD_SIZE) & PMD_MASK;

1403

this_end = (vstart + PMD_SIZE) & PMD_MASK;

1400

if (this_end > vend)

1404

if (this_end > vend)

1401

this_end = vend;

1405

this_end = vend;

1402

1406

1403

while (vstart < this_end) {

1407

while (vstart < this_end) {

1404

pte_val(*pte) = (paddr | pgprot_val(prot));

1408

pte_val(*pte) = (paddr | pgprot_val(prot));

1405

1409

1406

vstart += PAGE_SIZE;

1410

vstart += PAGE_SIZE;

1407

paddr += PAGE_SIZE;

1411

paddr += PAGE_SIZE;

1408

pte++;

1412

pte++;

1409

}

1413

}

1410

}

1414

}

1411

1415

1412

return alloc_bytes;

1416

return alloc_bytes;

1413

}

1417

}

1414

1418

1415

extern unsigned int kvmap_linear_patch[1];

1419

extern unsigned int kvmap_linear_patch[1];

1416

#endif /* CONFIG_DEBUG_PAGEALLOC */

1420

#endif /* CONFIG_DEBUG_PAGEALLOC */

1417

1421

1418

static void __init kpte_set_val(unsigned long index, unsigned long val)

1422

static void __init kpte_set_val(unsigned long index, unsigned long val)

1419

{

1423

{

1420

unsigned long *ptr = kpte_linear_bitmap;

1424

unsigned long *ptr = kpte_linear_bitmap;

1421

1425

1422

val <<= ((index % (BITS_PER_LONG / 2)) * 2);

1426

val <<= ((index % (BITS_PER_LONG / 2)) * 2);

1423

ptr += (index / (BITS_PER_LONG / 2));

1427

ptr += (index / (BITS_PER_LONG / 2));

1424

1428

1425

*ptr |= val;

1429

*ptr |= val;

1426

}

1430

}

1427

1431

1428

static const unsigned long kpte_shift_min = 28; /* 256MB */

1432

static const unsigned long kpte_shift_min = 28; /* 256MB */

1429

static const unsigned long kpte_shift_max = 34; /* 16GB */

1433

static const unsigned long kpte_shift_max = 34; /* 16GB */

1430

static const unsigned long kpte_shift_incr = 3;

1434

static const unsigned long kpte_shift_incr = 3;

1431

1435

1432

static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,

1436

static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,

1433

unsigned long shift)

1437

unsigned long shift)

1434

{

1438

{

1435

unsigned long size = (1UL << shift);

1439

unsigned long size = (1UL << shift);

1436

unsigned long mask = (size - 1UL);

1440

unsigned long mask = (size - 1UL);

1437

unsigned long remains = end - start;

1441

unsigned long remains = end - start;

1438

unsigned long val;

1442

unsigned long val;

1439

1443

1440

if (remains < size || (start & mask))

1444

if (remains < size || (start & mask))

1441

return start;

1445

return start;

1442

1446

1443

/* VAL maps:

1447

/* VAL maps:

1444

*

1448

*

1445

* shift 28 --> kern_linear_pte_xor index 1

1449

* shift 28 --> kern_linear_pte_xor index 1

1446

* shift 31 --> kern_linear_pte_xor index 2

1450

* shift 31 --> kern_linear_pte_xor index 2

1447

* shift 34 --> kern_linear_pte_xor index 3

1451

* shift 34 --> kern_linear_pte_xor index 3

1448

*/

1452

*/

1449

val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;

1453

val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;

1450

1454

1451

remains &= ~mask;

1455

remains &= ~mask;

1452

if (shift != kpte_shift_max)

1456

if (shift != kpte_shift_max)

1453

remains = size;

1457

remains = size;

1454

1458

1455

while (remains) {

1459

while (remains) {

1456

unsigned long index = start >> kpte_shift_min;

1460

unsigned long index = start >> kpte_shift_min;

1457

1461

1458

kpte_set_val(index, val);

1462

kpte_set_val(index, val);

1459

1463

1460

start += 1UL << kpte_shift_min;

1464

start += 1UL << kpte_shift_min;

1461

remains -= 1UL << kpte_shift_min;

1465

remains -= 1UL << kpte_shift_min;

1462

}

1466

}

1463

1467

1464

return start;

1468

return start;

1465

}

1469

}

1466

1470

1467

static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)

1471

static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)

1468

{

1472

{

1469

unsigned long smallest_size, smallest_mask;

1473

unsigned long smallest_size, smallest_mask;

1470

unsigned long s;

1474

unsigned long s;

1471

1475

1472

smallest_size = (1UL << kpte_shift_min);

1476

smallest_size = (1UL << kpte_shift_min);

1473

smallest_mask = (smallest_size - 1UL);

1477

smallest_mask = (smallest_size - 1UL);

1474

1478

1475

while (start < end) {

1479

while (start < end) {

1476

unsigned long orig_start = start;

1480

unsigned long orig_start = start;

1477

1481

1478

for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {

1482

for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {

1479

start = kpte_mark_using_shift(start, end, s);

1483

start = kpte_mark_using_shift(start, end, s);

1480

1484

1481

if (start != orig_start)

1485

if (start != orig_start)

1482

break;

1486

break;

1483

}

1487

}

1484

1488

1485

if (start == orig_start)

1489

if (start == orig_start)

1486

start = (start + smallest_size) & ~smallest_mask;

1490

start = (start + smallest_size) & ~smallest_mask;

1487

}

1491

}

1488

}

1492

}

1489

1493

1490

static void __init init_kpte_bitmap(void)

1494

static void __init init_kpte_bitmap(void)

1491

{

1495

{

1492

unsigned long i;

1496

unsigned long i;

1493

1497

1494

for (i = 0; i < pall_ents; i++) {

1498

for (i = 0; i < pall_ents; i++) {

1495

unsigned long phys_start, phys_end;

1499

unsigned long phys_start, phys_end;

1496

1500

1497

phys_start = pall[i].phys_addr;

1501

phys_start = pall[i].phys_addr;

1498

phys_end = phys_start + pall[i].reg_size;

1502

phys_end = phys_start + pall[i].reg_size;

1499

1503

1500

mark_kpte_bitmap(phys_start, phys_end);

1504

mark_kpte_bitmap(phys_start, phys_end);

1501

}

1505

}

1502

}

1506

}

1503

1507

1504

static void __init kernel_physical_mapping_init(void)

1508

static void __init kernel_physical_mapping_init(void)

1505

{

1509

{

1506

#ifdef CONFIG_DEBUG_PAGEALLOC

1510

#ifdef CONFIG_DEBUG_PAGEALLOC

1507

unsigned long i, mem_alloced = 0UL;

1511

unsigned long i, mem_alloced = 0UL;

1508

1512

1509

for (i = 0; i < pall_ents; i++) {

1513

for (i = 0; i < pall_ents; i++) {

1510

unsigned long phys_start, phys_end;

1514

unsigned long phys_start, phys_end;

1511

1515

1512

phys_start = pall[i].phys_addr;

1516

phys_start = pall[i].phys_addr;

1513

phys_end = phys_start + pall[i].reg_size;

1517

phys_end = phys_start + pall[i].reg_size;

1514

1518

1515

mem_alloced += kernel_map_range(phys_start, phys_end,

1519

mem_alloced += kernel_map_range(phys_start, phys_end,

1516

PAGE_KERNEL);

1520

PAGE_KERNEL);

1517

}

1521

}

1518

1522

1519

printk("Allocated %ld bytes for kernel page tables.\n",

1523

printk("Allocated %ld bytes for kernel page tables.\n",

1520

mem_alloced);

1524

mem_alloced);

1521

1525

1522

kvmap_linear_patch[0] = 0x01000000; /* nop */

1526

kvmap_linear_patch[0] = 0x01000000; /* nop */

1523

flushi(&kvmap_linear_patch[0]);

1527

flushi(&kvmap_linear_patch[0]);

1524

1528

1525

__flush_tlb_all();

1529

__flush_tlb_all();

1526

#endif

1530

#endif

1527

}

1531

}

1528

1532

1529

#ifdef CONFIG_DEBUG_PAGEALLOC

1533

#ifdef CONFIG_DEBUG_PAGEALLOC

1530

void kernel_map_pages(struct page *page, int numpages, int enable)

1534

void kernel_map_pages(struct page *page, int numpages, int enable)

1531

{

1535

{

1532

unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;

1536

unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;

1533

unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);

1537

unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);

1534

1538

1535

kernel_map_range(phys_start, phys_end,

1539

kernel_map_range(phys_start, phys_end,

1536

(enable ? PAGE_KERNEL : __pgprot(0)));

1540

(enable ? PAGE_KERNEL : __pgprot(0)));

1537

1541

1538

flush_tsb_kernel_range(PAGE_OFFSET + phys_start,

1542

flush_tsb_kernel_range(PAGE_OFFSET + phys_start,

1539

PAGE_OFFSET + phys_end);

1543

PAGE_OFFSET + phys_end);

1540

1544

1541

/* we should perform an IPI and flush all tlbs,

1545

/* we should perform an IPI and flush all tlbs,

1542

* but that can deadlock->flush only current cpu.

1546

* but that can deadlock->flush only current cpu.

1543

*/

1547

*/

1544

__flush_tlb_kernel_range(PAGE_OFFSET + phys_start,

1548

__flush_tlb_kernel_range(PAGE_OFFSET + phys_start,

1545

PAGE_OFFSET + phys_end);

1549

PAGE_OFFSET + phys_end);

1546

}

1550

}

1547

#endif

1551

#endif

1548

1552

1549

unsigned long __init find_ecache_flush_span(unsigned long size)

1553

unsigned long __init find_ecache_flush_span(unsigned long size)

1550

{

1554

{

1551

int i;

1555

int i;

1552

1556

1553

for (i = 0; i < pavail_ents; i++) {

1557

for (i = 0; i < pavail_ents; i++) {

1554

if (pavail[i].reg_size >= size)

1558

if (pavail[i].reg_size >= size)

1555

return pavail[i].phys_addr;

1559

return pavail[i].phys_addr;

1556

}

1560

}

1557

1561

1558

return ~0UL;

1562

return ~0UL;

1559

}

1563

}

1560

1564

1561

unsigned long PAGE_OFFSET;

1565

unsigned long PAGE_OFFSET;

1562

EXPORT_SYMBOL(PAGE_OFFSET);

1566

EXPORT_SYMBOL(PAGE_OFFSET);

1563

1567

1564

static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)

1568

static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)

1565

{

1569

{

1566

unsigned long final_shift;

1570

unsigned long final_shift;

1567

unsigned int val = *insn;

1571

unsigned int val = *insn;

1568

unsigned int cnt;

1572

unsigned int cnt;

1569

1573

1570

/* We are patching in ilog2(max_supported_phys_address), and

1574

/* We are patching in ilog2(max_supported_phys_address), and

1571

* we are doing so in a manner similar to a relocation addend.

1575

* we are doing so in a manner similar to a relocation addend.

1572

* That is, we are adding the shift value to whatever value

1576

* That is, we are adding the shift value to whatever value

1573

* is in the shift instruction count field already.

1577

* is in the shift instruction count field already.

1574

*/

1578

*/

1575

cnt = (val & 0x3f);

1579

cnt = (val & 0x3f);

1576

val &= ~0x3f;

1580

val &= ~0x3f;

1577

1581

1578

/* If we are trying to shift >= 64 bits, clear the destination

1582

/* If we are trying to shift >= 64 bits, clear the destination

1579

* register. This can happen when phys_bits ends up being equal

1583

* register. This can happen when phys_bits ends up being equal

1580

* to MAX_PHYS_ADDRESS_BITS.

1584

* to MAX_PHYS_ADDRESS_BITS.

1581

*/

1585

*/

1582

final_shift = (cnt + (64 - phys_bits));

1586

final_shift = (cnt + (64 - phys_bits));

1583

if (final_shift >= 64) {

1587

if (final_shift >= 64) {

1584

unsigned int rd = (val >> 25) & 0x1f;

1588

unsigned int rd = (val >> 25) & 0x1f;

1585

1589

1586

val = 0x80100000 | (rd << 25);

1590

val = 0x80100000 | (rd << 25);

1587

} else {

1591

} else {

1588

val |= final_shift;

1592

val |= final_shift;

1589

}

1593

}

1590

*insn = val;

1594

*insn = val;

1591

1595

1592

__asm__ __volatile__("flush %0"

1596

__asm__ __volatile__("flush %0"

1593

: /* no outputs */

1597

: /* no outputs */

1594

: "r" (insn));

1598

: "r" (insn));

1595

}

1599

}

1596

1600

1597

static void __init page_offset_shift_patch(unsigned long phys_bits)

1601

static void __init page_offset_shift_patch(unsigned long phys_bits)

1598

{

1602

{

1599

extern unsigned int __page_offset_shift_patch;

1603

extern unsigned int __page_offset_shift_patch;

1600

extern unsigned int __page_offset_shift_patch_end;

1604

extern unsigned int __page_offset_shift_patch_end;

1601

unsigned int *p;

1605

unsigned int *p;

1602

1606

1603

p = &__page_offset_shift_patch;

1607

p = &__page_offset_shift_patch;

1604

while (p < &__page_offset_shift_patch_end) {

1608

while (p < &__page_offset_shift_patch_end) {

1605

unsigned int *insn = (unsigned int *)(unsigned long)*p;

1609

unsigned int *insn = (unsigned int *)(unsigned long)*p;

1606

1610

1607

page_offset_shift_patch_one(insn, phys_bits);

1611

page_offset_shift_patch_one(insn, phys_bits);

1608

1612

1609

p++;

1613

p++;

1610

}

1614

}

1611

}

1615

}

1612

1616

1613

static void __init setup_page_offset(void)

1617

static void __init setup_page_offset(void)

1614

{

1618

{

1615

unsigned long max_phys_bits = 40;

1619

unsigned long max_phys_bits = 40;

1616

1620

1617

if (tlb_type == cheetah || tlb_type == cheetah_plus) {

1621

if (tlb_type == cheetah || tlb_type == cheetah_plus) {

1618

max_phys_bits = 42;

1622

max_phys_bits = 42;

1619

} else if (tlb_type == hypervisor) {

1623

} else if (tlb_type == hypervisor) {

1620

switch (sun4v_chip_type) {

1624

switch (sun4v_chip_type) {

1621

case SUN4V_CHIP_NIAGARA1:

1625

case SUN4V_CHIP_NIAGARA1:

1622

case SUN4V_CHIP_NIAGARA2:

1626

case SUN4V_CHIP_NIAGARA2:

1623

max_phys_bits = 39;

1627

max_phys_bits = 39;

1624

break;

1628

break;

1625

case SUN4V_CHIP_NIAGARA3:

1629

case SUN4V_CHIP_NIAGARA3:

1626

max_phys_bits = 43;

1630

max_phys_bits = 43;

1627

break;

1631

break;

1628

case SUN4V_CHIP_NIAGARA4:

1632

case SUN4V_CHIP_NIAGARA4:

1629

case SUN4V_CHIP_NIAGARA5:

1633

case SUN4V_CHIP_NIAGARA5:

1630

case SUN4V_CHIP_SPARC64X:

1634

case SUN4V_CHIP_SPARC64X:

1631

default:

1635

default:

1632

max_phys_bits = 47;

1636

max_phys_bits = 47;

1633

break;

1637

break;

1634

}

1638

}

1635

}

1639

}

1636

1640

1637

if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {

1641

if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {

1638

prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",

1642

prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",

1639

max_phys_bits);

1643

max_phys_bits);

1640

prom_halt();

1644

prom_halt();

1641

}

1645

}

1642

1646

1643

PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);

1647

PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);

1644

1648

1645

pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",

1649

pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",

1646

PAGE_OFFSET, max_phys_bits);

1650

PAGE_OFFSET, max_phys_bits);

1647

1651

1648

page_offset_shift_patch(max_phys_bits);

1652

page_offset_shift_patch(max_phys_bits);

1649

}

1653

}

1650

1654

1651

static void __init tsb_phys_patch(void)

1655

static void __init tsb_phys_patch(void)

1652

{

1656

{

1653

struct tsb_ldquad_phys_patch_entry *pquad;

1657

struct tsb_ldquad_phys_patch_entry *pquad;

1654

struct tsb_phys_patch_entry *p;

1658

struct tsb_phys_patch_entry *p;

1655

1659

1656

pquad = &__tsb_ldquad_phys_patch;

1660

pquad = &__tsb_ldquad_phys_patch;

1657

while (pquad < &__tsb_ldquad_phys_patch_end) {

1661

while (pquad < &__tsb_ldquad_phys_patch_end) {

1658

unsigned long addr = pquad->addr;

1662

unsigned long addr = pquad->addr;

1659

1663

1660

if (tlb_type == hypervisor)

1664

if (tlb_type == hypervisor)

1661

*(unsigned int *) addr = pquad->sun4v_insn;

1665

*(unsigned int *) addr = pquad->sun4v_insn;

1662

else

1666

else

1663

*(unsigned int *) addr = pquad->sun4u_insn;

1667

*(unsigned int *) addr = pquad->sun4u_insn;

1664

wmb();

1668

wmb();

1665

__asm__ __volatile__("flush %0"

1669

__asm__ __volatile__("flush %0"

1666

: /* no outputs */

1670

: /* no outputs */

1667

: "r" (addr));

1671

: "r" (addr));

1668

1672

1669

pquad++;

1673

pquad++;

1670

}

1674

}

1671

1675

1672

p = &__tsb_phys_patch;

1676

p = &__tsb_phys_patch;

1673

while (p < &__tsb_phys_patch_end) {

1677

while (p < &__tsb_phys_patch_end) {

1674

unsigned long addr = p->addr;

1678

unsigned long addr = p->addr;

1675

1679

1676

*(unsigned int *) addr = p->insn;

1680

*(unsigned int *) addr = p->insn;

1677

wmb();

1681

wmb();

1678

__asm__ __volatile__("flush %0"

1682

__asm__ __volatile__("flush %0"

1679

: /* no outputs */

1683

: /* no outputs */

1680

: "r" (addr));

1684

: "r" (addr));

1681

1685

1682

p++;

1686

p++;

1683

}

1687

}

1684

}

1688

}

1685

1689

1686

/* Don't mark as init, we give this to the Hypervisor. */

1690

/* Don't mark as init, we give this to the Hypervisor. */

1687

#ifndef CONFIG_DEBUG_PAGEALLOC

1691

#ifndef CONFIG_DEBUG_PAGEALLOC

1688

#define NUM_KTSB_DESCR 2

1692

#define NUM_KTSB_DESCR 2

1689

#else

1693

#else

1690

#define NUM_KTSB_DESCR 1

1694

#define NUM_KTSB_DESCR 1

1691

#endif

1695

#endif

1692

static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];

1696

static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];

1693

extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];

1697

extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];

1694

1698

1695

static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)

1699

static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)

1696

{

1700

{

1697

pa >>= KTSB_PHYS_SHIFT;

1701

pa >>= KTSB_PHYS_SHIFT;

1698

1702

1699

while (start < end) {

1703

while (start < end) {

1700

unsigned int *ia = (unsigned int *)(unsigned long)*start;

1704

unsigned int *ia = (unsigned int *)(unsigned long)*start;

1701

1705

1702

ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10);

1706

ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10);

1703

__asm__ __volatile__("flush %0" : : "r" (ia));

1707

__asm__ __volatile__("flush %0" : : "r" (ia));

1704

1708

1705

ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff);

1709

ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff);

1706

__asm__ __volatile__("flush %0" : : "r" (ia + 1));

1710

__asm__ __volatile__("flush %0" : : "r" (ia + 1));

1707

1711

1708

start++;

1712

start++;

1709

}

1713

}

1710

}

1714

}

1711

1715

1712

static void ktsb_phys_patch(void)

1716

static void ktsb_phys_patch(void)

1713

{

1717

{

1714

extern unsigned int __swapper_tsb_phys_patch;

1718

extern unsigned int __swapper_tsb_phys_patch;

1715

extern unsigned int __swapper_tsb_phys_patch_end;

1719

extern unsigned int __swapper_tsb_phys_patch_end;

1716

unsigned long ktsb_pa;

1720

unsigned long ktsb_pa;

1717

1721

1718

ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);

1722

ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);

1719

patch_one_ktsb_phys(&__swapper_tsb_phys_patch,

1723

patch_one_ktsb_phys(&__swapper_tsb_phys_patch,

1720

&__swapper_tsb_phys_patch_end, ktsb_pa);

1724

&__swapper_tsb_phys_patch_end, ktsb_pa);

1721

#ifndef CONFIG_DEBUG_PAGEALLOC

1725

#ifndef CONFIG_DEBUG_PAGEALLOC

1722

{

1726

{

1723

extern unsigned int __swapper_4m_tsb_phys_patch;

1727

extern unsigned int __swapper_4m_tsb_phys_patch;

1724

extern unsigned int __swapper_4m_tsb_phys_patch_end;

1728

extern unsigned int __swapper_4m_tsb_phys_patch_end;

1725

ktsb_pa = (kern_base +

1729

ktsb_pa = (kern_base +

1726

((unsigned long)&swapper_4m_tsb[0] - KERNBASE));

1730

((unsigned long)&swapper_4m_tsb[0] - KERNBASE));

1727

patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,

1731

patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,

1728

&__swapper_4m_tsb_phys_patch_end, ktsb_pa);

1732

&__swapper_4m_tsb_phys_patch_end, ktsb_pa);

1729

}

1733

}

1730

#endif

1734

#endif

1731

}

1735

}

1732

1736

1733

static void __init sun4v_ktsb_init(void)

1737

static void __init sun4v_ktsb_init(void)

1734

{

1738

{

1735

unsigned long ktsb_pa;

1739

unsigned long ktsb_pa;

1736

1740

1737

/* First KTSB for PAGE_SIZE mappings. */

1741

/* First KTSB for PAGE_SIZE mappings. */

1738

ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);

1742

ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);

1739

1743

1740

switch (PAGE_SIZE) {

1744

switch (PAGE_SIZE) {

1741

case 8 * 1024:

1745

case 8 * 1024:

1742

default:

1746

default:

1743

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;

1747

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;

1744

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;

1748

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;

1745

break;

1749

break;

1746

1750

1747

case 64 * 1024:

1751

case 64 * 1024:

1748

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;

1752

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;

1749

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;

1753

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;

1750

break;

1754

break;

1751

1755

1752

case 512 * 1024:

1756

case 512 * 1024:

1753

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;

1757

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;

1754

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;

1758

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;

1755

break;

1759

break;

1756

1760

1757

case 4 * 1024 * 1024:

1761

case 4 * 1024 * 1024:

1758

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;

1762

ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;

1759

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;

1763

ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;

1760

break;

1764

break;

1761

}

1765

}

1762

1766

1763

ktsb_descr[0].assoc = 1;

1767

ktsb_descr[0].assoc = 1;

1764

ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;

1768

ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;

1765

ktsb_descr[0].ctx_idx = 0;

1769

ktsb_descr[0].ctx_idx = 0;

1766

ktsb_descr[0].tsb_base = ktsb_pa;

1770

ktsb_descr[0].tsb_base = ktsb_pa;

1767

ktsb_descr[0].resv = 0;

1771

ktsb_descr[0].resv = 0;

1768

1772

1769

#ifndef CONFIG_DEBUG_PAGEALLOC

1773

#ifndef CONFIG_DEBUG_PAGEALLOC

1770

/* Second KTSB for 4MB/256MB/2GB/16GB mappings. */

1774

/* Second KTSB for 4MB/256MB/2GB/16GB mappings. */

1771

ktsb_pa = (kern_base +

1775

ktsb_pa = (kern_base +

1772

((unsigned long)&swapper_4m_tsb[0] - KERNBASE));

1776

((unsigned long)&swapper_4m_tsb[0] - KERNBASE));

1773

1777

1774

ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;

1778

ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;

1775

ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |

1779

ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |

1776

HV_PGSZ_MASK_256MB |

1780

HV_PGSZ_MASK_256MB |

1777

HV_PGSZ_MASK_2GB |

1781

HV_PGSZ_MASK_2GB |

1778

HV_PGSZ_MASK_16GB) &

1782

HV_PGSZ_MASK_16GB) &

1779

cpu_pgsz_mask);

1783

cpu_pgsz_mask);

1780

ktsb_descr[1].assoc = 1;

1784

ktsb_descr[1].assoc = 1;

1781

ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;

1785

ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;

1782

ktsb_descr[1].ctx_idx = 0;

1786

ktsb_descr[1].ctx_idx = 0;

1783

ktsb_descr[1].tsb_base = ktsb_pa;

1787

ktsb_descr[1].tsb_base = ktsb_pa;

1784

ktsb_descr[1].resv = 0;

1788

ktsb_descr[1].resv = 0;

1785

#endif

1789

#endif

1786

}

1790

}

1787

1791

1788

void sun4v_ktsb_register(void)

1792

void sun4v_ktsb_register(void)

1789

{

1793

{

1790

unsigned long pa, ret;

1794

unsigned long pa, ret;

1791

1795

1792

pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);

1796

pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);

1793

1797

1794

ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);

1798

ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);

1795

if (ret != 0) {

1799

if (ret != 0) {

1796

prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "

1800

prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "

1797

"errors with %lx\n", pa, ret);

1801

"errors with %lx\n", pa, ret);

1798

prom_halt();

1802

prom_halt();

1799

}

1803

}

1800

}

1804

}

1801

1805

1802

static void __init sun4u_linear_pte_xor_finalize(void)

1806

static void __init sun4u_linear_pte_xor_finalize(void)

1803

{

1807

{

1804

#ifndef CONFIG_DEBUG_PAGEALLOC

1808

#ifndef CONFIG_DEBUG_PAGEALLOC

1805

/* This is where we would add Panther support for

1809

/* This is where we would add Panther support for

1806

* 32MB and 256MB pages.

1810

* 32MB and 256MB pages.

1807

*/

1811

*/

1808

#endif

1812

#endif

1809

}

1813

}

1810

1814

1811

static void __init sun4v_linear_pte_xor_finalize(void)

1815

static void __init sun4v_linear_pte_xor_finalize(void)

1812

{

1816

{

1813

#ifndef CONFIG_DEBUG_PAGEALLOC

1817

#ifndef CONFIG_DEBUG_PAGEALLOC

1814

if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {

1818

if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {

1815

kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^

1819

kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^

1816

PAGE_OFFSET;

1820

PAGE_OFFSET;

1817

kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1821

kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1818

_PAGE_P_4V | _PAGE_W_4V);

1822

_PAGE_P_4V | _PAGE_W_4V);

1819

} else {

1823

} else {

1820

kern_linear_pte_xor[1] = kern_linear_pte_xor[0];

1824

kern_linear_pte_xor[1] = kern_linear_pte_xor[0];

1821

}

1825

}

1822

1826

1823

if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {

1827

if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {

1824

kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^

1828

kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^

1825

PAGE_OFFSET;

1829

PAGE_OFFSET;

1826

kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1830

kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1827

_PAGE_P_4V | _PAGE_W_4V);

1831

_PAGE_P_4V | _PAGE_W_4V);

1828

} else {

1832

} else {

1829

kern_linear_pte_xor[2] = kern_linear_pte_xor[1];

1833

kern_linear_pte_xor[2] = kern_linear_pte_xor[1];

1830

}

1834

}

1831

1835

1832

if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {

1836

if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {

1833

kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^

1837

kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^

1834

PAGE_OFFSET;

1838

PAGE_OFFSET;

1835

kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1839

kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |

1836

_PAGE_P_4V | _PAGE_W_4V);

1840

_PAGE_P_4V | _PAGE_W_4V);

1837

} else {

1841

} else {

1838

kern_linear_pte_xor[3] = kern_linear_pte_xor[2];

1842

kern_linear_pte_xor[3] = kern_linear_pte_xor[2];

1839

}

1843

}

1840

#endif

1844

#endif

1841

}

1845

}

1842

1846

1843

/* paging_init() sets up the page tables */

1847

/* paging_init() sets up the page tables */

1844

1848

1845

static unsigned long last_valid_pfn;

1849

static unsigned long last_valid_pfn;

1846

pgd_t swapper_pg_dir[PTRS_PER_PGD];

1850

pgd_t swapper_pg_dir[PTRS_PER_PGD];

1847

1851

1848

static void sun4u_pgprot_init(void);

1852

static void sun4u_pgprot_init(void);

1849

static void sun4v_pgprot_init(void);

1853

static void sun4v_pgprot_init(void);

1850

1854

1851

void __init paging_init(void)

1855

void __init paging_init(void)

1852

{

1856

{

1853

unsigned long end_pfn, shift, phys_base;

1857

unsigned long end_pfn, shift, phys_base;

1854

unsigned long real_end, i;

1858

unsigned long real_end, i;

1855

int node;

1859

int node;

1856

1860

1857

setup_page_offset();

1861

setup_page_offset();

1858

1862

1859

/* These build time checkes make sure that the dcache_dirty_cpu()

1863

/* These build time checkes make sure that the dcache_dirty_cpu()

1860

* page->flags usage will work.

1864

* page->flags usage will work.

1861

*

1865

*

1862

* When a page gets marked as dcache-dirty, we store the

1866

* When a page gets marked as dcache-dirty, we store the

1863

* cpu number starting at bit 32 in the page->flags. Also,

1867

* cpu number starting at bit 32 in the page->flags. Also,

1864

* functions like clear_dcache_dirty_cpu use the cpu mask

1868

* functions like clear_dcache_dirty_cpu use the cpu mask

1865

* in 13-bit signed-immediate instruction fields.

1869

* in 13-bit signed-immediate instruction fields.

1866

*/

1870

*/

1867

1871

1868

/*

1872

/*

1869

* Page flags must not reach into upper 32 bits that are used

1873

* Page flags must not reach into upper 32 bits that are used

1870

* for the cpu number

1874

* for the cpu number

1871

*/

1875

*/

1872

BUILD_BUG_ON(NR_PAGEFLAGS > 32);

1876

BUILD_BUG_ON(NR_PAGEFLAGS > 32);

1873

1877

1874

/*

1878

/*

1875

* The bit fields placed in the high range must not reach below

1879

* The bit fields placed in the high range must not reach below

1876

* the 32 bit boundary. Otherwise we cannot place the cpu field

1880

* the 32 bit boundary. Otherwise we cannot place the cpu field

1877

* at the 32 bit boundary.

1881

* at the 32 bit boundary.

1878

*/

1882

*/

1879

BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +

1883

BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +

1880

ilog2(roundup_pow_of_two(NR_CPUS)) > 32);

1884

ilog2(roundup_pow_of_two(NR_CPUS)) > 32);

1881

1885

1882

BUILD_BUG_ON(NR_CPUS > 4096);

1886

BUILD_BUG_ON(NR_CPUS > 4096);

1883

1887

1884

kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;

1888

kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;

1885

kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;

1889

kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;

1886

1890

1887

/* Invalidate both kernel TSBs. */

1891

/* Invalidate both kernel TSBs. */

1888

memset(swapper_tsb, 0x40, sizeof(swapper_tsb));

1892

memset(swapper_tsb, 0x40, sizeof(swapper_tsb));

1889

#ifndef CONFIG_DEBUG_PAGEALLOC

1893

#ifndef CONFIG_DEBUG_PAGEALLOC

1890

memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));

1894

memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));

1891

#endif

1895

#endif

1892

1896

1893

if (tlb_type == hypervisor)

1897

if (tlb_type == hypervisor)

1894

sun4v_pgprot_init();

1898

sun4v_pgprot_init();

1895

else

1899

else

1896

sun4u_pgprot_init();

1900

sun4u_pgprot_init();

1897

1901

1898

if (tlb_type == cheetah_plus ||

1902

if (tlb_type == cheetah_plus ||

1899

tlb_type == hypervisor) {

1903

tlb_type == hypervisor) {

1900

tsb_phys_patch();

1904

tsb_phys_patch();

1901

ktsb_phys_patch();

1905

ktsb_phys_patch();

1902

}

1906

}

1903

1907

1904

if (tlb_type == hypervisor)

1908

if (tlb_type == hypervisor)

1905

sun4v_patch_tlb_handlers();

1909

sun4v_patch_tlb_handlers();

1906

1910

1907

/* Find available physical memory...

1911

/* Find available physical memory...

1908

*

1912

*

1909

* Read it twice in order to work around a bug in openfirmware.

1913

* Read it twice in order to work around a bug in openfirmware.

1910

* The call to grab this table itself can cause openfirmware to

1914

* The call to grab this table itself can cause openfirmware to

1911

* allocate memory, which in turn can take away some space from

1915

* allocate memory, which in turn can take away some space from

1912

* the list of available memory. Reading it twice makes sure

1916

* the list of available memory. Reading it twice makes sure

1913

* we really do get the final value.

1917

* we really do get the final value.

1914

*/

1918

*/

1915

read_obp_translations();

1919

read_obp_translations();

1916

read_obp_memory("reg", &pall[0], &pall_ents);

1920

read_obp_memory("reg", &pall[0], &pall_ents);

1917

read_obp_memory("available", &pavail[0], &pavail_ents);

1921

read_obp_memory("available", &pavail[0], &pavail_ents);

1918

read_obp_memory("available", &pavail[0], &pavail_ents);

1922

read_obp_memory("available", &pavail[0], &pavail_ents);

1919

1923

1920

phys_base = 0xffffffffffffffffUL;

1924

phys_base = 0xffffffffffffffffUL;

1921

for (i = 0; i < pavail_ents; i++) {

1925

for (i = 0; i < pavail_ents; i++) {

1922

phys_base = min(phys_base, pavail[i].phys_addr);

1926

phys_base = min(phys_base, pavail[i].phys_addr);

1923

memblock_add(pavail[i].phys_addr, pavail[i].reg_size);

1927

memblock_add(pavail[i].phys_addr, pavail[i].reg_size);

1924

}

1928

}

1925

1929

1926

memblock_reserve(kern_base, kern_size);

1930

memblock_reserve(kern_base, kern_size);

1927

1931

1928

find_ramdisk(phys_base);

1932

find_ramdisk(phys_base);

1929

1933

1930

memblock_enforce_memory_limit(cmdline_memory_size);

1934

memblock_enforce_memory_limit(cmdline_memory_size);

1931

1935

1932

memblock_allow_resize();

1936

memblock_allow_resize();

1933

memblock_dump_all();

1937

memblock_dump_all();

1934

1938

1935

set_bit(0, mmu_context_bmap);

1939

set_bit(0, mmu_context_bmap);

1936

1940

1937

shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);

1941

shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);

1938

1942

1939

real_end = (unsigned long)_end;

1943

real_end = (unsigned long)_end;

1940

num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);

1944

num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);

1941

printk("Kernel: Using %d locked TLB entries for main kernel image.\n",

1945

printk("Kernel: Using %d locked TLB entries for main kernel image.\n",

1942

num_kernel_image_mappings);

1946

num_kernel_image_mappings);

1943

1947

1944

/* Set kernel pgd to upper alias so physical page computations

1948

/* Set kernel pgd to upper alias so physical page computations

1945

* work.

1949

* work.

1946

*/

1950

*/

1947

init_mm.pgd += ((shift) / (sizeof(pgd_t)));

1951

init_mm.pgd += ((shift) / (sizeof(pgd_t)));

1948

1952

1949

memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));

1953

memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));

1950

1954

1951

/* Now can init the kernel/bad page tables. */

1955

/* Now can init the kernel/bad page tables. */

1952

pud_set(pud_offset(&swapper_pg_dir[0], 0),

1956

pud_set(pud_offset(&swapper_pg_dir[0], 0),

1953

swapper_low_pmd_dir + (shift / sizeof(pgd_t)));

1957

swapper_low_pmd_dir + (shift / sizeof(pgd_t)));

1954

1958

1955

inherit_prom_mappings();

1959

inherit_prom_mappings();

1956

1960

1957

init_kpte_bitmap();

1961

init_kpte_bitmap();

1958

1962

1959

/* Ok, we can use our TLB miss and window trap handlers safely. */

1963

/* Ok, we can use our TLB miss and window trap handlers safely. */

1960

setup_tba();

1964

setup_tba();

1961

1965

1962

__flush_tlb_all();

1966

__flush_tlb_all();

1963

1967

1964

prom_build_devicetree();

1968

prom_build_devicetree();

1965

of_populate_present_mask();

1969

of_populate_present_mask();

1966

#ifndef CONFIG_SMP

1970

#ifndef CONFIG_SMP

1967

of_fill_in_cpu_data();

1971

of_fill_in_cpu_data();

1968

#endif

1972

#endif

1969

1973

1970

if (tlb_type == hypervisor) {

1974

if (tlb_type == hypervisor) {

1971

sun4v_mdesc_init();

1975

sun4v_mdesc_init();

1972

mdesc_populate_present_mask(cpu_all_mask);

1976

mdesc_populate_present_mask(cpu_all_mask);

1973

#ifndef CONFIG_SMP

1977

#ifndef CONFIG_SMP

1974

mdesc_fill_in_cpu_data(cpu_all_mask);

1978

mdesc_fill_in_cpu_data(cpu_all_mask);

1975

#endif

1979

#endif

1976

mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);

1980

mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);

1977

1981

1978

sun4v_linear_pte_xor_finalize();

1982

sun4v_linear_pte_xor_finalize();

1979

1983

1980

sun4v_ktsb_init();

1984

sun4v_ktsb_init();

1981

sun4v_ktsb_register();

1985

sun4v_ktsb_register();

1982

} else {

1986

} else {

1983

unsigned long impl, ver;

1987

unsigned long impl, ver;

1984

1988

1985

cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |

1989

cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |

1986

HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);

1990

HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);

1987

1991

1988

__asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));

1992

__asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));

1989

impl = ((ver >> 32) & 0xffff);

1993

impl = ((ver >> 32) & 0xffff);

1990

if (impl == PANTHER_IMPL)

1994

if (impl == PANTHER_IMPL)

1991

cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |

1995

cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |

1992

HV_PGSZ_MASK_256MB);

1996

HV_PGSZ_MASK_256MB);

1993

1997

1994

sun4u_linear_pte_xor_finalize();

1998

sun4u_linear_pte_xor_finalize();

1995

}

1999

}

1996

2000

1997

/* Flush the TLBs and the 4M TSB so that the updated linear

2001

/* Flush the TLBs and the 4M TSB so that the updated linear

1998

* pte XOR settings are realized for all mappings.

2002

* pte XOR settings are realized for all mappings.

1999

*/

2003

*/

2000

__flush_tlb_all();

2004

__flush_tlb_all();

2001

#ifndef CONFIG_DEBUG_PAGEALLOC

2005

#ifndef CONFIG_DEBUG_PAGEALLOC

2002

memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));

2006

memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));

2003

#endif

2007

#endif

2004

__flush_tlb_all();

2008

__flush_tlb_all();

2005

2009

2006

/* Setup bootmem... */

2010

/* Setup bootmem... */

2007

last_valid_pfn = end_pfn = bootmem_init(phys_base);

2011

last_valid_pfn = end_pfn = bootmem_init(phys_base);

2008

2012

2009

/* Once the OF device tree and MDESC have been setup, we know

2013

/* Once the OF device tree and MDESC have been setup, we know

2010

* the list of possible cpus. Therefore we can allocate the

2014

* the list of possible cpus. Therefore we can allocate the

2011

* IRQ stacks.

2015

* IRQ stacks.

2012

*/

2016

*/

2013

for_each_possible_cpu(i) {

2017

for_each_possible_cpu(i) {

2014

node = cpu_to_node(i);

2018

node = cpu_to_node(i);

2015

2019

2016

softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),

2020

softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),

2017

THREAD_SIZE,

2021

THREAD_SIZE,

2018

THREAD_SIZE, 0);

2022

THREAD_SIZE, 0);

2019

hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),

2023

hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),

2020

THREAD_SIZE,

2024

THREAD_SIZE,

2021

THREAD_SIZE, 0);

2025

THREAD_SIZE, 0);

2022

}

2026

}

2023

2027

2024

kernel_physical_mapping_init();

2028

kernel_physical_mapping_init();

2025

2029

2026

{

2030

{

2027

unsigned long max_zone_pfns[MAX_NR_ZONES];

2031

unsigned long max_zone_pfns[MAX_NR_ZONES];

2028

2032

2029

memset(max_zone_pfns, 0, sizeof(max_zone_pfns));

2033

memset(max_zone_pfns, 0, sizeof(max_zone_pfns));

2030

2034

2031

max_zone_pfns[ZONE_NORMAL] = end_pfn;

2035

max_zone_pfns[ZONE_NORMAL] = end_pfn;

2032

2036

2033

free_area_init_nodes(max_zone_pfns);

2037

free_area_init_nodes(max_zone_pfns);

2034

}

2038

}

2035

2039

2036

printk("Booting Linux...\n");

2040

printk("Booting Linux...\n");

2037

}

2041

}

2038

2042

2039

int page_in_phys_avail(unsigned long paddr)

2043

int page_in_phys_avail(unsigned long paddr)

2040

{

2044

{

2041

int i;

2045

int i;

2042

2046

2043

paddr &= PAGE_MASK;

2047

paddr &= PAGE_MASK;

2044

2048

2045

for (i = 0; i < pavail_ents; i++) {

2049

for (i = 0; i < pavail_ents; i++) {

2046

unsigned long start, end;

2050

unsigned long start, end;

2047

2051

2048

start = pavail[i].phys_addr;

2052

start = pavail[i].phys_addr;

2049

end = start + pavail[i].reg_size;

2053

end = start + pavail[i].reg_size;

2050

2054

2051

if (paddr >= start && paddr < end)

2055

if (paddr >= start && paddr < end)

2052

return 1;

2056

return 1;

2053

}

2057

}

2054

if (paddr >= kern_base && paddr < (kern_base + kern_size))

2058

if (paddr >= kern_base && paddr < (kern_base + kern_size))

2055

return 1;

2059

return 1;

2056

#ifdef CONFIG_BLK_DEV_INITRD

2060

#ifdef CONFIG_BLK_DEV_INITRD

2057

if (paddr >= __pa(initrd_start) &&

2061

if (paddr >= __pa(initrd_start) &&

2058

paddr < __pa(PAGE_ALIGN(initrd_end)))

2062

paddr < __pa(PAGE_ALIGN(initrd_end)))

2059

return 1;

2063

return 1;

2060

#endif

2064

#endif

2061

2065

2062

return 0;

2066

return 0;

2063

}

2067

}

2064

2068

2065

static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;

2069

static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;

2066

static int pavail_rescan_ents __initdata;

2070

static int pavail_rescan_ents __initdata;

2067

2071

2068

/* Certain OBP calls, such as fetching "available" properties, can

2072

/* Certain OBP calls, such as fetching "available" properties, can

2069

* claim physical memory. So, along with initializing the valid

2073

* claim physical memory. So, along with initializing the valid

2070

* address bitmap, what we do here is refetch the physical available

2074

* address bitmap, what we do here is refetch the physical available

2071

* memory list again, and make sure it provides at least as much

2075

* memory list again, and make sure it provides at least as much

2072

* memory as 'pavail' does.

2076

* memory as 'pavail' does.

2073

*/

2077

*/

2074

static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)

2078

static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)

2075

{

2079

{

2076

int i;

2080

int i;

2077

2081

2078

read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);

2082

read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);

2079

2083

2080

for (i = 0; i < pavail_ents; i++) {

2084

for (i = 0; i < pavail_ents; i++) {

2081

unsigned long old_start, old_end;

2085

unsigned long old_start, old_end;

2082

2086

2083

old_start = pavail[i].phys_addr;

2087

old_start = pavail[i].phys_addr;

2084

old_end = old_start + pavail[i].reg_size;

2088

old_end = old_start + pavail[i].reg_size;

2085

while (old_start < old_end) {

2089

while (old_start < old_end) {

2086

int n;

2090

int n;

2087

2091

2088

for (n = 0; n < pavail_rescan_ents; n++) {

2092

for (n = 0; n < pavail_rescan_ents; n++) {

2089

unsigned long new_start, new_end;

2093

unsigned long new_start, new_end;

2090

2094

2091

new_start = pavail_rescan[n].phys_addr;

2095

new_start = pavail_rescan[n].phys_addr;

2092

new_end = new_start +

2096

new_end = new_start +

2093

pavail_rescan[n].reg_size;

2097

pavail_rescan[n].reg_size;

2094

2098

2095

if (new_start <= old_start &&

2099

if (new_start <= old_start &&

2096

new_end >= (old_start + PAGE_SIZE)) {

2100

new_end >= (old_start + PAGE_SIZE)) {

2097

set_bit(old_start >> ILOG2_4MB, bitmap);

2101

set_bit(old_start >> ILOG2_4MB, bitmap);

2098

goto do_next_page;

2102

goto do_next_page;

2099

}

2103

}

2100

}

2104

}

2101

2105

2102

prom_printf("mem_init: Lost memory in pavail\n");

2106

prom_printf("mem_init: Lost memory in pavail\n");

2103

prom_printf("mem_init: OLD start[%lx] size[%lx]\n",

2107

prom_printf("mem_init: OLD start[%lx] size[%lx]\n",

2104

pavail[i].phys_addr,

2108

pavail[i].phys_addr,

2105

pavail[i].reg_size);

2109

pavail[i].reg_size);

2106

prom_printf("mem_init: NEW start[%lx] size[%lx]\n",

2110

prom_printf("mem_init: NEW start[%lx] size[%lx]\n",

2107

pavail_rescan[i].phys_addr,

2111

pavail_rescan[i].phys_addr,

2108

pavail_rescan[i].reg_size);

2112

pavail_rescan[i].reg_size);

2109

prom_printf("mem_init: Cannot continue, aborting.\n");

2113

prom_printf("mem_init: Cannot continue, aborting.\n");

2110

prom_halt();

2114

prom_halt();

2111

2115

2112

do_next_page:

2116

do_next_page:

2113

old_start += PAGE_SIZE;

2117

old_start += PAGE_SIZE;

2114

}

2118

}

2115

}

2119

}

2116

}

2120

}

2117

2121

2118

static void __init patch_tlb_miss_handler_bitmap(void)

2122

static void __init patch_tlb_miss_handler_bitmap(void)

2119

{

2123

{

2120

extern unsigned int valid_addr_bitmap_insn[];

2124

extern unsigned int valid_addr_bitmap_insn[];

2121

extern unsigned int valid_addr_bitmap_patch[];

2125

extern unsigned int valid_addr_bitmap_patch[];

2122

2126

2123

valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];

2127

valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];

2124

mb();

2128

mb();

2125

valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];

2129

valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];

2126

flushi(&valid_addr_bitmap_insn[0]);

2130

flushi(&valid_addr_bitmap_insn[0]);

2127

}

2131

}

2128

2132

2129

static void __init register_page_bootmem_info(void)

2133

static void __init register_page_bootmem_info(void)

2130

{

2134

{

2131

#ifdef CONFIG_NEED_MULTIPLE_NODES

2135

#ifdef CONFIG_NEED_MULTIPLE_NODES

2132

int i;

2136

int i;

2133

2137

2134

for_each_online_node(i)

2138

for_each_online_node(i)

2135

if (NODE_DATA(i)->node_spanned_pages)

2139

if (NODE_DATA(i)->node_spanned_pages)

2136

register_page_bootmem_info_node(NODE_DATA(i));

2140

register_page_bootmem_info_node(NODE_DATA(i));

2137

#endif

2141

#endif

2138

}

2142

}

2139

void __init mem_init(void)

2143

void __init mem_init(void)

2140

{

2144

{

2141

unsigned long addr, last;

2145

unsigned long addr, last;

2142

2146

2143

addr = PAGE_OFFSET + kern_base;

2147

addr = PAGE_OFFSET + kern_base;

2144

last = PAGE_ALIGN(kern_size) + addr;

2148

last = PAGE_ALIGN(kern_size) + addr;

2145

while (addr < last) {

2149

while (addr < last) {

2146

set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);

2150

set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);

2147

addr += PAGE_SIZE;

2151

addr += PAGE_SIZE;

2148

}

2152

}

2149

2153

2150

setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);

2154

setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);

2151

patch_tlb_miss_handler_bitmap();

2155

patch_tlb_miss_handler_bitmap();

2152

2156

2153

high_memory = __va(last_valid_pfn << PAGE_SHIFT);

2157

high_memory = __va(last_valid_pfn << PAGE_SHIFT);

2154

2158

2155

register_page_bootmem_info();

2159

register_page_bootmem_info();

2156

free_all_bootmem();

2160

free_all_bootmem();

2157

2161

2158

/*

2162

/*

2159

* Set up the zero page, mark it reserved, so that page count

2163

* Set up the zero page, mark it reserved, so that page count

2160

* is not manipulated when freeing the page from user ptes.

2164

* is not manipulated when freeing the page from user ptes.

2161

*/

2165

*/

2162

mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);

2166

mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);

2163

if (mem_map_zero == NULL) {

2167

if (mem_map_zero == NULL) {

2164

prom_printf("paging_init: Cannot alloc zero page.\n");

2168

prom_printf("paging_init: Cannot alloc zero page.\n");

2165

prom_halt();

2169

prom_halt();

2166

}

2170

}

2167

mark_page_reserved(mem_map_zero);

2171

mark_page_reserved(mem_map_zero);

2168

2172

2169

mem_init_print_info(NULL);

2173

mem_init_print_info(NULL);

2170

2174

2171

if (tlb_type == cheetah || tlb_type == cheetah_plus)

2175

if (tlb_type == cheetah || tlb_type == cheetah_plus)

2172

cheetah_ecache_flush_init();

2176

cheetah_ecache_flush_init();

2173

}

2177

}

2174

2178

2175

void free_initmem(void)

2179

void free_initmem(void)

2176

{

2180

{

2177

unsigned long addr, initend;

2181

unsigned long addr, initend;

2178

int do_free = 1;

2182

int do_free = 1;

2179

2183

2180

/* If the physical memory maps were trimmed by kernel command

2184

/* If the physical memory maps were trimmed by kernel command

2181

* line options, don't even try freeing this initmem stuff up.

2185

* line options, don't even try freeing this initmem stuff up.

2182

* The kernel image could have been in the trimmed out region

2186

* The kernel image could have been in the trimmed out region

2183

* and if so the freeing below will free invalid page structs.

2187

* and if so the freeing below will free invalid page structs.

2184

*/

2188

*/

2185

if (cmdline_memory_size)

2189

if (cmdline_memory_size)

2186

do_free = 0;

2190

do_free = 0;

2187

2191

2188

/*

2192

/*

2189

* The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.

2193

* The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.

2190

*/

2194

*/

2191

addr = PAGE_ALIGN((unsigned long)(__init_begin));

2195

addr = PAGE_ALIGN((unsigned long)(__init_begin));

2192

initend = (unsigned long)(__init_end) & PAGE_MASK;

2196

initend = (unsigned long)(__init_end) & PAGE_MASK;

2193

for (; addr < initend; addr += PAGE_SIZE) {

2197

for (; addr < initend; addr += PAGE_SIZE) {

2194

unsigned long page;

2198

unsigned long page;

2195

2199

2196

page = (addr +

2200

page = (addr +

2197

((unsigned long) __va(kern_base)) -

2201

((unsigned long) __va(kern_base)) -

2198

((unsigned long) KERNBASE));

2202

((unsigned long) KERNBASE));

2199

memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);

2203

memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);

2200

2204

2201

if (do_free)

2205

if (do_free)

2202

free_reserved_page(virt_to_page(page));

2206

free_reserved_page(virt_to_page(page));

2203

}

2207

}

2204

}

2208

}

2205

2209

2206

#ifdef CONFIG_BLK_DEV_INITRD

2210

#ifdef CONFIG_BLK_DEV_INITRD

2207

void free_initrd_mem(unsigned long start, unsigned long end)

2211

void free_initrd_mem(unsigned long start, unsigned long end)

2208

{

2212

{

2209

free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,

2213

free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,

2210

"initrd");

2214

"initrd");

2211

}

2215

}

2212

#endif

2216

#endif

2213

2217

2214

#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)

2218

#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U)

2215

#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)

2219

#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V)

2216

#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)

2220

#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)

2217

#define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)

2221

#define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)

2218

#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)

2222

#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)

2219

#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)

2223

#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)

2220

2224

2221

pgprot_t PAGE_KERNEL __read_mostly;

2225

pgprot_t PAGE_KERNEL __read_mostly;

2222

EXPORT_SYMBOL(PAGE_KERNEL);

2226

EXPORT_SYMBOL(PAGE_KERNEL);

2223

2227

2224

pgprot_t PAGE_KERNEL_LOCKED __read_mostly;

2228

pgprot_t PAGE_KERNEL_LOCKED __read_mostly;

2225

pgprot_t PAGE_COPY __read_mostly;

2229

pgprot_t PAGE_COPY __read_mostly;

2226

2230

2227

pgprot_t PAGE_SHARED __read_mostly;

2231

pgprot_t PAGE_SHARED __read_mostly;

2228

EXPORT_SYMBOL(PAGE_SHARED);

2232

EXPORT_SYMBOL(PAGE_SHARED);

2229

2233

2230

unsigned long pg_iobits __read_mostly;

2234

unsigned long pg_iobits __read_mostly;

2231

2235

2232

unsigned long _PAGE_IE __read_mostly;

2236

unsigned long _PAGE_IE __read_mostly;

2233

EXPORT_SYMBOL(_PAGE_IE);

2237

EXPORT_SYMBOL(_PAGE_IE);

2234

2238

2235

unsigned long _PAGE_E __read_mostly;

2239

unsigned long _PAGE_E __read_mostly;

2236

EXPORT_SYMBOL(_PAGE_E);

2240

EXPORT_SYMBOL(_PAGE_E);

2237

2241

2238

unsigned long _PAGE_CACHE __read_mostly;

2242

unsigned long _PAGE_CACHE __read_mostly;

2239

EXPORT_SYMBOL(_PAGE_CACHE);

2243

EXPORT_SYMBOL(_PAGE_CACHE);

2240

2244

2241

#ifdef CONFIG_SPARSEMEM_VMEMMAP

2245

#ifdef CONFIG_SPARSEMEM_VMEMMAP

2242

unsigned long vmemmap_table[VMEMMAP_SIZE];

2246

unsigned long vmemmap_table[VMEMMAP_SIZE];

2243

2247

2244

static long __meminitdata addr_start, addr_end;

2248

static long __meminitdata addr_start, addr_end;

2245

static int __meminitdata node_start;

2249

static int __meminitdata node_start;

2246

2250

2247

int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,

2251

int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,

2248

int node)

2252

int node)

2249

{

2253

{

2250

unsigned long phys_start = (vstart - VMEMMAP_BASE);

2254

unsigned long phys_start = (vstart - VMEMMAP_BASE);

2251

unsigned long phys_end = (vend - VMEMMAP_BASE);

2255

unsigned long phys_end = (vend - VMEMMAP_BASE);

2252

unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;

2256

unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;

2253

unsigned long end = VMEMMAP_ALIGN(phys_end);

2257

unsigned long end = VMEMMAP_ALIGN(phys_end);

2254

unsigned long pte_base;

2258

unsigned long pte_base;

2255

2259

2256

pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |

2260

pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |

2257

_PAGE_CP_4U | _PAGE_CV_4U |

2261

_PAGE_CP_4U | _PAGE_CV_4U |

2258

_PAGE_P_4U | _PAGE_W_4U);

2262

_PAGE_P_4U | _PAGE_W_4U);

2259

if (tlb_type == hypervisor)

2263

if (tlb_type == hypervisor)

2260

pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |

2264

pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |

2261

_PAGE_CP_4V | _PAGE_CV_4V |

2265

_PAGE_CP_4V | _PAGE_CV_4V |

2262

_PAGE_P_4V | _PAGE_W_4V);

2266

_PAGE_P_4V | _PAGE_W_4V);

2263

2267

2264

for (; addr < end; addr += VMEMMAP_CHUNK) {

2268

for (; addr < end; addr += VMEMMAP_CHUNK) {

2265

unsigned long *vmem_pp =

2269

unsigned long *vmem_pp =

2266

vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);

2270

vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);

2267

void *block;

2271

void *block;

2268

2272

2269

if (!(*vmem_pp & _PAGE_VALID)) {

2273

if (!(*vmem_pp & _PAGE_VALID)) {

2270

block = vmemmap_alloc_block(1UL << ILOG2_4MB, node);

2274

block = vmemmap_alloc_block(1UL << ILOG2_4MB, node);

2271

if (!block)

2275

if (!block)

2272

return -ENOMEM;

2276

return -ENOMEM;

2273

2277

2274

*vmem_pp = pte_base | __pa(block);

2278

*vmem_pp = pte_base | __pa(block);

2275

2279

2276

/* check to see if we have contiguous blocks */

2280

/* check to see if we have contiguous blocks */

2277

if (addr_end != addr || node_start != node) {

2281

if (addr_end != addr || node_start != node) {

2278

if (addr_start)

2282

if (addr_start)

2279

printk(KERN_DEBUG " [%lx-%lx] on node %d\n",

2283

printk(KERN_DEBUG " [%lx-%lx] on node %d\n",

2280

addr_start, addr_end-1, node_start);

2284

addr_start, addr_end-1, node_start);

2281

addr_start = addr;

2285

addr_start = addr;

2282

node_start = node;

2286

node_start = node;

2283

}

2287

}

2284

addr_end = addr + VMEMMAP_CHUNK;

2288

addr_end = addr + VMEMMAP_CHUNK;

2285

}

2289

}

2286

}

2290

}

2287

return 0;

2291

return 0;

2288

}

2292

}

2289

2293

2290

void __meminit vmemmap_populate_print_last(void)

2294

void __meminit vmemmap_populate_print_last(void)

2291

{

2295

{

2292

if (addr_start) {

2296

if (addr_start) {

2293

printk(KERN_DEBUG " [%lx-%lx] on node %d\n",

2297

printk(KERN_DEBUG " [%lx-%lx] on node %d\n",

2294

addr_start, addr_end-1, node_start);

2298

addr_start, addr_end-1, node_start);

2295

addr_start = 0;

2299

addr_start = 0;

2296

addr_end = 0;

2300

addr_end = 0;

2297

node_start = 0;

2301

node_start = 0;

2298

}

2302

}

2299

}

2303

}

2300

2304

2301

void vmemmap_free(unsigned long start, unsigned long end)

2305

void vmemmap_free(unsigned long start, unsigned long end)

2302

{

2306

{

2303

}

2307

}

2304

2308

2305

#endif /* CONFIG_SPARSEMEM_VMEMMAP */

2309

#endif /* CONFIG_SPARSEMEM_VMEMMAP */

2306

2310

2307

static void prot_init_common(unsigned long page_none,

2311

static void prot_init_common(unsigned long page_none,

2308

unsigned long page_shared,

2312

unsigned long page_shared,

2309

unsigned long page_copy,

2313

unsigned long page_copy,

2310

unsigned long page_readonly,

2314

unsigned long page_readonly,

2311

unsigned long page_exec_bit)

2315

unsigned long page_exec_bit)

2312

{

2316

{

2313

PAGE_COPY = __pgprot(page_copy);

2317

PAGE_COPY = __pgprot(page_copy);

2314

PAGE_SHARED = __pgprot(page_shared);

2318

PAGE_SHARED = __pgprot(page_shared);

2315

2319

2316

protection_map[0x0] = __pgprot(page_none);

2320

protection_map[0x0] = __pgprot(page_none);

2317

protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);

2321

protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);

2318

protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);

2322

protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);

2319

protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);

2323

protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);

2320

protection_map[0x4] = __pgprot(page_readonly);

2324

protection_map[0x4] = __pgprot(page_readonly);

2321

protection_map[0x5] = __pgprot(page_readonly);

2325

protection_map[0x5] = __pgprot(page_readonly);

2322

protection_map[0x6] = __pgprot(page_copy);

2326

protection_map[0x6] = __pgprot(page_copy);

2323

protection_map[0x7] = __pgprot(page_copy);

2327

protection_map[0x7] = __pgprot(page_copy);

2324

protection_map[0x8] = __pgprot(page_none);

2328

protection_map[0x8] = __pgprot(page_none);

2325

protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);

2329

protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);

2326

protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);

2330

protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);

2327

protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);

2331

protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);

2328

protection_map[0xc] = __pgprot(page_readonly);

2332

protection_map[0xc] = __pgprot(page_readonly);

2329

protection_map[0xd] = __pgprot(page_readonly);

2333

protection_map[0xd] = __pgprot(page_readonly);

2330

protection_map[0xe] = __pgprot(page_shared);

2334

protection_map[0xe] = __pgprot(page_shared);

2331

protection_map[0xf] = __pgprot(page_shared);

2335

protection_map[0xf] = __pgprot(page_shared);

2332

}

2336

}

2333

2337

2334

static void __init sun4u_pgprot_init(void)

2338

static void __init sun4u_pgprot_init(void)

2335

{

2339

{

2336

unsigned long page_none, page_shared, page_copy, page_readonly;

2340

unsigned long page_none, page_shared, page_copy, page_readonly;

2337

unsigned long page_exec_bit;

2341

unsigned long page_exec_bit;

2338

int i;

2342

int i;

2339

2343

2340

PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |

2344

PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |

2341

_PAGE_CACHE_4U | _PAGE_P_4U |

2345

_PAGE_CACHE_4U | _PAGE_P_4U |

2342

__ACCESS_BITS_4U | __DIRTY_BITS_4U |

2346

__ACCESS_BITS_4U | __DIRTY_BITS_4U |

2343

_PAGE_EXEC_4U);

2347

_PAGE_EXEC_4U);

2344

PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |

2348

PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |

2345

_PAGE_CACHE_4U | _PAGE_P_4U |

2349

_PAGE_CACHE_4U | _PAGE_P_4U |

2346

__ACCESS_BITS_4U | __DIRTY_BITS_4U |

2350

__ACCESS_BITS_4U | __DIRTY_BITS_4U |

2347

_PAGE_EXEC_4U | _PAGE_L_4U);

2351

_PAGE_EXEC_4U | _PAGE_L_4U);

2348

2352

2349

_PAGE_IE = _PAGE_IE_4U;

2353

_PAGE_IE = _PAGE_IE_4U;

2350

_PAGE_E = _PAGE_E_4U;

2354

_PAGE_E = _PAGE_E_4U;

2351

_PAGE_CACHE = _PAGE_CACHE_4U;

2355

_PAGE_CACHE = _PAGE_CACHE_4U;

2352

2356

2353

pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |

2357

pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |

2354

__ACCESS_BITS_4U | _PAGE_E_4U);

2358

__ACCESS_BITS_4U | _PAGE_E_4U);

2355

2359

2356

#ifdef CONFIG_DEBUG_PAGEALLOC

2360

#ifdef CONFIG_DEBUG_PAGEALLOC

2357

kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;

2361

kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;

2358

#else

2362

#else

2359

kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^

2363

kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^

2360

PAGE_OFFSET;

2364

PAGE_OFFSET;

2361

#endif

2365

#endif

2362

kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |

2366

kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |

2363

_PAGE_P_4U | _PAGE_W_4U);

2367

_PAGE_P_4U | _PAGE_W_4U);

2364

2368

2365

for (i = 1; i < 4; i++)

2369

for (i = 1; i < 4; i++)

2366

kern_linear_pte_xor[i] = kern_linear_pte_xor[0];

2370

kern_linear_pte_xor[i] = kern_linear_pte_xor[0];

2367

2371

2368

_PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |

2372

_PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |

2369

_PAGE_SZ64K_4U | _PAGE_SZ8K_4U |

2373

_PAGE_SZ64K_4U | _PAGE_SZ8K_4U |

2370

_PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);

2374

_PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);

2371

2375

2372

2376

2373

page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;

2377

page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;

2374

page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2378

page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2375

__ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);

2379

__ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);

2376

page_copy = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2380

page_copy = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2377

__ACCESS_BITS_4U | _PAGE_EXEC_4U);

2381

__ACCESS_BITS_4U | _PAGE_EXEC_4U);

2378

page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2382

page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |

2379

__ACCESS_BITS_4U | _PAGE_EXEC_4U);

2383

__ACCESS_BITS_4U | _PAGE_EXEC_4U);

2380

2384

2381

page_exec_bit = _PAGE_EXEC_4U;

2385

page_exec_bit = _PAGE_EXEC_4U;

2382

2386

2383

prot_init_common(page_none, page_shared, page_copy, page_readonly,

2387

prot_init_common(page_none, page_shared, page_copy, page_readonly,

2384

page_exec_bit);

2388

page_exec_bit);

2385

}

2389

}

2386

2390

2387

static void __init sun4v_pgprot_init(void)

2391

static void __init sun4v_pgprot_init(void)

2388

{

2392

{

2389

unsigned long page_none, page_shared, page_copy, page_readonly;

2393

unsigned long page_none, page_shared, page_copy, page_readonly;

2390

unsigned long page_exec_bit;

2394

unsigned long page_exec_bit;

2391

int i;

2395

int i;

2392

2396

2393

PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |

2397

PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |

2394

_PAGE_CACHE_4V | _PAGE_P_4V |

2398

_PAGE_CACHE_4V | _PAGE_P_4V |

2395

__ACCESS_BITS_4V | __DIRTY_BITS_4V |

2399

__ACCESS_BITS_4V | __DIRTY_BITS_4V |

2396

_PAGE_EXEC_4V);

2400

_PAGE_EXEC_4V);

2397

PAGE_KERNEL_LOCKED = PAGE_KERNEL;

2401

PAGE_KERNEL_LOCKED = PAGE_KERNEL;

2398

2402

2399

_PAGE_IE = _PAGE_IE_4V;

2403

_PAGE_IE = _PAGE_IE_4V;

2400

_PAGE_E = _PAGE_E_4V;

2404

_PAGE_E = _PAGE_E_4V;

2401

_PAGE_CACHE = _PAGE_CACHE_4V;

2405

_PAGE_CACHE = _PAGE_CACHE_4V;

2402

2406

2403

#ifdef CONFIG_DEBUG_PAGEALLOC

2407

#ifdef CONFIG_DEBUG_PAGEALLOC

2404

kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;

2408

kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;

2405

#else

2409

#else

2406

kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^

2410

kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^

2407

PAGE_OFFSET;

2411

PAGE_OFFSET;

2408

#endif

2412

#endif

2409

kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |

2413

kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |

2410

_PAGE_P_4V | _PAGE_W_4V);

2414

_PAGE_P_4V | _PAGE_W_4V);

2411

2415

2412

for (i = 1; i < 4; i++)

2416

for (i = 1; i < 4; i++)

2413

kern_linear_pte_xor[i] = kern_linear_pte_xor[0];

2417

kern_linear_pte_xor[i] = kern_linear_pte_xor[0];

2414

2418

2415

pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |

2419

pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |

2416

__ACCESS_BITS_4V | _PAGE_E_4V);

2420

__ACCESS_BITS_4V | _PAGE_E_4V);

2417

2421

2418

_PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |

2422

_PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |

2419

_PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |

2423

_PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |

2420

_PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |

2424

_PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |

2421

_PAGE_SZ64K_4V | _PAGE_SZ8K_4V);

2425

_PAGE_SZ64K_4V | _PAGE_SZ8K_4V);

2422

2426

2423

page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;

2427

page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;

2424

page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2428

page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2425

__ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);

2429

__ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);

2426

page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2430

page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2427

__ACCESS_BITS_4V | _PAGE_EXEC_4V);

2431

__ACCESS_BITS_4V | _PAGE_EXEC_4V);

2428

page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2432

page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |

2429

__ACCESS_BITS_4V | _PAGE_EXEC_4V);

2433

__ACCESS_BITS_4V | _PAGE_EXEC_4V);

2430

2434

2431

page_exec_bit = _PAGE_EXEC_4V;

2435

page_exec_bit = _PAGE_EXEC_4V;

2432

2436

2433

prot_init_common(page_none, page_shared, page_copy, page_readonly,

2437

prot_init_common(page_none, page_shared, page_copy, page_readonly,

2434

page_exec_bit);

2438

page_exec_bit);

2435

}

2439

}

2436

2440

2437

unsigned long pte_sz_bits(unsigned long sz)

2441

unsigned long pte_sz_bits(unsigned long sz)

2438

{

2442

{

2439

if (tlb_type == hypervisor) {

2443

if (tlb_type == hypervisor) {

2440

switch (sz) {

2444

switch (sz) {

2441

case 8 * 1024:

2445

case 8 * 1024:

2442

default:

2446

default:

2443

return _PAGE_SZ8K_4V;

2447

return _PAGE_SZ8K_4V;

2444

case 64 * 1024:

2448

case 64 * 1024:

2445

return _PAGE_SZ64K_4V;

2449

return _PAGE_SZ64K_4V;

2446

case 512 * 1024:

2450

case 512 * 1024:

2447

return _PAGE_SZ512K_4V;

2451

return _PAGE_SZ512K_4V;

2448

case 4 * 1024 * 1024:

2452

case 4 * 1024 * 1024:

2449

return _PAGE_SZ4MB_4V;

2453

return _PAGE_SZ4MB_4V;

2450

}

2454

}

2451

} else {

2455

} else {

2452

switch (sz) {

2456

switch (sz) {

2453

case 8 * 1024:

2457

case 8 * 1024:

2454

default:

2458

default:

2455

return _PAGE_SZ8K_4U;

2459

return _PAGE_SZ8K_4U;

2456

case 64 * 1024:

2460

case 64 * 1024:

2457

return _PAGE_SZ64K_4U;

2461

return _PAGE_SZ64K_4U;

2458

case 512 * 1024:

2462

case 512 * 1024:

2459

return _PAGE_SZ512K_4U;

2463

return _PAGE_SZ512K_4U;

2460

case 4 * 1024 * 1024:

2464

case 4 * 1024 * 1024:

2461

return _PAGE_SZ4MB_4U;

2465

return _PAGE_SZ4MB_4U;

2462

}

2466

}

2463

}

2467

}

2464

}

2468

}

2465

2469

2466

pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)

2470

pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)

2467

{

2471

{

2468

pte_t pte;

2472

pte_t pte;

2469

2473

2470

pte_val(pte) = page | pgprot_val(pgprot_noncached(prot));

2474

pte_val(pte) = page | pgprot_val(pgprot_noncached(prot));

2471

pte_val(pte) |= (((unsigned long)space) << 32);

2475

pte_val(pte) |= (((unsigned long)space) << 32);

2472

pte_val(pte) |= pte_sz_bits(page_size);

2476

pte_val(pte) |= pte_sz_bits(page_size);

2473

2477

2474

return pte;

2478

return pte;

2475

}

2479

}

2476

2480

2477

static unsigned long kern_large_tte(unsigned long paddr)

2481

static unsigned long kern_large_tte(unsigned long paddr)

2478

{

2482

{

2479

unsigned long val;

2483

unsigned long val;

2480

2484

2481

val = (_PAGE_VALID | _PAGE_SZ4MB_4U |

2485

val = (_PAGE_VALID | _PAGE_SZ4MB_4U |

2482

_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |

2486

_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |

2483

_PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);

2487

_PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);

2484

if (tlb_type == hypervisor)

2488

if (tlb_type == hypervisor)

2485

val = (_PAGE_VALID | _PAGE_SZ4MB_4V |

2489

val = (_PAGE_VALID | _PAGE_SZ4MB_4V |

2486

_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |

2490

_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |

2487

_PAGE_EXEC_4V | _PAGE_W_4V);

2491

_PAGE_EXEC_4V | _PAGE_W_4V);

2488

2492

2489

return val | paddr;

2493

return val | paddr;

2490

}

2494

}

2491

2495

2492

/* If not locked, zap it. */

2496

/* If not locked, zap it. */

2493

void __flush_tlb_all(void)

2497

void __flush_tlb_all(void)

2494

{

2498

{

2495

unsigned long pstate;

2499

unsigned long pstate;

2496

int i;

2500

int i;

2497

2501

2498

__asm__ __volatile__("flushw\n\t"

2502

__asm__ __volatile__("flushw\n\t"

2499

"rdpr %%pstate, %0\n\t"

2503

"rdpr %%pstate, %0\n\t"

2500

"wrpr %0, %1, %%pstate"

2504

"wrpr %0, %1, %%pstate"

2501

: "=r" (pstate)

2505

: "=r" (pstate)

2502

: "i" (PSTATE_IE));

2506

: "i" (PSTATE_IE));

2503

if (tlb_type == hypervisor) {

2507

if (tlb_type == hypervisor) {

2504

sun4v_mmu_demap_all();

2508

sun4v_mmu_demap_all();

2505

} else if (tlb_type == spitfire) {

2509

} else if (tlb_type == spitfire) {

2506

for (i = 0; i < 64; i++) {

2510

for (i = 0; i < 64; i++) {

2507

/* Spitfire Errata #32 workaround */

2511

/* Spitfire Errata #32 workaround */

2508

/* NOTE: Always runs on spitfire, so no

2512

/* NOTE: Always runs on spitfire, so no

2509

* cheetah+ page size encodings.

2513

* cheetah+ page size encodings.

2510

*/

2514

*/

2511

__asm__ __volatile__("stxa %0, [%1] %2\n\t"

2515

__asm__ __volatile__("stxa %0, [%1] %2\n\t"

2512

"flush %%g6"

2516

"flush %%g6"

2513

: /* No outputs */

2517

: /* No outputs */

2514

: "r" (0),

2518

: "r" (0),

2515

"r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

2519

"r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

2516

2520

2517

if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {

2521

if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {

2518

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

2522

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

2519

"membar #Sync"

2523

"membar #Sync"

2520

: /* no outputs */

2524

: /* no outputs */

2521

: "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));

2525

: "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));

2522

spitfire_put_dtlb_data(i, 0x0UL);

2526

spitfire_put_dtlb_data(i, 0x0UL);

2523

}

2527

}

2524

2528

2525

/* Spitfire Errata #32 workaround */

2529

/* Spitfire Errata #32 workaround */

2526

/* NOTE: Always runs on spitfire, so no

2530

/* NOTE: Always runs on spitfire, so no

2527

* cheetah+ page size encodings.

2531

* cheetah+ page size encodings.

2528

*/

2532

*/

2529

__asm__ __volatile__("stxa %0, [%1] %2\n\t"

2533

__asm__ __volatile__("stxa %0, [%1] %2\n\t"

2530

"flush %%g6"

2534

"flush %%g6"

2531

: /* No outputs */

2535

: /* No outputs */

2532

: "r" (0),

2536

: "r" (0),

2533

"r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

2537

"r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));

2534

2538

2535

if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {

2539

if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {

2536

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

2540

__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"

2537

"membar #Sync"

2541

"membar #Sync"

2538

: /* no outputs */

2542

: /* no outputs */

2539

: "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));

2543

: "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));

2540

spitfire_put_itlb_data(i, 0x0UL);

2544

spitfire_put_itlb_data(i, 0x0UL);

2541

}

2545

}

2542

}

2546

}

2543

} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {

2547

} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {

2544

cheetah_flush_dtlb_all();

2548

cheetah_flush_dtlb_all();

2545

cheetah_flush_itlb_all();

2549

cheetah_flush_itlb_all();

2546

}

2550

}

2547

__asm__ __volatile__("wrpr %0, 0, %%pstate"

2551

__asm__ __volatile__("wrpr %0, 0, %%pstate"

2548

: : "r" (pstate));

2552

: : "r" (pstate));

2549

}

2553

}

2550

2554

2551

pte_t *pte_alloc_one_kernel(struct mm_struct *mm,

2555

pte_t *pte_alloc_one_kernel(struct mm_struct *mm,

2552

unsigned long address)

2556

unsigned long address)

2553

{

2557

{

2554

struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |

2558

struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |

2555

__GFP_REPEAT | __GFP_ZERO);

2559

__GFP_REPEAT | __GFP_ZERO);

2556

pte_t *pte = NULL;

2560

pte_t *pte = NULL;

2557

2561

2558

if (page)

2562

if (page)

2559

pte = (pte_t *) page_address(page);

2563

pte = (pte_t *) page_address(page);

2560

2564

2561

return pte;

2565

return pte;

2562

}

2566

}

2563

2567

2564

pgtable_t pte_alloc_one(struct mm_struct *mm,

2568

pgtable_t pte_alloc_one(struct mm_struct *mm,

2565

unsigned long address)

2569

unsigned long address)

2566

{

2570

{

2567

struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |

2571

struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |

2568

__GFP_REPEAT | __GFP_ZERO);

2572

__GFP_REPEAT | __GFP_ZERO);

2569

if (!page)

2573

if (!page)

2570

return NULL;

2574

return NULL;

2571

if (!pgtable_page_ctor(page)) {

2575

if (!pgtable_page_ctor(page)) {

2572

free_hot_cold_page(page, 0);

2576

free_hot_cold_page(page, 0);

2573

return NULL;

2577

return NULL;

2574

}

2578

}

2575

return (pte_t *) page_address(page);

2579

return (pte_t *) page_address(page);

2576

}

2580

}

2577

2581

2578

void pte_free_kernel(struct mm_struct *mm, pte_t *pte)

2582

void pte_free_kernel(struct mm_struct *mm, pte_t *pte)

2579

{

2583

{

2580

free_page((unsigned long)pte);

2584

free_page((unsigned long)pte);

2581

}

2585

}

2582

2586

2583

static void __pte_free(pgtable_t pte)

2587

static void __pte_free(pgtable_t pte)

2584

{

2588

{

2585

struct page *page = virt_to_page(pte);

2589

struct page *page = virt_to_page(pte);

2586

2590

2587

pgtable_page_dtor(page);

2591

pgtable_page_dtor(page);

2588

__free_page(page);

2592

__free_page(page);

2589

}

2593

}

2590

2594

2591

void pte_free(struct mm_struct *mm, pgtable_t pte)

2595

void pte_free(struct mm_struct *mm, pgtable_t pte)

2592

{

2596

{

2593

__pte_free(pte);

2597

__pte_free(pte);

2594

}

2598

}

2595

2599

2596

void pgtable_free(void *table, bool is_page)

2600

void pgtable_free(void *table, bool is_page)

2597

{

2601

{

2598

if (is_page)

2602

if (is_page)

2599

__pte_free(table);

2603

__pte_free(table);

2600

else

2604

else

2601

kmem_cache_free(pgtable_cache, table);

2605

kmem_cache_free(pgtable_cache, table);

2602

}

2606

}

2603

2607

2604

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

2608

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

2605

void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,

2609

void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,

2606

pmd_t *pmd)

2610

pmd_t *pmd)

2607

{

2611

{

2608

unsigned long pte, flags;

2612

unsigned long pte, flags;

2609

struct mm_struct *mm;

2613

struct mm_struct *mm;

2610

pmd_t entry = *pmd;

2614

pmd_t entry = *pmd;

2611

2615

2612

if (!pmd_large(entry) || !pmd_young(entry))

2616

if (!pmd_large(entry) || !pmd_young(entry))

2613

return;

2617

return;

2614

2618

2615

pte = pmd_val(entry);

2619

pte = pmd_val(entry);

2620

2621

/* Don't insert a non-valid PMD into the TSB, we'll deadlock. */

2622

if (!(pte & _PAGE_VALID))

2623

return;

2616

2624

2617

/* We are fabricating 8MB pages using 4MB real hw pages. */

2625

/* We are fabricating 8MB pages using 4MB real hw pages. */

2618

pte |= (addr & (1UL << REAL_HPAGE_SHIFT));

2626

pte |= (addr & (1UL << REAL_HPAGE_SHIFT));

2619

2627

2620

mm = vma->vm_mm;

2628

mm = vma->vm_mm;

2621

2629

2622

spin_lock_irqsave(&mm->context.lock, flags);

2630

spin_lock_irqsave(&mm->context.lock, flags);

2623

2631

2624

if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)

2632

if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)

2625

__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,

2633

__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,

2626

addr, pte);

2634

addr, pte);

2627

2635

2628

spin_unlock_irqrestore(&mm->context.lock, flags);

2636

spin_unlock_irqrestore(&mm->context.lock, flags);

2629

}

2637

}

2630

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

2638

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

2631

2639

2632

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

2640

#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)

2633

static void context_reload(void *__data)

2641

static void context_reload(void *__data)

2634

{

2642

{

2635

struct mm_struct *mm = __data;

2643

struct mm_struct *mm = __data;

2636

2644

2637

if (mm == current->mm)

2645

if (mm == current->mm)

2638

load_secondary_context(mm);

2646

load_secondary_context(mm);

2639

}

2647

}

2640

2648

2641

void hugetlb_setup(struct pt_regs *regs)

2649

void hugetlb_setup(struct pt_regs *regs)

2642

{

2650

{

2643

struct mm_struct *mm = current->mm;

2651

struct mm_struct *mm = current->mm;

2644

struct tsb_config *tp;

2652

struct tsb_config *tp;

2645

2653

2646

if (in_atomic() || !mm) {

2654

if (in_atomic() || !mm) {

2647

const struct exception_table_entry *entry;

2655

const struct exception_table_entry *entry;

2648

2656

2649

entry = search_exception_tables(regs->tpc);

2657

entry = search_exception_tables(regs->tpc);

2650

if (entry) {

2658

if (entry) {

2651

regs->tpc = entry->fixup;

2659

regs->tpc = entry->fixup;

2652

regs->tnpc = regs->tpc + 4;

2660

regs->tnpc = regs->tpc + 4;

2653

return;

2661

return;

2654

}

2662

}

2655

pr_alert("Unexpected HugeTLB setup in atomic context.\n");

2663

pr_alert("Unexpected HugeTLB setup in atomic context.\n");

2656

die_if_kernel("HugeTSB in atomic", regs);

2664

die_if_kernel("HugeTSB in atomic", regs);

2657

}

2665

}

2658

2666

2659

tp = &mm->context.tsb_block[MM_TSB_HUGE];

2667

tp = &mm->context.tsb_block[MM_TSB_HUGE];

2660

if (likely(tp->tsb == NULL))

2668

if (likely(tp->tsb == NULL))

2661

tsb_grow(mm, MM_TSB_HUGE, 0);

2669

tsb_grow(mm, MM_TSB_HUGE, 0);

2662

2670

2663

tsb_context_switch(mm);

2671

tsb_context_switch(mm);

2664

smp_tsb_sync(mm);

2672

smp_tsb_sync(mm);

2665

2673

2666

/* On UltraSPARC-III+ and later, configure the second half of

2674

/* On UltraSPARC-III+ and later, configure the second half of

2667

* the Data-TLB for huge pages.

2675

* the Data-TLB for huge pages.

2668

*/

2676

*/

2669

if (tlb_type == cheetah_plus) {

2677

if (tlb_type == cheetah_plus) {

2670

unsigned long ctx;

2678

unsigned long ctx;

2671

2679

2672

spin_lock(&ctx_alloc_lock);

2680

spin_lock(&ctx_alloc_lock);

2673

ctx = mm->context.sparc64_ctx_val;

2681

ctx = mm->context.sparc64_ctx_val;

2674

ctx &= ~CTX_PGSZ_MASK;

2682

ctx &= ~CTX_PGSZ_MASK;

2675

ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;

2683

ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;

2676

ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;

2684

ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;

2677

2685

2678

if (ctx != mm->context.sparc64_ctx_val) {

2686

if (ctx != mm->context.sparc64_ctx_val) {

2679

/* When changing the page size fields, we

2687

/* When changing the page size fields, we

2680

* must perform a context flush so that no

2688

* must perform a context flush so that no

2681

* stale entries match. This flush must

2689

* stale entries match. This flush must

2682

* occur with the original context register

2690

* occur with the original context register

2683

* settings.

2691

* settings.

2684

*/

2692

*/

2685

do_flush_tlb_mm(mm);

2693

do_flush_tlb_mm(mm);

2686

2694

2687

/* Reload the context register of all processors

2695

/* Reload the context register of all processors

2688

* also executing in this address space.

2696

* also executing in this address space.

2689

*/

2697

*/

2690

mm->context.sparc64_ctx_val = ctx;

2698

mm->context.sparc64_ctx_val = ctx;

2691

on_each_cpu(context_reload, mm, 0);

2699

on_each_cpu(context_reload, mm, 0);

2692

}

2700

}

2693

spin_unlock(&ctx_alloc_lock);

2701

spin_unlock(&ctx_alloc_lock);

2694

}

2702

}

2695

}

2703

}

2696

#endif

2704

#endif

2697

2705

GITLAB

sparc64: Do not insert non-valid PTEs into the TSB hash table.

 /*
  *  arch/sparc64/mm/init.c
  *
  *  Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu)
  *  Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
  */
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/string.h>
 #include <linux/init.h>
 #include <linux/bootmem.h>
 #include <linux/mm.h>
 #include <linux/hugetlb.h>
 #include <linux/initrd.h>
 #include <linux/swap.h>
 #include <linux/pagemap.h>
 #include <linux/poison.h>
 #include <linux/fs.h>
 #include <linux/seq_file.h>
 #include <linux/kprobes.h>
 #include <linux/cache.h>
 #include <linux/sort.h>
 #include <linux/percpu.h>
 #include <linux/memblock.h>
 #include <linux/mmzone.h>
 #include <linux/gfp.h>
 #include <asm/head.h>
 #include <asm/page.h>
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 #include <asm/oplib.h>
 #include <asm/iommu.h>
 #include <asm/io.h>
 #include <asm/uaccess.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
 #include <asm/dma.h>
 #include <asm/starfire.h>
 #include <asm/tlb.h>
 #include <asm/spitfire.h>
 #include <asm/sections.h>
 #include <asm/tsb.h>
 #include <asm/hypervisor.h>
 #include <asm/prom.h>
 #include <asm/mdesc.h>
 #include <asm/cpudata.h>
 #include <asm/irq.h>
 #include "init_64.h"
 unsigned long kern_linear_pte_xor[4] __read_mostly;
 /* A bitmap, two bits for every 256MB of physical memory.  These two
  * bits determine what page size we use for kernel linear
  * translations.  They form an index into kern_linear_pte_xor[].  The
  * value in the indexed slot is XOR'd with the TLB miss virtual
  * address to form the resulting TTE.  The mapping is:
  *
  *	0	==>	4MB
  *	1	==>	256MB
  *	2	==>	2GB
  *	3	==>	16GB
  *
  * All sun4v chips support 256MB pages.  Only SPARC-T4 and later
  * support 2GB pages, and hopefully future cpus will support the 16GB
  * pages as well.  For slots 2 and 3, we encode a 256MB TTE xor there
  * if these larger page sizes are not supported by the cpu.
  *
  * It would be nice to determine this from the machine description
  * 'cpu' properties, but we need to have this table setup before the
  * MDESC is initialized.
  */
 unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];
 #ifndef CONFIG_DEBUG_PAGEALLOC
 /* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings.
  * Space is allocated for this right after the trap table in
  * arch/sparc64/kernel/head.S
  */
 extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES];
 #endif
 static unsigned long cpu_pgsz_mask;
 #define MAX_BANKS	32
 static struct linux_prom64_registers pavail[MAX_BANKS];
 static int pavail_ents;
 static int cmp_p64(const void *a, const void *b)
 {
 	const struct linux_prom64_registers *x = a, *y = b;
 	if (x->phys_addr > y->phys_addr)
 		return 1;
 	if (x->phys_addr < y->phys_addr)
 		return -1;
 	return 0;
 }
 static void __init read_obp_memory(const char *property,
 				   struct linux_prom64_registers *regs,
 				   int *num_ents)
 {
 	phandle node = prom_finddevice("/memory");
 	int prop_size = prom_getproplen(node, property);
 	int ents, ret, i;
 	ents = prop_size / sizeof(struct linux_prom64_registers);
 	if (ents > MAX_BANKS) {
 		prom_printf("The machine has more %s property entries than "
 			    "this kernel can support (%d).\n",
 			    property, MAX_BANKS);
 		prom_halt();
 	}
 	ret = prom_getproperty(node, property, (char *) regs, prop_size);
 	if (ret == -1) {
 		prom_printf("Couldn't get %s property from /memory.\n",
 				property);
 		prom_halt();
 	}
 	/* Sanitize what we got from the firmware, by page aligning
 	 * everything.
 	 */
 	for (i = 0; i < ents; i++) {
 		unsigned long base, size;
 		base = regs[i].phys_addr;
 		size = regs[i].reg_size;
 		size &= PAGE_MASK;
 		if (base & ~PAGE_MASK) {
 			unsigned long new_base = PAGE_ALIGN(base);
 			size -= new_base - base;
 			if ((long) size < 0L)
 				size = 0UL;
 			base = new_base;
 		}
 		if (size == 0UL) {
 			/* If it is empty, simply get rid of it.
 			 * This simplifies the logic of the other
 			 * functions that process these arrays.
 			 */
 			memmove(&regs[i], &regs[i + 1],
 				(ents - i - 1) * sizeof(regs[0]));
 			i--;
 			ents--;
 			continue;
 		}
 		regs[i].phys_addr = base;
 		regs[i].reg_size = size;
 	}
 	*num_ents = ents;
 	sort(regs, ents, sizeof(struct linux_prom64_registers),
 	     cmp_p64, NULL);
 }
 unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
 					sizeof(unsigned long)];
 EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
 /* Kernel physical address base and size in bytes.  */
 unsigned long kern_base __read_mostly;
 unsigned long kern_size __read_mostly;
 /* Initial ramdisk setup */
 extern unsigned long sparc_ramdisk_image64;
 extern unsigned int sparc_ramdisk_image;
 extern unsigned int sparc_ramdisk_size;
 struct page *mem_map_zero __read_mostly;
 EXPORT_SYMBOL(mem_map_zero);
 unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly;
 unsigned long sparc64_kern_pri_context __read_mostly;
 unsigned long sparc64_kern_pri_nuc_bits __read_mostly;
 unsigned long sparc64_kern_sec_context __read_mostly;
 int num_kernel_image_mappings;
 #ifdef CONFIG_DEBUG_DCFLUSH
 atomic_t dcpage_flushes = ATOMIC_INIT(0);
 #ifdef CONFIG_SMP
 atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0);
 #endif
 #endif
 inline void flush_dcache_page_impl(struct page *page)
 {
 	BUG_ON(tlb_type == hypervisor);
 #ifdef CONFIG_DEBUG_DCFLUSH
 	atomic_inc(&dcpage_flushes);
 #endif
 #ifdef DCACHE_ALIASING_POSSIBLE
 	__flush_dcache_page(page_address(page),
 			    ((tlb_type == spitfire) &&
 			     page_mapping(page) != NULL));
 #else
 	if (page_mapping(page) != NULL &&
 	    tlb_type == spitfire)
 		__flush_icache_page(__pa(page_address(page)));
 #endif
 }
 #define PG_dcache_dirty		PG_arch_1
 #define PG_dcache_cpu_shift	32UL
 #define PG_dcache_cpu_mask	\
 	((1UL<<ilog2(roundup_pow_of_two(NR_CPUS)))-1UL)
 #define dcache_dirty_cpu(page) \
 	(((page)->flags >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask)
 static inline void set_dcache_dirty(struct page *page, int this_cpu)
 {
 	unsigned long mask = this_cpu;
 	unsigned long non_cpu_bits;
 	non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift);
 	mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty);
 	__asm__ __volatile__("1:\n\t"
 			     "ldx	[%2], %%g7\n\t"
 			     "and	%%g7, %1, %%g1\n\t"
 			     "or	%%g1, %0, %%g1\n\t"
 			     "casx	[%2], %%g7, %%g1\n\t"
 			     "cmp	%%g7, %%g1\n\t"
 			     "bne,pn	%%xcc, 1b\n\t"
 			     " nop"
 			     : /* no outputs */
 			     : "r" (mask), "r" (non_cpu_bits), "r" (&page->flags)
 			     : "g1", "g7");
 }
 static inline void clear_dcache_dirty_cpu(struct page *page, unsigned long cpu)
 {
 	unsigned long mask = (1UL << PG_dcache_dirty);
 	__asm__ __volatile__("! test_and_clear_dcache_dirty\n"
 			     "1:\n\t"
 			     "ldx	[%2], %%g7\n\t"
 			     "srlx	%%g7, %4, %%g1\n\t"
 			     "and	%%g1, %3, %%g1\n\t"
 			     "cmp	%%g1, %0\n\t"
 			     "bne,pn	%%icc, 2f\n\t"
 			     " andn	%%g7, %1, %%g1\n\t"
 			     "casx	[%2], %%g7, %%g1\n\t"
 			     "cmp	%%g7, %%g1\n\t"
 			     "bne,pn	%%xcc, 1b\n\t"
 			     " nop\n"
 			     "2:"
 			     : /* no outputs */
 			     : "r" (cpu), "r" (mask), "r" (&page->flags),
 			       "i" (PG_dcache_cpu_mask),
 			       "i" (PG_dcache_cpu_shift)
 			     : "g1", "g7");
 }
 static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte)
 {
 	unsigned long tsb_addr = (unsigned long) ent;
 	if (tlb_type == cheetah_plus || tlb_type == hypervisor)
 		tsb_addr = __pa(tsb_addr);
 	__tsb_insert(tsb_addr, tag, pte);
 }
 unsigned long _PAGE_ALL_SZ_BITS __read_mostly;
 static void flush_dcache(unsigned long pfn)
 {
 	struct page *page;
 	page = pfn_to_page(pfn);
 	if (page) {
 		unsigned long pg_flags;
 		pg_flags = page->flags;
 		if (pg_flags & (1UL << PG_dcache_dirty)) {
 			int cpu = ((pg_flags >> PG_dcache_cpu_shift) &
 				   PG_dcache_cpu_mask);
 			int this_cpu = get_cpu();
 			/* This is just to optimize away some function calls
 			 * in the SMP case.
 			 */
 			if (cpu == this_cpu)
 				flush_dcache_page_impl(page);
 			else
 				smp_flush_dcache_page_impl(page, cpu);
 			clear_dcache_dirty_cpu(page, cpu);
 			put_cpu();
 		}
 	}
 }
 /* mm->context.lock must be held */
 static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
 				    unsigned long tsb_hash_shift, unsigned long address,
 				    unsigned long tte)
 {
 	struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
 	unsigned long tag;
 	if (unlikely(!tsb))
 		return;
 	tsb += ((address >> tsb_hash_shift) &
 		(mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
 	tag = (address >> 22UL);
 	tsb_insert(tsb, tag, tte);
 }
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 static inline bool is_hugetlb_pte(pte_t pte)
 {
 	if ((tlb_type == hypervisor &&
 	     (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
 	    (tlb_type != hypervisor &&
 	     (pte_val(pte) & _PAGE_SZALL_4U) == _PAGE_SZHUGE_4U))
 		return true;
 	return false;
 }
 #endif
 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 {
 	struct mm_struct *mm;
 	unsigned long flags;
 	pte_t pte = *ptep;
 	if (tlb_type != hypervisor) {
 		unsigned long pfn = pte_pfn(pte);
 		if (pfn_valid(pfn))
 			flush_dcache(pfn);
 	}
 	mm = vma->vm_mm;
+	/* Don't insert a non-valid PTE into the TSB, we'll deadlock.  */
+	if (!pte_accessible(mm, pte))
+		return;
 	spin_lock_irqsave(&mm->context.lock, flags);
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 	if (mm->context.huge_pte_count && is_hugetlb_pte(pte))
 		__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
 					address, pte_val(pte));
 	else
 #endif
 		__update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT,
 					address, pte_val(pte));
 	spin_unlock_irqrestore(&mm->context.lock, flags);
 }
 void flush_dcache_page(struct page *page)
 {
 	struct address_space *mapping;
 	int this_cpu;
 	if (tlb_type == hypervisor)
 		return;
 	/* Do not bother with the expensive D-cache flush if it
 	 * is merely the zero page.  The 'bigcore' testcase in GDB
 	 * causes this case to run millions of times.
 	 */
 	if (page == ZERO_PAGE(0))
 		return;
 	this_cpu = get_cpu();
 	mapping = page_mapping(page);
 	if (mapping && !mapping_mapped(mapping)) {
 		int dirty = test_bit(PG_dcache_dirty, &page->flags);
 		if (dirty) {
 			int dirty_cpu = dcache_dirty_cpu(page);
 			if (dirty_cpu == this_cpu)
 				goto out;
 			smp_flush_dcache_page_impl(page, dirty_cpu);
 		}
 		set_dcache_dirty(page, this_cpu);
 	} else {
 		/* We could delay the flush for the !page_mapping
 		 * case too.  But that case is for exec env/arg
 		 * pages and those are %99 certainly going to get
 		 * faulted into the tlb (and thus flushed) anyways.
 		 */
 		flush_dcache_page_impl(page);
 	}
 out:
 	put_cpu();
 }
 EXPORT_SYMBOL(flush_dcache_page);
 void __kprobes flush_icache_range(unsigned long start, unsigned long end)
 {
 	/* Cheetah and Hypervisor platform cpus have coherent I-cache. */
 	if (tlb_type == spitfire) {
 		unsigned long kaddr;
 		/* This code only runs on Spitfire cpus so this is
 		 * why we can assume _PAGE_PADDR_4U.
 		 */
 		for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) {
 			unsigned long paddr, mask = _PAGE_PADDR_4U;
 			if (kaddr >= PAGE_OFFSET)
 				paddr = kaddr & mask;
 			else {
 				pgd_t *pgdp = pgd_offset_k(kaddr);
 				pud_t *pudp = pud_offset(pgdp, kaddr);
 				pmd_t *pmdp = pmd_offset(pudp, kaddr);
 				pte_t *ptep = pte_offset_kernel(pmdp, kaddr);
 				paddr = pte_val(*ptep) & mask;
 			}
 			__flush_icache_page(paddr);
 		}
 	}
 }
 EXPORT_SYMBOL(flush_icache_range);
 void mmu_info(struct seq_file *m)
 {
 	static const char *pgsz_strings[] = {
 		"8K", "64K", "512K", "4MB", "32MB",
 		"256MB", "2GB", "16GB",
 	};
 	int i, printed;
 	if (tlb_type == cheetah)
 		seq_printf(m, "MMU Type\t: Cheetah\n");
 	else if (tlb_type == cheetah_plus)
 		seq_printf(m, "MMU Type\t: Cheetah+\n");
 	else if (tlb_type == spitfire)
 		seq_printf(m, "MMU Type\t: Spitfire\n");
 	else if (tlb_type == hypervisor)
 		seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n");
 	else
 		seq_printf(m, "MMU Type\t: ???\n");
 	seq_printf(m, "MMU PGSZs\t: ");
 	printed = 0;
 	for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) {
 		if (cpu_pgsz_mask & (1UL << i)) {
 			seq_printf(m, "%s%s",
 				   printed ? "," : "", pgsz_strings[i]);
 			printed++;
 		}
 	}
 	seq_putc(m, '\n');
 #ifdef CONFIG_DEBUG_DCFLUSH
 	seq_printf(m, "DCPageFlushes\t: %d\n",
 		   atomic_read(&dcpage_flushes));
 #ifdef CONFIG_SMP
 	seq_printf(m, "DCPageFlushesXC\t: %d\n",
 		   atomic_read(&dcpage_flushes_xcall));
 #endif /* CONFIG_SMP */
 #endif /* CONFIG_DEBUG_DCFLUSH */
 }
 struct linux_prom_translation prom_trans[512] __read_mostly;
 unsigned int prom_trans_ents __read_mostly;
 unsigned long kern_locked_tte_data;
 /* The obp translations are saved based on 8k pagesize, since obp can
  * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS ->
  * HI_OBP_ADDRESS range are handled in ktlb.S.
  */
 static inline int in_obp_range(unsigned long vaddr)
 {
 	return (vaddr >= LOW_OBP_ADDRESS &&
 		vaddr < HI_OBP_ADDRESS);
 }
 static int cmp_ptrans(const void *a, const void *b)
 {
 	const struct linux_prom_translation *x = a, *y = b;
 	if (x->virt > y->virt)
 		return 1;
 	if (x->virt < y->virt)
 		return -1;
 	return 0;
 }
 /* Read OBP translations property into 'prom_trans[]'.  */
 static void __init read_obp_translations(void)
 {
 	int n, node, ents, first, last, i;
 	node = prom_finddevice("/virtual-memory");
 	n = prom_getproplen(node, "translations");
 	if (unlikely(n == 0 || n == -1)) {
 		prom_printf("prom_mappings: Couldn't get size.\n");
 		prom_halt();
 	}
 	if (unlikely(n > sizeof(prom_trans))) {
 		prom_printf("prom_mappings: Size %d is too big.\n", n);
 		prom_halt();
 	}
 	if ((n = prom_getproperty(node, "translations",
 				  (char *)&prom_trans[0],
 				  sizeof(prom_trans))) == -1) {
 		prom_printf("prom_mappings: Couldn't get property.\n");
 		prom_halt();
 	}
 	n = n / sizeof(struct linux_prom_translation);
 	ents = n;
 	sort(prom_trans, ents, sizeof(struct linux_prom_translation),
 	     cmp_ptrans, NULL);
 	/* Now kick out all the non-OBP entries.  */
 	for (i = 0; i < ents; i++) {
 		if (in_obp_range(prom_trans[i].virt))
 			break;
 	}
 	first = i;
 	for (; i < ents; i++) {
 		if (!in_obp_range(prom_trans[i].virt))
 			break;
 	}
 	last = i;
 	for (i = 0; i < (last - first); i++) {
 		struct linux_prom_translation *src = &prom_trans[i + first];
 		struct linux_prom_translation *dest = &prom_trans[i];
 		*dest = *src;
 	}
 	for (; i < ents; i++) {
 		struct linux_prom_translation *dest = &prom_trans[i];
 		dest->virt = dest->size = dest->data = 0x0UL;
 	}
 	prom_trans_ents = last - first;
 	if (tlb_type == spitfire) {
 		/* Clear diag TTE bits. */
 		for (i = 0; i < prom_trans_ents; i++)
 			prom_trans[i].data &= ~0x0003fe0000000000UL;
 	}
 	/* Force execute bit on.  */
 	for (i = 0; i < prom_trans_ents; i++)
 		prom_trans[i].data |= (tlb_type == hypervisor ?
 				       _PAGE_EXEC_4V : _PAGE_EXEC_4U);
 }
 static void __init hypervisor_tlb_lock(unsigned long vaddr,
 				       unsigned long pte,
 				       unsigned long mmu)
 {
 	unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu);
 	if (ret != 0) {
 		prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: "
 			    "errors with %lx\n", vaddr, 0, pte, mmu, ret);
 		prom_halt();
 	}
 }
 static unsigned long kern_large_tte(unsigned long paddr);
 static void __init remap_kernel(void)
 {
 	unsigned long phys_page, tte_vaddr, tte_data;
 	int i, tlb_ent = sparc64_highest_locked_tlbent();
 	tte_vaddr = (unsigned long) KERNBASE;
 	phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
 	tte_data = kern_large_tte(phys_page);
 	kern_locked_tte_data = tte_data;
 	/* Now lock us into the TLBs via Hypervisor or OBP. */
 	if (tlb_type == hypervisor) {
 		for (i = 0; i < num_kernel_image_mappings; i++) {
 			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU);
 			hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU);
 			tte_vaddr += 0x400000;
 			tte_data += 0x400000;
 		}
 	} else {
 		for (i = 0; i < num_kernel_image_mappings; i++) {
 			prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr);
 			prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr);
 			tte_vaddr += 0x400000;
 			tte_data += 0x400000;
 		}
 		sparc64_highest_unlocked_tlb_ent = tlb_ent - i;
 	}
 	if (tlb_type == cheetah_plus) {
 		sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 |
 					    CTX_CHEETAH_PLUS_NUC);
 		sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC;
 		sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0;
 	}
 }
 static void __init inherit_prom_mappings(void)
 {
 	/* Now fixup OBP's idea about where we really are mapped. */
 	printk("Remapping the kernel... ");
 	remap_kernel();
 	printk("done.\n");
 }
 void prom_world(int enter)
 {
 	if (!enter)
 		set_fs(get_fs());
 	__asm__ __volatile__("flushw");
 }
 void __flush_dcache_range(unsigned long start, unsigned long end)
 {
 	unsigned long va;
 	if (tlb_type == spitfire) {
 		int n = 0;
 		for (va = start; va < end; va += 32) {
 			spitfire_put_dcache_tag(va & 0x3fe0, 0x0);
 			if (++n >= 512)
 				break;
 		}
 	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		start = __pa(start);
 		end = __pa(end);
 		for (va = start; va < end; va += 32)
 			__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 					     "membar #Sync"
 					     : /* no outputs */
 					     : "r" (va),
 					       "i" (ASI_DCACHE_INVALIDATE));
 	}
 }
 EXPORT_SYMBOL(__flush_dcache_range);
 /* get_new_mmu_context() uses "cache + 1".  */
 DEFINE_SPINLOCK(ctx_alloc_lock);
 unsigned long tlb_context_cache = CTX_FIRST_VERSION - 1;
 #define MAX_CTX_NR	(1UL << CTX_NR_BITS)
 #define CTX_BMAP_SLOTS	BITS_TO_LONGS(MAX_CTX_NR)
 DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR);
 /* Caller does TLB context flushing on local CPU if necessary.
  * The caller also ensures that CTX_VALID(mm->context) is false.
  *
  * We must be careful about boundary cases so that we never
  * let the user have CTX 0 (nucleus) or we ever use a CTX
  * version of zero (and thus NO_CONTEXT would not be caught
  * by version mis-match tests in mmu_context.h).
  *
  * Always invoked with interrupts disabled.
  */
 void get_new_mmu_context(struct mm_struct *mm)
 {
 	unsigned long ctx, new_ctx;
 	unsigned long orig_pgsz_bits;
 	int new_version;
 	spin_lock(&ctx_alloc_lock);
 	orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK);
 	ctx = (tlb_context_cache + 1) & CTX_NR_MASK;
 	new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx);
 	new_version = 0;
 	if (new_ctx >= (1 << CTX_NR_BITS)) {
 		new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1);
 		if (new_ctx >= ctx) {
 			int i;
 			new_ctx = (tlb_context_cache & CTX_VERSION_MASK) +
 				CTX_FIRST_VERSION;
 			if (new_ctx == 1)
 				new_ctx = CTX_FIRST_VERSION;
 			/* Don't call memset, for 16 entries that's just
 			 * plain silly...
 			 */
 			mmu_context_bmap[0] = 3;
 			mmu_context_bmap[1] = 0;
 			mmu_context_bmap[2] = 0;
 			mmu_context_bmap[3] = 0;
 			for (i = 4; i < CTX_BMAP_SLOTS; i += 4) {
 				mmu_context_bmap[i + 0] = 0;
 				mmu_context_bmap[i + 1] = 0;
 				mmu_context_bmap[i + 2] = 0;
 				mmu_context_bmap[i + 3] = 0;
 			}
 			new_version = 1;
 			goto out;
 		}
 	}
 	mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63));
 	new_ctx |= (tlb_context_cache & CTX_VERSION_MASK);
 out:
 	tlb_context_cache = new_ctx;
 	mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits;
 	spin_unlock(&ctx_alloc_lock);
 	if (unlikely(new_version))
 		smp_new_mmu_context_version();
 }
 static int numa_enabled = 1;
 static int numa_debug;
 static int __init early_numa(char *p)
 {
 	if (!p)
 		return 0;
 	if (strstr(p, "off"))
 		numa_enabled = 0;
 	if (strstr(p, "debug"))
 		numa_debug = 1;
 	return 0;
 }
 early_param("numa", early_numa);
 #define numadbg(f, a...) \
 do {	if (numa_debug) \
 		printk(KERN_INFO f, ## a); \
 } while (0)
 static void __init find_ramdisk(unsigned long phys_base)
 {
 #ifdef CONFIG_BLK_DEV_INITRD
 	if (sparc_ramdisk_image || sparc_ramdisk_image64) {
 		unsigned long ramdisk_image;
 		/* Older versions of the bootloader only supported a
 		 * 32-bit physical address for the ramdisk image
 		 * location, stored at sparc_ramdisk_image.  Newer
 		 * SILO versions set sparc_ramdisk_image to zero and
 		 * provide a full 64-bit physical address at
 		 * sparc_ramdisk_image64.
 		 */
 		ramdisk_image = sparc_ramdisk_image;
 		if (!ramdisk_image)
 			ramdisk_image = sparc_ramdisk_image64;
 		/* Another bootloader quirk.  The bootloader normalizes
 		 * the physical address to KERNBASE, so we have to
 		 * factor that back out and add in the lowest valid
 		 * physical page address to get the true physical address.
 		 */
 		ramdisk_image -= KERNBASE;
 		ramdisk_image += phys_base;
 		numadbg("Found ramdisk at physical address 0x%lx, size %u\n",
 			ramdisk_image, sparc_ramdisk_size);
 		initrd_start = ramdisk_image;
 		initrd_end = ramdisk_image + sparc_ramdisk_size;
 		memblock_reserve(initrd_start, sparc_ramdisk_size);
 		initrd_start += PAGE_OFFSET;
 		initrd_end += PAGE_OFFSET;
 	}
 #endif
 }
 struct node_mem_mask {
 	unsigned long mask;
 	unsigned long val;
 };
 static struct node_mem_mask node_masks[MAX_NUMNODES];
 static int num_node_masks;
 int numa_cpu_lookup_table[NR_CPUS];
 cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES];
 #ifdef CONFIG_NEED_MULTIPLE_NODES
 struct mdesc_mblock {
 	u64	base;
 	u64	size;
 	u64	offset; /* RA-to-PA */
 };
 static struct mdesc_mblock *mblocks;
 static int num_mblocks;
 static unsigned long ra_to_pa(unsigned long addr)
 {
 	int i;
 	for (i = 0; i < num_mblocks; i++) {
 		struct mdesc_mblock *m = &mblocks[i];
 		if (addr >= m->base &&
 		    addr < (m->base + m->size)) {
 			addr += m->offset;
 			break;
 		}
 	}
 	return addr;
 }
 static int find_node(unsigned long addr)
 {
 	int i;
 	addr = ra_to_pa(addr);
 	for (i = 0; i < num_node_masks; i++) {
 		struct node_mem_mask *p = &node_masks[i];
 		if ((addr & p->mask) == p->val)
 			return i;
 	}
 	return -1;
 }
 static u64 memblock_nid_range(u64 start, u64 end, int *nid)
 {
 	*nid = find_node(start);
 	start += PAGE_SIZE;
 	while (start < end) {
 		int n = find_node(start);
 		if (n != *nid)
 			break;
 		start += PAGE_SIZE;
 	}
 	if (start > end)
 		start = end;
 	return start;
 }
 #endif
 /* This must be invoked after performing all of the necessary
  * memblock_set_node() calls for 'nid'.  We need to be able to get
  * correct data from get_pfn_range_for_nid().
  */
 static void __init allocate_node_data(int nid)
 {
 	struct pglist_data *p;
 	unsigned long start_pfn, end_pfn;
 #ifdef CONFIG_NEED_MULTIPLE_NODES
 	unsigned long paddr;
 	paddr = memblock_alloc_try_nid(sizeof(struct pglist_data), SMP_CACHE_BYTES, nid);
 	if (!paddr) {
 		prom_printf("Cannot allocate pglist_data for nid[%d]\n", nid);
 		prom_halt();
 	}
 	NODE_DATA(nid) = __va(paddr);
 	memset(NODE_DATA(nid), 0, sizeof(struct pglist_data));
 	NODE_DATA(nid)->node_id = nid;
 #endif
 	p = NODE_DATA(nid);
 	get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
 	p->node_start_pfn = start_pfn;
 	p->node_spanned_pages = end_pfn - start_pfn;
 }
 static void init_node_masks_nonnuma(void)
 {
 	int i;
 	numadbg("Initializing tables for non-numa.\n");
 	node_masks[0].mask = node_masks[0].val = 0;
 	num_node_masks = 1;
 	for (i = 0; i < NR_CPUS; i++)
 		numa_cpu_lookup_table[i] = 0;
 	cpumask_setall(&numa_cpumask_lookup_table[0]);
 }
 #ifdef CONFIG_NEED_MULTIPLE_NODES
 struct pglist_data *node_data[MAX_NUMNODES];
 EXPORT_SYMBOL(numa_cpu_lookup_table);
 EXPORT_SYMBOL(numa_cpumask_lookup_table);
 EXPORT_SYMBOL(node_data);
 struct mdesc_mlgroup {
 	u64	node;
 	u64	latency;
 	u64	match;
 	u64	mask;
 };
 static struct mdesc_mlgroup *mlgroups;
 static int num_mlgroups;
 static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio,
 				   u32 cfg_handle)
 {
 	u64 arc;
 	mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) {
 		u64 target = mdesc_arc_target(md, arc);
 		const u64 *val;
 		val = mdesc_get_property(md, target,
 					 "cfg-handle", NULL);
 		if (val && *val == cfg_handle)
 			return 0;
 	}
 	return -ENODEV;
 }
 static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp,
 				    u32 cfg_handle)
 {
 	u64 arc, candidate, best_latency = ~(u64)0;
 	candidate = MDESC_NODE_NULL;
 	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
 		u64 target = mdesc_arc_target(md, arc);
 		const char *name = mdesc_node_name(md, target);
 		const u64 *val;
 		if (strcmp(name, "pio-latency-group"))
 			continue;
 		val = mdesc_get_property(md, target, "latency", NULL);
 		if (!val)
 			continue;
 		if (*val < best_latency) {
 			candidate = target;
 			best_latency = *val;
 		}
 	}
 	if (candidate == MDESC_NODE_NULL)
 		return -ENODEV;
 	return scan_pio_for_cfg_handle(md, candidate, cfg_handle);
 }
 int of_node_to_nid(struct device_node *dp)
 {
 	const struct linux_prom64_registers *regs;
 	struct mdesc_handle *md;
 	u32 cfg_handle;
 	int count, nid;
 	u64 grp;
 	/* This is the right thing to do on currently supported
 	 * SUN4U NUMA platforms as well, as the PCI controller does
 	 * not sit behind any particular memory controller.
 	 */
 	if (!mlgroups)
 		return -1;
 	regs = of_get_property(dp, "reg", NULL);
 	if (!regs)
 		return -1;
 	cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff;
 	md = mdesc_grab();
 	count = 0;
 	nid = -1;
 	mdesc_for_each_node_by_name(md, grp, "group") {
 		if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) {
 			nid = count;
 			break;
 		}
 		count++;
 	}
 	mdesc_release(md);
 	return nid;
 }
 static void __init add_node_ranges(void)
 {
 	struct memblock_region *reg;
 	for_each_memblock(memory, reg) {
 		unsigned long size = reg->size;
 		unsigned long start, end;
 		start = reg->base;
 		end = start + size;
 		while (start < end) {
 			unsigned long this_end;
 			int nid;
 			this_end = memblock_nid_range(start, end, &nid);
 			numadbg("Setting memblock NUMA node nid[%d] "
 				"start[%lx] end[%lx]\n",
 				nid, start, this_end);
 			memblock_set_node(start, this_end - start,
 					  &memblock.memory, nid);
 			start = this_end;
 		}
 	}
 }
 static int __init grab_mlgroups(struct mdesc_handle *md)
 {
 	unsigned long paddr;
 	int count = 0;
 	u64 node;
 	mdesc_for_each_node_by_name(md, node, "memory-latency-group")
 		count++;
 	if (!count)
 		return -ENOENT;
 	paddr = memblock_alloc(count * sizeof(struct mdesc_mlgroup),
 			  SMP_CACHE_BYTES);
 	if (!paddr)
 		return -ENOMEM;
 	mlgroups = __va(paddr);
 	num_mlgroups = count;
 	count = 0;
 	mdesc_for_each_node_by_name(md, node, "memory-latency-group") {
 		struct mdesc_mlgroup *m = &mlgroups[count++];
 		const u64 *val;
 		m->node = node;
 		val = mdesc_get_property(md, node, "latency", NULL);
 		m->latency = *val;
 		val = mdesc_get_property(md, node, "address-match", NULL);
 		m->match = *val;
 		val = mdesc_get_property(md, node, "address-mask", NULL);
 		m->mask = *val;
 		numadbg("MLGROUP[%d]: node[%llx] latency[%llx] "
 			"match[%llx] mask[%llx]\n",
 			count - 1, m->node, m->latency, m->match, m->mask);
 	}
 	return 0;
 }
 static int __init grab_mblocks(struct mdesc_handle *md)
 {
 	unsigned long paddr;
 	int count = 0;
 	u64 node;
 	mdesc_for_each_node_by_name(md, node, "mblock")
 		count++;
 	if (!count)
 		return -ENOENT;
 	paddr = memblock_alloc(count * sizeof(struct mdesc_mblock),
 			  SMP_CACHE_BYTES);
 	if (!paddr)
 		return -ENOMEM;
 	mblocks = __va(paddr);
 	num_mblocks = count;
 	count = 0;
 	mdesc_for_each_node_by_name(md, node, "mblock") {
 		struct mdesc_mblock *m = &mblocks[count++];
 		const u64 *val;
 		val = mdesc_get_property(md, node, "base", NULL);
 		m->base = *val;
 		val = mdesc_get_property(md, node, "size", NULL);
 		m->size = *val;
 		val = mdesc_get_property(md, node,
 					 "address-congruence-offset", NULL);
 		/* The address-congruence-offset property is optional.
 		 * Explicity zero it be identifty this.
 		 */
 		if (val)
 			m->offset = *val;
 		else
 			m->offset = 0UL;
 		numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n",
 			count - 1, m->base, m->size, m->offset);
 	}
 	return 0;
 }
 static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md,
 					       u64 grp, cpumask_t *mask)
 {
 	u64 arc;
 	cpumask_clear(mask);
 	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) {
 		u64 target = mdesc_arc_target(md, arc);
 		const char *name = mdesc_node_name(md, target);
 		const u64 *id;
 		if (strcmp(name, "cpu"))
 			continue;
 		id = mdesc_get_property(md, target, "id", NULL);
 		if (*id < nr_cpu_ids)
 			cpumask_set_cpu(*id, mask);
 	}
 }
 static struct mdesc_mlgroup * __init find_mlgroup(u64 node)
 {
 	int i;
 	for (i = 0; i < num_mlgroups; i++) {
 		struct mdesc_mlgroup *m = &mlgroups[i];
 		if (m->node == node)
 			return m;
 	}
 	return NULL;
 }
 static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp,
 				      int index)
 {
 	struct mdesc_mlgroup *candidate = NULL;
 	u64 arc, best_latency = ~(u64)0;
 	struct node_mem_mask *n;
 	mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) {
 		u64 target = mdesc_arc_target(md, arc);
 		struct mdesc_mlgroup *m = find_mlgroup(target);
 		if (!m)
 			continue;
 		if (m->latency < best_latency) {
 			candidate = m;
 			best_latency = m->latency;
 		}
 	}
 	if (!candidate)
 		return -ENOENT;
 	if (num_node_masks != index) {
 		printk(KERN_ERR "Inconsistent NUMA state, "
 		       "index[%d] != num_node_masks[%d]\n",
 		       index, num_node_masks);
 		return -EINVAL;
 	}
 	n = &node_masks[num_node_masks++];
 	n->mask = candidate->mask;
 	n->val = candidate->match;
 	numadbg("NUMA NODE[%d]: mask[%lx] val[%lx] (latency[%llx])\n",
 		index, n->mask, n->val, candidate->latency);
 	return 0;
 }
 static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp,
 					 int index)
 {
 	cpumask_t mask;
 	int cpu;
 	numa_parse_mdesc_group_cpus(md, grp, &mask);
 	for_each_cpu(cpu, &mask)
 		numa_cpu_lookup_table[cpu] = index;
 	cpumask_copy(&numa_cpumask_lookup_table[index], &mask);
 	if (numa_debug) {
 		printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index);
 		for_each_cpu(cpu, &mask)
 			printk("%d ", cpu);
 		printk("]\n");
 	}
 	return numa_attach_mlgroup(md, grp, index);
 }
 static int __init numa_parse_mdesc(void)
 {
 	struct mdesc_handle *md = mdesc_grab();
 	int i, err, count;
 	u64 node;
 	node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups");
 	if (node == MDESC_NODE_NULL) {
 		mdesc_release(md);
 		return -ENOENT;
 	}
 	err = grab_mblocks(md);
 	if (err < 0)
 		goto out;
 	err = grab_mlgroups(md);
 	if (err < 0)
 		goto out;
 	count = 0;
 	mdesc_for_each_node_by_name(md, node, "group") {
 		err = numa_parse_mdesc_group(md, node, count);
 		if (err < 0)
 			break;
 		count++;
 	}
 	add_node_ranges();
 	for (i = 0; i < num_node_masks; i++) {
 		allocate_node_data(i);
 		node_set_online(i);
 	}
 	err = 0;
 out:
 	mdesc_release(md);
 	return err;
 }
 static int __init numa_parse_jbus(void)
 {
 	unsigned long cpu, index;
 	/* NUMA node id is encoded in bits 36 and higher, and there is
 	 * a 1-to-1 mapping from CPU ID to NUMA node ID.
 	 */
 	index = 0;
 	for_each_present_cpu(cpu) {
 		numa_cpu_lookup_table[cpu] = index;
 		cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu));
 		node_masks[index].mask = ~((1UL << 36UL) - 1UL);
 		node_masks[index].val = cpu << 36UL;
 		index++;
 	}
 	num_node_masks = index;
 	add_node_ranges();
 	for (index = 0; index < num_node_masks; index++) {
 		allocate_node_data(index);
 		node_set_online(index);
 	}
 	return 0;
 }
 static int __init numa_parse_sun4u(void)
 {
 	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		unsigned long ver;
 		__asm__ ("rdpr %%ver, %0" : "=r" (ver));
 		if ((ver >> 32UL) == __JALAPENO_ID ||
 		    (ver >> 32UL) == __SERRANO_ID)
 			return numa_parse_jbus();
 	}
 	return -1;
 }
 static int __init bootmem_init_numa(void)
 {
 	int err = -1;
 	numadbg("bootmem_init_numa()\n");
 	if (numa_enabled) {
 		if (tlb_type == hypervisor)
 			err = numa_parse_mdesc();
 		else
 			err = numa_parse_sun4u();
 	}
 	return err;
 }
 #else
 static int bootmem_init_numa(void)
 {
 	return -1;
 }
 #endif
 static void __init bootmem_init_nonnuma(void)
 {
 	unsigned long top_of_ram = memblock_end_of_DRAM();
 	unsigned long total_ram = memblock_phys_mem_size();
 	numadbg("bootmem_init_nonnuma()\n");
 	printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n",
 	       top_of_ram, total_ram);
 	printk(KERN_INFO "Memory hole size: %ldMB\n",
 	       (top_of_ram - total_ram) >> 20);
 	init_node_masks_nonnuma();
 	memblock_set_node(0, (phys_addr_t)ULLONG_MAX, &memblock.memory, 0);
 	allocate_node_data(0);
 	node_set_online(0);
 }
 static unsigned long __init bootmem_init(unsigned long phys_base)
 {
 	unsigned long end_pfn;
 	end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
 	max_pfn = max_low_pfn = end_pfn;
 	min_low_pfn = (phys_base >> PAGE_SHIFT);
 	if (bootmem_init_numa() < 0)
 		bootmem_init_nonnuma();
 	/* Dump memblock with node info. */
 	memblock_dump_all();
 	/* XXX cpu notifier XXX */
 	sparse_memory_present_with_active_regions(MAX_NUMNODES);
 	sparse_init();
 	return end_pfn;
 }
 static struct linux_prom64_registers pall[MAX_BANKS] __initdata;
 static int pall_ents __initdata;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 static unsigned long __ref kernel_map_range(unsigned long pstart,
 					    unsigned long pend, pgprot_t prot)
 {
 	unsigned long vstart = PAGE_OFFSET + pstart;
 	unsigned long vend = PAGE_OFFSET + pend;
 	unsigned long alloc_bytes = 0UL;
 	if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) {
 		prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n",
 			    vstart, vend);
 		prom_halt();
 	}
 	while (vstart < vend) {
 		unsigned long this_end, paddr = __pa(vstart);
 		pgd_t *pgd = pgd_offset_k(vstart);
 		pud_t *pud;
 		pmd_t *pmd;
 		pte_t *pte;
 		pud = pud_offset(pgd, vstart);
 		if (pud_none(*pud)) {
 			pmd_t *new;
 			new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
 			alloc_bytes += PAGE_SIZE;
 			pud_populate(&init_mm, pud, new);
 		}
 		pmd = pmd_offset(pud, vstart);
 		if (!pmd_present(*pmd)) {
 			pte_t *new;
 			new = __alloc_bootmem(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
 			alloc_bytes += PAGE_SIZE;
 			pmd_populate_kernel(&init_mm, pmd, new);
 		}
 		pte = pte_offset_kernel(pmd, vstart);
 		this_end = (vstart + PMD_SIZE) & PMD_MASK;
 		if (this_end > vend)
 			this_end = vend;
 		while (vstart < this_end) {
 			pte_val(*pte) = (paddr | pgprot_val(prot));
 			vstart += PAGE_SIZE;
 			paddr += PAGE_SIZE;
 			pte++;
 		}
 	}
 	return alloc_bytes;
 }
 extern unsigned int kvmap_linear_patch[1];
 #endif /* CONFIG_DEBUG_PAGEALLOC */
 static void __init kpte_set_val(unsigned long index, unsigned long val)
 {
 	unsigned long *ptr = kpte_linear_bitmap;
 	val <<= ((index % (BITS_PER_LONG / 2)) * 2);
 	ptr += (index / (BITS_PER_LONG / 2));
 	*ptr |= val;
 }
 static const unsigned long kpte_shift_min = 28; /* 256MB */
 static const unsigned long kpte_shift_max = 34; /* 16GB */
 static const unsigned long kpte_shift_incr = 3;
 static unsigned long kpte_mark_using_shift(unsigned long start, unsigned long end,
 					   unsigned long shift)
 {
 	unsigned long size = (1UL << shift);
 	unsigned long mask = (size - 1UL);
 	unsigned long remains = end - start;
 	unsigned long val;
 	if (remains < size || (start & mask))
 		return start;
 	/* VAL maps:
 	 *
 	 *	shift 28 --> kern_linear_pte_xor index 1
 	 *	shift 31 --> kern_linear_pte_xor index 2
 	 *	shift 34 --> kern_linear_pte_xor index 3
 	 */
 	val = ((shift - kpte_shift_min) / kpte_shift_incr) + 1;
 	remains &= ~mask;
 	if (shift != kpte_shift_max)
 		remains = size;
 	while (remains) {
 		unsigned long index = start >> kpte_shift_min;
 		kpte_set_val(index, val);
 		start += 1UL << kpte_shift_min;
 		remains -= 1UL << kpte_shift_min;
 	}
 	return start;
 }
 static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
 {
 	unsigned long smallest_size, smallest_mask;
 	unsigned long s;
 	smallest_size = (1UL << kpte_shift_min);
 	smallest_mask = (smallest_size - 1UL);
 	while (start < end) {
 		unsigned long orig_start = start;
 		for (s = kpte_shift_max; s >= kpte_shift_min; s -= kpte_shift_incr) {
 			start = kpte_mark_using_shift(start, end, s);
 			if (start != orig_start)
 				break;
 		}
 		if (start == orig_start)
 			start = (start + smallest_size) & ~smallest_mask;
 	}
 }
 static void __init init_kpte_bitmap(void)
 {
 	unsigned long i;
 	for (i = 0; i < pall_ents; i++) {
 		unsigned long phys_start, phys_end;
 		phys_start = pall[i].phys_addr;
 		phys_end = phys_start + pall[i].reg_size;
 		mark_kpte_bitmap(phys_start, phys_end);
 	}
 }
 static void __init kernel_physical_mapping_init(void)
 {
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	unsigned long i, mem_alloced = 0UL;
 	for (i = 0; i < pall_ents; i++) {
 		unsigned long phys_start, phys_end;
 		phys_start = pall[i].phys_addr;
 		phys_end = phys_start + pall[i].reg_size;
 		mem_alloced += kernel_map_range(phys_start, phys_end,
 						PAGE_KERNEL);
 	}
 	printk("Allocated %ld bytes for kernel page tables.\n",
 	       mem_alloced);
 	kvmap_linear_patch[0] = 0x01000000; /* nop */
 	flushi(&kvmap_linear_patch[0]);
 	__flush_tlb_all();
 #endif
 }
 #ifdef CONFIG_DEBUG_PAGEALLOC
 void kernel_map_pages(struct page *page, int numpages, int enable)
 {
 	unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT;
 	unsigned long phys_end = phys_start + (numpages * PAGE_SIZE);
 	kernel_map_range(phys_start, phys_end,
 			 (enable ? PAGE_KERNEL : __pgprot(0)));
 	flush_tsb_kernel_range(PAGE_OFFSET + phys_start,
 			       PAGE_OFFSET + phys_end);
 	/* we should perform an IPI and flush all tlbs,
 	 * but that can deadlock->flush only current cpu.
 	 */
 	__flush_tlb_kernel_range(PAGE_OFFSET + phys_start,
 				 PAGE_OFFSET + phys_end);
 }
 #endif
 unsigned long __init find_ecache_flush_span(unsigned long size)
 {
 	int i;
 	for (i = 0; i < pavail_ents; i++) {
 		if (pavail[i].reg_size >= size)
 			return pavail[i].phys_addr;
 	}
 	return ~0UL;
 }
 unsigned long PAGE_OFFSET;
 EXPORT_SYMBOL(PAGE_OFFSET);
 static void __init page_offset_shift_patch_one(unsigned int *insn, unsigned long phys_bits)
 {
 	unsigned long final_shift;
 	unsigned int val = *insn;
 	unsigned int cnt;
 	/* We are patching in ilog2(max_supported_phys_address), and
 	 * we are doing so in a manner similar to a relocation addend.
 	 * That is, we are adding the shift value to whatever value
 	 * is in the shift instruction count field already.
 	 */
 	cnt = (val & 0x3f);
 	val &= ~0x3f;
 	/* If we are trying to shift >= 64 bits, clear the destination
 	 * register.  This can happen when phys_bits ends up being equal
 	 * to MAX_PHYS_ADDRESS_BITS.
 	 */
 	final_shift = (cnt + (64 - phys_bits));
 	if (final_shift >= 64) {
 		unsigned int rd = (val >> 25) & 0x1f;
 		val = 0x80100000 | (rd << 25);
 	} else {
 		val |= final_shift;
 	}
 	*insn = val;
 	__asm__ __volatile__("flush	%0"
 			     : /* no outputs */
 			     : "r" (insn));
 }
 static void __init page_offset_shift_patch(unsigned long phys_bits)
 {
 	extern unsigned int __page_offset_shift_patch;
 	extern unsigned int __page_offset_shift_patch_end;
 	unsigned int *p;
 	p = &__page_offset_shift_patch;
 	while (p < &__page_offset_shift_patch_end) {
 		unsigned int *insn = (unsigned int *)(unsigned long)*p;
 		page_offset_shift_patch_one(insn, phys_bits);
 		p++;
 	}
 }
 static void __init setup_page_offset(void)
 {
 	unsigned long max_phys_bits = 40;
 	if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		max_phys_bits = 42;
 	} else if (tlb_type == hypervisor) {
 		switch (sun4v_chip_type) {
 		case SUN4V_CHIP_NIAGARA1:
 		case SUN4V_CHIP_NIAGARA2:
 			max_phys_bits = 39;
 			break;
 		case SUN4V_CHIP_NIAGARA3:
 			max_phys_bits = 43;
 			break;
 		case SUN4V_CHIP_NIAGARA4:
 		case SUN4V_CHIP_NIAGARA5:
 		case SUN4V_CHIP_SPARC64X:
 		default:
 			max_phys_bits = 47;
 			break;
 		}
 	}
 	if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) {
 		prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n",
 			    max_phys_bits);
 		prom_halt();
 	}
 	PAGE_OFFSET = PAGE_OFFSET_BY_BITS(max_phys_bits);
 	pr_info("PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n",
 		PAGE_OFFSET, max_phys_bits);
 	page_offset_shift_patch(max_phys_bits);
 }
 static void __init tsb_phys_patch(void)
 {
 	struct tsb_ldquad_phys_patch_entry *pquad;
 	struct tsb_phys_patch_entry *p;
 	pquad = &__tsb_ldquad_phys_patch;
 	while (pquad < &__tsb_ldquad_phys_patch_end) {
 		unsigned long addr = pquad->addr;
 		if (tlb_type == hypervisor)
 			*(unsigned int *) addr = pquad->sun4v_insn;
 		else
 			*(unsigned int *) addr = pquad->sun4u_insn;
 		wmb();
 		__asm__ __volatile__("flush	%0"
 				     : /* no outputs */
 				     : "r" (addr));
 		pquad++;
 	}
 	p = &__tsb_phys_patch;
 	while (p < &__tsb_phys_patch_end) {
 		unsigned long addr = p->addr;
 		*(unsigned int *) addr = p->insn;
 		wmb();
 		__asm__ __volatile__("flush	%0"
 				     : /* no outputs */
 				     : "r" (addr));
 		p++;
 	}
 }
 /* Don't mark as init, we give this to the Hypervisor.  */
 #ifndef CONFIG_DEBUG_PAGEALLOC
 #define NUM_KTSB_DESCR	2
 #else
 #define NUM_KTSB_DESCR	1
 #endif
 static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR];
 extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
 static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa)
 {
 	pa >>= KTSB_PHYS_SHIFT;
 	while (start < end) {
 		unsigned int *ia = (unsigned int *)(unsigned long)*start;
 		ia[0] = (ia[0] & ~0x3fffff) | (pa >> 10);
 		__asm__ __volatile__("flush	%0" : : "r" (ia));
 		ia[1] = (ia[1] & ~0x3ff) | (pa & 0x3ff);
 		__asm__ __volatile__("flush	%0" : : "r" (ia + 1));
 		start++;
 	}
 }
 static void ktsb_phys_patch(void)
 {
 	extern unsigned int __swapper_tsb_phys_patch;
 	extern unsigned int __swapper_tsb_phys_patch_end;
 	unsigned long ktsb_pa;
 	ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
 	patch_one_ktsb_phys(&__swapper_tsb_phys_patch,
 			    &__swapper_tsb_phys_patch_end, ktsb_pa);
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	{
 	extern unsigned int __swapper_4m_tsb_phys_patch;
 	extern unsigned int __swapper_4m_tsb_phys_patch_end;
 	ktsb_pa = (kern_base +
 		   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
 	patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch,
 			    &__swapper_4m_tsb_phys_patch_end, ktsb_pa);
 	}
 #endif
 }
 static void __init sun4v_ktsb_init(void)
 {
 	unsigned long ktsb_pa;
 	/* First KTSB for PAGE_SIZE mappings.  */
 	ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE);
 	switch (PAGE_SIZE) {
 	case 8 * 1024:
 	default:
 		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K;
 		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K;
 		break;
 	case 64 * 1024:
 		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K;
 		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K;
 		break;
 	case 512 * 1024:
 		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K;
 		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K;
 		break;
 	case 4 * 1024 * 1024:
 		ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB;
 		ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB;
 		break;
 	}
 	ktsb_descr[0].assoc = 1;
 	ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES;
 	ktsb_descr[0].ctx_idx = 0;
 	ktsb_descr[0].tsb_base = ktsb_pa;
 	ktsb_descr[0].resv = 0;
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	/* Second KTSB for 4MB/256MB/2GB/16GB mappings.  */
 	ktsb_pa = (kern_base +
 		   ((unsigned long)&swapper_4m_tsb[0] - KERNBASE));
 	ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB;
 	ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB |
 				    HV_PGSZ_MASK_256MB |
 				    HV_PGSZ_MASK_2GB |
 				    HV_PGSZ_MASK_16GB) &
 				   cpu_pgsz_mask);
 	ktsb_descr[1].assoc = 1;
 	ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES;
 	ktsb_descr[1].ctx_idx = 0;
 	ktsb_descr[1].tsb_base = ktsb_pa;
 	ktsb_descr[1].resv = 0;
 #endif
 }
 void sun4v_ktsb_register(void)
 {
 	unsigned long pa, ret;
 	pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE);
 	ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa);
 	if (ret != 0) {
 		prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: "
 			    "errors with %lx\n", pa, ret);
 		prom_halt();
 	}
 }
 static void __init sun4u_linear_pte_xor_finalize(void)
 {
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	/* This is where we would add Panther support for
 	 * 32MB and 256MB pages.
 	 */
 #endif
 }
 static void __init sun4v_linear_pte_xor_finalize(void)
 {
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) {
 		kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^
 			PAGE_OFFSET;
 		kern_linear_pte_xor[1] |= (_PAGE_CP_4V | _PAGE_CV_4V |
 					   _PAGE_P_4V | _PAGE_W_4V);
 	} else {
 		kern_linear_pte_xor[1] = kern_linear_pte_xor[0];
 	}
 	if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) {
 		kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^
 			PAGE_OFFSET;
 		kern_linear_pte_xor[2] |= (_PAGE_CP_4V | _PAGE_CV_4V |
 					   _PAGE_P_4V | _PAGE_W_4V);
 	} else {
 		kern_linear_pte_xor[2] = kern_linear_pte_xor[1];
 	}
 	if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) {
 		kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^
 			PAGE_OFFSET;
 		kern_linear_pte_xor[3] |= (_PAGE_CP_4V | _PAGE_CV_4V |
 					   _PAGE_P_4V | _PAGE_W_4V);
 	} else {
 		kern_linear_pte_xor[3] = kern_linear_pte_xor[2];
 	}
 #endif
 }
 /* paging_init() sets up the page tables */
 static unsigned long last_valid_pfn;
 pgd_t swapper_pg_dir[PTRS_PER_PGD];
 static void sun4u_pgprot_init(void);
 static void sun4v_pgprot_init(void);
 void __init paging_init(void)
 {
 	unsigned long end_pfn, shift, phys_base;
 	unsigned long real_end, i;
 	int node;
 	setup_page_offset();
 	/* These build time checkes make sure that the dcache_dirty_cpu()
 	 * page->flags usage will work.
 	 *
 	 * When a page gets marked as dcache-dirty, we store the
 	 * cpu number starting at bit 32 in the page->flags.  Also,
 	 * functions like clear_dcache_dirty_cpu use the cpu mask
 	 * in 13-bit signed-immediate instruction fields.
 	 */
 	/*
 	 * Page flags must not reach into upper 32 bits that are used
 	 * for the cpu number
 	 */
 	BUILD_BUG_ON(NR_PAGEFLAGS > 32);
 	/*
 	 * The bit fields placed in the high range must not reach below
 	 * the 32 bit boundary. Otherwise we cannot place the cpu field
 	 * at the 32 bit boundary.
 	 */
 	BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH +
 		ilog2(roundup_pow_of_two(NR_CPUS)) > 32);
 	BUILD_BUG_ON(NR_CPUS > 4096);
 	kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB;
 	kern_size = (unsigned long)&_end - (unsigned long)KERNBASE;
 	/* Invalidate both kernel TSBs.  */
 	memset(swapper_tsb, 0x40, sizeof(swapper_tsb));
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
 #endif
 	if (tlb_type == hypervisor)
 		sun4v_pgprot_init();
 	else
 		sun4u_pgprot_init();
 	if (tlb_type == cheetah_plus ||
 	    tlb_type == hypervisor) {
 		tsb_phys_patch();
 		ktsb_phys_patch();
 	}
 	if (tlb_type == hypervisor)
 		sun4v_patch_tlb_handlers();
 	/* Find available physical memory...
 	 *
 	 * Read it twice in order to work around a bug in openfirmware.
 	 * The call to grab this table itself can cause openfirmware to
 	 * allocate memory, which in turn can take away some space from
 	 * the list of available memory.  Reading it twice makes sure
 	 * we really do get the final value.
 	 */
 	read_obp_translations();
 	read_obp_memory("reg", &pall[0], &pall_ents);
 	read_obp_memory("available", &pavail[0], &pavail_ents);
 	read_obp_memory("available", &pavail[0], &pavail_ents);
 	phys_base = 0xffffffffffffffffUL;
 	for (i = 0; i < pavail_ents; i++) {
 		phys_base = min(phys_base, pavail[i].phys_addr);
 		memblock_add(pavail[i].phys_addr, pavail[i].reg_size);
 	}
 	memblock_reserve(kern_base, kern_size);
 	find_ramdisk(phys_base);
 	memblock_enforce_memory_limit(cmdline_memory_size);
 	memblock_allow_resize();
 	memblock_dump_all();
 	set_bit(0, mmu_context_bmap);
 	shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE);
 	real_end = (unsigned long)_end;
 	num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB);
 	printk("Kernel: Using %d locked TLB entries for main kernel image.\n",
 	       num_kernel_image_mappings);
 	/* Set kernel pgd to upper alias so physical page computations
 	 * work.
 	 */
 	init_mm.pgd += ((shift) / (sizeof(pgd_t)));
 	memset(swapper_low_pmd_dir, 0, sizeof(swapper_low_pmd_dir));
 	/* Now can init the kernel/bad page tables. */
 	pud_set(pud_offset(&swapper_pg_dir[0], 0),
 		swapper_low_pmd_dir + (shift / sizeof(pgd_t)));
 	inherit_prom_mappings();
 	init_kpte_bitmap();
 	/* Ok, we can use our TLB miss and window trap handlers safely.  */
 	setup_tba();
 	__flush_tlb_all();
 	prom_build_devicetree();
 	of_populate_present_mask();
 #ifndef CONFIG_SMP
 	of_fill_in_cpu_data();
 #endif
 	if (tlb_type == hypervisor) {
 		sun4v_mdesc_init();
 		mdesc_populate_present_mask(cpu_all_mask);
 #ifndef CONFIG_SMP
 		mdesc_fill_in_cpu_data(cpu_all_mask);
 #endif
 		mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask);
 		sun4v_linear_pte_xor_finalize();
 		sun4v_ktsb_init();
 		sun4v_ktsb_register();
 	} else {
 		unsigned long impl, ver;
 		cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K |
 				 HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB);
 		__asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver));
 		impl = ((ver >> 32) & 0xffff);
 		if (impl == PANTHER_IMPL)
 			cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB |
 					  HV_PGSZ_MASK_256MB);
 		sun4u_linear_pte_xor_finalize();
 	}
 	/* Flush the TLBs and the 4M TSB so that the updated linear
 	 * pte XOR settings are realized for all mappings.
 	 */
 	__flush_tlb_all();
 #ifndef CONFIG_DEBUG_PAGEALLOC
 	memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb));
 #endif
 	__flush_tlb_all();
 	/* Setup bootmem... */
 	last_valid_pfn = end_pfn = bootmem_init(phys_base);
 	/* Once the OF device tree and MDESC have been setup, we know
 	 * the list of possible cpus.  Therefore we can allocate the
 	 * IRQ stacks.
 	 */
 	for_each_possible_cpu(i) {
 		node = cpu_to_node(i);
 		softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),
 							THREAD_SIZE,
 							THREAD_SIZE, 0);
 		hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node),
 							THREAD_SIZE,
 							THREAD_SIZE, 0);
 	}
 	kernel_physical_mapping_init();
 	{
 		unsigned long max_zone_pfns[MAX_NR_ZONES];
 		memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
 		max_zone_pfns[ZONE_NORMAL] = end_pfn;
 		free_area_init_nodes(max_zone_pfns);
 	}
 	printk("Booting Linux...\n");
 }
 int page_in_phys_avail(unsigned long paddr)
 {
 	int i;
 	paddr &= PAGE_MASK;
 	for (i = 0; i < pavail_ents; i++) {
 		unsigned long start, end;
 		start = pavail[i].phys_addr;
 		end = start + pavail[i].reg_size;
 		if (paddr >= start && paddr < end)
 			return 1;
 	}
 	if (paddr >= kern_base && paddr < (kern_base + kern_size))
 		return 1;
 #ifdef CONFIG_BLK_DEV_INITRD
 	if (paddr >= __pa(initrd_start) &&
 	    paddr < __pa(PAGE_ALIGN(initrd_end)))
 		return 1;
 #endif
 	return 0;
 }
 static struct linux_prom64_registers pavail_rescan[MAX_BANKS] __initdata;
 static int pavail_rescan_ents __initdata;
 /* Certain OBP calls, such as fetching "available" properties, can
  * claim physical memory.  So, along with initializing the valid
  * address bitmap, what we do here is refetch the physical available
  * memory list again, and make sure it provides at least as much
  * memory as 'pavail' does.
  */
 static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
 {
 	int i;
 	read_obp_memory("available", &pavail_rescan[0], &pavail_rescan_ents);
 	for (i = 0; i < pavail_ents; i++) {
 		unsigned long old_start, old_end;
 		old_start = pavail[i].phys_addr;
 		old_end = old_start + pavail[i].reg_size;
 		while (old_start < old_end) {
 			int n;
 			for (n = 0; n < pavail_rescan_ents; n++) {
 				unsigned long new_start, new_end;
 				new_start = pavail_rescan[n].phys_addr;
 				new_end = new_start +
 					pavail_rescan[n].reg_size;
 				if (new_start <= old_start &&
 				    new_end >= (old_start + PAGE_SIZE)) {
 					set_bit(old_start >> ILOG2_4MB, bitmap);
 					goto do_next_page;
 				}
 			}
 			prom_printf("mem_init: Lost memory in pavail\n");
 			prom_printf("mem_init: OLD start[%lx] size[%lx]\n",
 				    pavail[i].phys_addr,
 				    pavail[i].reg_size);
 			prom_printf("mem_init: NEW start[%lx] size[%lx]\n",
 				    pavail_rescan[i].phys_addr,
 				    pavail_rescan[i].reg_size);
 			prom_printf("mem_init: Cannot continue, aborting.\n");
 			prom_halt();
 		do_next_page:
 			old_start += PAGE_SIZE;
 		}
 	}
 }
 static void __init patch_tlb_miss_handler_bitmap(void)
 {
 	extern unsigned int valid_addr_bitmap_insn[];
 	extern unsigned int valid_addr_bitmap_patch[];
 	valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
 	mb();
 	valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
 	flushi(&valid_addr_bitmap_insn[0]);
 }
 static void __init register_page_bootmem_info(void)
 {
 #ifdef CONFIG_NEED_MULTIPLE_NODES
 	int i;
 	for_each_online_node(i)
 		if (NODE_DATA(i)->node_spanned_pages)
 			register_page_bootmem_info_node(NODE_DATA(i));
 #endif
 }
 void __init mem_init(void)
 {
 	unsigned long addr, last;
 	addr = PAGE_OFFSET + kern_base;
 	last = PAGE_ALIGN(kern_size) + addr;
 	while (addr < last) {
 		set_bit(__pa(addr) >> ILOG2_4MB, sparc64_valid_addr_bitmap);
 		addr += PAGE_SIZE;
 	}
 	setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
 	patch_tlb_miss_handler_bitmap();
 	high_memory = __va(last_valid_pfn << PAGE_SHIFT);
 	register_page_bootmem_info();
 	free_all_bootmem();
 	/*
 	 * Set up the zero page, mark it reserved, so that page count
 	 * is not manipulated when freeing the page from user ptes.
 	 */
 	mem_map_zero = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0);
 	if (mem_map_zero == NULL) {
 		prom_printf("paging_init: Cannot alloc zero page.\n");
 		prom_halt();
 	}
 	mark_page_reserved(mem_map_zero);
 	mem_init_print_info(NULL);
 	if (tlb_type == cheetah || tlb_type == cheetah_plus)
 		cheetah_ecache_flush_init();
 }
 void free_initmem(void)
 {
 	unsigned long addr, initend;
 	int do_free = 1;
 	/* If the physical memory maps were trimmed by kernel command
 	 * line options, don't even try freeing this initmem stuff up.
 	 * The kernel image could have been in the trimmed out region
 	 * and if so the freeing below will free invalid page structs.
 	 */
 	if (cmdline_memory_size)
 		do_free = 0;
 	/*
 	 * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes.
 	 */
 	addr = PAGE_ALIGN((unsigned long)(__init_begin));
 	initend = (unsigned long)(__init_end) & PAGE_MASK;
 	for (; addr < initend; addr += PAGE_SIZE) {
 		unsigned long page;
 		page = (addr +
 			((unsigned long) __va(kern_base)) -
 			((unsigned long) KERNBASE));
 		memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
 		if (do_free)
 			free_reserved_page(virt_to_page(page));
 	}
 }
 #ifdef CONFIG_BLK_DEV_INITRD
 void free_initrd_mem(unsigned long start, unsigned long end)
 {
 	free_reserved_area((void *)start, (void *)end, POISON_FREE_INITMEM,
 			   "initrd");
 }
 #endif
 #define _PAGE_CACHE_4U	(_PAGE_CP_4U | _PAGE_CV_4U)
 #define _PAGE_CACHE_4V	(_PAGE_CP_4V | _PAGE_CV_4V)
 #define __DIRTY_BITS_4U	 (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U)
 #define __DIRTY_BITS_4V	 (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V)
 #define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R)
 #define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R)
 pgprot_t PAGE_KERNEL __read_mostly;
 EXPORT_SYMBOL(PAGE_KERNEL);
 pgprot_t PAGE_KERNEL_LOCKED __read_mostly;
 pgprot_t PAGE_COPY __read_mostly;
 pgprot_t PAGE_SHARED __read_mostly;
 EXPORT_SYMBOL(PAGE_SHARED);
 unsigned long pg_iobits __read_mostly;
 unsigned long _PAGE_IE __read_mostly;
 EXPORT_SYMBOL(_PAGE_IE);
 unsigned long _PAGE_E __read_mostly;
 EXPORT_SYMBOL(_PAGE_E);
 unsigned long _PAGE_CACHE __read_mostly;
 EXPORT_SYMBOL(_PAGE_CACHE);
 #ifdef CONFIG_SPARSEMEM_VMEMMAP
 unsigned long vmemmap_table[VMEMMAP_SIZE];
 static long __meminitdata addr_start, addr_end;
 static int __meminitdata node_start;
 int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend,
 			       int node)
 {
 	unsigned long phys_start = (vstart - VMEMMAP_BASE);
 	unsigned long phys_end = (vend - VMEMMAP_BASE);
 	unsigned long addr = phys_start & VMEMMAP_CHUNK_MASK;
 	unsigned long end = VMEMMAP_ALIGN(phys_end);
 	unsigned long pte_base;
 	pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U |
 		    _PAGE_CP_4U | _PAGE_CV_4U |
 		    _PAGE_P_4U | _PAGE_W_4U);
 	if (tlb_type == hypervisor)
 		pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V |
 			    _PAGE_CP_4V | _PAGE_CV_4V |
 			    _PAGE_P_4V | _PAGE_W_4V);
 	for (; addr < end; addr += VMEMMAP_CHUNK) {
 		unsigned long *vmem_pp =
 			vmemmap_table + (addr >> VMEMMAP_CHUNK_SHIFT);
 		void *block;
 		if (!(*vmem_pp & _PAGE_VALID)) {
 			block = vmemmap_alloc_block(1UL << ILOG2_4MB, node);
 			if (!block)
 				return -ENOMEM;
 			*vmem_pp = pte_base | __pa(block);
 			/* check to see if we have contiguous blocks */
 			if (addr_end != addr || node_start != node) {
 				if (addr_start)
 					printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
 					       addr_start, addr_end-1, node_start);
 				addr_start = addr;
 				node_start = node;
 			}
 			addr_end = addr + VMEMMAP_CHUNK;
 		}
 	}
 	return 0;
 }
 void __meminit vmemmap_populate_print_last(void)
 {
 	if (addr_start) {
 		printk(KERN_DEBUG " [%lx-%lx] on node %d\n",
 		       addr_start, addr_end-1, node_start);
 		addr_start = 0;
 		addr_end = 0;
 		node_start = 0;
 	}
 }
 void vmemmap_free(unsigned long start, unsigned long end)
 {
 }
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 static void prot_init_common(unsigned long page_none,
 			     unsigned long page_shared,
 			     unsigned long page_copy,
 			     unsigned long page_readonly,
 			     unsigned long page_exec_bit)
 {
 	PAGE_COPY = __pgprot(page_copy);
 	PAGE_SHARED = __pgprot(page_shared);
 	protection_map[0x0] = __pgprot(page_none);
 	protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit);
 	protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit);
 	protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit);
 	protection_map[0x4] = __pgprot(page_readonly);
 	protection_map[0x5] = __pgprot(page_readonly);
 	protection_map[0x6] = __pgprot(page_copy);
 	protection_map[0x7] = __pgprot(page_copy);
 	protection_map[0x8] = __pgprot(page_none);
 	protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit);
 	protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit);
 	protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit);
 	protection_map[0xc] = __pgprot(page_readonly);
 	protection_map[0xd] = __pgprot(page_readonly);
 	protection_map[0xe] = __pgprot(page_shared);
 	protection_map[0xf] = __pgprot(page_shared);
 }
 static void __init sun4u_pgprot_init(void)
 {
 	unsigned long page_none, page_shared, page_copy, page_readonly;
 	unsigned long page_exec_bit;
 	int i;
 	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
 				_PAGE_CACHE_4U | _PAGE_P_4U |
 				__ACCESS_BITS_4U | __DIRTY_BITS_4U |
 				_PAGE_EXEC_4U);
 	PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID |
 				       _PAGE_CACHE_4U | _PAGE_P_4U |
 				       __ACCESS_BITS_4U | __DIRTY_BITS_4U |
 				       _PAGE_EXEC_4U | _PAGE_L_4U);
 	_PAGE_IE = _PAGE_IE_4U;
 	_PAGE_E = _PAGE_E_4U;
 	_PAGE_CACHE = _PAGE_CACHE_4U;
 	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U |
 		     __ACCESS_BITS_4U | _PAGE_E_4U);
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
 #else
 	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^
 		PAGE_OFFSET;
 #endif
 	kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U |
 				   _PAGE_P_4U | _PAGE_W_4U);
 	for (i = 1; i < 4; i++)
 		kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
 	_PAGE_ALL_SZ_BITS =  (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U |
 			      _PAGE_SZ64K_4U | _PAGE_SZ8K_4U |
 			      _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U);
 	page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U;
 	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
 		       __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U);
 	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
 		       __ACCESS_BITS_4U | _PAGE_EXEC_4U);
 	page_readonly   = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U |
 			   __ACCESS_BITS_4U | _PAGE_EXEC_4U);
 	page_exec_bit = _PAGE_EXEC_4U;
 	prot_init_common(page_none, page_shared, page_copy, page_readonly,
 			 page_exec_bit);
 }
 static void __init sun4v_pgprot_init(void)
 {
 	unsigned long page_none, page_shared, page_copy, page_readonly;
 	unsigned long page_exec_bit;
 	int i;
 	PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID |
 				_PAGE_CACHE_4V | _PAGE_P_4V |
 				__ACCESS_BITS_4V | __DIRTY_BITS_4V |
 				_PAGE_EXEC_4V);
 	PAGE_KERNEL_LOCKED = PAGE_KERNEL;
 	_PAGE_IE = _PAGE_IE_4V;
 	_PAGE_E = _PAGE_E_4V;
 	_PAGE_CACHE = _PAGE_CACHE_4V;
 #ifdef CONFIG_DEBUG_PAGEALLOC
 	kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET;
 #else
 	kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^
 		PAGE_OFFSET;
 #endif
 	kern_linear_pte_xor[0] |= (_PAGE_CP_4V | _PAGE_CV_4V |
 				   _PAGE_P_4V | _PAGE_W_4V);
 	for (i = 1; i < 4; i++)
 		kern_linear_pte_xor[i] = kern_linear_pte_xor[0];
 	pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V |
 		     __ACCESS_BITS_4V | _PAGE_E_4V);
 	_PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V |
 			     _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V |
 			     _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V |
 			     _PAGE_SZ64K_4V | _PAGE_SZ8K_4V);
 	page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | _PAGE_CACHE_4V;
 	page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
 		       __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V);
 	page_copy   = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
 		       __ACCESS_BITS_4V | _PAGE_EXEC_4V);
 	page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | _PAGE_CACHE_4V |
 			 __ACCESS_BITS_4V | _PAGE_EXEC_4V);
 	page_exec_bit = _PAGE_EXEC_4V;
 	prot_init_common(page_none, page_shared, page_copy, page_readonly,
 			 page_exec_bit);
 }
 unsigned long pte_sz_bits(unsigned long sz)
 {
 	if (tlb_type == hypervisor) {
 		switch (sz) {
 		case 8 * 1024:
 		default:
 			return _PAGE_SZ8K_4V;
 		case 64 * 1024:
 			return _PAGE_SZ64K_4V;
 		case 512 * 1024:
 			return _PAGE_SZ512K_4V;
 		case 4 * 1024 * 1024:
 			return _PAGE_SZ4MB_4V;
 		}
 	} else {
 		switch (sz) {
 		case 8 * 1024:
 		default:
 			return _PAGE_SZ8K_4U;
 		case 64 * 1024:
 			return _PAGE_SZ64K_4U;
 		case 512 * 1024:
 			return _PAGE_SZ512K_4U;
 		case 4 * 1024 * 1024:
 			return _PAGE_SZ4MB_4U;
 		}
 	}
 }
 pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size)
 {
 	pte_t pte;
 	pte_val(pte)  = page | pgprot_val(pgprot_noncached(prot));
 	pte_val(pte) |= (((unsigned long)space) << 32);
 	pte_val(pte) |= pte_sz_bits(page_size);
 	return pte;
 }
 static unsigned long kern_large_tte(unsigned long paddr)
 {
 	unsigned long val;
 	val = (_PAGE_VALID | _PAGE_SZ4MB_4U |
 	       _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U |
 	       _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U);
 	if (tlb_type == hypervisor)
 		val = (_PAGE_VALID | _PAGE_SZ4MB_4V |
 		       _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_P_4V |
 		       _PAGE_EXEC_4V | _PAGE_W_4V);
 	return val | paddr;
 }
 /* If not locked, zap it. */
 void __flush_tlb_all(void)
 {
 	unsigned long pstate;
 	int i;
 	__asm__ __volatile__("flushw\n\t"
 			     "rdpr	%%pstate, %0\n\t"
 			     "wrpr	%0, %1, %%pstate"
 			     : "=r" (pstate)
 			     : "i" (PSTATE_IE));
 	if (tlb_type == hypervisor) {
 		sun4v_mmu_demap_all();
 	} else if (tlb_type == spitfire) {
 		for (i = 0; i < 64; i++) {
 			/* Spitfire Errata #32 workaround */
 			/* NOTE: Always runs on spitfire, so no
 			 *       cheetah+ page size encodings.
 			 */
 			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
 					     "flush	%%g6"
 					     : /* No outputs */
 					     : "r" (0),
 					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 			if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) {
 				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 						     "membar #Sync"
 						     : /* no outputs */
 						     : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU));
 				spitfire_put_dtlb_data(i, 0x0UL);
 			}
 			/* Spitfire Errata #32 workaround */
 			/* NOTE: Always runs on spitfire, so no
 			 *       cheetah+ page size encodings.
 			 */
 			__asm__ __volatile__("stxa	%0, [%1] %2\n\t"
 					     "flush	%%g6"
 					     : /* No outputs */
 					     : "r" (0),
 					     "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU));
 			if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) {
 				__asm__ __volatile__("stxa %%g0, [%0] %1\n\t"
 						     "membar #Sync"
 						     : /* no outputs */
 						     : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU));
 				spitfire_put_itlb_data(i, 0x0UL);
 			}
 		}
 	} else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
 		cheetah_flush_dtlb_all();
 		cheetah_flush_itlb_all();
 	}
 	__asm__ __volatile__("wrpr	%0, 0, %%pstate"
 			     : : "r" (pstate));
 }
 pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 			    unsigned long address)
 {
 	struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
 				       __GFP_REPEAT | __GFP_ZERO);
 	pte_t *pte = NULL;
 	if (page)
 		pte = (pte_t *) page_address(page);
 	return pte;
 }
 pgtable_t pte_alloc_one(struct mm_struct *mm,
 			unsigned long address)
 {
 	struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
 				       __GFP_REPEAT | __GFP_ZERO);
 	if (!page)
 		return NULL;
 	if (!pgtable_page_ctor(page)) {
 		free_hot_cold_page(page, 0);
 		return NULL;
 	}
 	return (pte_t *) page_address(page);
 }
 void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
 	free_page((unsigned long)pte);
 }
 static void __pte_free(pgtable_t pte)
 {
 	struct page *page = virt_to_page(pte);
 	pgtable_page_dtor(page);
 	__free_page(page);
 }
 void pte_free(struct mm_struct *mm, pgtable_t pte)
 {
 	__pte_free(pte);
 }
 void pgtable_free(void *table, bool is_page)
 {
 	if (is_page)
 		__pte_free(table);
 	else
 		kmem_cache_free(pgtable_cache, table);
 }
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
 			  pmd_t *pmd)
 {
 	unsigned long pte, flags;
 	struct mm_struct *mm;
 	pmd_t entry = *pmd;
 	if (!pmd_large(entry) || !pmd_young(entry))
 		return;
 	pte = pmd_val(entry);
+	/* Don't insert a non-valid PMD into the TSB, we'll deadlock.  */
+	if (!(pte & _PAGE_VALID))
+		return;
 	/* We are fabricating 8MB pages using 4MB real hw pages.  */
 	pte |= (addr & (1UL << REAL_HPAGE_SHIFT));
 	mm = vma->vm_mm;
 	spin_lock_irqsave(&mm->context.lock, flags);
 	if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
 		__update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT,
 					addr, pte);
 	spin_unlock_irqrestore(&mm->context.lock, flags);
 }
 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 static void context_reload(void *__data)
 {
 	struct mm_struct *mm = __data;
 	if (mm == current->mm)
 		load_secondary_context(mm);
 }
 void hugetlb_setup(struct pt_regs *regs)
 {
 	struct mm_struct *mm = current->mm;
 	struct tsb_config *tp;
 	if (in_atomic() || !mm) {
 		const struct exception_table_entry *entry;
 		entry = search_exception_tables(regs->tpc);
 		if (entry) {
 			regs->tpc = entry->fixup;
 			regs->tnpc = regs->tpc + 4;
 			return;
 		}
 		pr_alert("Unexpected HugeTLB setup in atomic context.\n");
 		die_if_kernel("HugeTSB in atomic", regs);
 	}
 	tp = &mm->context.tsb_block[MM_TSB_HUGE];
 	if (likely(tp->tsb == NULL))
 		tsb_grow(mm, MM_TSB_HUGE, 0);
 	tsb_context_switch(mm);
 	smp_tsb_sync(mm);
 	/* On UltraSPARC-III+ and later, configure the second half of
 	 * the Data-TLB for huge pages.
 	 */
 	if (tlb_type == cheetah_plus) {
 		unsigned long ctx;
 		spin_lock(&ctx_alloc_lock);
 		ctx = mm->context.sparc64_ctx_val;
 		ctx &= ~CTX_PGSZ_MASK;
 		ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
 		ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
 		if (ctx != mm->context.sparc64_ctx_val) {
 			/* When changing the page size fields, we
 			 * must perform a context flush so that no
 			 * stale entries match.  This flush must
 			 * occur with the original context register
 			 * settings.
 			 */
 			do_flush_tlb_mm(mm);
 			/* Reload the context register of all processors
 			 * also executing in this address space.
 			 */
 			mm->context.sparc64_ctx_val = ctx;
 			on_each_cpu(context_reload, mm, 0);
 		}
 		spin_unlock(&ctx_alloc_lock);
 	}
 }
 #endif