Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 325f8a0a31df567dbafafc48f8e60f3c1f101a46

Authored by David Daney 2009-12-05 05:52:36 +0800

Committed by Ralf Baechle 2010-02-27 19:53:03 +0800

Exists in master and in 7 other branches

MIPS: Two-level pagetables for 64-bit kernels with 64KB pages.

For 64-bit kernels with 64KB pages and two level page tables, there are
42 bits worth of virtual address space This is larger than the 40 bits of
virtual address space obtained with the default 4KB Page size and three
levels, so there are no draw backs for using two level tables with this
configuration.

Signed-off-by: David Daney <ddaney@caviumnetworks.com>
Cc: linux-mips@linux-mips.org
Patchwork: http://patchwork.linux-mips.org/patch/761/
Signed-off-by: Ralf Baechle <ralf@linux-mips.org>

Showing 8 changed files with 71 additions and 35 deletions Inline Diff

arch/mips/include/asm/page.h
arch/mips/include/asm/pgalloc.h
arch/mips/include/asm/pgtable-64.h
arch/mips/include/asm/pgtable.h
arch/mips/kernel/asm-offsets.c
arch/mips/mm/init.c
arch/mips/mm/pgtable-64.c
arch/mips/mm/tlbex.c

arch/mips/include/asm/page.h

Diff comments View file @ 325f8a0

 /*
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
  * Copyright (C) 1994 - 1999, 2000, 03 Ralf Baechle
  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
  */
 #ifndef _ASM_PAGE_H
 #define _ASM_PAGE_H
 #include <spaces.h>
 #include <linux/const.h>
 /*
  * PAGE_SHIFT determines the page size
  */
 #ifdef CONFIG_PAGE_SIZE_4KB
 #define PAGE_SHIFT	12
 #endif
 #ifdef CONFIG_PAGE_SIZE_8KB
 #define PAGE_SHIFT	13
 #endif
 #ifdef CONFIG_PAGE_SIZE_16KB
 #define PAGE_SHIFT	14
 #endif
 #ifdef CONFIG_PAGE_SIZE_32KB
 #define PAGE_SHIFT	15
 #endif
 #ifdef CONFIG_PAGE_SIZE_64KB
 #define PAGE_SHIFT	16
 #endif
 #define PAGE_SIZE	(_AC(1,UL) << PAGE_SHIFT)
 #define PAGE_MASK       (~((1 << PAGE_SHIFT) - 1))
 #ifdef CONFIG_HUGETLB_PAGE
 #define HPAGE_SHIFT	(PAGE_SHIFT + PAGE_SHIFT - 3)
 #define HPAGE_SIZE	(_AC(1,UL) << HPAGE_SHIFT)
 #define HPAGE_MASK	(~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
 #endif /* CONFIG_HUGETLB_PAGE */
 #ifndef __ASSEMBLY__
 #include <linux/pfn.h>
 #include <asm/io.h>
 extern void build_clear_page(void);
 extern void build_copy_page(void);
 /*
  * It's normally defined only for FLATMEM config but it's
  * used in our early mem init code for all memory models.
  * So always define it.
  */
 #define ARCH_PFN_OFFSET		PFN_UP(PHYS_OFFSET)
 extern void clear_page(void * page);
 extern void copy_page(void * to, void * from);
 extern unsigned long shm_align_mask;
 static inline unsigned long pages_do_alias(unsigned long addr1,
 	unsigned long addr2)
 {
 	return (addr1 ^ addr2) & shm_align_mask;
 }
 struct page;
 static inline void clear_user_page(void *addr, unsigned long vaddr,
 	struct page *page)
 {
 	extern void (*flush_data_cache_page)(unsigned long addr);
 	clear_page(addr);
 	if (pages_do_alias((unsigned long) addr, vaddr & PAGE_MASK))
 		flush_data_cache_page((unsigned long)addr);
 }
 extern void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
 	struct page *to);
 struct vm_area_struct;
 extern void copy_user_highpage(struct page *to, struct page *from,
 	unsigned long vaddr, struct vm_area_struct *vma);
 #define __HAVE_ARCH_COPY_USER_HIGHPAGE
 /*
  * These are used to make use of C type-checking..
  */
 #ifdef CONFIG_64BIT_PHYS_ADDR
   #ifdef CONFIG_CPU_MIPS32
     typedef struct { unsigned long pte_low, pte_high; } pte_t;
     #define pte_val(x)    ((x).pte_low | ((unsigned long long)(x).pte_high << 32))
     #define __pte(x)      ({ pte_t __pte = {(x), ((unsigned long long)(x)) >> 32}; __pte; })
   #else
      typedef struct { unsigned long long pte; } pte_t;
      #define pte_val(x)	((x).pte)
      #define __pte(x)	((pte_t) { (x) } )
   #endif
 #else
 typedef struct { unsigned long pte; } pte_t;
 #define pte_val(x)	((x).pte)
 #define __pte(x)	((pte_t) { (x) } )
 #endif
 typedef struct page *pgtable_t;
 /*
- * For 3-level pagetables we defines these ourselves, for 2-level the
- * definitions are supplied by <asm-generic/pgtable-nopmd.h>.
- */
-#ifdef CONFIG_64BIT
-typedef struct { unsigned long pmd; } pmd_t;
-#define pmd_val(x)	((x).pmd)
-#define __pmd(x)	((pmd_t) { (x) } )
-#endif
-/*
  * Right now we don't support 4-level pagetables, so all pud-related
  * definitions come from <asm-generic/pgtable-nopud.h>.
  */
 /*
  * Finall the top of the hierarchy, the pgd
  */
 typedef struct { unsigned long pgd; } pgd_t;
 #define pgd_val(x)	((x).pgd)
 #define __pgd(x)	((pgd_t) { (x) } )
 /*
  * Manipulate page protection bits
  */
 typedef struct { unsigned long pgprot; } pgprot_t;
 #define pgprot_val(x)	((x).pgprot)
 #define __pgprot(x)	((pgprot_t) { (x) } )
 /*
  * On R4000-style MMUs where a TLB entry is mapping a adjacent even / odd
  * pair of pages we only have a single global bit per pair of pages.  When
  * writing to the TLB make sure we always have the bit set for both pages
  * or none.  This macro is used to access the `buddy' of the pte we're just
  * working on.
  */
 #define ptep_buddy(x)	((pte_t *)((unsigned long)(x) ^ sizeof(pte_t)))
 #endif /* !__ASSEMBLY__ */
 /*
  * __pa()/__va() should be used only during mem init.
  */
 #ifdef CONFIG_64BIT
 #define __pa(x)								\
 ({									\
     unsigned long __x = (unsigned long)(x);				\
     __x < CKSEG0 ? XPHYSADDR(__x) : CPHYSADDR(__x);			\
 })
 #else
 #define __pa(x)								\
     ((unsigned long)(x) - PAGE_OFFSET + PHYS_OFFSET)
 #endif
 #define __va(x)		((void *)((unsigned long)(x) + PAGE_OFFSET - PHYS_OFFSET))
 #define __pa_symbol(x)	__pa(RELOC_HIDE((unsigned long)(x), 0))
 #define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
 #ifdef CONFIG_FLATMEM
 #define pfn_valid(pfn)							\
 ({									\
 	unsigned long __pfn = (pfn);					\
 	/* avoid <linux/bootmem.h> include hell */			\
 	extern unsigned long min_low_pfn;				\
 									\
 	__pfn >= min_low_pfn && __pfn < max_mapnr;			\
 })
 #elif defined(CONFIG_SPARSEMEM)
 /* pfn_valid is defined in linux/mmzone.h */
 #elif defined(CONFIG_NEED_MULTIPLE_NODES)
 #define pfn_valid(pfn)							\
 ({									\
 	unsigned long __pfn = (pfn);					\
 	int __n = pfn_to_nid(__pfn);					\
 	((__n >= 0) ? (__pfn < NODE_DATA(__n)->node_start_pfn +		\
 	                       NODE_DATA(__n)->node_spanned_pages)	\
 	            : 0);						\
 })
 #endif
 #define virt_to_page(kaddr)	pfn_to_page(PFN_DOWN(virt_to_phys(kaddr)))
 #define virt_addr_valid(kaddr)	pfn_valid(PFN_DOWN(virt_to_phys(kaddr)))
 #define VM_DATA_DEFAULT_FLAGS	(VM_READ | VM_WRITE | VM_EXEC | \
 				 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 #define UNCAC_ADDR(addr)	((addr) - PAGE_OFFSET + UNCAC_BASE)
 #define CAC_ADDR(addr)		((addr) - UNCAC_BASE + PAGE_OFFSET)
 #include <asm-generic/memory_model.h>
 #include <asm-generic/getorder.h>
 #endif /* _ASM_PAGE_H */

arch/mips/include/asm/pgalloc.h

Diff comments View file @ 325f8a0

1	/*	1	/*
2	* This file is subject to the terms and conditions of the GNU General Public	2	* This file is subject to the terms and conditions of the GNU General Public
3	* License. See the file "COPYING" in the main directory of this archive	3	* License. See the file "COPYING" in the main directory of this archive
4	* for more details.	4	* for more details.
5	*	5	*
6	* Copyright (C) 1994 - 2001, 2003 by Ralf Baechle	6	* Copyright (C) 1994 - 2001, 2003 by Ralf Baechle
7	* Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc.	7	* Copyright (C) 1999, 2000, 2001 Silicon Graphics, Inc.
8	*/	8	*/
9	#ifndef _ASM_PGALLOC_H	9	#ifndef _ASM_PGALLOC_H
10	#define _ASM_PGALLOC_H	10	#define _ASM_PGALLOC_H
11		11
12	#include <linux/highmem.h>	12	#include <linux/highmem.h>
13	#include <linux/mm.h>	13	#include <linux/mm.h>
14	#include <linux/sched.h>	14	#include <linux/sched.h>
15		15
16	static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,	16	static inline void pmd_populate_kernel(struct mm_struct mm, pmd_t pmd,
17	pte_t *pte)	17	pte_t *pte)
18	{	18	{
19	set_pmd(pmd, __pmd((unsigned long)pte));	19	set_pmd(pmd, __pmd((unsigned long)pte));
20	}	20	}
21		21
22	static inline void pmd_populate(struct mm_struct mm, pmd_t pmd,	22	static inline void pmd_populate(struct mm_struct mm, pmd_t pmd,
23	pgtable_t pte)	23	pgtable_t pte)
24	{	24	{
25	set_pmd(pmd, __pmd((unsigned long)page_address(pte)));	25	set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
26	}	26	}
27	#define pmd_pgtable(pmd) pmd_page(pmd)	27	#define pmd_pgtable(pmd) pmd_page(pmd)
28		28
29	/*	29	/*
30	* Initialize a new pmd table with invalid pointers.	30	* Initialize a new pmd table with invalid pointers.
31	*/	31	*/
32	extern void pmd_init(unsigned long page, unsigned long pagetable);	32	extern void pmd_init(unsigned long page, unsigned long pagetable);
33		33
34	#ifdef CONFIG_64BIT	34	#ifndef __PAGETABLE_PMD_FOLDED
35		35
36	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)	36	static inline void pud_populate(struct mm_struct mm, pud_t pud, pmd_t *pmd)
37	{	37	{
38	set_pud(pud, __pud((unsigned long)pmd));	38	set_pud(pud, __pud((unsigned long)pmd));
39	}	39	}
40	#endif	40	#endif
41		41
42	/*	42	/*
43	* Initialize a new pgd / pmd table with invalid pointers.	43	* Initialize a new pgd / pmd table with invalid pointers.
44	*/	44	*/
45	extern void pgd_init(unsigned long page);	45	extern void pgd_init(unsigned long page);
46		46
47	static inline pgd_t pgd_alloc(struct mm_struct mm)	47	static inline pgd_t pgd_alloc(struct mm_struct mm)
48	{	48	{
49	pgd_t ret, init;	49	pgd_t ret, init;
50		50
51	ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER);	51	ret = (pgd_t *) __get_free_pages(GFP_KERNEL, PGD_ORDER);
52	if (ret) {	52	if (ret) {
53	init = pgd_offset(&init_mm, 0UL);	53	init = pgd_offset(&init_mm, 0UL);
54	pgd_init((unsigned long)ret);	54	pgd_init((unsigned long)ret);
55	memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,	55	memcpy(ret + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD,
56	(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));	56	(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
57	}	57	}
58		58
59	return ret;	59	return ret;
60	}	60	}
61		61
62	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)	62	static inline void pgd_free(struct mm_struct mm, pgd_t pgd)
63	{	63	{
64	free_pages((unsigned long)pgd, PGD_ORDER);	64	free_pages((unsigned long)pgd, PGD_ORDER);
65	}	65	}
66		66
67	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,	67	static inline pte_t pte_alloc_one_kernel(struct mm_struct mm,
68	unsigned long address)	68	unsigned long address)
69	{	69	{
70	pte_t *pte;	70	pte_t *pte;
71		71
72	pte = (pte_t *) __get_free_pages(GFP_KERNEL\|__GFP_REPEAT\|__GFP_ZERO, PTE_ORDER);	72	pte = (pte_t *) __get_free_pages(GFP_KERNEL\|__GFP_REPEAT\|__GFP_ZERO, PTE_ORDER);
73		73
74	return pte;	74	return pte;
75	}	75	}
76		76
77	static inline struct page pte_alloc_one(struct mm_struct mm,	77	static inline struct page pte_alloc_one(struct mm_struct mm,
78	unsigned long address)	78	unsigned long address)
79	{	79	{
80	struct page *pte;	80	struct page *pte;
81		81
82	pte = alloc_pages(GFP_KERNEL \| __GFP_REPEAT, PTE_ORDER);	82	pte = alloc_pages(GFP_KERNEL \| __GFP_REPEAT, PTE_ORDER);
83	if (pte) {	83	if (pte) {
84	clear_highpage(pte);	84	clear_highpage(pte);
85	pgtable_page_ctor(pte);	85	pgtable_page_ctor(pte);
86	}	86	}
87	return pte;	87	return pte;
88	}	88	}
89		89
90	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)	90	static inline void pte_free_kernel(struct mm_struct mm, pte_t pte)
91	{	91	{
92	free_pages((unsigned long)pte, PTE_ORDER);	92	free_pages((unsigned long)pte, PTE_ORDER);
93	}	93	}
94		94
95	static inline void pte_free(struct mm_struct *mm, pgtable_t pte)	95	static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
96	{	96	{
97	pgtable_page_dtor(pte);	97	pgtable_page_dtor(pte);
98	__free_pages(pte, PTE_ORDER);	98	__free_pages(pte, PTE_ORDER);
99	}	99	}
100		100
101	#define __pte_free_tlb(tlb,pte,address) \	101	#define __pte_free_tlb(tlb,pte,address) \
102	do { \	102	do { \
103	pgtable_page_dtor(pte); \	103	pgtable_page_dtor(pte); \
104	tlb_remove_page((tlb), pte); \	104	tlb_remove_page((tlb), pte); \
105	} while (0)	105	} while (0)
106		106
107	#ifdef CONFIG_64BIT	107	#ifndef __PAGETABLE_PMD_FOLDED
108		108
109	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long address)	109	static inline pmd_t pmd_alloc_one(struct mm_struct mm, unsigned long address)
110	{	110	{
111	pmd_t *pmd;	111	pmd_t *pmd;
112		112
113	pmd = (pmd_t *) __get_free_pages(GFP_KERNEL\|__GFP_REPEAT, PMD_ORDER);	113	pmd = (pmd_t *) __get_free_pages(GFP_KERNEL\|__GFP_REPEAT, PMD_ORDER);
114	if (pmd)	114	if (pmd)
115	pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table);	115	pmd_init((unsigned long)pmd, (unsigned long)invalid_pte_table);
116	return pmd;	116	return pmd;
117	}	117	}
118		118
119	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)	119	static inline void pmd_free(struct mm_struct mm, pmd_t pmd)
120	{	120	{
121	free_pages((unsigned long)pmd, PMD_ORDER);	121	free_pages((unsigned long)pmd, PMD_ORDER);
122	}	122	}
123		123
124	#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)	124	#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
125		125
126	#endif	126	#endif
127		127
128	#define check_pgt_cache() do { } while (0)	128	#define check_pgt_cache() do { } while (0)
129		129
130	extern void pagetable_init(void);	130	extern void pagetable_init(void);
131		131
132	#endif /* _ASM_PGALLOC_H */	132	#endif /* _ASM_PGALLOC_H */
133		133

arch/mips/include/asm/pgtable-64.h

Diff comments View file @ 325f8a0

arch/mips/include/asm/pgtable.h

Diff comments View file @ 325f8a0

arch/mips/kernel/asm-offsets.c

Diff comments View file @ 325f8a0

 /*
  * offset.c: Calculate pt_regs and task_struct offsets.
  *
  * Copyright (C) 1996 David S. Miller
  * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003 Ralf Baechle
  * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
  *
  * Kevin Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
  * Copyright (C) 2000 MIPS Technologies, Inc.
  */
 #include <linux/compat.h>
 #include <linux/types.h>
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/interrupt.h>
 #include <linux/kbuild.h>
 #include <linux/suspend.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
 void output_ptreg_defines(void)
 {
 	COMMENT("MIPS pt_regs offsets.");
 	OFFSET(PT_R0, pt_regs, regs[0]);
 	OFFSET(PT_R1, pt_regs, regs[1]);
 	OFFSET(PT_R2, pt_regs, regs[2]);
 	OFFSET(PT_R3, pt_regs, regs[3]);
 	OFFSET(PT_R4, pt_regs, regs[4]);
 	OFFSET(PT_R5, pt_regs, regs[5]);
 	OFFSET(PT_R6, pt_regs, regs[6]);
 	OFFSET(PT_R7, pt_regs, regs[7]);
 	OFFSET(PT_R8, pt_regs, regs[8]);
 	OFFSET(PT_R9, pt_regs, regs[9]);
 	OFFSET(PT_R10, pt_regs, regs[10]);
 	OFFSET(PT_R11, pt_regs, regs[11]);
 	OFFSET(PT_R12, pt_regs, regs[12]);
 	OFFSET(PT_R13, pt_regs, regs[13]);
 	OFFSET(PT_R14, pt_regs, regs[14]);
 	OFFSET(PT_R15, pt_regs, regs[15]);
 	OFFSET(PT_R16, pt_regs, regs[16]);
 	OFFSET(PT_R17, pt_regs, regs[17]);
 	OFFSET(PT_R18, pt_regs, regs[18]);
 	OFFSET(PT_R19, pt_regs, regs[19]);
 	OFFSET(PT_R20, pt_regs, regs[20]);
 	OFFSET(PT_R21, pt_regs, regs[21]);
 	OFFSET(PT_R22, pt_regs, regs[22]);
 	OFFSET(PT_R23, pt_regs, regs[23]);
 	OFFSET(PT_R24, pt_regs, regs[24]);
 	OFFSET(PT_R25, pt_regs, regs[25]);
 	OFFSET(PT_R26, pt_regs, regs[26]);
 	OFFSET(PT_R27, pt_regs, regs[27]);
 	OFFSET(PT_R28, pt_regs, regs[28]);
 	OFFSET(PT_R29, pt_regs, regs[29]);
 	OFFSET(PT_R30, pt_regs, regs[30]);
 	OFFSET(PT_R31, pt_regs, regs[31]);
 	OFFSET(PT_LO, pt_regs, lo);
 	OFFSET(PT_HI, pt_regs, hi);
 #ifdef CONFIG_CPU_HAS_SMARTMIPS
 	OFFSET(PT_ACX, pt_regs, acx);
 #endif
 	OFFSET(PT_EPC, pt_regs, cp0_epc);
 	OFFSET(PT_BVADDR, pt_regs, cp0_badvaddr);
 	OFFSET(PT_STATUS, pt_regs, cp0_status);
 	OFFSET(PT_CAUSE, pt_regs, cp0_cause);
 #ifdef CONFIG_MIPS_MT_SMTC
 	OFFSET(PT_TCSTATUS, pt_regs, cp0_tcstatus);
 #endif /* CONFIG_MIPS_MT_SMTC */
 #ifdef CONFIG_CPU_CAVIUM_OCTEON
 	OFFSET(PT_MPL, pt_regs, mpl);
 	OFFSET(PT_MTP, pt_regs, mtp);
 #endif /* CONFIG_CPU_CAVIUM_OCTEON */
 	DEFINE(PT_SIZE, sizeof(struct pt_regs));
 	BLANK();
 }
 void output_task_defines(void)
 {
 	COMMENT("MIPS task_struct offsets.");
 	OFFSET(TASK_STATE, task_struct, state);
 	OFFSET(TASK_THREAD_INFO, task_struct, stack);
 	OFFSET(TASK_FLAGS, task_struct, flags);
 	OFFSET(TASK_MM, task_struct, mm);
 	OFFSET(TASK_PID, task_struct, pid);
 	DEFINE(TASK_STRUCT_SIZE, sizeof(struct task_struct));
 	BLANK();
 }
 void output_thread_info_defines(void)
 {
 	COMMENT("MIPS thread_info offsets.");
 	OFFSET(TI_TASK, thread_info, task);
 	OFFSET(TI_EXEC_DOMAIN, thread_info, exec_domain);
 	OFFSET(TI_FLAGS, thread_info, flags);
 	OFFSET(TI_TP_VALUE, thread_info, tp_value);
 	OFFSET(TI_CPU, thread_info, cpu);
 	OFFSET(TI_PRE_COUNT, thread_info, preempt_count);
 	OFFSET(TI_ADDR_LIMIT, thread_info, addr_limit);
 	OFFSET(TI_RESTART_BLOCK, thread_info, restart_block);
 	OFFSET(TI_REGS, thread_info, regs);
 	DEFINE(_THREAD_SIZE, THREAD_SIZE);
 	DEFINE(_THREAD_MASK, THREAD_MASK);
 	BLANK();
 }
 void output_thread_defines(void)
 {
 	COMMENT("MIPS specific thread_struct offsets.");
 	OFFSET(THREAD_REG16, task_struct, thread.reg16);
 	OFFSET(THREAD_REG17, task_struct, thread.reg17);
 	OFFSET(THREAD_REG18, task_struct, thread.reg18);
 	OFFSET(THREAD_REG19, task_struct, thread.reg19);
 	OFFSET(THREAD_REG20, task_struct, thread.reg20);
 	OFFSET(THREAD_REG21, task_struct, thread.reg21);
 	OFFSET(THREAD_REG22, task_struct, thread.reg22);
 	OFFSET(THREAD_REG23, task_struct, thread.reg23);
 	OFFSET(THREAD_REG29, task_struct, thread.reg29);
 	OFFSET(THREAD_REG30, task_struct, thread.reg30);
 	OFFSET(THREAD_REG31, task_struct, thread.reg31);
 	OFFSET(THREAD_STATUS, task_struct,
 	       thread.cp0_status);
 	OFFSET(THREAD_FPU, task_struct, thread.fpu);
 	OFFSET(THREAD_BVADDR, task_struct, \
 	       thread.cp0_badvaddr);
 	OFFSET(THREAD_BUADDR, task_struct, \
 	       thread.cp0_baduaddr);
 	OFFSET(THREAD_ECODE, task_struct, \
 	       thread.error_code);
 	OFFSET(THREAD_TRAPNO, task_struct, thread.trap_no);
 	OFFSET(THREAD_TRAMP, task_struct, \
 	       thread.irix_trampoline);
 	OFFSET(THREAD_OLDCTX, task_struct, \
 	       thread.irix_oldctx);
 	BLANK();
 }
 void output_thread_fpu_defines(void)
 {
 	OFFSET(THREAD_FPR0, task_struct, thread.fpu.fpr[0]);
 	OFFSET(THREAD_FPR1, task_struct, thread.fpu.fpr[1]);
 	OFFSET(THREAD_FPR2, task_struct, thread.fpu.fpr[2]);
 	OFFSET(THREAD_FPR3, task_struct, thread.fpu.fpr[3]);
 	OFFSET(THREAD_FPR4, task_struct, thread.fpu.fpr[4]);
 	OFFSET(THREAD_FPR5, task_struct, thread.fpu.fpr[5]);
 	OFFSET(THREAD_FPR6, task_struct, thread.fpu.fpr[6]);
 	OFFSET(THREAD_FPR7, task_struct, thread.fpu.fpr[7]);
 	OFFSET(THREAD_FPR8, task_struct, thread.fpu.fpr[8]);
 	OFFSET(THREAD_FPR9, task_struct, thread.fpu.fpr[9]);
 	OFFSET(THREAD_FPR10, task_struct, thread.fpu.fpr[10]);
 	OFFSET(THREAD_FPR11, task_struct, thread.fpu.fpr[11]);
 	OFFSET(THREAD_FPR12, task_struct, thread.fpu.fpr[12]);
 	OFFSET(THREAD_FPR13, task_struct, thread.fpu.fpr[13]);
 	OFFSET(THREAD_FPR14, task_struct, thread.fpu.fpr[14]);
 	OFFSET(THREAD_FPR15, task_struct, thread.fpu.fpr[15]);
 	OFFSET(THREAD_FPR16, task_struct, thread.fpu.fpr[16]);
 	OFFSET(THREAD_FPR17, task_struct, thread.fpu.fpr[17]);
 	OFFSET(THREAD_FPR18, task_struct, thread.fpu.fpr[18]);
 	OFFSET(THREAD_FPR19, task_struct, thread.fpu.fpr[19]);
 	OFFSET(THREAD_FPR20, task_struct, thread.fpu.fpr[20]);
 	OFFSET(THREAD_FPR21, task_struct, thread.fpu.fpr[21]);
 	OFFSET(THREAD_FPR22, task_struct, thread.fpu.fpr[22]);
 	OFFSET(THREAD_FPR23, task_struct, thread.fpu.fpr[23]);
 	OFFSET(THREAD_FPR24, task_struct, thread.fpu.fpr[24]);
 	OFFSET(THREAD_FPR25, task_struct, thread.fpu.fpr[25]);
 	OFFSET(THREAD_FPR26, task_struct, thread.fpu.fpr[26]);
 	OFFSET(THREAD_FPR27, task_struct, thread.fpu.fpr[27]);
 	OFFSET(THREAD_FPR28, task_struct, thread.fpu.fpr[28]);
 	OFFSET(THREAD_FPR29, task_struct, thread.fpu.fpr[29]);
 	OFFSET(THREAD_FPR30, task_struct, thread.fpu.fpr[30]);
 	OFFSET(THREAD_FPR31, task_struct, thread.fpu.fpr[31]);
 	OFFSET(THREAD_FCR31, task_struct, thread.fpu.fcr31);
 	BLANK();
 }
 void output_mm_defines(void)
 {
 	COMMENT("Size of struct page");
 	DEFINE(STRUCT_PAGE_SIZE, sizeof(struct page));
 	BLANK();
 	COMMENT("Linux mm_struct offsets.");
 	OFFSET(MM_USERS, mm_struct, mm_users);
 	OFFSET(MM_PGD, mm_struct, pgd);
 	OFFSET(MM_CONTEXT, mm_struct, context);
 	BLANK();
 	DEFINE(_PGD_T_SIZE, sizeof(pgd_t));
 	DEFINE(_PMD_T_SIZE, sizeof(pmd_t));
 	DEFINE(_PTE_T_SIZE, sizeof(pte_t));
 	BLANK();
 	DEFINE(_PGD_T_LOG2, PGD_T_LOG2);
+#ifndef __PAGETABLE_PMD_FOLDED
 	DEFINE(_PMD_T_LOG2, PMD_T_LOG2);
+#endif
 	DEFINE(_PTE_T_LOG2, PTE_T_LOG2);
 	BLANK();
 	DEFINE(_PGD_ORDER, PGD_ORDER);
+#ifndef __PAGETABLE_PMD_FOLDED
 	DEFINE(_PMD_ORDER, PMD_ORDER);
+#endif
 	DEFINE(_PTE_ORDER, PTE_ORDER);
 	BLANK();
 	DEFINE(_PMD_SHIFT, PMD_SHIFT);
 	DEFINE(_PGDIR_SHIFT, PGDIR_SHIFT);
 	BLANK();
 	DEFINE(_PTRS_PER_PGD, PTRS_PER_PGD);
 	DEFINE(_PTRS_PER_PMD, PTRS_PER_PMD);
 	DEFINE(_PTRS_PER_PTE, PTRS_PER_PTE);
 	BLANK();
 }
 #ifdef CONFIG_32BIT
 void output_sc_defines(void)
 {
 	COMMENT("Linux sigcontext offsets.");
 	OFFSET(SC_REGS, sigcontext, sc_regs);
 	OFFSET(SC_FPREGS, sigcontext, sc_fpregs);
 	OFFSET(SC_ACX, sigcontext, sc_acx);
 	OFFSET(SC_MDHI, sigcontext, sc_mdhi);
 	OFFSET(SC_MDLO, sigcontext, sc_mdlo);
 	OFFSET(SC_PC, sigcontext, sc_pc);
 	OFFSET(SC_FPC_CSR, sigcontext, sc_fpc_csr);
 	OFFSET(SC_FPC_EIR, sigcontext, sc_fpc_eir);
 	OFFSET(SC_HI1, sigcontext, sc_hi1);
 	OFFSET(SC_LO1, sigcontext, sc_lo1);
 	OFFSET(SC_HI2, sigcontext, sc_hi2);
 	OFFSET(SC_LO2, sigcontext, sc_lo2);
 	OFFSET(SC_HI3, sigcontext, sc_hi3);
 	OFFSET(SC_LO3, sigcontext, sc_lo3);
 	BLANK();
 }
 #endif
 #ifdef CONFIG_64BIT
 void output_sc_defines(void)
 {
 	COMMENT("Linux sigcontext offsets.");
 	OFFSET(SC_REGS, sigcontext, sc_regs);
 	OFFSET(SC_FPREGS, sigcontext, sc_fpregs);
 	OFFSET(SC_MDHI, sigcontext, sc_mdhi);
 	OFFSET(SC_MDLO, sigcontext, sc_mdlo);
 	OFFSET(SC_PC, sigcontext, sc_pc);
 	OFFSET(SC_FPC_CSR, sigcontext, sc_fpc_csr);
 	BLANK();
 }
 #endif
 #ifdef CONFIG_MIPS32_COMPAT
 void output_sc32_defines(void)
 {
 	COMMENT("Linux 32-bit sigcontext offsets.");
 	OFFSET(SC32_FPREGS, sigcontext32, sc_fpregs);
 	OFFSET(SC32_FPC_CSR, sigcontext32, sc_fpc_csr);
 	OFFSET(SC32_FPC_EIR, sigcontext32, sc_fpc_eir);
 	BLANK();
 }
 #endif
 void output_signal_defined(void)
 {
 	COMMENT("Linux signal numbers.");
 	DEFINE(_SIGHUP, SIGHUP);
 	DEFINE(_SIGINT, SIGINT);
 	DEFINE(_SIGQUIT, SIGQUIT);
 	DEFINE(_SIGILL, SIGILL);
 	DEFINE(_SIGTRAP, SIGTRAP);
 	DEFINE(_SIGIOT, SIGIOT);
 	DEFINE(_SIGABRT, SIGABRT);
 	DEFINE(_SIGEMT, SIGEMT);
 	DEFINE(_SIGFPE, SIGFPE);
 	DEFINE(_SIGKILL, SIGKILL);
 	DEFINE(_SIGBUS, SIGBUS);
 	DEFINE(_SIGSEGV, SIGSEGV);
 	DEFINE(_SIGSYS, SIGSYS);
 	DEFINE(_SIGPIPE, SIGPIPE);
 	DEFINE(_SIGALRM, SIGALRM);
 	DEFINE(_SIGTERM, SIGTERM);
 	DEFINE(_SIGUSR1, SIGUSR1);
 	DEFINE(_SIGUSR2, SIGUSR2);
 	DEFINE(_SIGCHLD, SIGCHLD);
 	DEFINE(_SIGPWR, SIGPWR);
 	DEFINE(_SIGWINCH, SIGWINCH);
 	DEFINE(_SIGURG, SIGURG);
 	DEFINE(_SIGIO, SIGIO);
 	DEFINE(_SIGSTOP, SIGSTOP);
 	DEFINE(_SIGTSTP, SIGTSTP);
 	DEFINE(_SIGCONT, SIGCONT);
 	DEFINE(_SIGTTIN, SIGTTIN);
 	DEFINE(_SIGTTOU, SIGTTOU);
 	DEFINE(_SIGVTALRM, SIGVTALRM);
 	DEFINE(_SIGPROF, SIGPROF);
 	DEFINE(_SIGXCPU, SIGXCPU);
 	DEFINE(_SIGXFSZ, SIGXFSZ);
 	BLANK();
 }
 void output_irq_cpustat_t_defines(void)
 {
 	COMMENT("Linux irq_cpustat_t offsets.");
 	DEFINE(IC_SOFTIRQ_PENDING,
 			offsetof(irq_cpustat_t, __softirq_pending));
 	DEFINE(IC_IRQ_CPUSTAT_T, sizeof(irq_cpustat_t));
 	BLANK();
 }
 #ifdef CONFIG_CPU_CAVIUM_OCTEON
 void output_octeon_cop2_state_defines(void)
 {
 	COMMENT("Octeon specific octeon_cop2_state offsets.");
 	OFFSET(OCTEON_CP2_CRC_IV,	octeon_cop2_state, cop2_crc_iv);
 	OFFSET(OCTEON_CP2_CRC_LENGTH,	octeon_cop2_state, cop2_crc_length);
 	OFFSET(OCTEON_CP2_CRC_POLY,	octeon_cop2_state, cop2_crc_poly);
 	OFFSET(OCTEON_CP2_LLM_DAT,	octeon_cop2_state, cop2_llm_dat);
 	OFFSET(OCTEON_CP2_3DES_IV,	octeon_cop2_state, cop2_3des_iv);
 	OFFSET(OCTEON_CP2_3DES_KEY,	octeon_cop2_state, cop2_3des_key);
 	OFFSET(OCTEON_CP2_3DES_RESULT,	octeon_cop2_state, cop2_3des_result);
 	OFFSET(OCTEON_CP2_AES_INP0,	octeon_cop2_state, cop2_aes_inp0);
 	OFFSET(OCTEON_CP2_AES_IV,	octeon_cop2_state, cop2_aes_iv);
 	OFFSET(OCTEON_CP2_AES_KEY,	octeon_cop2_state, cop2_aes_key);
 	OFFSET(OCTEON_CP2_AES_KEYLEN,	octeon_cop2_state, cop2_aes_keylen);
 	OFFSET(OCTEON_CP2_AES_RESULT,	octeon_cop2_state, cop2_aes_result);
 	OFFSET(OCTEON_CP2_GFM_MULT,	octeon_cop2_state, cop2_gfm_mult);
 	OFFSET(OCTEON_CP2_GFM_POLY,	octeon_cop2_state, cop2_gfm_poly);
 	OFFSET(OCTEON_CP2_GFM_RESULT,	octeon_cop2_state, cop2_gfm_result);
 	OFFSET(OCTEON_CP2_HSH_DATW,	octeon_cop2_state, cop2_hsh_datw);
 	OFFSET(OCTEON_CP2_HSH_IVW,	octeon_cop2_state, cop2_hsh_ivw);
 	OFFSET(THREAD_CP2,	task_struct, thread.cp2);
 	OFFSET(THREAD_CVMSEG,	task_struct, thread.cvmseg.cvmseg);
 	BLANK();
 }
 #endif
 #ifdef CONFIG_HIBERNATION
 void output_pbe_defines(void)
 {
 	COMMENT(" Linux struct pbe offsets. ");
 	OFFSET(PBE_ADDRESS, pbe, address);
 	OFFSET(PBE_ORIG_ADDRESS, pbe, orig_address);
 	OFFSET(PBE_NEXT, pbe, next);
 	DEFINE(PBE_SIZE, sizeof(struct pbe));
 	BLANK();
 }
 #endif

arch/mips/mm/init.c

Diff comments View file @ 325f8a0

1	/*	1	/*
2	* This file is subject to the terms and conditions of the GNU General Public	2	* This file is subject to the terms and conditions of the GNU General Public
3	* License. See the file "COPYING" in the main directory of this archive	3	* License. See the file "COPYING" in the main directory of this archive
4	* for more details.	4	* for more details.
5	*	5	*
6	* Copyright (C) 1994 - 2000 Ralf Baechle	6	* Copyright (C) 1994 - 2000 Ralf Baechle
7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.	7	* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
8	* Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com	8	* Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
9	* Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved.	9	* Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved.
10	*/	10	*/
11	#include <linux/bug.h>	11	#include <linux/bug.h>
12	#include <linux/init.h>	12	#include <linux/init.h>
13	#include <linux/module.h>	13	#include <linux/module.h>
14	#include <linux/signal.h>	14	#include <linux/signal.h>
15	#include <linux/sched.h>	15	#include <linux/sched.h>
16	#include <linux/smp.h>	16	#include <linux/smp.h>
17	#include <linux/kernel.h>	17	#include <linux/kernel.h>
18	#include <linux/errno.h>	18	#include <linux/errno.h>
19	#include <linux/string.h>	19	#include <linux/string.h>
20	#include <linux/types.h>	20	#include <linux/types.h>
21	#include <linux/pagemap.h>	21	#include <linux/pagemap.h>
22	#include <linux/ptrace.h>	22	#include <linux/ptrace.h>
23	#include <linux/mman.h>	23	#include <linux/mman.h>
24	#include <linux/mm.h>	24	#include <linux/mm.h>
25	#include <linux/bootmem.h>	25	#include <linux/bootmem.h>
26	#include <linux/highmem.h>	26	#include <linux/highmem.h>
27	#include <linux/swap.h>	27	#include <linux/swap.h>
28	#include <linux/proc_fs.h>	28	#include <linux/proc_fs.h>
29	#include <linux/pfn.h>	29	#include <linux/pfn.h>
30	#include <linux/hardirq.h>	30	#include <linux/hardirq.h>
31		31
32	#include <asm/asm-offsets.h>	32	#include <asm/asm-offsets.h>
33	#include <asm/bootinfo.h>	33	#include <asm/bootinfo.h>
34	#include <asm/cachectl.h>	34	#include <asm/cachectl.h>
35	#include <asm/cpu.h>	35	#include <asm/cpu.h>
36	#include <asm/dma.h>	36	#include <asm/dma.h>
37	#include <asm/kmap_types.h>	37	#include <asm/kmap_types.h>
38	#include <asm/mmu_context.h>	38	#include <asm/mmu_context.h>
39	#include <asm/sections.h>	39	#include <asm/sections.h>
40	#include <asm/pgtable.h>	40	#include <asm/pgtable.h>
41	#include <asm/pgalloc.h>	41	#include <asm/pgalloc.h>
42	#include <asm/tlb.h>	42	#include <asm/tlb.h>
43	#include <asm/fixmap.h>	43	#include <asm/fixmap.h>
44		44
45	/* Atomicity and interruptability */	45	/* Atomicity and interruptability */
46	#ifdef CONFIG_MIPS_MT_SMTC	46	#ifdef CONFIG_MIPS_MT_SMTC
47		47
48	#include <asm/mipsmtregs.h>	48	#include <asm/mipsmtregs.h>
49		49
50	#define ENTER_CRITICAL(flags) \	50	#define ENTER_CRITICAL(flags) \
51	{ \	51	{ \
52	unsigned int mvpflags; \	52	unsigned int mvpflags; \
53	local_irq_save(flags);\	53	local_irq_save(flags);\
54	mvpflags = dvpe()	54	mvpflags = dvpe()
55	#define EXIT_CRITICAL(flags) \	55	#define EXIT_CRITICAL(flags) \
56	evpe(mvpflags); \	56	evpe(mvpflags); \
57	local_irq_restore(flags); \	57	local_irq_restore(flags); \
58	}	58	}
59	#else	59	#else
60		60
61	#define ENTER_CRITICAL(flags) local_irq_save(flags)	61	#define ENTER_CRITICAL(flags) local_irq_save(flags)
62	#define EXIT_CRITICAL(flags) local_irq_restore(flags)	62	#define EXIT_CRITICAL(flags) local_irq_restore(flags)
63		63
64	#endif /* CONFIG_MIPS_MT_SMTC */	64	#endif /* CONFIG_MIPS_MT_SMTC */
65		65
66	DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);	66	DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
67		67
68	/*	68	/*
69	* We have up to 8 empty zeroed pages so we can map one of the right colour	69	* We have up to 8 empty zeroed pages so we can map one of the right colour
70	* when needed. This is necessary only on R4000 / R4400 SC and MC versions	70	* when needed. This is necessary only on R4000 / R4400 SC and MC versions
71	* where we have to avoid VCED / VECI exceptions for good performance at	71	* where we have to avoid VCED / VECI exceptions for good performance at
72	* any price. Since page is never written to after the initialization we	72	* any price. Since page is never written to after the initialization we
73	* don't have to care about aliases on other CPUs.	73	* don't have to care about aliases on other CPUs.
74	*/	74	*/
75	unsigned long empty_zero_page, zero_page_mask;	75	unsigned long empty_zero_page, zero_page_mask;
76	EXPORT_SYMBOL_GPL(empty_zero_page);	76	EXPORT_SYMBOL_GPL(empty_zero_page);
77		77
78	/*	78	/*
79	* Not static inline because used by IP27 special magic initialization code	79	* Not static inline because used by IP27 special magic initialization code
80	*/	80	*/
81	unsigned long setup_zero_pages(void)	81	unsigned long setup_zero_pages(void)
82	{	82	{
83	unsigned int order;	83	unsigned int order;
84	unsigned long size;	84	unsigned long size;
85	struct page *page;	85	struct page *page;
86		86
87	if (cpu_has_vce)	87	if (cpu_has_vce)
88	order = 3;	88	order = 3;
89	else	89	else
90	order = 0;	90	order = 0;
91		91
92	empty_zero_page = __get_free_pages(GFP_KERNEL \| __GFP_ZERO, order);	92	empty_zero_page = __get_free_pages(GFP_KERNEL \| __GFP_ZERO, order);
93	if (!empty_zero_page)	93	if (!empty_zero_page)
94	panic("Oh boy, that early out of memory?");	94	panic("Oh boy, that early out of memory?");
95		95
96	page = virt_to_page((void *)empty_zero_page);	96	page = virt_to_page((void *)empty_zero_page);
97	split_page(page, order);	97	split_page(page, order);
98	while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {	98	while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {
99	SetPageReserved(page);	99	SetPageReserved(page);
100	page++;	100	page++;
101	}	101	}
102		102
103	size = PAGE_SIZE << order;	103	size = PAGE_SIZE << order;
104	zero_page_mask = (size - 1) & PAGE_MASK;	104	zero_page_mask = (size - 1) & PAGE_MASK;
105		105
106	return 1UL << order;	106	return 1UL << order;
107	}	107	}
108		108
109	#ifdef CONFIG_MIPS_MT_SMTC	109	#ifdef CONFIG_MIPS_MT_SMTC
110	static pte_t *kmap_coherent_pte;	110	static pte_t *kmap_coherent_pte;
111	static void __init kmap_coherent_init(void)	111	static void __init kmap_coherent_init(void)
112	{	112	{
113	unsigned long vaddr;	113	unsigned long vaddr;
114		114
115	/* cache the first coherent kmap pte */	115	/* cache the first coherent kmap pte */
116	vaddr = __fix_to_virt(FIX_CMAP_BEGIN);	116	vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
117	kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);	117	kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
118	}	118	}
119	#else	119	#else
120	static inline void kmap_coherent_init(void) {}	120	static inline void kmap_coherent_init(void) {}
121	#endif	121	#endif
122		122
123	void kmap_coherent(struct page page, unsigned long addr)	123	void kmap_coherent(struct page page, unsigned long addr)
124	{	124	{
125	enum fixed_addresses idx;	125	enum fixed_addresses idx;
126	unsigned long vaddr, flags, entrylo;	126	unsigned long vaddr, flags, entrylo;
127	unsigned long old_ctx;	127	unsigned long old_ctx;
128	pte_t pte;	128	pte_t pte;
129	int tlbidx;	129	int tlbidx;
130		130
131	BUG_ON(Page_dcache_dirty(page));	131	BUG_ON(Page_dcache_dirty(page));
132		132
133	inc_preempt_count();	133	inc_preempt_count();
134	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);	134	idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
135	#ifdef CONFIG_MIPS_MT_SMTC	135	#ifdef CONFIG_MIPS_MT_SMTC
136	idx += FIX_N_COLOURS * smp_processor_id() +	136	idx += FIX_N_COLOURS * smp_processor_id() +
137	(in_interrupt() ? (FIX_N_COLOURS * NR_CPUS) : 0);	137	(in_interrupt() ? (FIX_N_COLOURS * NR_CPUS) : 0);
138	#else	138	#else
139	idx += in_interrupt() ? FIX_N_COLOURS : 0;	139	idx += in_interrupt() ? FIX_N_COLOURS : 0;
140	#endif	140	#endif
141	vaddr = __fix_to_virt(FIX_CMAP_END - idx);	141	vaddr = __fix_to_virt(FIX_CMAP_END - idx);
142	pte = mk_pte(page, PAGE_KERNEL);	142	pte = mk_pte(page, PAGE_KERNEL);
143	#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)	143	#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
144	entrylo = pte.pte_high;	144	entrylo = pte.pte_high;
145	#else	145	#else
146	entrylo = pte_val(pte) >> 6;	146	entrylo = pte_val(pte) >> 6;
147	#endif	147	#endif
148		148
149	ENTER_CRITICAL(flags);	149	ENTER_CRITICAL(flags);
150	old_ctx = read_c0_entryhi();	150	old_ctx = read_c0_entryhi();
151	write_c0_entryhi(vaddr & (PAGE_MASK << 1));	151	write_c0_entryhi(vaddr & (PAGE_MASK << 1));
152	write_c0_entrylo0(entrylo);	152	write_c0_entrylo0(entrylo);
153	write_c0_entrylo1(entrylo);	153	write_c0_entrylo1(entrylo);
154	#ifdef CONFIG_MIPS_MT_SMTC	154	#ifdef CONFIG_MIPS_MT_SMTC
155	set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);	155	set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
156	/* preload TLB instead of local_flush_tlb_one() */	156	/* preload TLB instead of local_flush_tlb_one() */
157	mtc0_tlbw_hazard();	157	mtc0_tlbw_hazard();
158	tlb_probe();	158	tlb_probe();
159	tlb_probe_hazard();	159	tlb_probe_hazard();
160	tlbidx = read_c0_index();	160	tlbidx = read_c0_index();
161	mtc0_tlbw_hazard();	161	mtc0_tlbw_hazard();
162	if (tlbidx < 0)	162	if (tlbidx < 0)
163	tlb_write_random();	163	tlb_write_random();
164	else	164	else
165	tlb_write_indexed();	165	tlb_write_indexed();
166	#else	166	#else
167	tlbidx = read_c0_wired();	167	tlbidx = read_c0_wired();
168	write_c0_wired(tlbidx + 1);	168	write_c0_wired(tlbidx + 1);
169	write_c0_index(tlbidx);	169	write_c0_index(tlbidx);
170	mtc0_tlbw_hazard();	170	mtc0_tlbw_hazard();
171	tlb_write_indexed();	171	tlb_write_indexed();
172	#endif	172	#endif
173	tlbw_use_hazard();	173	tlbw_use_hazard();
174	write_c0_entryhi(old_ctx);	174	write_c0_entryhi(old_ctx);
175	EXIT_CRITICAL(flags);	175	EXIT_CRITICAL(flags);
176		176
177	return (void*) vaddr;	177	return (void*) vaddr;
178	}	178	}
179		179
180	#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))	180	#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
181		181
182	void kunmap_coherent(void)	182	void kunmap_coherent(void)
183	{	183	{
184	#ifndef CONFIG_MIPS_MT_SMTC	184	#ifndef CONFIG_MIPS_MT_SMTC
185	unsigned int wired;	185	unsigned int wired;
186	unsigned long flags, old_ctx;	186	unsigned long flags, old_ctx;
187		187
188	ENTER_CRITICAL(flags);	188	ENTER_CRITICAL(flags);
189	old_ctx = read_c0_entryhi();	189	old_ctx = read_c0_entryhi();
190	wired = read_c0_wired() - 1;	190	wired = read_c0_wired() - 1;
191	write_c0_wired(wired);	191	write_c0_wired(wired);
192	write_c0_index(wired);	192	write_c0_index(wired);
193	write_c0_entryhi(UNIQUE_ENTRYHI(wired));	193	write_c0_entryhi(UNIQUE_ENTRYHI(wired));
194	write_c0_entrylo0(0);	194	write_c0_entrylo0(0);
195	write_c0_entrylo1(0);	195	write_c0_entrylo1(0);
196	mtc0_tlbw_hazard();	196	mtc0_tlbw_hazard();
197	tlb_write_indexed();	197	tlb_write_indexed();
198	tlbw_use_hazard();	198	tlbw_use_hazard();
199	write_c0_entryhi(old_ctx);	199	write_c0_entryhi(old_ctx);
200	EXIT_CRITICAL(flags);	200	EXIT_CRITICAL(flags);
201	#endif	201	#endif
202	dec_preempt_count();	202	dec_preempt_count();
203	preempt_check_resched();	203	preempt_check_resched();
204	}	204	}
205		205
206	void copy_user_highpage(struct page to, struct page from,	206	void copy_user_highpage(struct page to, struct page from,
207	unsigned long vaddr, struct vm_area_struct *vma)	207	unsigned long vaddr, struct vm_area_struct *vma)
208	{	208	{
209	void vfrom, vto;	209	void vfrom, vto;
210		210
211	vto = kmap_atomic(to, KM_USER1);	211	vto = kmap_atomic(to, KM_USER1);
212	if (cpu_has_dc_aliases &&	212	if (cpu_has_dc_aliases &&
213	page_mapped(from) && !Page_dcache_dirty(from)) {	213	page_mapped(from) && !Page_dcache_dirty(from)) {
214	vfrom = kmap_coherent(from, vaddr);	214	vfrom = kmap_coherent(from, vaddr);
215	copy_page(vto, vfrom);	215	copy_page(vto, vfrom);
216	kunmap_coherent();	216	kunmap_coherent();
217	} else {	217	} else {
218	vfrom = kmap_atomic(from, KM_USER0);	218	vfrom = kmap_atomic(from, KM_USER0);
219	copy_page(vto, vfrom);	219	copy_page(vto, vfrom);
220	kunmap_atomic(vfrom, KM_USER0);	220	kunmap_atomic(vfrom, KM_USER0);
221	}	221	}
222	if ((!cpu_has_ic_fills_f_dc) \|\|	222	if ((!cpu_has_ic_fills_f_dc) \|\|
223	pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))	223	pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
224	flush_data_cache_page((unsigned long)vto);	224	flush_data_cache_page((unsigned long)vto);
225	kunmap_atomic(vto, KM_USER1);	225	kunmap_atomic(vto, KM_USER1);
226	/* Make sure this page is cleared on other CPU's too before using it */	226	/* Make sure this page is cleared on other CPU's too before using it */
227	smp_wmb();	227	smp_wmb();
228	}	228	}
229		229
230	void copy_to_user_page(struct vm_area_struct *vma,	230	void copy_to_user_page(struct vm_area_struct *vma,
231	struct page page, unsigned long vaddr, void dst, const void *src,	231	struct page page, unsigned long vaddr, void dst, const void *src,
232	unsigned long len)	232	unsigned long len)
233	{	233	{
234	if (cpu_has_dc_aliases &&	234	if (cpu_has_dc_aliases &&
235	page_mapped(page) && !Page_dcache_dirty(page)) {	235	page_mapped(page) && !Page_dcache_dirty(page)) {
236	void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);	236	void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
237	memcpy(vto, src, len);	237	memcpy(vto, src, len);
238	kunmap_coherent();	238	kunmap_coherent();
239	} else {	239	} else {
240	memcpy(dst, src, len);	240	memcpy(dst, src, len);
241	if (cpu_has_dc_aliases)	241	if (cpu_has_dc_aliases)
242	SetPageDcacheDirty(page);	242	SetPageDcacheDirty(page);
243	}	243	}
244	if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)	244	if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
245	flush_cache_page(vma, vaddr, page_to_pfn(page));	245	flush_cache_page(vma, vaddr, page_to_pfn(page));
246	}	246	}
247		247
248	void copy_from_user_page(struct vm_area_struct *vma,	248	void copy_from_user_page(struct vm_area_struct *vma,
249	struct page page, unsigned long vaddr, void dst, const void *src,	249	struct page page, unsigned long vaddr, void dst, const void *src,
250	unsigned long len)	250	unsigned long len)
251	{	251	{
252	if (cpu_has_dc_aliases &&	252	if (cpu_has_dc_aliases &&
253	page_mapped(page) && !Page_dcache_dirty(page)) {	253	page_mapped(page) && !Page_dcache_dirty(page)) {
254	void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);	254	void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
255	memcpy(dst, vfrom, len);	255	memcpy(dst, vfrom, len);
256	kunmap_coherent();	256	kunmap_coherent();
257	} else {	257	} else {
258	memcpy(dst, src, len);	258	memcpy(dst, src, len);
259	if (cpu_has_dc_aliases)	259	if (cpu_has_dc_aliases)
260	SetPageDcacheDirty(page);	260	SetPageDcacheDirty(page);
261	}	261	}
262	}	262	}
263		263
264	void __init fixrange_init(unsigned long start, unsigned long end,	264	void __init fixrange_init(unsigned long start, unsigned long end,
265	pgd_t *pgd_base)	265	pgd_t *pgd_base)
266	{	266	{
267	#if defined(CONFIG_HIGHMEM) \|\| defined(CONFIG_MIPS_MT_SMTC)	267	#if defined(CONFIG_HIGHMEM) \|\| defined(CONFIG_MIPS_MT_SMTC)
268	pgd_t *pgd;	268	pgd_t *pgd;
269	pud_t *pud;	269	pud_t *pud;
270	pmd_t *pmd;	270	pmd_t *pmd;
271	pte_t *pte;	271	pte_t *pte;
272	int i, j, k;	272	int i, j, k;
273	unsigned long vaddr;	273	unsigned long vaddr;
274		274
275	vaddr = start;	275	vaddr = start;
276	i = __pgd_offset(vaddr);	276	i = __pgd_offset(vaddr);
277	j = __pud_offset(vaddr);	277	j = __pud_offset(vaddr);
278	k = __pmd_offset(vaddr);	278	k = __pmd_offset(vaddr);
279	pgd = pgd_base + i;	279	pgd = pgd_base + i;
280		280
281	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {	281	for ( ; (i < PTRS_PER_PGD) && (vaddr != end); pgd++, i++) {
282	pud = (pud_t *)pgd;	282	pud = (pud_t *)pgd;
283	for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {	283	for ( ; (j < PTRS_PER_PUD) && (vaddr != end); pud++, j++) {
284	pmd = (pmd_t *)pud;	284	pmd = (pmd_t *)pud;
285	for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {	285	for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
286	if (pmd_none(*pmd)) {	286	if (pmd_none(*pmd)) {
287	pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);	287	pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
288	set_pmd(pmd, __pmd((unsigned long)pte));	288	set_pmd(pmd, __pmd((unsigned long)pte));
289	BUG_ON(pte != pte_offset_kernel(pmd, 0));	289	BUG_ON(pte != pte_offset_kernel(pmd, 0));
290	}	290	}
291	vaddr += PMD_SIZE;	291	vaddr += PMD_SIZE;
292	}	292	}
293	k = 0;	293	k = 0;
294	}	294	}
295	j = 0;	295	j = 0;
296	}	296	}
297	#endif	297	#endif
298	}	298	}
299		299
300	#ifndef CONFIG_NEED_MULTIPLE_NODES	300	#ifndef CONFIG_NEED_MULTIPLE_NODES
301	static int __init page_is_ram(unsigned long pagenr)	301	static int __init page_is_ram(unsigned long pagenr)
302	{	302	{
303	int i;	303	int i;
304		304
305	for (i = 0; i < boot_mem_map.nr_map; i++) {	305	for (i = 0; i < boot_mem_map.nr_map; i++) {
306	unsigned long addr, end;	306	unsigned long addr, end;
307		307
308	if (boot_mem_map.map[i].type != BOOT_MEM_RAM)	308	if (boot_mem_map.map[i].type != BOOT_MEM_RAM)
309	/* not usable memory */	309	/* not usable memory */
310	continue;	310	continue;
311		311
312	addr = PFN_UP(boot_mem_map.map[i].addr);	312	addr = PFN_UP(boot_mem_map.map[i].addr);
313	end = PFN_DOWN(boot_mem_map.map[i].addr +	313	end = PFN_DOWN(boot_mem_map.map[i].addr +
314	boot_mem_map.map[i].size);	314	boot_mem_map.map[i].size);
315		315
316	if (pagenr >= addr && pagenr < end)	316	if (pagenr >= addr && pagenr < end)
317	return 1;	317	return 1;
318	}	318	}
319		319
320	return 0;	320	return 0;
321	}	321	}
322		322
323	void __init paging_init(void)	323	void __init paging_init(void)
324	{	324	{
325	unsigned long max_zone_pfns[MAX_NR_ZONES];	325	unsigned long max_zone_pfns[MAX_NR_ZONES];
326	unsigned long lastpfn;	326	unsigned long lastpfn;
327		327
328	pagetable_init();	328	pagetable_init();
329		329
330	#ifdef CONFIG_HIGHMEM	330	#ifdef CONFIG_HIGHMEM
331	kmap_init();	331	kmap_init();
332	#endif	332	#endif
333	kmap_coherent_init();	333	kmap_coherent_init();
334		334
335	#ifdef CONFIG_ZONE_DMA	335	#ifdef CONFIG_ZONE_DMA
336	max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;	336	max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
337	#endif	337	#endif
338	#ifdef CONFIG_ZONE_DMA32	338	#ifdef CONFIG_ZONE_DMA32
339	max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;	339	max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
340	#endif	340	#endif
341	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;	341	max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
342	lastpfn = max_low_pfn;	342	lastpfn = max_low_pfn;
343	#ifdef CONFIG_HIGHMEM	343	#ifdef CONFIG_HIGHMEM
344	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;	344	max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
345	lastpfn = highend_pfn;	345	lastpfn = highend_pfn;
346		346
347	if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {	347	if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
348	printk(KERN_WARNING "This processor doesn't support highmem."	348	printk(KERN_WARNING "This processor doesn't support highmem."
349	" %ldk highmem ignored\n",	349	" %ldk highmem ignored\n",
350	(highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));	350	(highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
351	max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;	351	max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
352	lastpfn = max_low_pfn;	352	lastpfn = max_low_pfn;
353	}	353	}
354	#endif	354	#endif
355		355
356	free_area_init_nodes(max_zone_pfns);	356	free_area_init_nodes(max_zone_pfns);
357	}	357	}
358		358
359	#ifdef CONFIG_64BIT	359	#ifdef CONFIG_64BIT
360	static struct kcore_list kcore_kseg0;	360	static struct kcore_list kcore_kseg0;
361	#endif	361	#endif
362		362
363	void __init mem_init(void)	363	void __init mem_init(void)
364	{	364	{
365	unsigned long codesize, reservedpages, datasize, initsize;	365	unsigned long codesize, reservedpages, datasize, initsize;
366	unsigned long tmp, ram;	366	unsigned long tmp, ram;
367		367
368	#ifdef CONFIG_HIGHMEM	368	#ifdef CONFIG_HIGHMEM
369	#ifdef CONFIG_DISCONTIGMEM	369	#ifdef CONFIG_DISCONTIGMEM
370	#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"	370	#error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
371	#endif	371	#endif
372	max_mapnr = highend_pfn;	372	max_mapnr = highend_pfn;
373	#else	373	#else
374	max_mapnr = max_low_pfn;	374	max_mapnr = max_low_pfn;
375	#endif	375	#endif
376	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);	376	high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
377		377
378	totalram_pages += free_all_bootmem();	378	totalram_pages += free_all_bootmem();
379	totalram_pages -= setup_zero_pages(); /* Setup zeroed pages. */	379	totalram_pages -= setup_zero_pages(); /* Setup zeroed pages. */
380		380
381	reservedpages = ram = 0;	381	reservedpages = ram = 0;
382	for (tmp = 0; tmp < max_low_pfn; tmp++)	382	for (tmp = 0; tmp < max_low_pfn; tmp++)
383	if (page_is_ram(tmp)) {	383	if (page_is_ram(tmp)) {
384	ram++;	384	ram++;
385	if (PageReserved(pfn_to_page(tmp)))	385	if (PageReserved(pfn_to_page(tmp)))
386	reservedpages++;	386	reservedpages++;
387	}	387	}
388	num_physpages = ram;	388	num_physpages = ram;
389		389
390	#ifdef CONFIG_HIGHMEM	390	#ifdef CONFIG_HIGHMEM
391	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {	391	for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
392	struct page *page = pfn_to_page(tmp);	392	struct page *page = pfn_to_page(tmp);
393		393
394	if (!page_is_ram(tmp)) {	394	if (!page_is_ram(tmp)) {
395	SetPageReserved(page);	395	SetPageReserved(page);
396	continue;	396	continue;
397	}	397	}
398	ClearPageReserved(page);	398	ClearPageReserved(page);
399	init_page_count(page);	399	init_page_count(page);
400	__free_page(page);	400	__free_page(page);
401	totalhigh_pages++;	401	totalhigh_pages++;
402	}	402	}
403	totalram_pages += totalhigh_pages;	403	totalram_pages += totalhigh_pages;
404	num_physpages += totalhigh_pages;	404	num_physpages += totalhigh_pages;
405	#endif	405	#endif
406		406
407	codesize = (unsigned long) &_etext - (unsigned long) &_text;	407	codesize = (unsigned long) &_etext - (unsigned long) &_text;
408	datasize = (unsigned long) &_edata - (unsigned long) &_etext;	408	datasize = (unsigned long) &_edata - (unsigned long) &_etext;
409	initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;	409	initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
410		410
411	#ifdef CONFIG_64BIT	411	#ifdef CONFIG_64BIT
412	if ((unsigned long) &_text > (unsigned long) CKSEG0)	412	if ((unsigned long) &_text > (unsigned long) CKSEG0)
413	/* The -4 is a hack so that user tools don't have to handle	413	/* The -4 is a hack so that user tools don't have to handle
414	the overflow. */	414	the overflow. */
415	kclist_add(&kcore_kseg0, (void *) CKSEG0,	415	kclist_add(&kcore_kseg0, (void *) CKSEG0,
416	0x80000000 - 4, KCORE_TEXT);	416	0x80000000 - 4, KCORE_TEXT);
417	#endif	417	#endif
418		418
419	printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "	419	printk(KERN_INFO "Memory: %luk/%luk available (%ldk kernel code, "
420	"%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",	420	"%ldk reserved, %ldk data, %ldk init, %ldk highmem)\n",
421	nr_free_pages() << (PAGE_SHIFT-10),	421	nr_free_pages() << (PAGE_SHIFT-10),
422	ram << (PAGE_SHIFT-10),	422	ram << (PAGE_SHIFT-10),
423	codesize >> 10,	423	codesize >> 10,
424	reservedpages << (PAGE_SHIFT-10),	424	reservedpages << (PAGE_SHIFT-10),
425	datasize >> 10,	425	datasize >> 10,
426	initsize >> 10,	426	initsize >> 10,
427	totalhigh_pages << (PAGE_SHIFT-10));	427	totalhigh_pages << (PAGE_SHIFT-10));
428	}	428	}
429	#endif /* !CONFIG_NEED_MULTIPLE_NODES */	429	#endif /* !CONFIG_NEED_MULTIPLE_NODES */
430		430
431	void free_init_pages(const char *what, unsigned long begin, unsigned long end)	431	void free_init_pages(const char *what, unsigned long begin, unsigned long end)
432	{	432	{
433	unsigned long pfn;	433	unsigned long pfn;
434		434
435	for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {	435	for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
436	struct page *page = pfn_to_page(pfn);	436	struct page *page = pfn_to_page(pfn);
437	void *addr = phys_to_virt(PFN_PHYS(pfn));	437	void *addr = phys_to_virt(PFN_PHYS(pfn));
438		438
439	ClearPageReserved(page);	439	ClearPageReserved(page);
440	init_page_count(page);	440	init_page_count(page);
441	memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);	441	memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
442	__free_page(page);	442	__free_page(page);
443	totalram_pages++;	443	totalram_pages++;
444	}	444	}
445	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);	445	printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
446	}	446	}
447		447
448	#ifdef CONFIG_BLK_DEV_INITRD	448	#ifdef CONFIG_BLK_DEV_INITRD
449	void free_initrd_mem(unsigned long start, unsigned long end)	449	void free_initrd_mem(unsigned long start, unsigned long end)
450	{	450	{
451	free_init_pages("initrd memory",	451	free_init_pages("initrd memory",
452	virt_to_phys((void *)start),	452	virt_to_phys((void *)start),
453	virt_to_phys((void *)end));	453	virt_to_phys((void *)end));
454	}	454	}
455	#endif	455	#endif
456		456
457	void __init_refok free_initmem(void)	457	void __init_refok free_initmem(void)
458	{	458	{
459	prom_free_prom_memory();	459	prom_free_prom_memory();
460	free_init_pages("unused kernel memory",	460	free_init_pages("unused kernel memory",
461	__pa_symbol(&__init_begin),	461	__pa_symbol(&__init_begin),
462	__pa_symbol(&__init_end));	462	__pa_symbol(&__init_end));
463	}	463	}
464		464
465	#ifndef CONFIG_MIPS_PGD_C0_CONTEXT	465	#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
466	unsigned long pgd_current[NR_CPUS];	466	unsigned long pgd_current[NR_CPUS];
467	#endif	467	#endif
468	/*	468	/*
469	* On 64-bit we've got three-level pagetables with a slightly	469	* On 64-bit we've got three-level pagetables with a slightly
470	* different layout ...	470	* different layout ...
471	*/	471	*/
472	#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))	472	#define __page_aligned(order) __attribute__((__aligned__(PAGE_SIZE<<order)))
473		473
474	/*	474	/*
475	* gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER	475	* gcc 3.3 and older have trouble determining that PTRS_PER_PGD and PGD_ORDER
476	* are constants. So we use the variants from asm-offset.h until that gcc	476	* are constants. So we use the variants from asm-offset.h until that gcc
477	* will officially be retired.	477	* will officially be retired.
478	*/	478	*/
479	pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);	479	pgd_t swapper_pg_dir[_PTRS_PER_PGD] __page_aligned(_PGD_ORDER);
480	#ifdef CONFIG_64BIT	480	#ifndef __PAGETABLE_PMD_FOLDED
481	pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);	481	pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned(PMD_ORDER);
482	#endif	482	#endif
483	pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);	483	pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned(PTE_ORDER);
484		484

arch/mips/mm/pgtable-64.c

Diff comments View file @ 325f8a0

1	/*	1	/*
2	* This file is subject to the terms and conditions of the GNU General Public	2	* This file is subject to the terms and conditions of the GNU General Public
3	* License. See the file "COPYING" in the main directory of this archive	3	* License. See the file "COPYING" in the main directory of this archive
4	* for more details.	4	* for more details.
5	*	5	*
6	* Copyright (C) 1999, 2000 by Silicon Graphics	6	* Copyright (C) 1999, 2000 by Silicon Graphics
7	* Copyright (C) 2003 by Ralf Baechle	7	* Copyright (C) 2003 by Ralf Baechle
8	*/	8	*/
9	#include <linux/init.h>	9	#include <linux/init.h>
10	#include <linux/mm.h>	10	#include <linux/mm.h>
11	#include <asm/fixmap.h>	11	#include <asm/fixmap.h>
12	#include <asm/pgtable.h>	12	#include <asm/pgtable.h>
13	#include <asm/pgalloc.h>	13	#include <asm/pgalloc.h>
14		14
15	void pgd_init(unsigned long page)	15	void pgd_init(unsigned long page)
16	{	16	{
17	unsigned long p, end;	17	unsigned long p, end;
		18	unsigned long entry;
18		19
		20	#ifdef __PAGETABLE_PMD_FOLDED
		21	entry = (unsigned long)invalid_pte_table;
		22	#else
		23	entry = (unsigned long)invalid_pmd_table;
		24	#endif
		25
19	p = (unsigned long *) page;	26	p = (unsigned long *) page;
20	end = p + PTRS_PER_PGD;	27	end = p + PTRS_PER_PGD;
21		28
22	while (p < end) {	29	while (p < end) {
23	p[0] = (unsigned long) invalid_pmd_table;	30	p[0] = entry;
24	p[1] = (unsigned long) invalid_pmd_table;	31	p[1] = entry;
25	p[2] = (unsigned long) invalid_pmd_table;	32	p[2] = entry;
26	p[3] = (unsigned long) invalid_pmd_table;	33	p[3] = entry;
27	p[4] = (unsigned long) invalid_pmd_table;	34	p[4] = entry;
28	p[5] = (unsigned long) invalid_pmd_table;	35	p[5] = entry;
29	p[6] = (unsigned long) invalid_pmd_table;	36	p[6] = entry;
30	p[7] = (unsigned long) invalid_pmd_table;	37	p[7] = entry;
31	p += 8;	38	p += 8;
32	}	39	}
33	}	40	}
34		41
		42	#ifndef __PAGETABLE_PMD_FOLDED
35	void pmd_init(unsigned long addr, unsigned long pagetable)	43	void pmd_init(unsigned long addr, unsigned long pagetable)
36	{	44	{
37	unsigned long p, end;	45	unsigned long p, end;
38		46
39	p = (unsigned long *) addr;	47	p = (unsigned long *) addr;
40	end = p + PTRS_PER_PMD;	48	end = p + PTRS_PER_PMD;
41		49
42	while (p < end) {	50	while (p < end) {
43	p[0] = (unsigned long)pagetable;	51	p[0] = pagetable;
44	p[1] = (unsigned long)pagetable;	52	p[1] = pagetable;
45	p[2] = (unsigned long)pagetable;	53	p[2] = pagetable;
46	p[3] = (unsigned long)pagetable;	54	p[3] = pagetable;
47	p[4] = (unsigned long)pagetable;	55	p[4] = pagetable;
48	p[5] = (unsigned long)pagetable;	56	p[5] = pagetable;
49	p[6] = (unsigned long)pagetable;	57	p[6] = pagetable;
50	p[7] = (unsigned long)pagetable;	58	p[7] = pagetable;
51	p += 8;	59	p += 8;
52	}	60	}
53	}	61	}
		62	#endif
54		63
55	void __init pagetable_init(void)	64	void __init pagetable_init(void)
56	{	65	{
57	unsigned long vaddr;	66	unsigned long vaddr;
58	pgd_t *pgd_base;	67	pgd_t *pgd_base;
59		68
60	/* Initialize the entire pgd. */	69	/* Initialize the entire pgd. */
61	pgd_init((unsigned long)swapper_pg_dir);	70	pgd_init((unsigned long)swapper_pg_dir);
		71	#ifndef __PAGETABLE_PMD_FOLDED
62	pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);	72	pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
63		73	#endif
64	pgd_base = swapper_pg_dir;	74	pgd_base = swapper_pg_dir;
65	/*	75	/*
66	* Fixed mappings:	76	* Fixed mappings:
67	*/	77	*/
68	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;	78	vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
69	fixrange_init(vaddr, 0, pgd_base);	79	fixrange_init(vaddr, 0, pgd_base);
70	}	80	}
71		81

arch/mips/mm/tlbex.c

Diff comments View file @ 325f8a0

 /*
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file "COPYING" in the main directory of this archive
  * for more details.
  *
  * Synthesize TLB refill handlers at runtime.
  *
  * Copyright (C) 2004, 2005, 2006, 2008  Thiemo Seufer
  * Copyright (C) 2005, 2007, 2008, 2009  Maciej W. Rozycki
  * Copyright (C) 2006  Ralf Baechle (ralf@linux-mips.org)
  * Copyright (C) 2008, 2009 Cavium Networks, Inc.
  *
  * ... and the days got worse and worse and now you see
  * I've gone completly out of my mind.
  *
  * They're coming to take me a away haha
  * they're coming to take me a away hoho hihi haha
  * to the funny farm where code is beautiful all the time ...
  *
  * (Condolences to Napoleon XIV)
  */
 #include <linux/bug.h>
 #include <linux/kernel.h>
 #include <linux/types.h>
 #include <linux/smp.h>
 #include <linux/string.h>
 #include <linux/init.h>
 #include <asm/mmu_context.h>
 #include <asm/war.h>
 #include "uasm.h"
 static inline int r45k_bvahwbug(void)
 {
 	/* XXX: We should probe for the presence of this bug, but we don't. */
 	return 0;
 }
 static inline int r4k_250MHZhwbug(void)
 {
 	/* XXX: We should probe for the presence of this bug, but we don't. */
 	return 0;
 }
 static inline int __maybe_unused bcm1250_m3_war(void)
 {
 	return BCM1250_M3_WAR;
 }
 static inline int __maybe_unused r10000_llsc_war(void)
 {
 	return R10000_LLSC_WAR;
 }
 /*
  * Found by experiment: At least some revisions of the 4kc throw under
  * some circumstances a machine check exception, triggered by invalid
  * values in the index register.  Delaying the tlbp instruction until
  * after the next branch,  plus adding an additional nop in front of
  * tlbwi/tlbwr avoids the invalid index register values. Nobody knows
  * why; it's not an issue caused by the core RTL.
  *
  */
 static int __cpuinit m4kc_tlbp_war(void)
 {
 	return (current_cpu_data.processor_id & 0xffff00) ==
 	       (PRID_COMP_MIPS | PRID_IMP_4KC);
 }
 /* Handle labels (which must be positive integers). */
 enum label_id {
 	label_second_part = 1,
 	label_leave,
 	label_vmalloc,
 	label_vmalloc_done,
 	label_tlbw_hazard,
 	label_split,
 	label_nopage_tlbl,
 	label_nopage_tlbs,
 	label_nopage_tlbm,
 	label_smp_pgtable_change,
 	label_r3000_write_probe_fail,
 #ifdef CONFIG_HUGETLB_PAGE
 	label_tlb_huge_update,
 #endif
 };
 UASM_L_LA(_second_part)
 UASM_L_LA(_leave)
 UASM_L_LA(_vmalloc)
 UASM_L_LA(_vmalloc_done)
 UASM_L_LA(_tlbw_hazard)
 UASM_L_LA(_split)
 UASM_L_LA(_nopage_tlbl)
 UASM_L_LA(_nopage_tlbs)
 UASM_L_LA(_nopage_tlbm)
 UASM_L_LA(_smp_pgtable_change)
 UASM_L_LA(_r3000_write_probe_fail)
 #ifdef CONFIG_HUGETLB_PAGE
 UASM_L_LA(_tlb_huge_update)
 #endif
 /*
  * For debug purposes.
  */
 static inline void dump_handler(const u32 *handler, int count)
 {
 	int i;
 	pr_debug("\t.set push\n");
 	pr_debug("\t.set noreorder\n");
 	for (i = 0; i < count; i++)
 		pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
 	pr_debug("\t.set pop\n");
 }
 /* The only general purpose registers allowed in TLB handlers. */
 #define K0		26
 #define K1		27
 /* Some CP0 registers */
 #define C0_INDEX	0, 0
 #define C0_ENTRYLO0	2, 0
 #define C0_TCBIND	2, 2
 #define C0_ENTRYLO1	3, 0
 #define C0_CONTEXT	4, 0
 #define C0_PAGEMASK	5, 0
 #define C0_BADVADDR	8, 0
 #define C0_ENTRYHI	10, 0
 #define C0_EPC		14, 0
 #define C0_XCONTEXT	20, 0
 #ifdef CONFIG_64BIT
 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
 #else
 # define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
 #endif
 /* The worst case length of the handler is around 18 instructions for
  * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
  * Maximum space available is 32 instructions for R3000 and 64
  * instructions for R4000.
  *
  * We deliberately chose a buffer size of 128, so we won't scribble
  * over anything important on overflow before we panic.
  */
 static u32 tlb_handler[128] __cpuinitdata;
 /* simply assume worst case size for labels and relocs */
 static struct uasm_label labels[128] __cpuinitdata;
 static struct uasm_reloc relocs[128] __cpuinitdata;
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 /*
  * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
  * we cannot do r3000 under these circumstances.
  */
 /*
  * The R3000 TLB handler is simple.
  */
 static void __cpuinit build_r3000_tlb_refill_handler(void)
 {
 	long pgdc = (long)pgd_current;
 	u32 *p;
 	memset(tlb_handler, 0, sizeof(tlb_handler));
 	p = tlb_handler;
 	uasm_i_mfc0(&p, K0, C0_BADVADDR);
 	uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
 	uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
 	uasm_i_srl(&p, K0, K0, 22); /* load delay */
 	uasm_i_sll(&p, K0, K0, 2);
 	uasm_i_addu(&p, K1, K1, K0);
 	uasm_i_mfc0(&p, K0, C0_CONTEXT);
 	uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
 	uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
 	uasm_i_addu(&p, K1, K1, K0);
 	uasm_i_lw(&p, K0, 0, K1);
 	uasm_i_nop(&p); /* load delay */
 	uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
 	uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
 	uasm_i_tlbwr(&p); /* cp0 delay */
 	uasm_i_jr(&p, K1);
 	uasm_i_rfe(&p); /* branch delay */
 	if (p > tlb_handler + 32)
 		panic("TLB refill handler space exceeded");
 	pr_debug("Wrote TLB refill handler (%u instructions).\n",
 		 (unsigned int)(p - tlb_handler));
 	memcpy((void *)ebase, tlb_handler, 0x80);
 	dump_handler((u32 *)ebase, 32);
 }
 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 /*
  * The R4000 TLB handler is much more complicated. We have two
  * consecutive handler areas with 32 instructions space each.
  * Since they aren't used at the same time, we can overflow in the
  * other one.To keep things simple, we first assume linear space,
  * then we relocate it to the final handler layout as needed.
  */
 static u32 final_handler[64] __cpuinitdata;
 /*
  * Hazards
  *
  * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0:
  * 2. A timing hazard exists for the TLBP instruction.
  *
  *      stalling_instruction
  *      TLBP
  *
  * The JTLB is being read for the TLBP throughout the stall generated by the
  * previous instruction. This is not really correct as the stalling instruction
  * can modify the address used to access the JTLB.  The failure symptom is that
  * the TLBP instruction will use an address created for the stalling instruction
  * and not the address held in C0_ENHI and thus report the wrong results.
  *
  * The software work-around is to not allow the instruction preceding the TLBP
  * to stall - make it an NOP or some other instruction guaranteed not to stall.
  *
  * Errata 2 will not be fixed.  This errata is also on the R5000.
  *
  * As if we MIPS hackers wouldn't know how to nop pipelines happy ...
  */
 static void __cpuinit __maybe_unused build_tlb_probe_entry(u32 **p)
 {
 	switch (current_cpu_type()) {
 	/* Found by experiment: R4600 v2.0/R4700 needs this, too.  */
 	case CPU_R4600:
 	case CPU_R4700:
 	case CPU_R5000:
 	case CPU_R5000A:
 	case CPU_NEVADA:
 		uasm_i_nop(p);
 		uasm_i_tlbp(p);
 		break;
 	default:
 		uasm_i_tlbp(p);
 		break;
 	}
 }
 /*
  * Write random or indexed TLB entry, and care about the hazards from
  * the preceeding mtc0 and for the following eret.
  */
 enum tlb_write_entry { tlb_random, tlb_indexed };
 static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
 					 struct uasm_reloc **r,
 					 enum tlb_write_entry wmode)
 {
 	void(*tlbw)(u32 **) = NULL;
 	switch (wmode) {
 	case tlb_random: tlbw = uasm_i_tlbwr; break;
 	case tlb_indexed: tlbw = uasm_i_tlbwi; break;
 	}
 	if (cpu_has_mips_r2) {
 		if (cpu_has_mips_r2_exec_hazard)
 			uasm_i_ehb(p);
 		tlbw(p);
 		return;
 	}
 	switch (current_cpu_type()) {
 	case CPU_R4000PC:
 	case CPU_R4000SC:
 	case CPU_R4000MC:
 	case CPU_R4400PC:
 	case CPU_R4400SC:
 	case CPU_R4400MC:
 		/*
 		 * This branch uses up a mtc0 hazard nop slot and saves
 		 * two nops after the tlbw instruction.
 		 */
 		uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
 		tlbw(p);
 		uasm_l_tlbw_hazard(l, *p);
 		uasm_i_nop(p);
 		break;
 	case CPU_R4600:
 	case CPU_R4700:
 	case CPU_R5000:
 	case CPU_R5000A:
 		uasm_i_nop(p);
 		tlbw(p);
 		uasm_i_nop(p);
 		break;
 	case CPU_R4300:
 	case CPU_5KC:
 	case CPU_TX49XX:
 	case CPU_PR4450:
 		uasm_i_nop(p);
 		tlbw(p);
 		break;
 	case CPU_R10000:
 	case CPU_R12000:
 	case CPU_R14000:
 	case CPU_4KC:
 	case CPU_4KEC:
 	case CPU_SB1:
 	case CPU_SB1A:
 	case CPU_4KSC:
 	case CPU_20KC:
 	case CPU_25KF:
 	case CPU_BCM3302:
 	case CPU_BCM4710:
 	case CPU_LOONGSON2:
 	case CPU_BCM6338:
 	case CPU_BCM6345:
 	case CPU_BCM6348:
 	case CPU_BCM6358:
 	case CPU_R5500:
 		if (m4kc_tlbp_war())
 			uasm_i_nop(p);
 	case CPU_ALCHEMY:
 		tlbw(p);
 		break;
 	case CPU_NEVADA:
 		uasm_i_nop(p); /* QED specifies 2 nops hazard */
 		/*
 		 * This branch uses up a mtc0 hazard nop slot and saves
 		 * a nop after the tlbw instruction.
 		 */
 		uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
 		tlbw(p);
 		uasm_l_tlbw_hazard(l, *p);
 		break;
 	case CPU_RM7000:
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		tlbw(p);
 		break;
 	case CPU_RM9000:
 		/*
 		 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
 		 * use of the JTLB for instructions should not occur for 4
 		 * cpu cycles and use for data translations should not occur
 		 * for 3 cpu cycles.
 		 */
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		tlbw(p);
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		uasm_i_ssnop(p);
 		break;
 	case CPU_VR4111:
 	case CPU_VR4121:
 	case CPU_VR4122:
 	case CPU_VR4181:
 	case CPU_VR4181A:
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		tlbw(p);
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		break;
 	case CPU_VR4131:
 	case CPU_VR4133:
 	case CPU_R5432:
 		uasm_i_nop(p);
 		uasm_i_nop(p);
 		tlbw(p);
 		break;
 	default:
 		panic("No TLB refill handler yet (CPU type: %d)",
 		      current_cpu_data.cputype);
 		break;
 	}
 }
 #ifdef CONFIG_HUGETLB_PAGE
 static __cpuinit void build_huge_tlb_write_entry(u32 **p,
 						 struct uasm_label **l,
 						 struct uasm_reloc **r,
 						 unsigned int tmp,
 						 enum tlb_write_entry wmode)
 {
 	/* Set huge page tlb entry size */
 	uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
 	uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
 	uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 	build_tlb_write_entry(p, l, r, wmode);
 	/* Reset default page size */
 	if (PM_DEFAULT_MASK >> 16) {
 		uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
 		uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
 		uasm_il_b(p, r, label_leave);
 		uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 	} else if (PM_DEFAULT_MASK) {
 		uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
 		uasm_il_b(p, r, label_leave);
 		uasm_i_mtc0(p, tmp, C0_PAGEMASK);
 	} else {
 		uasm_il_b(p, r, label_leave);
 		uasm_i_mtc0(p, 0, C0_PAGEMASK);
 	}
 }
 /*
  * Check if Huge PTE is present, if so then jump to LABEL.
  */
 static void __cpuinit
 build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp,
 		unsigned int pmd, int lid)
 {
 	UASM_i_LW(p, tmp, 0, pmd);
 	uasm_i_andi(p, tmp, tmp, _PAGE_HUGE);
 	uasm_il_bnez(p, r, tmp, lid);
 }
 static __cpuinit void build_huge_update_entries(u32 **p,
 						unsigned int pte,
 						unsigned int tmp)
 {
 	int small_sequence;
 	/*
 	 * A huge PTE describes an area the size of the
 	 * configured huge page size. This is twice the
 	 * of the large TLB entry size we intend to use.
 	 * A TLB entry half the size of the configured
 	 * huge page size is configured into entrylo0
 	 * and entrylo1 to cover the contiguous huge PTE
 	 * address space.
 	 */
 	small_sequence = (HPAGE_SIZE >> 7) < 0x10000;
 	/* We can clobber tmp.  It isn't used after this.*/
 	if (!small_sequence)
 		uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
 	UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */
 	uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */
 	/* convert to entrylo1 */
 	if (small_sequence)
 		UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
 	else
 		UASM_i_ADDU(p, pte, pte, tmp);
 	uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */
 }
 static __cpuinit void build_huge_handler_tail(u32 **p,
 					      struct uasm_reloc **r,
 					      struct uasm_label **l,
 					      unsigned int pte,
 					      unsigned int ptr)
 {
 #ifdef CONFIG_SMP
 	UASM_i_SC(p, pte, 0, ptr);
 	uasm_il_beqz(p, r, pte, label_tlb_huge_update);
 	UASM_i_LW(p, pte, 0, ptr); /* Needed because SC killed our PTE */
 #else
 	UASM_i_SW(p, pte, 0, ptr);
 #endif
 	build_huge_update_entries(p, pte, ptr);
 	build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed);
 }
 #endif /* CONFIG_HUGETLB_PAGE */
 #ifdef CONFIG_64BIT
 /*
  * TMP and PTR are scratch.
  * TMP will be clobbered, PTR will hold the pmd entry.
  */
 static void __cpuinit
 build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 		 unsigned int tmp, unsigned int ptr)
 {
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 	long pgdc = (long)pgd_current;
 #endif
 	/*
 	 * The vmalloc handling is not in the hotpath.
 	 */
 	uasm_i_dmfc0(p, tmp, C0_BADVADDR);
 	uasm_il_bltz(p, r, tmp, label_vmalloc);
 	/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
 #ifdef CONFIG_MIPS_PGD_C0_CONTEXT
 	/*
 	 * &pgd << 11 stored in CONTEXT [23..63].
 	 */
 	UASM_i_MFC0(p, ptr, C0_CONTEXT);
 	uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */
 	uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0  1 0 1  << 6  xkphys cached */
 	uasm_i_drotr(p, ptr, ptr, 11);
 #elif defined(CONFIG_SMP)
 # ifdef  CONFIG_MIPS_MT_SMTC
 	/*
 	 * SMTC uses TCBind value as "CPU" index
 	 */
 	uasm_i_mfc0(p, ptr, C0_TCBIND);
 	uasm_i_dsrl(p, ptr, ptr, 19);
 # else
 	/*
 	 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
 	 * stored in CONTEXT.
 	 */
 	uasm_i_dmfc0(p, ptr, C0_CONTEXT);
 	uasm_i_dsrl(p, ptr, ptr, 23);
 # endif
 	UASM_i_LA_mostly(p, tmp, pgdc);
 	uasm_i_daddu(p, ptr, ptr, tmp);
 	uasm_i_dmfc0(p, tmp, C0_BADVADDR);
 	uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 #else
 	UASM_i_LA_mostly(p, ptr, pgdc);
 	uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
 #endif
 	uasm_l_vmalloc_done(l, *p);
 	if (PGDIR_SHIFT - 3 < 32)		/* get pgd offset in bytes */
 		uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
 	else
 		uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
 	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
 	uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
+#ifndef __PAGETABLE_PMD_FOLDED
 	uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
 	uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
 	uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
 	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
 	uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+#endif
 }
 /*
  * BVADDR is the faulting address, PTR is scratch.
  * PTR will hold the pgd for vmalloc.
  */
 static void __cpuinit
 build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
 			unsigned int bvaddr, unsigned int ptr)
 {
 	long swpd = (long)swapper_pg_dir;
 	uasm_l_vmalloc(l, *p);
 	if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
 		uasm_il_b(p, r, label_vmalloc_done);
 		uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
 	} else {
 		UASM_i_LA_mostly(p, ptr, swpd);
 		uasm_il_b(p, r, label_vmalloc_done);
 		if (uasm_in_compat_space_p(swpd))
 			uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
 		else
 			uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
 	}
 }
 #else /* !CONFIG_64BIT */
 /*
  * TMP and PTR are scratch.
  * TMP will be clobbered, PTR will hold the pgd entry.
  */
 static void __cpuinit __maybe_unused
 build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
 {
 	long pgdc = (long)pgd_current;
 	/* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
 #ifdef CONFIG_SMP
 #ifdef  CONFIG_MIPS_MT_SMTC
 	/*
 	 * SMTC uses TCBind value as "CPU" index
 	 */
 	uasm_i_mfc0(p, ptr, C0_TCBIND);
 	UASM_i_LA_mostly(p, tmp, pgdc);
 	uasm_i_srl(p, ptr, ptr, 19);
 #else
 	/*
 	 * smp_processor_id() << 3 is stored in CONTEXT.
          */
 	uasm_i_mfc0(p, ptr, C0_CONTEXT);
 	UASM_i_LA_mostly(p, tmp, pgdc);
 	uasm_i_srl(p, ptr, ptr, 23);
 #endif
 	uasm_i_addu(p, ptr, tmp, ptr);
 #else
 	UASM_i_LA_mostly(p, ptr, pgdc);
 #endif
 	uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
 	uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
 	uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
 	uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
 	uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
 }
 #endif /* !CONFIG_64BIT */
 static void __cpuinit build_adjust_context(u32 **p, unsigned int ctx)
 {
 	unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
 	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
 	switch (current_cpu_type()) {
 	case CPU_VR41XX:
 	case CPU_VR4111:
 	case CPU_VR4121:
 	case CPU_VR4122:
 	case CPU_VR4131:
 	case CPU_VR4181:
 	case CPU_VR4181A:
 	case CPU_VR4133:
 		shift += 2;
 		break;
 	default:
 		break;
 	}
 	if (shift)
 		UASM_i_SRL(p, ctx, ctx, shift);
 	uasm_i_andi(p, ctx, ctx, mask);
 }
 static void __cpuinit build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
 {
 	/*
 	 * Bug workaround for the Nevada. It seems as if under certain
 	 * circumstances the move from cp0_context might produce a
 	 * bogus result when the mfc0 instruction and its consumer are
 	 * in a different cacheline or a load instruction, probably any
 	 * memory reference, is between them.
 	 */
 	switch (current_cpu_type()) {
 	case CPU_NEVADA:
 		UASM_i_LW(p, ptr, 0, ptr);
 		GET_CONTEXT(p, tmp); /* get context reg */
 		break;
 	default:
 		GET_CONTEXT(p, tmp); /* get context reg */
 		UASM_i_LW(p, ptr, 0, ptr);
 		break;
 	}
 	build_adjust_context(p, tmp);
 	UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
 }
 static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
 					unsigned int ptep)
 {
 	/*
 	 * 64bit address support (36bit on a 32bit CPU) in a 32bit
 	 * Kernel is a special case. Only a few CPUs use it.
 	 */
 #ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits) {
 		uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
 		uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
 		uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
 		uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
 		uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
 		uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
 	} else {
 		int pte_off_even = sizeof(pte_t) / 2;
 		int pte_off_odd = pte_off_even + sizeof(pte_t);
 		/* The pte entries are pre-shifted */
 		uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
 		uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
 		uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
 		uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
 	}
 #else
 	UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
 	UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
 	if (r45k_bvahwbug())
 		build_tlb_probe_entry(p);
 	UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
 	if (r4k_250MHZhwbug())
 		uasm_i_mtc0(p, 0, C0_ENTRYLO0);
 	uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
 	UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
 	if (r45k_bvahwbug())
 		uasm_i_mfc0(p, tmp, C0_INDEX);
 	if (r4k_250MHZhwbug())
 		uasm_i_mtc0(p, 0, C0_ENTRYLO1);
 	uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
 #endif
 }
 /*
  * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
  * because EXL == 0.  If we wrap, we can also use the 32 instruction
  * slots before the XTLB refill exception handler which belong to the
  * unused TLB refill exception.
  */
 #define MIPS64_REFILL_INSNS 32
 static void __cpuinit build_r4000_tlb_refill_handler(void)
 {
 	u32 *p = tlb_handler;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	u32 *f;
 	unsigned int final_len;
 	memset(tlb_handler, 0, sizeof(tlb_handler));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	memset(final_handler, 0, sizeof(final_handler));
 	/*
 	 * create the plain linear handler
 	 */
 	if (bcm1250_m3_war()) {
 		UASM_i_MFC0(&p, K0, C0_BADVADDR);
 		UASM_i_MFC0(&p, K1, C0_ENTRYHI);
 		uasm_i_xor(&p, K0, K0, K1);
 		UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
 		uasm_il_bnez(&p, &r, K0, label_leave);
 		/* No need for uasm_i_nop */
 	}
 #ifdef CONFIG_64BIT
 	build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
 #else
 	build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
 	build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update);
 #endif
 	build_get_ptep(&p, K0, K1);
 	build_update_entries(&p, K0, K1);
 	build_tlb_write_entry(&p, &l, &r, tlb_random);
 	uasm_l_leave(&l, p);
 	uasm_i_eret(&p); /* return from trap */
 #ifdef CONFIG_HUGETLB_PAGE
 	uasm_l_tlb_huge_update(&l, p);
 	UASM_i_LW(&p, K0, 0, K1);
 	build_huge_update_entries(&p, K0, K1);
 	build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random);
 #endif
 #ifdef CONFIG_64BIT
 	build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
 #endif
 	/*
 	 * Overflow check: For the 64bit handler, we need at least one
 	 * free instruction slot for the wrap-around branch. In worst
 	 * case, if the intended insertion point is a delay slot, we
 	 * need three, with the second nop'ed and the third being
 	 * unused.
 	 */
 	/* Loongson2 ebase is different than r4k, we have more space */
 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
 	if ((p - tlb_handler) > 64)
 		panic("TLB refill handler space exceeded");
 #else
 	if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)
 	    || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)
 		&& uasm_insn_has_bdelay(relocs,
 					tlb_handler + MIPS64_REFILL_INSNS - 3)))
 		panic("TLB refill handler space exceeded");
 #endif
 	/*
 	 * Now fold the handler in the TLB refill handler space.
 	 */
 #if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
 	f = final_handler;
 	/* Simplest case, just copy the handler. */
 	uasm_copy_handler(relocs, labels, tlb_handler, p, f);
 	final_len = p - tlb_handler;
 #else /* CONFIG_64BIT */
 	f = final_handler + MIPS64_REFILL_INSNS;
 	if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {
 		/* Just copy the handler. */
 		uasm_copy_handler(relocs, labels, tlb_handler, p, f);
 		final_len = p - tlb_handler;
 	} else {
 #if defined(CONFIG_HUGETLB_PAGE)
 		const enum label_id ls = label_tlb_huge_update;
 #else
 		const enum label_id ls = label_vmalloc;
 #endif
 		u32 *split;
 		int ov = 0;
 		int i;
 		for (i = 0; i < ARRAY_SIZE(labels) && labels[i].lab != ls; i++)
 			;
 		BUG_ON(i == ARRAY_SIZE(labels));
 		split = labels[i].addr;
 		/*
 		 * See if we have overflown one way or the other.
 		 */
 		if (split > tlb_handler + MIPS64_REFILL_INSNS ||
 		    split < p - MIPS64_REFILL_INSNS)
 			ov = 1;
 		if (ov) {
 			/*
 			 * Split two instructions before the end.  One
 			 * for the branch and one for the instruction
 			 * in the delay slot.
 			 */
 			split = tlb_handler + MIPS64_REFILL_INSNS - 2;
 			/*
 			 * If the branch would fall in a delay slot,
 			 * we must back up an additional instruction
 			 * so that it is no longer in a delay slot.
 			 */
 			if (uasm_insn_has_bdelay(relocs, split - 1))
 				split--;
 		}
 		/* Copy first part of the handler. */
 		uasm_copy_handler(relocs, labels, tlb_handler, split, f);
 		f += split - tlb_handler;
 		if (ov) {
 			/* Insert branch. */
 			uasm_l_split(&l, final_handler);
 			uasm_il_b(&f, &r, label_split);
 			if (uasm_insn_has_bdelay(relocs, split))
 				uasm_i_nop(&f);
 			else {
 				uasm_copy_handler(relocs, labels,
 						  split, split + 1, f);
 				uasm_move_labels(labels, f, f + 1, -1);
 				f++;
 				split++;
 			}
 		}
 		/* Copy the rest of the handler. */
 		uasm_copy_handler(relocs, labels, split, p, final_handler);
 		final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +
 			    (p - split);
 	}
 #endif /* CONFIG_64BIT */
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB refill handler (%u instructions).\n",
 		 final_len);
 	memcpy((void *)ebase, final_handler, 0x100);
 	dump_handler((u32 *)ebase, 64);
 }
 /*
  * TLB load/store/modify handlers.
  *
  * Only the fastpath gets synthesized at runtime, the slowpath for
  * do_page_fault remains normal asm.
  */
 extern void tlb_do_page_fault_0(void);
 extern void tlb_do_page_fault_1(void);
 /*
  * 128 instructions for the fastpath handler is generous and should
  * never be exceeded.
  */
 #define FASTPATH_SIZE 128
 u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
 u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
 static void __cpuinit
 iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr)
 {
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
 		uasm_i_lld(p, pte, 0, ptr);
 	else
 # endif
 		UASM_i_LL(p, pte, 0, ptr);
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
 		uasm_i_ld(p, pte, 0, ptr);
 	else
 # endif
 		UASM_i_LW(p, pte, 0, ptr);
 #endif
 }
 static void __cpuinit
 iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
 	unsigned int mode)
 {
 #ifdef CONFIG_64BIT_PHYS_ADDR
 	unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
 #endif
 	uasm_i_ori(p, pte, pte, mode);
 #ifdef CONFIG_SMP
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
 		uasm_i_scd(p, pte, 0, ptr);
 	else
 # endif
 		UASM_i_SC(p, pte, 0, ptr);
 	if (r10000_llsc_war())
 		uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
 	else
 		uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (!cpu_has_64bits) {
 		/* no uasm_i_nop needed */
 		uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
 		uasm_i_ori(p, pte, pte, hwmode);
 		uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
 		uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
 		/* no uasm_i_nop needed */
 		uasm_i_lw(p, pte, 0, ptr);
 	} else
 		uasm_i_nop(p);
 # else
 	uasm_i_nop(p);
 # endif
 #else
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (cpu_has_64bits)
 		uasm_i_sd(p, pte, 0, ptr);
 	else
 # endif
 		UASM_i_SW(p, pte, 0, ptr);
 # ifdef CONFIG_64BIT_PHYS_ADDR
 	if (!cpu_has_64bits) {
 		uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
 		uasm_i_ori(p, pte, pte, hwmode);
 		uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
 		uasm_i_lw(p, pte, 0, ptr);
 	}
 # endif
 #endif
 }
 /*
  * Check if PTE is present, if not then jump to LABEL. PTR points to
  * the page table where this PTE is located, PTE will be re-loaded
  * with it's original value.
  */
 static void __cpuinit
 build_pte_present(u32 **p, struct uasm_reloc **r,
 		  unsigned int pte, unsigned int ptr, enum label_id lid)
 {
 	uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
 	uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
 	uasm_il_bnez(p, r, pte, lid);
 	iPTE_LW(p, pte, ptr);
 }
 /* Make PTE valid, store result in PTR. */
 static void __cpuinit
 build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
 		 unsigned int ptr)
 {
 	unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
 	iPTE_SW(p, r, pte, ptr, mode);
 }
 /*
  * Check if PTE can be written to, if not branch to LABEL. Regardless
  * restore PTE with value from PTR when done.
  */
 static void __cpuinit
 build_pte_writable(u32 **p, struct uasm_reloc **r,
 		   unsigned int pte, unsigned int ptr, enum label_id lid)
 {
 	uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
 	uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
 	uasm_il_bnez(p, r, pte, lid);
 	iPTE_LW(p, pte, ptr);
 }
 /* Make PTE writable, update software status bits as well, then store
  * at PTR.
  */
 static void __cpuinit
 build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
 		 unsigned int ptr)
 {
 	unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
 			     | _PAGE_DIRTY);
 	iPTE_SW(p, r, pte, ptr, mode);
 }
 /*
  * Check if PTE can be modified, if not branch to LABEL. Regardless
  * restore PTE with value from PTR when done.
  */
 static void __cpuinit
 build_pte_modifiable(u32 **p, struct uasm_reloc **r,
 		     unsigned int pte, unsigned int ptr, enum label_id lid)
 {
 	uasm_i_andi(p, pte, pte, _PAGE_WRITE);
 	uasm_il_beqz(p, r, pte, lid);
 	iPTE_LW(p, pte, ptr);
 }
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 /*
  * R3000 style TLB load/store/modify handlers.
  */
 /*
  * This places the pte into ENTRYLO0 and writes it with tlbwi.
  * Then it returns.
  */
 static void __cpuinit
 build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
 {
 	uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
 	uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
 	uasm_i_tlbwi(p);
 	uasm_i_jr(p, tmp);
 	uasm_i_rfe(p); /* branch delay */
 }
 /*
  * This places the pte into ENTRYLO0 and writes it with tlbwi
  * or tlbwr as appropriate.  This is because the index register
  * may have the probe fail bit set as a result of a trap on a
  * kseg2 access, i.e. without refill.  Then it returns.
  */
 static void __cpuinit
 build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
 			     struct uasm_reloc **r, unsigned int pte,
 			     unsigned int tmp)
 {
 	uasm_i_mfc0(p, tmp, C0_INDEX);
 	uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
 	uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
 	uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
 	uasm_i_tlbwi(p); /* cp0 delay */
 	uasm_i_jr(p, tmp);
 	uasm_i_rfe(p); /* branch delay */
 	uasm_l_r3000_write_probe_fail(l, *p);
 	uasm_i_tlbwr(p); /* cp0 delay */
 	uasm_i_jr(p, tmp);
 	uasm_i_rfe(p); /* branch delay */
 }
 static void __cpuinit
 build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
 				   unsigned int ptr)
 {
 	long pgdc = (long)pgd_current;
 	uasm_i_mfc0(p, pte, C0_BADVADDR);
 	uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
 	uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
 	uasm_i_srl(p, pte, pte, 22); /* load delay */
 	uasm_i_sll(p, pte, pte, 2);
 	uasm_i_addu(p, ptr, ptr, pte);
 	uasm_i_mfc0(p, pte, C0_CONTEXT);
 	uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
 	uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
 	uasm_i_addu(p, ptr, ptr, pte);
 	uasm_i_lw(p, pte, 0, ptr);
 	uasm_i_tlbp(p); /* load delay */
 }
 static void __cpuinit build_r3000_tlb_load_handler(void)
 {
 	u32 *p = handle_tlbl;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbl, 0, sizeof(handle_tlbl));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
 	uasm_i_nop(&p); /* load delay */
 	build_make_valid(&p, &r, K0, K1);
 	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 	uasm_l_nopage_tlbl(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbl) > FASTPATH_SIZE)
 		panic("TLB load handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbl));
 	dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
 }
 static void __cpuinit build_r3000_tlb_store_handler(void)
 {
 	u32 *p = handle_tlbs;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbs, 0, sizeof(handle_tlbs));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
 	uasm_i_nop(&p); /* load delay */
 	build_make_write(&p, &r, K0, K1);
 	build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
 	uasm_l_nopage_tlbs(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbs) > FASTPATH_SIZE)
 		panic("TLB store handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbs));
 	dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
 }
 static void __cpuinit build_r3000_tlb_modify_handler(void)
 {
 	u32 *p = handle_tlbm;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbm, 0, sizeof(handle_tlbm));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	build_r3000_tlbchange_handler_head(&p, K0, K1);
 	build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
 	uasm_i_nop(&p); /* load delay */
 	build_make_write(&p, &r, K0, K1);
 	build_r3000_pte_reload_tlbwi(&p, K0, K1);
 	uasm_l_nopage_tlbm(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbm) > FASTPATH_SIZE)
 		panic("TLB modify handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbm));
 	dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
 #endif /* CONFIG_MIPS_PGD_C0_CONTEXT */
 /*
  * R4000 style TLB load/store/modify handlers.
  */
 static void __cpuinit
 build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
 				   struct uasm_reloc **r, unsigned int pte,
 				   unsigned int ptr)
 {
 #ifdef CONFIG_64BIT
 	build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
 #else
 	build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
 #endif
 #ifdef CONFIG_HUGETLB_PAGE
 	/*
 	 * For huge tlb entries, pmd doesn't contain an address but
 	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
 	 * see if we need to jump to huge tlb processing.
 	 */
 	build_is_huge_pte(p, r, pte, ptr, label_tlb_huge_update);
 #endif
 	UASM_i_MFC0(p, pte, C0_BADVADDR);
 	UASM_i_LW(p, ptr, 0, ptr);
 	UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
 	uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
 	UASM_i_ADDU(p, ptr, ptr, pte);
 #ifdef CONFIG_SMP
 	uasm_l_smp_pgtable_change(l, *p);
 #endif
 	iPTE_LW(p, pte, ptr); /* get even pte */
 	if (!m4kc_tlbp_war())
 		build_tlb_probe_entry(p);
 }
 static void __cpuinit
 build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
 				   struct uasm_reloc **r, unsigned int tmp,
 				   unsigned int ptr)
 {
 	uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
 	uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
 	build_update_entries(p, tmp, ptr);
 	build_tlb_write_entry(p, l, r, tlb_indexed);
 	uasm_l_leave(l, *p);
 	uasm_i_eret(p); /* return from trap */
 #ifdef CONFIG_64BIT
 	build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
 #endif
 }
 static void __cpuinit build_r4000_tlb_load_handler(void)
 {
 	u32 *p = handle_tlbl;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbl, 0, sizeof(handle_tlbl));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	if (bcm1250_m3_war()) {
 		UASM_i_MFC0(&p, K0, C0_BADVADDR);
 		UASM_i_MFC0(&p, K1, C0_ENTRYHI);
 		uasm_i_xor(&p, K0, K0, K1);
 		UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
 		uasm_il_bnez(&p, &r, K0, label_leave);
 		/* No need for uasm_i_nop */
 	}
 	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
 	build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
 	if (m4kc_tlbp_war())
 		build_tlb_probe_entry(&p);
 	build_make_valid(&p, &r, K0, K1);
 	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
 #ifdef CONFIG_HUGETLB_PAGE
 	/*
 	 * This is the entry point when build_r4000_tlbchange_handler_head
 	 * spots a huge page.
 	 */
 	uasm_l_tlb_huge_update(&l, p);
 	iPTE_LW(&p, K0, K1);
 	build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
 	build_tlb_probe_entry(&p);
 	uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
 	build_huge_handler_tail(&p, &r, &l, K0, K1);
 #endif
 	uasm_l_nopage_tlbl(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbl) > FASTPATH_SIZE)
 		panic("TLB load handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbl));
 	dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
 }
 static void __cpuinit build_r4000_tlb_store_handler(void)
 {
 	u32 *p = handle_tlbs;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbs, 0, sizeof(handle_tlbs));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
 	build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
 	if (m4kc_tlbp_war())
 		build_tlb_probe_entry(&p);
 	build_make_write(&p, &r, K0, K1);
 	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
 #ifdef CONFIG_HUGETLB_PAGE
 	/*
 	 * This is the entry point when
 	 * build_r4000_tlbchange_handler_head spots a huge page.
 	 */
 	uasm_l_tlb_huge_update(&l, p);
 	iPTE_LW(&p, K0, K1);
 	build_pte_writable(&p, &r, K0, K1, label_nopage_tlbs);
 	build_tlb_probe_entry(&p);
 	uasm_i_ori(&p, K0, K0,
 		   _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
 	build_huge_handler_tail(&p, &r, &l, K0, K1);
 #endif
 	uasm_l_nopage_tlbs(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbs) > FASTPATH_SIZE)
 		panic("TLB store handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbs));
 	dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
 }
 static void __cpuinit build_r4000_tlb_modify_handler(void)
 {
 	u32 *p = handle_tlbm;
 	struct uasm_label *l = labels;
 	struct uasm_reloc *r = relocs;
 	memset(handle_tlbm, 0, sizeof(handle_tlbm));
 	memset(labels, 0, sizeof(labels));
 	memset(relocs, 0, sizeof(relocs));
 	build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
 	build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
 	if (m4kc_tlbp_war())
 		build_tlb_probe_entry(&p);
 	/* Present and writable bits set, set accessed and dirty bits. */
 	build_make_write(&p, &r, K0, K1);
 	build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
 #ifdef CONFIG_HUGETLB_PAGE
 	/*
 	 * This is the entry point when
 	 * build_r4000_tlbchange_handler_head spots a huge page.
 	 */
 	uasm_l_tlb_huge_update(&l, p);
 	iPTE_LW(&p, K0, K1);
 	build_pte_modifiable(&p, &r, K0, K1, label_nopage_tlbm);
 	build_tlb_probe_entry(&p);
 	uasm_i_ori(&p, K0, K0,
 		   _PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID | _PAGE_DIRTY);
 	build_huge_handler_tail(&p, &r, &l, K0, K1);
 #endif
 	uasm_l_nopage_tlbm(&l, p);
 	uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
 	uasm_i_nop(&p);
 	if ((p - handle_tlbm) > FASTPATH_SIZE)
 		panic("TLB modify handler fastpath space exceeded");
 	uasm_resolve_relocs(relocs, labels);
 	pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
 		 (unsigned int)(p - handle_tlbm));
 	dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
 }
 void __cpuinit build_tlb_refill_handler(void)
 {
 	/*
 	 * The refill handler is generated per-CPU, multi-node systems
 	 * may have local storage for it. The other handlers are only
 	 * needed once.
 	 */
 	static int run_once = 0;
 	switch (current_cpu_type()) {
 	case CPU_R2000:
 	case CPU_R3000:
 	case CPU_R3000A:
 	case CPU_R3081E:
 	case CPU_TX3912:
 	case CPU_TX3922:
 	case CPU_TX3927:
 #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
 		build_r3000_tlb_refill_handler();
 		if (!run_once) {
 			build_r3000_tlb_load_handler();
 			build_r3000_tlb_store_handler();
 			build_r3000_tlb_modify_handler();
 			run_once++;
 		}
 #else
 		panic("No R3000 TLB refill handler");
 #endif
 		break;
 	case CPU_R6000:
 	case CPU_R6000A:
 		panic("No R6000 TLB refill handler yet");
 		break;
 	case CPU_R8000:
 		panic("No R8000 TLB refill handler yet");
 		break;
 	default:
 		build_r4000_tlb_refill_handler();
 		if (!run_once) {
 			build_r4000_tlb_load_handler();
 			build_r4000_tlb_store_handler();
 			build_r4000_tlb_modify_handler();
 			run_once++;
 		}
 	}
 }
 void __cpuinit flush_tlb_handlers(void)
 {
 	local_flush_icache_range((unsigned long)handle_tlbl,
 			   (unsigned long)handle_tlbl + sizeof(handle_tlbl));
 	local_flush_icache_range((unsigned long)handle_tlbs,
 			   (unsigned long)handle_tlbs + sizeof(handle_tlbs));
 	local_flush_icache_range((unsigned long)handle_tlbm,
 			   (unsigned long)handle_tlbm + sizeof(handle_tlbm));
 }