Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 5c1f6ee9a31cbdac90bbb8ae1ba4475031ac74b4

Authored by Aneesh Kumar K.V 2013-04-28 17:37:33 +0800

Committed by Benjamin Herrenschmidt 2013-04-30 14:00:07 +0800

Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches

powerpc: Reduce PTE table memory wastage

We allocate one page for the last level of linux page table. With THP and
large page size of 16MB, that would mean we are wasting large part
of that page. To map 16MB area, we only need a PTE space of 2K with 64K
page size. This patch reduce the space wastage by sharing the page
allocated for the last level of linux page table with multiple pmd
entries. We call these smaller chunks PTE page fragments and allocated
page, PTE page.

In order to support systems which doesn't have 64K HPTE support, we also
add another 2K to PTE page fragment. The second half of the PTE fragments
is used for storing slot and secondary bit information of an HPTE. With this
we now have a 4K PTE fragment.

We use a simple approach to share the PTE page. On allocation, we bump the
PTE page refcount to 16 and share the PTE page with the next 16 pte alloc
request. This should help in the node locality of the PTE page fragment,
assuming that the immediate pte alloc request will mostly come from the
same NUMA node. We don't try to reuse the freed PTE page fragment. Hence
we could be waisting some space.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Acked-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

Showing 7 changed files with 195 additions and 58 deletions Inline Diff

arch/powerpc/include/asm/mmu-book3e.h
arch/powerpc/include/asm/mmu-hash64.h
arch/powerpc/include/asm/page.h
arch/powerpc/include/asm/pgalloc-64.h
arch/powerpc/kernel/setup_64.c
arch/powerpc/mm/mmu_context_hash64.c
arch/powerpc/mm/pgtable_64.c

arch/powerpc/include/asm/mmu-book3e.h

Diff comments View file @ 5c1f6ee

 #ifndef _ASM_POWERPC_MMU_BOOK3E_H_
 #define _ASM_POWERPC_MMU_BOOK3E_H_
 /*
  * Freescale Book-E/Book-3e (ISA 2.06+) MMU support
  */
 /* Book-3e defined page sizes */
 #define BOOK3E_PAGESZ_1K	0
 #define BOOK3E_PAGESZ_2K	1
 #define BOOK3E_PAGESZ_4K	2
 #define BOOK3E_PAGESZ_8K	3
 #define BOOK3E_PAGESZ_16K	4
 #define BOOK3E_PAGESZ_32K	5
 #define BOOK3E_PAGESZ_64K	6
 #define BOOK3E_PAGESZ_128K	7
 #define BOOK3E_PAGESZ_256K	8
 #define BOOK3E_PAGESZ_512K	9
 #define BOOK3E_PAGESZ_1M	10
 #define BOOK3E_PAGESZ_2M	11
 #define BOOK3E_PAGESZ_4M	12
 #define BOOK3E_PAGESZ_8M	13
 #define BOOK3E_PAGESZ_16M	14
 #define BOOK3E_PAGESZ_32M	15
 #define BOOK3E_PAGESZ_64M	16
 #define BOOK3E_PAGESZ_128M	17
 #define BOOK3E_PAGESZ_256M	18
 #define BOOK3E_PAGESZ_512M	19
 #define BOOK3E_PAGESZ_1GB	20
 #define BOOK3E_PAGESZ_2GB	21
 #define BOOK3E_PAGESZ_4GB	22
 #define BOOK3E_PAGESZ_8GB	23
 #define BOOK3E_PAGESZ_16GB	24
 #define BOOK3E_PAGESZ_32GB	25
 #define BOOK3E_PAGESZ_64GB	26
 #define BOOK3E_PAGESZ_128GB	27
 #define BOOK3E_PAGESZ_256GB	28
 #define BOOK3E_PAGESZ_512GB	29
 #define BOOK3E_PAGESZ_1TB	30
 #define BOOK3E_PAGESZ_2TB	31
 /* MAS registers bit definitions */
 #define MAS0_TLBSEL(x)		(((x) << 28) & 0x30000000)
 #define MAS0_ESEL_MASK		0x0FFF0000
 #define MAS0_ESEL_SHIFT		16
 #define MAS0_ESEL(x)		(((x) << MAS0_ESEL_SHIFT) & MAS0_ESEL_MASK)
 #define MAS0_NV(x)		((x) & 0x00000FFF)
 #define MAS0_HES		0x00004000
 #define MAS0_WQ_ALLWAYS		0x00000000
 #define MAS0_WQ_COND		0x00001000
 #define MAS0_WQ_CLR_RSRV       	0x00002000
 #define MAS1_VALID		0x80000000
 #define MAS1_IPROT		0x40000000
 #define MAS1_TID(x)		(((x) << 16) & 0x3FFF0000)
 #define MAS1_IND		0x00002000
 #define MAS1_TS			0x00001000
 #define MAS1_TSIZE_MASK		0x00000f80
 #define MAS1_TSIZE_SHIFT	7
 #define MAS1_TSIZE(x)		(((x) << MAS1_TSIZE_SHIFT) & MAS1_TSIZE_MASK)
 #define MAS2_EPN		(~0xFFFUL)
 #define MAS2_X0			0x00000040
 #define MAS2_X1			0x00000020
 #define MAS2_W			0x00000010
 #define MAS2_I			0x00000008
 #define MAS2_M			0x00000004
 #define MAS2_G			0x00000002
 #define MAS2_E			0x00000001
 #define MAS2_WIMGE_MASK		0x0000001f
 #define MAS2_EPN_MASK(size)		(~0 << (size + 10))
 #define MAS2_VAL(addr, size, flags)	((addr) & MAS2_EPN_MASK(size) | (flags))
 #define MAS3_RPN		0xFFFFF000
 #define MAS3_U0			0x00000200
 #define MAS3_U1			0x00000100
 #define MAS3_U2			0x00000080
 #define MAS3_U3			0x00000040
 #define MAS3_UX			0x00000020
 #define MAS3_SX			0x00000010
 #define MAS3_UW			0x00000008
 #define MAS3_SW			0x00000004
 #define MAS3_UR			0x00000002
 #define MAS3_SR			0x00000001
 #define MAS3_BAP_MASK		0x0000003f
 #define MAS3_SPSIZE		0x0000003e
 #define MAS3_SPSIZE_SHIFT	1
 #define MAS4_TLBSELD(x) 	MAS0_TLBSEL(x)
 #define MAS4_INDD		0x00008000	/* Default IND */
 #define MAS4_TSIZED(x)		MAS1_TSIZE(x)
 #define MAS4_X0D		0x00000040
 #define MAS4_X1D		0x00000020
 #define MAS4_WD			0x00000010
 #define MAS4_ID			0x00000008
 #define MAS4_MD			0x00000004
 #define MAS4_GD			0x00000002
 #define MAS4_ED			0x00000001
 #define MAS4_WIMGED_MASK	0x0000001f	/* Default WIMGE */
 #define MAS4_WIMGED_SHIFT	0
 #define MAS4_VLED		MAS4_X1D	/* Default VLE */
 #define MAS4_ACMD		0x000000c0	/* Default ACM */
 #define MAS4_ACMD_SHIFT		6
 #define MAS4_TSIZED_MASK	0x00000f80	/* Default TSIZE */
 #define MAS4_TSIZED_SHIFT	7
 #define MAS5_SGS		0x80000000
 #define MAS6_SPID0		0x3FFF0000
 #define MAS6_SPID1		0x00007FFE
 #define MAS6_ISIZE(x)		MAS1_TSIZE(x)
 #define MAS6_SAS		0x00000001
 #define MAS6_SPID		MAS6_SPID0
 #define MAS6_SIND 		0x00000002	/* Indirect page */
 #define MAS6_SIND_SHIFT		1
 #define MAS6_SPID_MASK		0x3fff0000
 #define MAS6_SPID_SHIFT		16
 #define MAS6_ISIZE_MASK		0x00000f80
 #define MAS6_ISIZE_SHIFT	7
 #define MAS7_RPN		0xFFFFFFFF
 #define MAS8_TGS		0x80000000 /* Guest space */
 #define MAS8_VF			0x40000000 /* Virtualization Fault */
 #define MAS8_TLPID		0x000000ff
 /* Bit definitions for MMUCFG */
 #define MMUCFG_MAVN	0x00000003	/* MMU Architecture Version Number */
 #define MMUCFG_MAVN_V1	0x00000000	/* v1.0 */
 #define MMUCFG_MAVN_V2	0x00000001	/* v2.0 */
 #define MMUCFG_NTLBS	0x0000000c	/* Number of TLBs */
 #define MMUCFG_PIDSIZE	0x000007c0	/* PID Reg Size */
 #define MMUCFG_TWC	0x00008000	/* TLB Write Conditional (v2.0) */
 #define MMUCFG_LRAT	0x00010000	/* LRAT Supported (v2.0) */
 #define MMUCFG_RASIZE	0x00fe0000	/* Real Addr Size */
 #define MMUCFG_LPIDSIZE	0x0f000000	/* LPID Reg Size */
 /* Bit definitions for MMUCSR0 */
 #define MMUCSR0_TLB1FI	0x00000002	/* TLB1 Flash invalidate */
 #define MMUCSR0_TLB0FI	0x00000004	/* TLB0 Flash invalidate */
 #define MMUCSR0_TLB2FI	0x00000040	/* TLB2 Flash invalidate */
 #define MMUCSR0_TLB3FI	0x00000020	/* TLB3 Flash invalidate */
 #define MMUCSR0_TLBFI	(MMUCSR0_TLB0FI | MMUCSR0_TLB1FI | \
 			 MMUCSR0_TLB2FI | MMUCSR0_TLB3FI)
 #define MMUCSR0_TLB0PS	0x00000780	/* TLB0 Page Size */
 #define MMUCSR0_TLB1PS	0x00007800	/* TLB1 Page Size */
 #define MMUCSR0_TLB2PS	0x00078000	/* TLB2 Page Size */
 #define MMUCSR0_TLB3PS	0x00780000	/* TLB3 Page Size */
 /* MMUCFG bits */
 #define MMUCFG_MAVN_NASK	0x00000003
 #define MMUCFG_MAVN_V1_0	0x00000000
 #define MMUCFG_MAVN_V2_0	0x00000001
 #define MMUCFG_NTLB_MASK	0x0000000c
 #define MMUCFG_NTLB_SHIFT	2
 #define MMUCFG_PIDSIZE_MASK	0x000007c0
 #define MMUCFG_PIDSIZE_SHIFT	6
 #define MMUCFG_TWC		0x00008000
 #define MMUCFG_LRAT		0x00010000
 #define MMUCFG_RASIZE_MASK	0x00fe0000
 #define MMUCFG_RASIZE_SHIFT	17
 #define MMUCFG_LPIDSIZE_MASK	0x0f000000
 #define MMUCFG_LPIDSIZE_SHIFT	24
 /* TLBnCFG encoding */
 #define TLBnCFG_N_ENTRY		0x00000fff	/* number of entries */
 #define TLBnCFG_HES		0x00002000	/* HW select supported */
 #define TLBnCFG_IPROT		0x00008000	/* IPROT supported */
 #define TLBnCFG_GTWE		0x00010000	/* Guest can write */
 #define TLBnCFG_IND		0x00020000	/* IND entries supported */
 #define TLBnCFG_PT		0x00040000	/* Can load from page table */
 #define TLBnCFG_MINSIZE		0x00f00000	/* Minimum Page Size (v1.0) */
 #define TLBnCFG_MINSIZE_SHIFT	20
 #define TLBnCFG_MAXSIZE		0x000f0000	/* Maximum Page Size (v1.0) */
 #define TLBnCFG_MAXSIZE_SHIFT	16
 #define TLBnCFG_ASSOC		0xff000000	/* Associativity */
 #define TLBnCFG_ASSOC_SHIFT	24
 /* TLBnPS encoding */
 #define TLBnPS_4K		0x00000004
 #define TLBnPS_8K		0x00000008
 #define TLBnPS_16K		0x00000010
 #define TLBnPS_32K		0x00000020
 #define TLBnPS_64K		0x00000040
 #define TLBnPS_128K		0x00000080
 #define TLBnPS_256K		0x00000100
 #define TLBnPS_512K		0x00000200
 #define TLBnPS_1M 		0x00000400
 #define TLBnPS_2M 		0x00000800
 #define TLBnPS_4M 		0x00001000
 #define TLBnPS_8M 		0x00002000
 #define TLBnPS_16M		0x00004000
 #define TLBnPS_32M		0x00008000
 #define TLBnPS_64M		0x00010000
 #define TLBnPS_128M		0x00020000
 #define TLBnPS_256M		0x00040000
 #define TLBnPS_512M		0x00080000
 #define TLBnPS_1G		0x00100000
 #define TLBnPS_2G		0x00200000
 #define TLBnPS_4G		0x00400000
 #define TLBnPS_8G		0x00800000
 #define TLBnPS_16G		0x01000000
 #define TLBnPS_32G		0x02000000
 #define TLBnPS_64G		0x04000000
 #define TLBnPS_128G		0x08000000
 #define TLBnPS_256G		0x10000000
 /* tlbilx action encoding */
 #define TLBILX_T_ALL			0
 #define TLBILX_T_TID			1
 #define TLBILX_T_FULLMATCH		3
 #define TLBILX_T_CLASS0			4
 #define TLBILX_T_CLASS1			5
 #define TLBILX_T_CLASS2			6
 #define TLBILX_T_CLASS3			7
 #ifndef __ASSEMBLY__
 extern unsigned int tlbcam_index;
 typedef struct {
 	unsigned int	id;
 	unsigned int	active;
 	unsigned long	vdso_base;
 #ifdef CONFIG_PPC_ICSWX
 	struct spinlock *cop_lockp;	/* guard cop related stuff */
 	unsigned long acop;		/* mask of enabled coprocessor types */
 #endif /* CONFIG_PPC_ICSWX */
 #ifdef CONFIG_PPC_MM_SLICES
 	u64 low_slices_psize;   /* SLB page size encodings */
 	u64 high_slices_psize;  /* 4 bits per slice for now */
 	u16 user_psize;         /* page size index */
 #endif
+#ifdef CONFIG_PPC_64K_PAGES
+	/* for 4K PTE fragment support */
+	void *pte_frag;
+#endif
 } mm_context_t;
 /* Page size definitions, common between 32 and 64-bit
  *
  *    shift : is the "PAGE_SHIFT" value for that page size
  *    penc  : is the pte encoding mask
  *
  */
 struct mmu_psize_def
 {
 	unsigned int	shift;	/* number of bits */
 	unsigned int	enc;	/* PTE encoding */
 	unsigned int    ind;    /* Corresponding indirect page size shift */
 	unsigned int	flags;
 #define MMU_PAGE_SIZE_DIRECT	0x1	/* Supported as a direct size */
 #define MMU_PAGE_SIZE_INDIRECT	0x2	/* Supported as an indirect size */
 };
 extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
 /* The page sizes use the same names as 64-bit hash but are
  * constants
  */
 #if defined(CONFIG_PPC_4K_PAGES)
 #define mmu_virtual_psize	MMU_PAGE_4K
 #elif defined(CONFIG_PPC_64K_PAGES)
 #define mmu_virtual_psize	MMU_PAGE_64K
 #else
 #error Unsupported page size
 #endif
 extern int mmu_linear_psize;
 extern int mmu_vmemmap_psize;
 #ifdef CONFIG_PPC64
 extern unsigned long linear_map_top;
 /*
  * 64-bit booke platforms don't load the tlb in the tlb miss handler code.
  * HUGETLB_NEED_PRELOAD handles this - it causes huge_ptep_set_access_flags to
  * return 1, indicating that the tlb requires preloading.
  */
 #define HUGETLB_NEED_PRELOAD
 #endif
 #endif /* !__ASSEMBLY__ */
 #endif /* _ASM_POWERPC_MMU_BOOK3E_H_ */

arch/powerpc/include/asm/mmu-hash64.h

Diff comments View file @ 5c1f6ee

 #ifndef _ASM_POWERPC_MMU_HASH64_H_
 #define _ASM_POWERPC_MMU_HASH64_H_
 /*
  * PowerPC64 memory management structures
  *
  * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com>
  *   PPC64 rework.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #include <asm/asm-compat.h>
 #include <asm/page.h>
 /*
  * This is necessary to get the definition of PGTABLE_RANGE which we
  * need for various slices related matters. Note that this isn't the
  * complete pgtable.h but only a portion of it.
  */
 #include <asm/pgtable-ppc64.h>
 #include <asm/bug.h>
 /*
  * Segment table
  */
 #define STE_ESID_V	0x80
 #define STE_ESID_KS	0x20
 #define STE_ESID_KP	0x10
 #define STE_ESID_N	0x08
 #define STE_VSID_SHIFT	12
 /* Location of cpu0's segment table */
 #define STAB0_PAGE	0x8
 #define STAB0_OFFSET	(STAB0_PAGE << 12)
 #define STAB0_PHYS_ADDR	(STAB0_OFFSET + PHYSICAL_START)
 #ifndef __ASSEMBLY__
 extern char initial_stab[];
 #endif /* ! __ASSEMBLY */
 /*
  * SLB
  */
 #define SLB_NUM_BOLTED		3
 #define SLB_CACHE_ENTRIES	8
 #define SLB_MIN_SIZE		32
 /* Bits in the SLB ESID word */
 #define SLB_ESID_V		ASM_CONST(0x0000000008000000) /* valid */
 /* Bits in the SLB VSID word */
 #define SLB_VSID_SHIFT		12
 #define SLB_VSID_SHIFT_1T	24
 #define SLB_VSID_SSIZE_SHIFT	62
 #define SLB_VSID_B		ASM_CONST(0xc000000000000000)
 #define SLB_VSID_B_256M		ASM_CONST(0x0000000000000000)
 #define SLB_VSID_B_1T		ASM_CONST(0x4000000000000000)
 #define SLB_VSID_KS		ASM_CONST(0x0000000000000800)
 #define SLB_VSID_KP		ASM_CONST(0x0000000000000400)
 #define SLB_VSID_N		ASM_CONST(0x0000000000000200) /* no-execute */
 #define SLB_VSID_L		ASM_CONST(0x0000000000000100)
 #define SLB_VSID_C		ASM_CONST(0x0000000000000080) /* class */
 #define SLB_VSID_LP		ASM_CONST(0x0000000000000030)
 #define SLB_VSID_LP_00		ASM_CONST(0x0000000000000000)
 #define SLB_VSID_LP_01		ASM_CONST(0x0000000000000010)
 #define SLB_VSID_LP_10		ASM_CONST(0x0000000000000020)
 #define SLB_VSID_LP_11		ASM_CONST(0x0000000000000030)
 #define SLB_VSID_LLP		(SLB_VSID_L|SLB_VSID_LP)
 #define SLB_VSID_KERNEL		(SLB_VSID_KP)
 #define SLB_VSID_USER		(SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C)
 #define SLBIE_C			(0x08000000)
 #define SLBIE_SSIZE_SHIFT	25
 /*
  * Hash table
  */
 #define HPTES_PER_GROUP 8
 #define HPTE_V_SSIZE_SHIFT	62
 #define HPTE_V_AVPN_SHIFT	7
 #define HPTE_V_AVPN		ASM_CONST(0x3fffffffffffff80)
 #define HPTE_V_AVPN_VAL(x)	(((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT)
 #define HPTE_V_COMPARE(x,y)	(!(((x) ^ (y)) & 0xffffffffffffff80UL))
 #define HPTE_V_BOLTED		ASM_CONST(0x0000000000000010)
 #define HPTE_V_LOCK		ASM_CONST(0x0000000000000008)
 #define HPTE_V_LARGE		ASM_CONST(0x0000000000000004)
 #define HPTE_V_SECONDARY	ASM_CONST(0x0000000000000002)
 #define HPTE_V_VALID		ASM_CONST(0x0000000000000001)
 #define HPTE_R_PP0		ASM_CONST(0x8000000000000000)
 #define HPTE_R_TS		ASM_CONST(0x4000000000000000)
 #define HPTE_R_KEY_HI		ASM_CONST(0x3000000000000000)
 #define HPTE_R_RPN_SHIFT	12
 #define HPTE_R_RPN		ASM_CONST(0x0ffffffffffff000)
 #define HPTE_R_PP		ASM_CONST(0x0000000000000003)
 #define HPTE_R_N		ASM_CONST(0x0000000000000004)
 #define HPTE_R_G		ASM_CONST(0x0000000000000008)
 #define HPTE_R_M		ASM_CONST(0x0000000000000010)
 #define HPTE_R_I		ASM_CONST(0x0000000000000020)
 #define HPTE_R_W		ASM_CONST(0x0000000000000040)
 #define HPTE_R_WIMG		ASM_CONST(0x0000000000000078)
 #define HPTE_R_C		ASM_CONST(0x0000000000000080)
 #define HPTE_R_R		ASM_CONST(0x0000000000000100)
 #define HPTE_R_KEY_LO		ASM_CONST(0x0000000000000e00)
 #define HPTE_V_1TB_SEG		ASM_CONST(0x4000000000000000)
 #define HPTE_V_VRMA_MASK	ASM_CONST(0x4001ffffff000000)
 /* Values for PP (assumes Ks=0, Kp=1) */
 #define PP_RWXX	0	/* Supervisor read/write, User none */
 #define PP_RWRX 1	/* Supervisor read/write, User read */
 #define PP_RWRW 2	/* Supervisor read/write, User read/write */
 #define PP_RXRX 3	/* Supervisor read,       User read */
 #define PP_RXXX	(HPTE_R_PP0 | 2)	/* Supervisor read, user none */
 /* Fields for tlbiel instruction in architecture 2.06 */
 #define TLBIEL_INVAL_SEL_MASK	0xc00	/* invalidation selector */
 #define  TLBIEL_INVAL_PAGE	0x000	/* invalidate a single page */
 #define  TLBIEL_INVAL_SET_LPID	0x800	/* invalidate a set for current LPID */
 #define  TLBIEL_INVAL_SET	0xc00	/* invalidate a set for all LPIDs */
 #define TLBIEL_INVAL_SET_MASK	0xfff000	/* set number to inval. */
 #define TLBIEL_INVAL_SET_SHIFT	12
 #define POWER7_TLB_SETS		128	/* # sets in POWER7 TLB */
 #ifndef __ASSEMBLY__
 struct hash_pte {
 	unsigned long v;
 	unsigned long r;
 };
 extern struct hash_pte *htab_address;
 extern unsigned long htab_size_bytes;
 extern unsigned long htab_hash_mask;
 /*
  * Page size definition
  *
  *    shift : is the "PAGE_SHIFT" value for that page size
  *    sllp  : is a bit mask with the value of SLB L || LP to be or'ed
  *            directly to a slbmte "vsid" value
  *    penc  : is the HPTE encoding mask for the "LP" field:
  *
  */
 struct mmu_psize_def
 {
 	unsigned int	shift;	/* number of bits */
 	unsigned int	penc;	/* HPTE encoding */
 	unsigned int	tlbiel;	/* tlbiel supported for that page size */
 	unsigned long	avpnm;	/* bits to mask out in AVPN in the HPTE */
 	unsigned long	sllp;	/* SLB L||LP (exact mask to use in slbmte) */
 };
 extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT];
 static inline int shift_to_mmu_psize(unsigned int shift)
 {
 	int psize;
 	for (psize = 0; psize < MMU_PAGE_COUNT; ++psize)
 		if (mmu_psize_defs[psize].shift == shift)
 			return psize;
 	return -1;
 }
 static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize)
 {
 	if (mmu_psize_defs[mmu_psize].shift)
 		return mmu_psize_defs[mmu_psize].shift;
 	BUG();
 }
 #endif /* __ASSEMBLY__ */
 /*
  * Segment sizes.
  * These are the values used by hardware in the B field of
  * SLB entries and the first dword of MMU hashtable entries.
  * The B field is 2 bits; the values 2 and 3 are unused and reserved.
  */
 #define MMU_SEGSIZE_256M	0
 #define MMU_SEGSIZE_1T		1
 /*
  * encode page number shift.
  * in order to fit the 78 bit va in a 64 bit variable we shift the va by
  * 12 bits. This enable us to address upto 76 bit va.
  * For hpt hash from a va we can ignore the page size bits of va and for
  * hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure
  * we work in all cases including 4k page size.
  */
 #define VPN_SHIFT	12
 #ifndef __ASSEMBLY__
 static inline int segment_shift(int ssize)
 {
 	if (ssize == MMU_SEGSIZE_256M)
 		return SID_SHIFT;
 	return SID_SHIFT_1T;
 }
 /*
  * The current system page and segment sizes
  */
 extern int mmu_linear_psize;
 extern int mmu_virtual_psize;
 extern int mmu_vmalloc_psize;
 extern int mmu_vmemmap_psize;
 extern int mmu_io_psize;
 extern int mmu_kernel_ssize;
 extern int mmu_highuser_ssize;
 extern u16 mmu_slb_size;
 extern unsigned long tce_alloc_start, tce_alloc_end;
 /*
  * If the processor supports 64k normal pages but not 64k cache
  * inhibited pages, we have to be prepared to switch processes
  * to use 4k pages when they create cache-inhibited mappings.
  * If this is the case, mmu_ci_restrictions will be set to 1.
  */
 extern int mmu_ci_restrictions;
 /*
  * This computes the AVPN and B fields of the first dword of a HPTE,
  * for use when we want to match an existing PTE.  The bottom 7 bits
  * of the returned value are zero.
  */
 static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize,
 					     int ssize)
 {
 	unsigned long v;
 	/*
 	 * The AVA field omits the low-order 23 bits of the 78 bits VA.
 	 * These bits are not needed in the PTE, because the
 	 * low-order b of these bits are part of the byte offset
 	 * into the virtual page and, if b < 23, the high-order
 	 * 23-b of these bits are always used in selecting the
 	 * PTEGs to be searched
 	 */
 	v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm);
 	v <<= HPTE_V_AVPN_SHIFT;
 	v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
 	return v;
 }
 /*
  * This function sets the AVPN and L fields of the HPTE  appropriately
  * for the page size
  */
 static inline unsigned long hpte_encode_v(unsigned long vpn,
 					  int psize, int ssize)
 {
 	unsigned long v;
 	v = hpte_encode_avpn(vpn, psize, ssize);
 	if (psize != MMU_PAGE_4K)
 		v |= HPTE_V_LARGE;
 	return v;
 }
 /*
  * This function sets the ARPN, and LP fields of the HPTE appropriately
  * for the page size. We assume the pa is already "clean" that is properly
  * aligned for the requested page size
  */
 static inline unsigned long hpte_encode_r(unsigned long pa, int psize)
 {
 	unsigned long r;
 	/* A 4K page needs no special encoding */
 	if (psize == MMU_PAGE_4K)
 		return pa & HPTE_R_RPN;
 	else {
 		unsigned int penc = mmu_psize_defs[psize].penc;
 		unsigned int shift = mmu_psize_defs[psize].shift;
 		return (pa & ~((1ul << shift) - 1)) | (penc << 12);
 	}
 	return r;
 }
 /*
  * Build a VPN_SHIFT bit shifted va given VSID, EA and segment size.
  */
 static inline unsigned long hpt_vpn(unsigned long ea,
 				    unsigned long vsid, int ssize)
 {
 	unsigned long mask;
 	int s_shift = segment_shift(ssize);
 	mask = (1ul << (s_shift - VPN_SHIFT)) - 1;
 	return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask);
 }
 /*
  * This hashes a virtual address
  */
 static inline unsigned long hpt_hash(unsigned long vpn,
 				     unsigned int shift, int ssize)
 {
 	int mask;
 	unsigned long hash, vsid;
 	/* VPN_SHIFT can be atmost 12 */
 	if (ssize == MMU_SEGSIZE_256M) {
 		mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1;
 		hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^
 			((vpn & mask) >> (shift - VPN_SHIFT));
 	} else {
 		mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1;
 		vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT);
 		hash = vsid ^ (vsid << 25) ^
 			((vpn & mask) >> (shift - VPN_SHIFT)) ;
 	}
 	return hash & 0x7fffffffffUL;
 }
 extern int __hash_page_4K(unsigned long ea, unsigned long access,
 			  unsigned long vsid, pte_t *ptep, unsigned long trap,
 			  unsigned int local, int ssize, int subpage_prot);
 extern int __hash_page_64K(unsigned long ea, unsigned long access,
 			   unsigned long vsid, pte_t *ptep, unsigned long trap,
 			   unsigned int local, int ssize);
 struct mm_struct;
 unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap);
 extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap);
 int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
 		     pte_t *ptep, unsigned long trap, int local, int ssize,
 		     unsigned int shift, unsigned int mmu_psize);
 extern void hash_failure_debug(unsigned long ea, unsigned long access,
 			       unsigned long vsid, unsigned long trap,
 			       int ssize, int psize, unsigned long pte);
 extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
 			     unsigned long pstart, unsigned long prot,
 			     int psize, int ssize);
 extern void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages);
 extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr);
 extern void hpte_init_native(void);
 extern void hpte_init_lpar(void);
 extern void hpte_init_beat(void);
 extern void hpte_init_beat_v3(void);
 extern void stabs_alloc(void);
 extern void slb_initialize(void);
 extern void slb_flush_and_rebolt(void);
 extern void stab_initialize(unsigned long stab);
 extern void slb_vmalloc_update(void);
 extern void slb_set_size(u16 size);
 #endif /* __ASSEMBLY__ */
 /*
  * VSID allocation (256MB segment)
  *
  * We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated
  * from mmu context id and effective segment id of the address.
  *
  * For user processes max context id is limited to ((1ul << 19) - 5)
  * for kernel space, we use the top 4 context ids to map address as below
  * NOTE: each context only support 64TB now.
  * 0x7fffc -  [ 0xc000000000000000 - 0xc0003fffffffffff ]
  * 0x7fffd -  [ 0xd000000000000000 - 0xd0003fffffffffff ]
  * 0x7fffe -  [ 0xe000000000000000 - 0xe0003fffffffffff ]
  * 0x7ffff -  [ 0xf000000000000000 - 0xf0003fffffffffff ]
  *
  * The proto-VSIDs are then scrambled into real VSIDs with the
  * multiplicative hash:
  *
  *	VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS
  *
  * VSID_MULTIPLIER is prime, so in particular it is
  * co-prime to VSID_MODULUS, making this a 1:1 scrambling function.
  * Because the modulus is 2^n-1 we can compute it efficiently without
  * a divide or extra multiply (see below). The scramble function gives
  * robust scattering in the hash table (at least based on some initial
  * results).
  *
  * We also consider VSID 0 special. We use VSID 0 for slb entries mapping
  * bad address. This enables us to consolidate bad address handling in
  * hash_page.
  *
  * We also need to avoid the last segment of the last context, because that
  * would give a protovsid of 0x1fffffffff. That will result in a VSID 0
  * because of the modulo operation in vsid scramble. But the vmemmap
  * (which is what uses region 0xf) will never be close to 64TB in size
  * (it's 56 bytes per page of system memory).
  */
 #define CONTEXT_BITS		19
 #define ESID_BITS		18
 #define ESID_BITS_1T		6
 /*
  * 256MB segment
  * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments
  * available for user + kernel mapping. The top 4 contexts are used for
  * kernel mapping. Each segment contains 2^28 bytes. Each
  * context maps 2^46 bytes (64TB) so we can support 2^19-1 contexts
  * (19 == 37 + 28 - 46).
  */
 #define MAX_USER_CONTEXT	((ASM_CONST(1) << CONTEXT_BITS) - 5)
 /*
  * This should be computed such that protovosid * vsid_mulitplier
  * doesn't overflow 64 bits. It should also be co-prime to vsid_modulus
  */
 #define VSID_MULTIPLIER_256M	ASM_CONST(12538073)	/* 24-bit prime */
 #define VSID_BITS_256M		(CONTEXT_BITS + ESID_BITS)
 #define VSID_MODULUS_256M	((1UL<<VSID_BITS_256M)-1)
 #define VSID_MULTIPLIER_1T	ASM_CONST(12538073)	/* 24-bit prime */
 #define VSID_BITS_1T		(CONTEXT_BITS + ESID_BITS_1T)
 #define VSID_MODULUS_1T		((1UL<<VSID_BITS_1T)-1)
 #define USER_VSID_RANGE	(1UL << (ESID_BITS + SID_SHIFT))
 /*
  * This macro generates asm code to compute the VSID scramble
  * function.  Used in slb_allocate() and do_stab_bolted.  The function
  * computed is: (protovsid*VSID_MULTIPLIER) % VSID_MODULUS
  *
  *	rt = register continaing the proto-VSID and into which the
  *		VSID will be stored
  *	rx = scratch register (clobbered)
  *
  * 	- rt and rx must be different registers
  * 	- The answer will end up in the low VSID_BITS bits of rt.  The higher
  * 	  bits may contain other garbage, so you may need to mask the
  * 	  result.
  */
 #define ASM_VSID_SCRAMBLE(rt, rx, size)					\
 	lis	rx,VSID_MULTIPLIER_##size@h;				\
 	ori	rx,rx,VSID_MULTIPLIER_##size@l;				\
 	mulld	rt,rt,rx;		/* rt = rt * MULTIPLIER */	\
 									\
 	srdi	rx,rt,VSID_BITS_##size;					\
 	clrldi	rt,rt,(64-VSID_BITS_##size);				\
 	add	rt,rt,rx;		/* add high and low bits */	\
 	/* NOTE: explanation based on VSID_BITS_##size = 36		\
 	 * Now, r3 == VSID (mod 2^36-1), and lies between 0 and		\
 	 * 2^36-1+2^28-1.  That in particular means that if r3 >=	\
 	 * 2^36-1, then r3+1 has the 2^36 bit set.  So, if r3+1 has	\
 	 * the bit clear, r3 already has the answer we want, if it	\
 	 * doesn't, the answer is the low 36 bits of r3+1.  So in all	\
 	 * cases the answer is the low 36 bits of (r3 + ((r3+1) >> 36))*/\
 	addi	rx,rt,1;						\
 	srdi	rx,rx,VSID_BITS_##size;	/* extract 2^VSID_BITS bit */	\
 	add	rt,rt,rx
 /* 4 bits per slice and we have one slice per 1TB */
 #define SLICE_ARRAY_SIZE  (PGTABLE_RANGE >> 41)
 #ifndef __ASSEMBLY__
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 /*
  * For the sub-page protection option, we extend the PGD with one of
  * these.  Basically we have a 3-level tree, with the top level being
  * the protptrs array.  To optimize speed and memory consumption when
  * only addresses < 4GB are being protected, pointers to the first
  * four pages of sub-page protection words are stored in the low_prot
  * array.
  * Each page of sub-page protection words protects 1GB (4 bytes
  * protects 64k).  For the 3-level tree, each page of pointers then
  * protects 8TB.
  */
 struct subpage_prot_table {
 	unsigned long maxaddr;	/* only addresses < this are protected */
 	unsigned int **protptrs[2];
 	unsigned int *low_prot[4];
 };
 #define SBP_L1_BITS		(PAGE_SHIFT - 2)
 #define SBP_L2_BITS		(PAGE_SHIFT - 3)
 #define SBP_L1_COUNT		(1 << SBP_L1_BITS)
 #define SBP_L2_COUNT		(1 << SBP_L2_BITS)
 #define SBP_L2_SHIFT		(PAGE_SHIFT + SBP_L1_BITS)
 #define SBP_L3_SHIFT		(SBP_L2_SHIFT + SBP_L2_BITS)
 extern void subpage_prot_free(struct mm_struct *mm);
 extern void subpage_prot_init_new_context(struct mm_struct *mm);
 #else
 static inline void subpage_prot_free(struct mm_struct *mm) {}
 static inline void subpage_prot_init_new_context(struct mm_struct *mm) { }
 #endif /* CONFIG_PPC_SUBPAGE_PROT */
 typedef unsigned long mm_context_id_t;
 struct spinlock;
 typedef struct {
 	mm_context_id_t id;
 	u16 user_psize;		/* page size index */
 #ifdef CONFIG_PPC_MM_SLICES
 	u64 low_slices_psize;	/* SLB page size encodings */
 	unsigned char high_slices_psize[SLICE_ARRAY_SIZE];
 #else
 	u16 sllp;		/* SLB page size encoding */
 #endif
 	unsigned long vdso_base;
 #ifdef CONFIG_PPC_SUBPAGE_PROT
 	struct subpage_prot_table spt;
 #endif /* CONFIG_PPC_SUBPAGE_PROT */
 #ifdef CONFIG_PPC_ICSWX
 	struct spinlock *cop_lockp; /* guard acop and cop_pid */
 	unsigned long acop;	/* mask of enabled coprocessor types */
 	unsigned int cop_pid;	/* pid value used with coprocessors */
 #endif /* CONFIG_PPC_ICSWX */
+#ifdef CONFIG_PPC_64K_PAGES
+	/* for 4K PTE fragment support */
+	void *pte_frag;
+#endif
 } mm_context_t;
 #if 0
 /*
  * The code below is equivalent to this function for arguments
  * < 2^VSID_BITS, which is all this should ever be called
  * with.  However gcc is not clever enough to compute the
  * modulus (2^n-1) without a second multiply.
  */
 #define vsid_scramble(protovsid, size) \
 	((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size))
 #else /* 1 */
 #define vsid_scramble(protovsid, size) \
 	({								 \
 		unsigned long x;					 \
 		x = (protovsid) * VSID_MULTIPLIER_##size;		 \
 		x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \
 		(x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \
 	})
 #endif /* 1 */
 /* Returns the segment size indicator for a user address */
 static inline int user_segment_size(unsigned long addr)
 {
 	/* Use 1T segments if possible for addresses >= 1T */
 	if (addr >= (1UL << SID_SHIFT_1T))
 		return mmu_highuser_ssize;
 	return MMU_SEGSIZE_256M;
 }
 static inline unsigned long get_vsid(unsigned long context, unsigned long ea,
 				     int ssize)
 {
 	/*
 	 * Bad address. We return VSID 0 for that
 	 */
 	if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
 		return 0;
 	if (ssize == MMU_SEGSIZE_256M)
 		return vsid_scramble((context << ESID_BITS)
 				     | (ea >> SID_SHIFT), 256M);
 	return vsid_scramble((context << ESID_BITS_1T)
 			     | (ea >> SID_SHIFT_1T), 1T);
 }
 /*
  * This is only valid for addresses >= PAGE_OFFSET
  *
  * For kernel space, we use the top 4 context ids to map address as below
  * 0x7fffc -  [ 0xc000000000000000 - 0xc0003fffffffffff ]
  * 0x7fffd -  [ 0xd000000000000000 - 0xd0003fffffffffff ]
  * 0x7fffe -  [ 0xe000000000000000 - 0xe0003fffffffffff ]
  * 0x7ffff -  [ 0xf000000000000000 - 0xf0003fffffffffff ]
  */
 static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize)
 {
 	unsigned long context;
 	/*
 	 * kernel take the top 4 context from the available range
 	 */
 	context = (MAX_USER_CONTEXT) + ((ea >> 60) - 0xc) + 1;
 	return get_vsid(context, ea, ssize);
 }
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_POWERPC_MMU_HASH64_H_ */

arch/powerpc/include/asm/page.h

Diff comments View file @ 5c1f6ee

 #ifndef _ASM_POWERPC_PAGE_H
 #define _ASM_POWERPC_PAGE_H
 /*
  * Copyright (C) 2001,2005 IBM Corporation.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #ifndef __ASSEMBLY__
 #include <linux/types.h>
 #else
 #include <asm/types.h>
 #endif
 #include <asm/asm-compat.h>
 #include <asm/kdump.h>
 /*
  * On regular PPC32 page size is 4K (but we support 4K/16K/64K/256K pages
  * on PPC44x). For PPC64 we support either 4K or 64K software
  * page size. When using 64K pages however, whether we are really supporting
  * 64K pages in HW or not is irrelevant to those definitions.
  */
 #if defined(CONFIG_PPC_256K_PAGES)
 #define PAGE_SHIFT		18
 #elif defined(CONFIG_PPC_64K_PAGES)
 #define PAGE_SHIFT		16
 #elif defined(CONFIG_PPC_16K_PAGES)
 #define PAGE_SHIFT		14
 #else
 #define PAGE_SHIFT		12
 #endif
 #define PAGE_SIZE		(ASM_CONST(1) << PAGE_SHIFT)
 #ifndef __ASSEMBLY__
 #ifdef CONFIG_HUGETLB_PAGE
 extern unsigned int HPAGE_SHIFT;
 #else
 #define HPAGE_SHIFT PAGE_SHIFT
 #endif
 #define HPAGE_SIZE		((1UL) << HPAGE_SHIFT)
 #define HPAGE_MASK		(~(HPAGE_SIZE - 1))
 #define HUGETLB_PAGE_ORDER	(HPAGE_SHIFT - PAGE_SHIFT)
 #define HUGE_MAX_HSTATE		(MMU_PAGE_COUNT-1)
 #endif
 /* We do define AT_SYSINFO_EHDR but don't use the gate mechanism */
 #define __HAVE_ARCH_GATE_AREA		1
 /*
  * Subtle: (1 << PAGE_SHIFT) is an int, not an unsigned long. So if we
  * assign PAGE_MASK to a larger type it gets extended the way we want
  * (i.e. with 1s in the high bits)
  */
 #define PAGE_MASK      (~((1 << PAGE_SHIFT) - 1))
 /*
  * KERNELBASE is the virtual address of the start of the kernel, it's often
  * the same as PAGE_OFFSET, but _might not be_.
  *
  * The kdump dump kernel is one example where KERNELBASE != PAGE_OFFSET.
  *
  * PAGE_OFFSET is the virtual address of the start of lowmem.
  *
  * PHYSICAL_START is the physical address of the start of the kernel.
  *
  * MEMORY_START is the physical address of the start of lowmem.
  *
  * KERNELBASE, PAGE_OFFSET, and PHYSICAL_START are all configurable on
  * ppc32 and based on how they are set we determine MEMORY_START.
  *
  * For the linear mapping the following equation should be true:
  * KERNELBASE - PAGE_OFFSET = PHYSICAL_START - MEMORY_START
  *
  * Also, KERNELBASE >= PAGE_OFFSET and PHYSICAL_START >= MEMORY_START
  *
  * There are two was to determine a physical address from a virtual one:
  * va = pa + PAGE_OFFSET - MEMORY_START
  * va = pa + KERNELBASE - PHYSICAL_START
  *
  * If you want to know something's offset from the start of the kernel you
  * should subtract KERNELBASE.
  *
  * If you want to test if something's a kernel address, use is_kernel_addr().
  */
 #define KERNELBASE      ASM_CONST(CONFIG_KERNEL_START)
 #define PAGE_OFFSET	ASM_CONST(CONFIG_PAGE_OFFSET)
 #define LOAD_OFFSET	ASM_CONST((CONFIG_KERNEL_START-CONFIG_PHYSICAL_START))
 #if defined(CONFIG_NONSTATIC_KERNEL)
 #ifndef __ASSEMBLY__
 extern phys_addr_t memstart_addr;
 extern phys_addr_t kernstart_addr;
 #ifdef CONFIG_RELOCATABLE_PPC32
 extern long long virt_phys_offset;
 #endif
 #endif /* __ASSEMBLY__ */
 #define PHYSICAL_START	kernstart_addr
 #else	/* !CONFIG_NONSTATIC_KERNEL */
 #define PHYSICAL_START	ASM_CONST(CONFIG_PHYSICAL_START)
 #endif
 /* See Description below for VIRT_PHYS_OFFSET */
 #ifdef CONFIG_RELOCATABLE_PPC32
 #define VIRT_PHYS_OFFSET virt_phys_offset
 #else
 #define VIRT_PHYS_OFFSET (KERNELBASE - PHYSICAL_START)
 #endif
 #ifdef CONFIG_PPC64
 #define MEMORY_START	0UL
 #elif defined(CONFIG_NONSTATIC_KERNEL)
 #define MEMORY_START	memstart_addr
 #else
 #define MEMORY_START	(PHYSICAL_START + PAGE_OFFSET - KERNELBASE)
 #endif
 #ifdef CONFIG_FLATMEM
 #define ARCH_PFN_OFFSET		((unsigned long)(MEMORY_START >> PAGE_SHIFT))
 #define pfn_valid(pfn)		((pfn) >= ARCH_PFN_OFFSET && (pfn) < max_mapnr)
 #endif
 #define virt_to_page(kaddr)	pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
 #define pfn_to_kaddr(pfn)	__va((pfn) << PAGE_SHIFT)
 #define virt_addr_valid(kaddr)	pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
 /*
  * On Book-E parts we need __va to parse the device tree and we can't
  * determine MEMORY_START until then.  However we can determine PHYSICAL_START
  * from information at hand (program counter, TLB lookup).
  *
  * On BookE with RELOCATABLE (RELOCATABLE_PPC32)
  *
  *   With RELOCATABLE_PPC32,  we support loading the kernel at any physical
  *   address without any restriction on the page alignment.
  *
  *   We find the runtime address of _stext and relocate ourselves based on
  *   the following calculation:
  *
  *  	  virtual_base = ALIGN_DOWN(KERNELBASE,256M) +
  *  				MODULO(_stext.run,256M)
  *   and create the following mapping:
  *
  * 	  ALIGN_DOWN(_stext.run,256M) => ALIGN_DOWN(KERNELBASE,256M)
  *
  *   When we process relocations, we cannot depend on the
  *   existing equation for the __va()/__pa() translations:
  *
  * 	   __va(x) = (x)  - PHYSICAL_START + KERNELBASE
  *
  *   Where:
  *   	 PHYSICAL_START = kernstart_addr = Physical address of _stext
  *  	 KERNELBASE = Compiled virtual address of _stext.
  *
  *   This formula holds true iff, kernel load address is TLB page aligned.
  *
  *   In our case, we need to also account for the shift in the kernel Virtual
  *   address.
  *
  *   E.g.,
  *
  *   Let the kernel be loaded at 64MB and KERNELBASE be 0xc0000000 (same as PAGE_OFFSET).
  *   In this case, we would be mapping 0 to 0xc0000000, and kernstart_addr = 64M
  *
  *   Now __va(1MB) = (0x100000) - (0x4000000) + 0xc0000000
  *                 = 0xbc100000 , which is wrong.
  *
  *   Rather, it should be : 0xc0000000 + 0x100000 = 0xc0100000
  *      	according to our mapping.
  *
  *   Hence we use the following formula to get the translations right:
  *
  * 	  __va(x) = (x) - [ PHYSICAL_START - Effective KERNELBASE ]
  *
  * 	  Where :
  * 		PHYSICAL_START = dynamic load address.(kernstart_addr variable)
  * 		Effective KERNELBASE = virtual_base =
  * 				     = ALIGN_DOWN(KERNELBASE,256M) +
  * 						MODULO(PHYSICAL_START,256M)
  *
  * 	To make the cost of __va() / __pa() more light weight, we introduce
  * 	a new variable virt_phys_offset, which will hold :
  *
  * 	virt_phys_offset = Effective KERNELBASE - PHYSICAL_START
  * 			 = ALIGN_DOWN(KERNELBASE,256M) -
  * 			 	ALIGN_DOWN(PHYSICALSTART,256M)
  *
  * 	Hence :
  *
  * 	__va(x) = x - PHYSICAL_START + Effective KERNELBASE
  * 		= x + virt_phys_offset
  *
  * 		and
  * 	__pa(x) = x + PHYSICAL_START - Effective KERNELBASE
  * 		= x - virt_phys_offset
  *
  * On non-Book-E PPC64 PAGE_OFFSET and MEMORY_START are constants so use
  * the other definitions for __va & __pa.
  */
 #ifdef CONFIG_BOOKE
 #define __va(x) ((void *)(unsigned long)((phys_addr_t)(x) + VIRT_PHYS_OFFSET))
 #define __pa(x) ((unsigned long)(x) - VIRT_PHYS_OFFSET)
 #else
 #define __va(x) ((void *)(unsigned long)((phys_addr_t)(x) + PAGE_OFFSET - MEMORY_START))
 #define __pa(x) ((unsigned long)(x) - PAGE_OFFSET + MEMORY_START)
 #endif
 /*
  * Unfortunately the PLT is in the BSS in the PPC32 ELF ABI,
  * and needs to be executable.  This means the whole heap ends
  * up being executable.
  */
 #define VM_DATA_DEFAULT_FLAGS32	(VM_READ | VM_WRITE | VM_EXEC | \
 				 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 #define VM_DATA_DEFAULT_FLAGS64	(VM_READ | VM_WRITE | \
 				 VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC)
 #ifdef __powerpc64__
 #include <asm/page_64.h>
 #else
 #include <asm/page_32.h>
 #endif
 /* align addr on a size boundary - adjust address up/down if needed */
 #define _ALIGN_UP(addr,size)	(((addr)+((size)-1))&(~((size)-1)))
 #define _ALIGN_DOWN(addr,size)	((addr)&(~((size)-1)))
 /* align addr on a size boundary - adjust address up if needed */
 #define _ALIGN(addr,size)     _ALIGN_UP(addr,size)
 /*
  * Don't compare things with KERNELBASE or PAGE_OFFSET to test for
  * "kernelness", use is_kernel_addr() - it should do what you want.
  */
 #ifdef CONFIG_PPC_BOOK3E_64
 #define is_kernel_addr(x)	((x) >= 0x8000000000000000ul)
 #else
 #define is_kernel_addr(x)	((x) >= PAGE_OFFSET)
 #endif
 #ifndef CONFIG_PPC_BOOK3S_64
 /*
  * Use the top bit of the higher-level page table entries to indicate whether
  * the entries we point to contain hugepages.  This works because we know that
  * the page tables live in kernel space.  If we ever decide to support having
  * page tables at arbitrary addresses, this breaks and will have to change.
  */
 #ifdef CONFIG_PPC64
 #define PD_HUGE 0x8000000000000000
 #else
 #define PD_HUGE 0x80000000
 #endif
 #endif /* CONFIG_PPC_BOOK3S_64 */
 /*
  * Some number of bits at the level of the page table that points to
  * a hugepte are used to encode the size.  This masks those bits.
  */
 #define HUGEPD_SHIFT_MASK     0x3f
 #ifndef __ASSEMBLY__
 #undef STRICT_MM_TYPECHECKS
 #ifdef STRICT_MM_TYPECHECKS
 /* These are used to make use of C type-checking. */
 /* PTE level */
 typedef struct { pte_basic_t pte; } pte_t;
 #define pte_val(x)	((x).pte)
 #define __pte(x)	((pte_t) { (x) })
 /* 64k pages additionally define a bigger "real PTE" type that gathers
  * the "second half" part of the PTE for pseudo 64k pages
  */
 #if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC_STD_MMU_64)
 typedef struct { pte_t pte; unsigned long hidx; } real_pte_t;
 #else
 typedef struct { pte_t pte; } real_pte_t;
 #endif
 /* PMD level */
 #ifdef CONFIG_PPC64
 typedef struct { unsigned long pmd; } pmd_t;
 #define pmd_val(x)	((x).pmd)
 #define __pmd(x)	((pmd_t) { (x) })
 /* PUD level exusts only on 4k pages */
 #ifndef CONFIG_PPC_64K_PAGES
 typedef struct { unsigned long pud; } pud_t;
 #define pud_val(x)	((x).pud)
 #define __pud(x)	((pud_t) { (x) })
 #endif /* !CONFIG_PPC_64K_PAGES */
 #endif /* CONFIG_PPC64 */
 /* PGD level */
 typedef struct { unsigned long pgd; } pgd_t;
 #define pgd_val(x)	((x).pgd)
 #define __pgd(x)	((pgd_t) { (x) })
 /* Page protection bits */
 typedef struct { unsigned long pgprot; } pgprot_t;
 #define pgprot_val(x)	((x).pgprot)
 #define __pgprot(x)	((pgprot_t) { (x) })
 #else
 /*
  * .. while these make it easier on the compiler
  */
 typedef pte_basic_t pte_t;
 #define pte_val(x)	(x)
 #define __pte(x)	(x)
 #if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC_STD_MMU_64)
 typedef struct { pte_t pte; unsigned long hidx; } real_pte_t;
 #else
 typedef pte_t real_pte_t;
 #endif
 #ifdef CONFIG_PPC64
 typedef unsigned long pmd_t;
 #define pmd_val(x)	(x)
 #define __pmd(x)	(x)
 #ifndef CONFIG_PPC_64K_PAGES
 typedef unsigned long pud_t;
 #define pud_val(x)	(x)
 #define __pud(x)	(x)
 #endif /* !CONFIG_PPC_64K_PAGES */
 #endif /* CONFIG_PPC64 */
 typedef unsigned long pgd_t;
 #define pgd_val(x)	(x)
 #define pgprot_val(x)	(x)
 typedef unsigned long pgprot_t;
 #define __pgd(x)	(x)
 #define __pgprot(x)	(x)
 #endif
 typedef struct { signed long pd; } hugepd_t;
 #ifdef CONFIG_HUGETLB_PAGE
 #ifdef CONFIG_PPC_BOOK3S_64
 static inline int hugepd_ok(hugepd_t hpd)
 {
 	/*
 	 * hugepd pointer, bottom two bits == 00 and next 4 bits
 	 * indicate size of table
 	 */
 	return (((hpd.pd & 0x3) == 0x0) && ((hpd.pd & HUGEPD_SHIFT_MASK) != 0));
 }
 #else
 static inline int hugepd_ok(hugepd_t hpd)
 {
 	return (hpd.pd > 0);
 }
 #endif
 #define is_hugepd(pdep)               (hugepd_ok(*((hugepd_t *)(pdep))))
 int pgd_huge(pgd_t pgd);
 #else /* CONFIG_HUGETLB_PAGE */
 #define is_hugepd(pdep)			0
 #define pgd_huge(pgd)			0
 #endif /* CONFIG_HUGETLB_PAGE */
 struct page;
 extern void clear_user_page(void *page, unsigned long vaddr, struct page *pg);
 extern void copy_user_page(void *to, void *from, unsigned long vaddr,
 		struct page *p);
 extern int page_is_ram(unsigned long pfn);
 extern int devmem_is_allowed(unsigned long pfn);
 #ifdef CONFIG_PPC_SMLPAR
 void arch_free_page(struct page *page, int order);
 #define HAVE_ARCH_FREE_PAGE
 #endif
 struct vm_area_struct;
+#ifdef CONFIG_PPC_64K_PAGES
+typedef pte_t *pgtable_t;
+#else
 typedef struct page *pgtable_t;
+#endif
 #include <asm-generic/memory_model.h>
 #endif /* __ASSEMBLY__ */
 #endif /* _ASM_POWERPC_PAGE_H */

arch/powerpc/include/asm/pgalloc-64.h

Diff comments View file @ 5c1f6ee

 #ifndef _ASM_POWERPC_PGALLOC_64_H
 #define _ASM_POWERPC_PGALLOC_64_H
 /*
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #include <linux/slab.h>
 #include <linux/cpumask.h>
 #include <linux/percpu.h>
 struct vmemmap_backing {
 	struct vmemmap_backing *list;
 	unsigned long phys;
 	unsigned long virt_addr;
 };
 /*
  * Functions that deal with pagetables that could be at any level of
  * the table need to be passed an "index_size" so they know how to
  * handle allocation.  For PTE pages (which are linked to a struct
  * page for now, and drawn from the main get_free_pages() pool), the
  * allocation size will be (2^index_size * sizeof(pointer)) and
  * allocations are drawn from the kmem_cache in PGT_CACHE(index_size).
  *
  * The maximum index size needs to be big enough to allow any
  * pagetable sizes we need, but small enough to fit in the low bits of
  * any page table pointer.  In other words all pagetables, even tiny
  * ones, must be aligned to allow at least enough low 0 bits to
  * contain this value.  This value is also used as a mask, so it must
  * be one less than a power of two.
  */
 #define MAX_PGTABLE_INDEX_SIZE	0xf
 extern struct kmem_cache *pgtable_cache[];
 #define PGT_CACHE(shift) ({				\
 			BUG_ON(!(shift));		\
 			pgtable_cache[(shift) - 1];	\
 		})
 static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 {
 	return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
 }
 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 {
 	kmem_cache_free(PGT_CACHE(PGD_INDEX_SIZE), pgd);
 }
 #ifndef CONFIG_PPC_64K_PAGES
 #define pgd_populate(MM, PGD, PUD)	pgd_set(PGD, PUD)
 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
 	return kmem_cache_alloc(PGT_CACHE(PUD_INDEX_SIZE),
 				GFP_KERNEL|__GFP_REPEAT);
 }
 static inline void pud_free(struct mm_struct *mm, pud_t *pud)
 {
 	kmem_cache_free(PGT_CACHE(PUD_INDEX_SIZE), pud);
 }
 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 {
 	pud_set(pud, (unsigned long)pmd);
 }
 #define pmd_populate(mm, pmd, pte_page) \
 	pmd_populate_kernel(mm, pmd, page_address(pte_page))
 #define pmd_populate_kernel(mm, pmd, pte) pmd_set(pmd, (unsigned long)(pte))
 #define pmd_pgtable(pmd) pmd_page(pmd)
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 					  unsigned long address)
 {
 	return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
 }
 static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
 				      unsigned long address)
 {
 	struct page *page;
 	pte_t *pte;
 	pte = pte_alloc_one_kernel(mm, address);
 	if (!pte)
 		return NULL;
 	page = virt_to_page(pte);
 	pgtable_page_ctor(page);
 	return page;
 }
 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
 	free_page((unsigned long)pte);
 }
 static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
 {
 	pgtable_page_dtor(ptepage);
 	__free_page(ptepage);
 }
 static inline void pgtable_free(void *table, unsigned index_size)
 {
 	if (!index_size)
 		free_page((unsigned long)table);
 	else {
 		BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
 		kmem_cache_free(PGT_CACHE(index_size), table);
 	}
 }
 #ifdef CONFIG_SMP
 static inline void pgtable_free_tlb(struct mmu_gather *tlb,
 				    void *table, int shift)
 {
 	unsigned long pgf = (unsigned long)table;
 	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
 	pgf |= shift;
 	tlb_remove_table(tlb, (void *)pgf);
 }
 static inline void __tlb_remove_table(void *_table)
 {
 	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
 	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
 	pgtable_free(table, shift);
 }
 #else /* !CONFIG_SMP */
 static inline void pgtable_free_tlb(struct mmu_gather *tlb,
 				    void *table, int shift)
 {
 	pgtable_free(table, shift);
 }
 #endif /* CONFIG_SMP */
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 				  unsigned long address)
 {
 	struct page *page = page_address(table);
 	tlb_flush_pgtable(tlb, address);
 	pgtable_page_dtor(page);
 	pgtable_free_tlb(tlb, page, 0);
 }
 #else /* if CONFIG_PPC_64K_PAGES */
+/*
+ * we support 16 fragments per PTE page.
+ */
+#define PTE_FRAG_NR	16
+/*
+ * We use a 2K PTE page fragment and another 2K for storing
+ * real_pte_t hash index
+ */
+#define PTE_FRAG_SIZE_SHIFT  12
+#define PTE_FRAG_SIZE (2 * PTRS_PER_PTE * sizeof(pte_t))
+extern pte_t *page_table_alloc(struct mm_struct *, unsigned long, int);
+extern void page_table_free(struct mm_struct *, unsigned long *, int);
+extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
+#ifdef CONFIG_SMP
+extern void __tlb_remove_table(void *_table);
+#endif
 #define pud_populate(mm, pud, pmd)	pud_set(pud, (unsigned long)pmd)
 static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
 				       pte_t *pte)
 {
 	pmd_set(pmd, (unsigned long)pte);
 }
 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
 				pgtable_t pte_page)
 {
-	pmd_populate_kernel(mm, pmd, page_address(pte_page));
+	pmd_set(pmd, (unsigned long)pte_page);
 }
 static inline pgtable_t pmd_pgtable(pmd_t pmd)
 {
-	return pmd_page(pmd);
+	return (pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE);
 }
 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
 					  unsigned long address)
 {
-	return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
+	return (pte_t *)page_table_alloc(mm, address, 1);
 }
 static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
-				      unsigned long address)
+					unsigned long address)
 {
-	struct page *page;
+	return (pgtable_t)page_table_alloc(mm, address, 0);
-	pte_t *pte;
-	pte = pte_alloc_one_kernel(mm, address);
-	if (!pte)
-		return NULL;
-	page = virt_to_page(pte);
-	pgtable_page_ctor(page);
-	return page;
 }
 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
 {
-	free_page((unsigned long)pte);
+	page_table_free(mm, (unsigned long *)pte, 1);
 }
 static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
 {
-	pgtable_page_dtor(ptepage);
+	page_table_free(mm, (unsigned long *)ptepage, 0);
-	__free_page(ptepage);
 }
-static inline void pgtable_free(void *table, unsigned index_size)
-{
-	if (!index_size)
-		free_page((unsigned long)table);
-	else {
-		BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
-		kmem_cache_free(PGT_CACHE(index_size), table);
-	}
-}
-#ifdef CONFIG_SMP
-static inline void pgtable_free_tlb(struct mmu_gather *tlb,
-				    void *table, int shift)
-{
-	unsigned long pgf = (unsigned long)table;
-	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
-	pgf |= shift;
-	tlb_remove_table(tlb, (void *)pgf);
-}
-static inline void __tlb_remove_table(void *_table)
-{
-	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
-	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
-	pgtable_free(table, shift);
-}
-#else /* !CONFIG_SMP */
-static inline void pgtable_free_tlb(struct mmu_gather *tlb,
-				    void *table, int shift)
-{
-	pgtable_free(table, shift);
-}
-#endif /* CONFIG_SMP */
 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
 				  unsigned long address)
 {
-	struct page *page = page_address(table);
 	tlb_flush_pgtable(tlb, address);
-	pgtable_page_dtor(page);
+	pgtable_free_tlb(tlb, table, 0);
-	pgtable_free_tlb(tlb, page, 0);
 }
 #endif /* CONFIG_PPC_64K_PAGES */
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
 	return kmem_cache_alloc(PGT_CACHE(PMD_INDEX_SIZE),
 				GFP_KERNEL|__GFP_REPEAT);
 }
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)

arch/powerpc/kernel/setup_64.c

Diff comments View file @ 5c1f6ee

1	/*	1	/*
2	*	2	*
3	* Common boot and setup code.	3	* Common boot and setup code.
4	*	4	*
5	* Copyright (C) 2001 PPC64 Team, IBM Corp	5	* Copyright (C) 2001 PPC64 Team, IBM Corp
6	*	6	*
7	* This program is free software; you can redistribute it and/or	7	* This program is free software; you can redistribute it and/or
8	* modify it under the terms of the GNU General Public License	8	* modify it under the terms of the GNU General Public License
9	* as published by the Free Software Foundation; either version	9	* as published by the Free Software Foundation; either version
10	* 2 of the License, or (at your option) any later version.	10	* 2 of the License, or (at your option) any later version.
11	*/	11	*/
12		12
13	#undef DEBUG	13	#undef DEBUG
14		14
15	#include <linux/export.h>	15	#include <linux/export.h>
16	#include <linux/string.h>	16	#include <linux/string.h>
17	#include <linux/sched.h>	17	#include <linux/sched.h>
18	#include <linux/init.h>	18	#include <linux/init.h>
19	#include <linux/kernel.h>	19	#include <linux/kernel.h>
20	#include <linux/reboot.h>	20	#include <linux/reboot.h>
21	#include <linux/delay.h>	21	#include <linux/delay.h>
22	#include <linux/initrd.h>	22	#include <linux/initrd.h>
23	#include <linux/seq_file.h>	23	#include <linux/seq_file.h>
24	#include <linux/ioport.h>	24	#include <linux/ioport.h>
25	#include <linux/console.h>	25	#include <linux/console.h>
26	#include <linux/utsname.h>	26	#include <linux/utsname.h>
27	#include <linux/tty.h>	27	#include <linux/tty.h>
28	#include <linux/root_dev.h>	28	#include <linux/root_dev.h>
29	#include <linux/notifier.h>	29	#include <linux/notifier.h>
30	#include <linux/cpu.h>	30	#include <linux/cpu.h>
31	#include <linux/unistd.h>	31	#include <linux/unistd.h>
32	#include <linux/serial.h>	32	#include <linux/serial.h>
33	#include <linux/serial_8250.h>	33	#include <linux/serial_8250.h>
34	#include <linux/bootmem.h>	34	#include <linux/bootmem.h>
35	#include <linux/pci.h>	35	#include <linux/pci.h>
36	#include <linux/lockdep.h>	36	#include <linux/lockdep.h>
37	#include <linux/memblock.h>	37	#include <linux/memblock.h>
38	#include <linux/hugetlb.h>	38	#include <linux/hugetlb.h>
39		39
40	#include <asm/io.h>	40	#include <asm/io.h>
41	#include <asm/kdump.h>	41	#include <asm/kdump.h>
42	#include <asm/prom.h>	42	#include <asm/prom.h>
43	#include <asm/processor.h>	43	#include <asm/processor.h>
44	#include <asm/pgtable.h>	44	#include <asm/pgtable.h>
45	#include <asm/smp.h>	45	#include <asm/smp.h>
46	#include <asm/elf.h>	46	#include <asm/elf.h>
47	#include <asm/machdep.h>	47	#include <asm/machdep.h>
48	#include <asm/paca.h>	48	#include <asm/paca.h>
49	#include <asm/time.h>	49	#include <asm/time.h>
50	#include <asm/cputable.h>	50	#include <asm/cputable.h>
51	#include <asm/sections.h>	51	#include <asm/sections.h>
52	#include <asm/btext.h>	52	#include <asm/btext.h>
53	#include <asm/nvram.h>	53	#include <asm/nvram.h>
54	#include <asm/setup.h>	54	#include <asm/setup.h>
55	#include <asm/rtas.h>	55	#include <asm/rtas.h>
56	#include <asm/iommu.h>	56	#include <asm/iommu.h>
57	#include <asm/serial.h>	57	#include <asm/serial.h>
58	#include <asm/cache.h>	58	#include <asm/cache.h>
59	#include <asm/page.h>	59	#include <asm/page.h>
60	#include <asm/mmu.h>	60	#include <asm/mmu.h>
61	#include <asm/firmware.h>	61	#include <asm/firmware.h>
62	#include <asm/xmon.h>	62	#include <asm/xmon.h>
63	#include <asm/udbg.h>	63	#include <asm/udbg.h>
64	#include <asm/kexec.h>	64	#include <asm/kexec.h>
65	#include <asm/mmu_context.h>	65	#include <asm/mmu_context.h>
66	#include <asm/code-patching.h>	66	#include <asm/code-patching.h>
67	#include <asm/kvm_ppc.h>	67	#include <asm/kvm_ppc.h>
68	#include <asm/hugetlb.h>	68	#include <asm/hugetlb.h>
69		69
70	#include "setup.h"	70	#include "setup.h"
71		71
72	#ifdef DEBUG	72	#ifdef DEBUG
73	#define DBG(fmt...) udbg_printf(fmt)	73	#define DBG(fmt...) udbg_printf(fmt)
74	#else	74	#else
75	#define DBG(fmt...)	75	#define DBG(fmt...)
76	#endif	76	#endif
77		77
78	int boot_cpuid = 0;	78	int boot_cpuid = 0;
79	int __initdata spinning_secondaries;	79	int __initdata spinning_secondaries;
80	u64 ppc64_pft_size;	80	u64 ppc64_pft_size;
81		81
82	/* Pick defaults since we might want to patch instructions	82	/* Pick defaults since we might want to patch instructions
83	* before we've read this from the device tree.	83	* before we've read this from the device tree.
84	*/	84	*/
85	struct ppc64_caches ppc64_caches = {	85	struct ppc64_caches ppc64_caches = {
86	.dline_size = 0x40,	86	.dline_size = 0x40,
87	.log_dline_size = 6,	87	.log_dline_size = 6,
88	.iline_size = 0x40,	88	.iline_size = 0x40,
89	.log_iline_size = 6	89	.log_iline_size = 6
90	};	90	};
91	EXPORT_SYMBOL_GPL(ppc64_caches);	91	EXPORT_SYMBOL_GPL(ppc64_caches);
92		92
93	/*	93	/*
94	* These are used in binfmt_elf.c to put aux entries on the stack	94	* These are used in binfmt_elf.c to put aux entries on the stack
95	* for each elf executable being started.	95	* for each elf executable being started.
96	*/	96	*/
97	int dcache_bsize;	97	int dcache_bsize;
98	int icache_bsize;	98	int icache_bsize;
99	int ucache_bsize;	99	int ucache_bsize;
100		100
101	#ifdef CONFIG_SMP	101	#ifdef CONFIG_SMP
102		102
103	static char *smt_enabled_cmdline;	103	static char *smt_enabled_cmdline;
104		104
105	/* Look for ibm,smt-enabled OF option */	105	/* Look for ibm,smt-enabled OF option */
106	static void check_smt_enabled(void)	106	static void check_smt_enabled(void)
107	{	107	{
108	struct device_node *dn;	108	struct device_node *dn;
109	const char *smt_option;	109	const char *smt_option;
110		110
111	/* Default to enabling all threads */	111	/* Default to enabling all threads */
112	smt_enabled_at_boot = threads_per_core;	112	smt_enabled_at_boot = threads_per_core;
113		113
114	/* Allow the command line to overrule the OF option */	114	/* Allow the command line to overrule the OF option */
115	if (smt_enabled_cmdline) {	115	if (smt_enabled_cmdline) {
116	if (!strcmp(smt_enabled_cmdline, "on"))	116	if (!strcmp(smt_enabled_cmdline, "on"))
117	smt_enabled_at_boot = threads_per_core;	117	smt_enabled_at_boot = threads_per_core;
118	else if (!strcmp(smt_enabled_cmdline, "off"))	118	else if (!strcmp(smt_enabled_cmdline, "off"))
119	smt_enabled_at_boot = 0;	119	smt_enabled_at_boot = 0;
120	else {	120	else {
121	long smt;	121	long smt;
122	int rc;	122	int rc;
123		123
124	rc = strict_strtol(smt_enabled_cmdline, 10, &smt);	124	rc = strict_strtol(smt_enabled_cmdline, 10, &smt);
125	if (!rc)	125	if (!rc)
126	smt_enabled_at_boot =	126	smt_enabled_at_boot =
127	min(threads_per_core, (int)smt);	127	min(threads_per_core, (int)smt);
128	}	128	}
129	} else {	129	} else {
130	dn = of_find_node_by_path("/options");	130	dn = of_find_node_by_path("/options");
131	if (dn) {	131	if (dn) {
132	smt_option = of_get_property(dn, "ibm,smt-enabled",	132	smt_option = of_get_property(dn, "ibm,smt-enabled",
133	NULL);	133	NULL);
134		134
135	if (smt_option) {	135	if (smt_option) {
136	if (!strcmp(smt_option, "on"))	136	if (!strcmp(smt_option, "on"))
137	smt_enabled_at_boot = threads_per_core;	137	smt_enabled_at_boot = threads_per_core;
138	else if (!strcmp(smt_option, "off"))	138	else if (!strcmp(smt_option, "off"))
139	smt_enabled_at_boot = 0;	139	smt_enabled_at_boot = 0;
140	}	140	}
141		141
142	of_node_put(dn);	142	of_node_put(dn);
143	}	143	}
144	}	144	}
145	}	145	}
146		146
147	/* Look for smt-enabled= cmdline option */	147	/* Look for smt-enabled= cmdline option */
148	static int __init early_smt_enabled(char *p)	148	static int __init early_smt_enabled(char *p)
149	{	149	{
150	smt_enabled_cmdline = p;	150	smt_enabled_cmdline = p;
151	return 0;	151	return 0;
152	}	152	}
153	early_param("smt-enabled", early_smt_enabled);	153	early_param("smt-enabled", early_smt_enabled);
154		154
155	#else	155	#else
156	#define check_smt_enabled()	156	#define check_smt_enabled()
157	#endif /* CONFIG_SMP */	157	#endif /* CONFIG_SMP */
158		158
159	/** Fix up paca fields required for the boot cpu */	159	/** Fix up paca fields required for the boot cpu */
160	static void fixup_boot_paca(void)	160	static void fixup_boot_paca(void)
161	{	161	{
162	/* The boot cpu is started */	162	/* The boot cpu is started */
163	get_paca()->cpu_start = 1;	163	get_paca()->cpu_start = 1;
164	/* Allow percpu accesses to work until we setup percpu data */	164	/* Allow percpu accesses to work until we setup percpu data */
165	get_paca()->data_offset = 0;	165	get_paca()->data_offset = 0;
166	}	166	}
167		167
168	/*	168	/*
169	* Early initialization entry point. This is called by head.S	169	* Early initialization entry point. This is called by head.S
170	* with MMU translation disabled. We rely on the "feature" of	170	* with MMU translation disabled. We rely on the "feature" of
171	* the CPU that ignores the top 2 bits of the address in real	171	* the CPU that ignores the top 2 bits of the address in real
172	* mode so we can access kernel globals normally provided we	172	* mode so we can access kernel globals normally provided we
173	* only toy with things in the RMO region. From here, we do	173	* only toy with things in the RMO region. From here, we do
174	* some early parsing of the device-tree to setup out MEMBLOCK	174	* some early parsing of the device-tree to setup out MEMBLOCK
175	* data structures, and allocate & initialize the hash table	175	* data structures, and allocate & initialize the hash table
176	* and segment tables so we can start running with translation	176	* and segment tables so we can start running with translation
177	* enabled.	177	* enabled.
178	*	178	*
179	* It is this function which will call the probe() callback of	179	* It is this function which will call the probe() callback of
180	* the various platform types and copy the matching one to the	180	* the various platform types and copy the matching one to the
181	* global ppc_md structure. Your platform can eventually do	181	* global ppc_md structure. Your platform can eventually do
182	* some very early initializations from the probe() routine, but	182	* some very early initializations from the probe() routine, but
183	* this is not recommended, be very careful as, for example, the	183	* this is not recommended, be very careful as, for example, the
184	* device-tree is not accessible via normal means at this point.	184	* device-tree is not accessible via normal means at this point.
185	*/	185	*/
186		186
187	void __init early_setup(unsigned long dt_ptr)	187	void __init early_setup(unsigned long dt_ptr)
188	{	188	{
189	static __initdata struct paca_struct boot_paca;	189	static __initdata struct paca_struct boot_paca;
190		190
191	/* -------- printk is _NOT_ safe to use here ! ------- */	191	/* -------- printk is _NOT_ safe to use here ! ------- */
192		192
193	/* Identify CPU type */	193	/* Identify CPU type */
194	identify_cpu(0, mfspr(SPRN_PVR));	194	identify_cpu(0, mfspr(SPRN_PVR));
195		195
196	/* Assume we're on cpu 0 for now. Don't write to the paca yet! */	196	/* Assume we're on cpu 0 for now. Don't write to the paca yet! */
197	initialise_paca(&boot_paca, 0);	197	initialise_paca(&boot_paca, 0);
198	setup_paca(&boot_paca);	198	setup_paca(&boot_paca);
199	fixup_boot_paca();	199	fixup_boot_paca();
200		200
201	/* Initialize lockdep early or else spinlocks will blow */	201	/* Initialize lockdep early or else spinlocks will blow */
202	lockdep_init();	202	lockdep_init();
203		203
204	/* -------- printk is now safe to use ------- */	204	/* -------- printk is now safe to use ------- */
205		205
206	/* Enable early debugging if any specified (see udbg.h) */	206	/* Enable early debugging if any specified (see udbg.h) */
207	udbg_early_init();	207	udbg_early_init();
208		208
209	DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr);	209	DBG(" -> early_setup(), dt_ptr: 0x%lx\n", dt_ptr);
210		210
211	/*	211	/*
212	* Do early initialization using the flattened device	212	* Do early initialization using the flattened device
213	* tree, such as retrieving the physical memory map or	213	* tree, such as retrieving the physical memory map or
214	* calculating/retrieving the hash table size.	214	* calculating/retrieving the hash table size.
215	*/	215	*/
216	early_init_devtree(__va(dt_ptr));	216	early_init_devtree(__va(dt_ptr));
217		217
218	/* Now we know the logical id of our boot cpu, setup the paca. */	218	/* Now we know the logical id of our boot cpu, setup the paca. */
219	setup_paca(&paca[boot_cpuid]);	219	setup_paca(&paca[boot_cpuid]);
220	fixup_boot_paca();	220	fixup_boot_paca();
221		221
222	/* Probe the machine type */	222	/* Probe the machine type */
223	probe_machine();	223	probe_machine();
224		224
225	setup_kdump_trampoline();	225	setup_kdump_trampoline();
226		226
227	DBG("Found, Initializing memory management...\n");	227	DBG("Found, Initializing memory management...\n");
228		228
229	/* Initialize the hash table or TLB handling */	229	/* Initialize the hash table or TLB handling */
230	early_init_mmu();	230	early_init_mmu();
231		231
232	/*	232	/*
233	* Reserve any gigantic pages requested on the command line.	233	* Reserve any gigantic pages requested on the command line.
234	* memblock needs to have been initialized by the time this is	234	* memblock needs to have been initialized by the time this is
235	* called since this will reserve memory.	235	* called since this will reserve memory.
236	*/	236	*/
237	reserve_hugetlb_gpages();	237	reserve_hugetlb_gpages();
238		238
239	DBG(" <- early_setup()\n");	239	DBG(" <- early_setup()\n");
240	}	240	}
241		241
242	#ifdef CONFIG_SMP	242	#ifdef CONFIG_SMP
243	void early_setup_secondary(void)	243	void early_setup_secondary(void)
244	{	244	{
245	/* Mark interrupts enabled in PACA */	245	/* Mark interrupts enabled in PACA */
246	get_paca()->soft_enabled = 0;	246	get_paca()->soft_enabled = 0;
247		247
248	/* Initialize the hash table or TLB handling */	248	/* Initialize the hash table or TLB handling */
249	early_init_mmu_secondary();	249	early_init_mmu_secondary();
250	}	250	}
251		251
252	#endif /* CONFIG_SMP */	252	#endif /* CONFIG_SMP */
253		253
254	#if defined(CONFIG_SMP) \|\| defined(CONFIG_KEXEC)	254	#if defined(CONFIG_SMP) \|\| defined(CONFIG_KEXEC)
255	void smp_release_cpus(void)	255	void smp_release_cpus(void)
256	{	256	{
257	unsigned long *ptr;	257	unsigned long *ptr;
258	int i;	258	int i;
259		259
260	DBG(" -> smp_release_cpus()\n");	260	DBG(" -> smp_release_cpus()\n");
261		261
262	/* All secondary cpus are spinning on a common spinloop, release them	262	/* All secondary cpus are spinning on a common spinloop, release them
263	* all now so they can start to spin on their individual paca	263	* all now so they can start to spin on their individual paca
264	* spinloops. For non SMP kernels, the secondary cpus never get out	264	* spinloops. For non SMP kernels, the secondary cpus never get out
265	* of the common spinloop.	265	* of the common spinloop.
266	*/	266	*/
267		267
268	ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop	268	ptr = (unsigned long *)((unsigned long)&__secondary_hold_spinloop
269	- PHYSICAL_START);	269	- PHYSICAL_START);
270	*ptr = __pa(generic_secondary_smp_init);	270	*ptr = __pa(generic_secondary_smp_init);
271		271
272	/* And wait a bit for them to catch up */	272	/* And wait a bit for them to catch up */
273	for (i = 0; i < 100000; i++) {	273	for (i = 0; i < 100000; i++) {
274	mb();	274	mb();
275	HMT_low();	275	HMT_low();
276	if (spinning_secondaries == 0)	276	if (spinning_secondaries == 0)
277	break;	277	break;
278	udelay(1);	278	udelay(1);
279	}	279	}
280	DBG("spinning_secondaries = %d\n", spinning_secondaries);	280	DBG("spinning_secondaries = %d\n", spinning_secondaries);
281		281
282	DBG(" <- smp_release_cpus()\n");	282	DBG(" <- smp_release_cpus()\n");
283	}	283	}
284	#endif /* CONFIG_SMP \|\| CONFIG_KEXEC */	284	#endif /* CONFIG_SMP \|\| CONFIG_KEXEC */
285		285
286	/*	286	/*
287	* Initialize some remaining members of the ppc64_caches and systemcfg	287	* Initialize some remaining members of the ppc64_caches and systemcfg
288	* structures	288	* structures
289	* (at least until we get rid of them completely). This is mostly some	289	* (at least until we get rid of them completely). This is mostly some
290	* cache informations about the CPU that will be used by cache flush	290	* cache informations about the CPU that will be used by cache flush
291	* routines and/or provided to userland	291	* routines and/or provided to userland
292	*/	292	*/
293	static void __init initialize_cache_info(void)	293	static void __init initialize_cache_info(void)
294	{	294	{
295	struct device_node *np;	295	struct device_node *np;
296	unsigned long num_cpus = 0;	296	unsigned long num_cpus = 0;
297		297
298	DBG(" -> initialize_cache_info()\n");	298	DBG(" -> initialize_cache_info()\n");
299		299
300	for_each_node_by_type(np, "cpu") {	300	for_each_node_by_type(np, "cpu") {
301	num_cpus += 1;	301	num_cpus += 1;
302		302
303	/*	303	/*
304	* We're assuming all of the CPUs have the same	304	* We're assuming all of the CPUs have the same
305	* d-cache and i-cache sizes... -Peter	305	* d-cache and i-cache sizes... -Peter
306	*/	306	*/
307	if (num_cpus == 1) {	307	if (num_cpus == 1) {
308	const u32 sizep, lsizep;	308	const u32 sizep, lsizep;
309	u32 size, lsize;	309	u32 size, lsize;
310		310
311	size = 0;	311	size = 0;
312	lsize = cur_cpu_spec->dcache_bsize;	312	lsize = cur_cpu_spec->dcache_bsize;
313	sizep = of_get_property(np, "d-cache-size", NULL);	313	sizep = of_get_property(np, "d-cache-size", NULL);
314	if (sizep != NULL)	314	if (sizep != NULL)
315	size = *sizep;	315	size = *sizep;
316	lsizep = of_get_property(np, "d-cache-block-size",	316	lsizep = of_get_property(np, "d-cache-block-size",
317	NULL);	317	NULL);
318	/* fallback if block size missing */	318	/* fallback if block size missing */
319	if (lsizep == NULL)	319	if (lsizep == NULL)
320	lsizep = of_get_property(np,	320	lsizep = of_get_property(np,
321	"d-cache-line-size",	321	"d-cache-line-size",
322	NULL);	322	NULL);
323	if (lsizep != NULL)	323	if (lsizep != NULL)
324	lsize = *lsizep;	324	lsize = *lsizep;
325	if (sizep == 0 \|\| lsizep == 0)	325	if (sizep == 0 \|\| lsizep == 0)
326	DBG("Argh, can't find dcache properties ! "	326	DBG("Argh, can't find dcache properties ! "
327	"sizep: %p, lsizep: %p\n", sizep, lsizep);	327	"sizep: %p, lsizep: %p\n", sizep, lsizep);
328		328
329	ppc64_caches.dsize = size;	329	ppc64_caches.dsize = size;
330	ppc64_caches.dline_size = lsize;	330	ppc64_caches.dline_size = lsize;
331	ppc64_caches.log_dline_size = __ilog2(lsize);	331	ppc64_caches.log_dline_size = __ilog2(lsize);
332	ppc64_caches.dlines_per_page = PAGE_SIZE / lsize;	332	ppc64_caches.dlines_per_page = PAGE_SIZE / lsize;
333		333
334	size = 0;	334	size = 0;
335	lsize = cur_cpu_spec->icache_bsize;	335	lsize = cur_cpu_spec->icache_bsize;
336	sizep = of_get_property(np, "i-cache-size", NULL);	336	sizep = of_get_property(np, "i-cache-size", NULL);
337	if (sizep != NULL)	337	if (sizep != NULL)
338	size = *sizep;	338	size = *sizep;
339	lsizep = of_get_property(np, "i-cache-block-size",	339	lsizep = of_get_property(np, "i-cache-block-size",
340	NULL);	340	NULL);
341	if (lsizep == NULL)	341	if (lsizep == NULL)
342	lsizep = of_get_property(np,	342	lsizep = of_get_property(np,
343	"i-cache-line-size",	343	"i-cache-line-size",
344	NULL);	344	NULL);
345	if (lsizep != NULL)	345	if (lsizep != NULL)
346	lsize = *lsizep;	346	lsize = *lsizep;
347	if (sizep == 0 \|\| lsizep == 0)	347	if (sizep == 0 \|\| lsizep == 0)
348	DBG("Argh, can't find icache properties ! "	348	DBG("Argh, can't find icache properties ! "
349	"sizep: %p, lsizep: %p\n", sizep, lsizep);	349	"sizep: %p, lsizep: %p\n", sizep, lsizep);
350		350
351	ppc64_caches.isize = size;	351	ppc64_caches.isize = size;
352	ppc64_caches.iline_size = lsize;	352	ppc64_caches.iline_size = lsize;
353	ppc64_caches.log_iline_size = __ilog2(lsize);	353	ppc64_caches.log_iline_size = __ilog2(lsize);
354	ppc64_caches.ilines_per_page = PAGE_SIZE / lsize;	354	ppc64_caches.ilines_per_page = PAGE_SIZE / lsize;
355	}	355	}
356	}	356	}
357		357
358	DBG(" <- initialize_cache_info()\n");	358	DBG(" <- initialize_cache_info()\n");
359	}	359	}
360		360
361		361
362	/*	362	/*
363	* Do some initial setup of the system. The parameters are those which	363	* Do some initial setup of the system. The parameters are those which
364	* were passed in from the bootloader.	364	* were passed in from the bootloader.
365	*/	365	*/
366	void __init setup_system(void)	366	void __init setup_system(void)
367	{	367	{
368	DBG(" -> setup_system()\n");	368	DBG(" -> setup_system()\n");
369		369
370	/* Apply the CPUs-specific and firmware specific fixups to kernel	370	/* Apply the CPUs-specific and firmware specific fixups to kernel
371	* text (nop out sections not relevant to this CPU or this firmware)	371	* text (nop out sections not relevant to this CPU or this firmware)
372	*/	372	*/
373	do_feature_fixups(cur_cpu_spec->cpu_features,	373	do_feature_fixups(cur_cpu_spec->cpu_features,
374	&__start___ftr_fixup, &__stop___ftr_fixup);	374	&__start___ftr_fixup, &__stop___ftr_fixup);
375	do_feature_fixups(cur_cpu_spec->mmu_features,	375	do_feature_fixups(cur_cpu_spec->mmu_features,
376	&__start___mmu_ftr_fixup, &__stop___mmu_ftr_fixup);	376	&__start___mmu_ftr_fixup, &__stop___mmu_ftr_fixup);
377	do_feature_fixups(powerpc_firmware_features,	377	do_feature_fixups(powerpc_firmware_features,
378	&__start___fw_ftr_fixup, &__stop___fw_ftr_fixup);	378	&__start___fw_ftr_fixup, &__stop___fw_ftr_fixup);
379	do_lwsync_fixups(cur_cpu_spec->cpu_features,	379	do_lwsync_fixups(cur_cpu_spec->cpu_features,
380	&__start___lwsync_fixup, &__stop___lwsync_fixup);	380	&__start___lwsync_fixup, &__stop___lwsync_fixup);
381	do_final_fixups();	381	do_final_fixups();
382		382
383	/*	383	/*
384	* Unflatten the device-tree passed by prom_init or kexec	384	* Unflatten the device-tree passed by prom_init or kexec
385	*/	385	*/
386	unflatten_device_tree();	386	unflatten_device_tree();
387		387
388	/*	388	/*
389	* Fill the ppc64_caches & systemcfg structures with informations	389	* Fill the ppc64_caches & systemcfg structures with informations
390	* retrieved from the device-tree.	390	* retrieved from the device-tree.
391	*/	391	*/
392	initialize_cache_info();	392	initialize_cache_info();
393		393
394	#ifdef CONFIG_PPC_RTAS	394	#ifdef CONFIG_PPC_RTAS
395	/*	395	/*
396	* Initialize RTAS if available	396	* Initialize RTAS if available
397	*/	397	*/
398	rtas_initialize();	398	rtas_initialize();
399	#endif /* CONFIG_PPC_RTAS */	399	#endif /* CONFIG_PPC_RTAS */
400		400
401	/*	401	/*
402	* Check if we have an initrd provided via the device-tree	402	* Check if we have an initrd provided via the device-tree
403	*/	403	*/
404	check_for_initrd();	404	check_for_initrd();
405		405
406	/*	406	/*
407	* Do some platform specific early initializations, that includes	407	* Do some platform specific early initializations, that includes
408	* setting up the hash table pointers. It also sets up some interrupt-mapping	408	* setting up the hash table pointers. It also sets up some interrupt-mapping
409	* related options that will be used by finish_device_tree()	409	* related options that will be used by finish_device_tree()
410	*/	410	*/
411	if (ppc_md.init_early)	411	if (ppc_md.init_early)
412	ppc_md.init_early();	412	ppc_md.init_early();
413		413
414	/*	414	/*
415	* We can discover serial ports now since the above did setup the	415	* We can discover serial ports now since the above did setup the
416	* hash table management for us, thus ioremap works. We do that early	416	* hash table management for us, thus ioremap works. We do that early
417	* so that further code can be debugged	417	* so that further code can be debugged
418	*/	418	*/
419	find_legacy_serial_ports();	419	find_legacy_serial_ports();
420		420
421	/*	421	/*
422	* Register early console	422	* Register early console
423	*/	423	*/
424	register_early_udbg_console();	424	register_early_udbg_console();
425		425
426	/*	426	/*
427	* Initialize xmon	427	* Initialize xmon
428	*/	428	*/
429	xmon_setup();	429	xmon_setup();
430		430
431	smp_setup_cpu_maps();	431	smp_setup_cpu_maps();
432	check_smt_enabled();	432	check_smt_enabled();
433		433
434	#ifdef CONFIG_SMP	434	#ifdef CONFIG_SMP
435	/* Release secondary cpus out of their spinloops at 0x60 now that	435	/* Release secondary cpus out of their spinloops at 0x60 now that
436	* we can map physical -> logical CPU ids	436	* we can map physical -> logical CPU ids
437	*/	437	*/
438	smp_release_cpus();	438	smp_release_cpus();
439	#endif	439	#endif
440		440
441	printk("Starting Linux PPC64 %s\n", init_utsname()->version);	441	printk("Starting Linux PPC64 %s\n", init_utsname()->version);
442		442
443	printk("-----------------------------------------------------\n");	443	printk("-----------------------------------------------------\n");
444	printk("ppc64_pft_size = 0x%llx\n", ppc64_pft_size);	444	printk("ppc64_pft_size = 0x%llx\n", ppc64_pft_size);
445	printk("physicalMemorySize = 0x%llx\n", memblock_phys_mem_size());	445	printk("physicalMemorySize = 0x%llx\n", memblock_phys_mem_size());
446	if (ppc64_caches.dline_size != 0x80)	446	if (ppc64_caches.dline_size != 0x80)
447	printk("ppc64_caches.dcache_line_size = 0x%x\n",	447	printk("ppc64_caches.dcache_line_size = 0x%x\n",
448	ppc64_caches.dline_size);	448	ppc64_caches.dline_size);
449	if (ppc64_caches.iline_size != 0x80)	449	if (ppc64_caches.iline_size != 0x80)
450	printk("ppc64_caches.icache_line_size = 0x%x\n",	450	printk("ppc64_caches.icache_line_size = 0x%x\n",
451	ppc64_caches.iline_size);	451	ppc64_caches.iline_size);
452	#ifdef CONFIG_PPC_STD_MMU_64	452	#ifdef CONFIG_PPC_STD_MMU_64
453	if (htab_address)	453	if (htab_address)
454	printk("htab_address = 0x%p\n", htab_address);	454	printk("htab_address = 0x%p\n", htab_address);
455	printk("htab_hash_mask = 0x%lx\n", htab_hash_mask);	455	printk("htab_hash_mask = 0x%lx\n", htab_hash_mask);
456	#endif /* CONFIG_PPC_STD_MMU_64 */	456	#endif /* CONFIG_PPC_STD_MMU_64 */
457	if (PHYSICAL_START > 0)	457	if (PHYSICAL_START > 0)
458	printk("physical_start = 0x%llx\n",	458	printk("physical_start = 0x%llx\n",
459	(unsigned long long)PHYSICAL_START);	459	(unsigned long long)PHYSICAL_START);
460	printk("-----------------------------------------------------\n");	460	printk("-----------------------------------------------------\n");
461		461
462	DBG(" <- setup_system()\n");	462	DBG(" <- setup_system()\n");
463	}	463	}
464		464
465	/* This returns the limit below which memory accesses to the linear	465	/* This returns the limit below which memory accesses to the linear
466	* mapping are guarnateed not to cause a TLB or SLB miss. This is	466	* mapping are guarnateed not to cause a TLB or SLB miss. This is
467	* used to allocate interrupt or emergency stacks for which our	467	* used to allocate interrupt or emergency stacks for which our
468	* exception entry path doesn't deal with being interrupted.	468	* exception entry path doesn't deal with being interrupted.
469	*/	469	*/
470	static u64 safe_stack_limit(void)	470	static u64 safe_stack_limit(void)
471	{	471	{
472	#ifdef CONFIG_PPC_BOOK3E	472	#ifdef CONFIG_PPC_BOOK3E
473	/* Freescale BookE bolts the entire linear mapping */	473	/* Freescale BookE bolts the entire linear mapping */
474	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))	474	if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))
475	return linear_map_top;	475	return linear_map_top;
476	/* Other BookE, we assume the first GB is bolted */	476	/* Other BookE, we assume the first GB is bolted */
477	return 1ul << 30;	477	return 1ul << 30;
478	#else	478	#else
479	/* BookS, the first segment is bolted */	479	/* BookS, the first segment is bolted */
480	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))	480	if (mmu_has_feature(MMU_FTR_1T_SEGMENT))
481	return 1UL << SID_SHIFT_1T;	481	return 1UL << SID_SHIFT_1T;
482	return 1UL << SID_SHIFT;	482	return 1UL << SID_SHIFT;
483	#endif	483	#endif
484	}	484	}
485		485
486	static void __init irqstack_early_init(void)	486	static void __init irqstack_early_init(void)
487	{	487	{
488	u64 limit = safe_stack_limit();	488	u64 limit = safe_stack_limit();
489	unsigned int i;	489	unsigned int i;
490		490
491	/*	491	/*
492	* Interrupt stacks must be in the first segment since we	492	* Interrupt stacks must be in the first segment since we
493	* cannot afford to take SLB misses on them.	493	* cannot afford to take SLB misses on them.
494	*/	494	*/
495	for_each_possible_cpu(i) {	495	for_each_possible_cpu(i) {
496	softirq_ctx[i] = (struct thread_info *)	496	softirq_ctx[i] = (struct thread_info *)
497	__va(memblock_alloc_base(THREAD_SIZE,	497	__va(memblock_alloc_base(THREAD_SIZE,
498	THREAD_SIZE, limit));	498	THREAD_SIZE, limit));
499	hardirq_ctx[i] = (struct thread_info *)	499	hardirq_ctx[i] = (struct thread_info *)
500	__va(memblock_alloc_base(THREAD_SIZE,	500	__va(memblock_alloc_base(THREAD_SIZE,
501	THREAD_SIZE, limit));	501	THREAD_SIZE, limit));
502	}	502	}
503	}	503	}
504		504
505	#ifdef CONFIG_PPC_BOOK3E	505	#ifdef CONFIG_PPC_BOOK3E
506	static void __init exc_lvl_early_init(void)	506	static void __init exc_lvl_early_init(void)
507	{	507	{
508	extern unsigned int interrupt_base_book3e;	508	extern unsigned int interrupt_base_book3e;
509	extern unsigned int exc_debug_debug_book3e;	509	extern unsigned int exc_debug_debug_book3e;
510		510
511	unsigned int i;	511	unsigned int i;
512		512
513	for_each_possible_cpu(i) {	513	for_each_possible_cpu(i) {
514	critirq_ctx[i] = (struct thread_info *)	514	critirq_ctx[i] = (struct thread_info *)
515	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));	515	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
516	dbgirq_ctx[i] = (struct thread_info *)	516	dbgirq_ctx[i] = (struct thread_info *)
517	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));	517	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
518	mcheckirq_ctx[i] = (struct thread_info *)	518	mcheckirq_ctx[i] = (struct thread_info *)
519	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));	519	__va(memblock_alloc(THREAD_SIZE, THREAD_SIZE));
520	}	520	}
521		521
522	if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))	522	if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
523	patch_branch(&interrupt_base_book3e + (0x040 / 4) + 1,	523	patch_branch(&interrupt_base_book3e + (0x040 / 4) + 1,
524	(unsigned long)&exc_debug_debug_book3e, 0);	524	(unsigned long)&exc_debug_debug_book3e, 0);
525	}	525	}
526	#else	526	#else
527	#define exc_lvl_early_init()	527	#define exc_lvl_early_init()
528	#endif	528	#endif
529		529
530	/*	530	/*
531	* Stack space used when we detect a bad kernel stack pointer, and	531	* Stack space used when we detect a bad kernel stack pointer, and
532	* early in SMP boots before relocation is enabled.	532	* early in SMP boots before relocation is enabled.
533	*/	533	*/
534	static void __init emergency_stack_init(void)	534	static void __init emergency_stack_init(void)
535	{	535	{
536	u64 limit;	536	u64 limit;
537	unsigned int i;	537	unsigned int i;
538		538
539	/*	539	/*
540	* Emergency stacks must be under 256MB, we cannot afford to take	540	* Emergency stacks must be under 256MB, we cannot afford to take
541	* SLB misses on them. The ABI also requires them to be 128-byte	541	* SLB misses on them. The ABI also requires them to be 128-byte
542	* aligned.	542	* aligned.
543	*	543	*
544	* Since we use these as temporary stacks during secondary CPU	544	* Since we use these as temporary stacks during secondary CPU
545	* bringup, we need to get at them in real mode. This means they	545	* bringup, we need to get at them in real mode. This means they
546	* must also be within the RMO region.	546	* must also be within the RMO region.
547	*/	547	*/
548	limit = min(safe_stack_limit(), ppc64_rma_size);	548	limit = min(safe_stack_limit(), ppc64_rma_size);
549		549
550	for_each_possible_cpu(i) {	550	for_each_possible_cpu(i) {
551	unsigned long sp;	551	unsigned long sp;
552	sp = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit);	552	sp = memblock_alloc_base(THREAD_SIZE, THREAD_SIZE, limit);
553	sp += THREAD_SIZE;	553	sp += THREAD_SIZE;
554	paca[i].emergency_sp = __va(sp);	554	paca[i].emergency_sp = __va(sp);
555	}	555	}
556	}	556	}
557		557
558	/*	558	/*
559	* Called into from start_kernel this initializes bootmem, which is used	559	* Called into from start_kernel this initializes bootmem, which is used
560	* to manage page allocation until mem_init is called.	560	* to manage page allocation until mem_init is called.
561	*/	561	*/
562	void __init setup_arch(char **cmdline_p)	562	void __init setup_arch(char **cmdline_p)
563	{	563	{
564	ppc64_boot_msg(0x12, "Setup Arch");	564	ppc64_boot_msg(0x12, "Setup Arch");
565		565
566	*cmdline_p = cmd_line;	566	*cmdline_p = cmd_line;
567		567
568	/*	568	/*
569	* Set cache line size based on type of cpu as a default.	569	* Set cache line size based on type of cpu as a default.
570	* Systems with OF can look in the properties on the cpu node(s)	570	* Systems with OF can look in the properties on the cpu node(s)
571	* for a possibly more accurate value.	571	* for a possibly more accurate value.
572	*/	572	*/
573	dcache_bsize = ppc64_caches.dline_size;	573	dcache_bsize = ppc64_caches.dline_size;
574	icache_bsize = ppc64_caches.iline_size;	574	icache_bsize = ppc64_caches.iline_size;
575		575
576	/* reboot on panic */	576	/* reboot on panic */
577	panic_timeout = 180;	577	panic_timeout = 180;
578		578
579	if (ppc_md.panic)	579	if (ppc_md.panic)
580	setup_panic();	580	setup_panic();
581		581
582	init_mm.start_code = (unsigned long)_stext;	582	init_mm.start_code = (unsigned long)_stext;
583	init_mm.end_code = (unsigned long) _etext;	583	init_mm.end_code = (unsigned long) _etext;
584	init_mm.end_data = (unsigned long) _edata;	584	init_mm.end_data = (unsigned long) _edata;
585	init_mm.brk = klimit;	585	init_mm.brk = klimit;
586		586	#ifdef CONFIG_PPC_64K_PAGES
		587	init_mm.context.pte_frag = NULL;
		588	#endif
587	irqstack_early_init();	589	irqstack_early_init();
588	exc_lvl_early_init();	590	exc_lvl_early_init();
589	emergency_stack_init();	591	emergency_stack_init();
590		592
591	#ifdef CONFIG_PPC_STD_MMU_64	593	#ifdef CONFIG_PPC_STD_MMU_64
592	stabs_alloc();	594	stabs_alloc();
593	#endif	595	#endif
594	/* set up the bootmem stuff with available memory */	596	/* set up the bootmem stuff with available memory */
595	do_init_bootmem();	597	do_init_bootmem();
596	sparse_init();	598	sparse_init();
597		599
598	#ifdef CONFIG_DUMMY_CONSOLE	600	#ifdef CONFIG_DUMMY_CONSOLE
599	conswitchp = &dummy_con;	601	conswitchp = &dummy_con;
600	#endif	602	#endif
601		603
602	if (ppc_md.setup_arch)	604	if (ppc_md.setup_arch)
603	ppc_md.setup_arch();	605	ppc_md.setup_arch();
604		606
605	paging_init();	607	paging_init();
606		608
607	/* Initialize the MMU context management stuff */	609	/* Initialize the MMU context management stuff */
608	mmu_context_init();	610	mmu_context_init();
609		611
610	kvm_linear_init();	612	kvm_linear_init();
611		613
612	/* Interrupt code needs to be 64K-aligned */	614	/* Interrupt code needs to be 64K-aligned */
613	if ((unsigned long)_stext & 0xffff)	615	if ((unsigned long)_stext & 0xffff)
614	panic("Kernelbase not 64K-aligned (0x%lx)!\n",	616	panic("Kernelbase not 64K-aligned (0x%lx)!\n",
615	(unsigned long)_stext);	617	(unsigned long)_stext);
616		618
617	ppc64_boot_msg(0x15, "Setup Done");	619	ppc64_boot_msg(0x15, "Setup Done");
618	}	620	}
619		621
620		622
621	/* ToDo: do something useful if ppc_md is not yet setup. */	623	/* ToDo: do something useful if ppc_md is not yet setup. */
622	#define PPC64_LINUX_FUNCTION 0x0f000000	624	#define PPC64_LINUX_FUNCTION 0x0f000000
623	#define PPC64_IPL_MESSAGE 0xc0000000	625	#define PPC64_IPL_MESSAGE 0xc0000000
624	#define PPC64_TERM_MESSAGE 0xb0000000	626	#define PPC64_TERM_MESSAGE 0xb0000000
625		627
626	static void ppc64_do_msg(unsigned int src, const char *msg)	628	static void ppc64_do_msg(unsigned int src, const char *msg)
627	{	629	{
628	if (ppc_md.progress) {	630	if (ppc_md.progress) {
629	char buf[128];	631	char buf[128];
630		632
631	sprintf(buf, "%08X\n", src);	633	sprintf(buf, "%08X\n", src);
632	ppc_md.progress(buf, 0);	634	ppc_md.progress(buf, 0);
633	snprintf(buf, 128, "%s", msg);	635	snprintf(buf, 128, "%s", msg);
634	ppc_md.progress(buf, 0);	636	ppc_md.progress(buf, 0);
635	}	637	}
636	}	638	}
637		639
638	/* Print a boot progress message. */	640	/* Print a boot progress message. */
639	void ppc64_boot_msg(unsigned int src, const char *msg)	641	void ppc64_boot_msg(unsigned int src, const char *msg)
640	{	642	{
641	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_IPL_MESSAGE\|src, msg);	643	ppc64_do_msg(PPC64_LINUX_FUNCTION\|PPC64_IPL_MESSAGE\|src, msg);
642	printk("[boot]%04x %s\n", src, msg);	644	printk("[boot]%04x %s\n", src, msg);
643	}	645	}
644		646
645	#ifdef CONFIG_SMP	647	#ifdef CONFIG_SMP
646	#define PCPU_DYN_SIZE ()	648	#define PCPU_DYN_SIZE ()
647		649
648	static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)	650	static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size, size_t align)
649	{	651	{
650	return __alloc_bootmem_node(NODE_DATA(cpu_to_node(cpu)), size, align,	652	return __alloc_bootmem_node(NODE_DATA(cpu_to_node(cpu)), size, align,
651	__pa(MAX_DMA_ADDRESS));	653	__pa(MAX_DMA_ADDRESS));
652	}	654	}
653		655
654	static void __init pcpu_fc_free(void *ptr, size_t size)	656	static void __init pcpu_fc_free(void *ptr, size_t size)
655	{	657	{
656	free_bootmem(__pa(ptr), size);	658	free_bootmem(__pa(ptr), size);
657	}	659	}
658		660
659	static int pcpu_cpu_distance(unsigned int from, unsigned int to)	661	static int pcpu_cpu_distance(unsigned int from, unsigned int to)
660	{	662	{
661	if (cpu_to_node(from) == cpu_to_node(to))	663	if (cpu_to_node(from) == cpu_to_node(to))
662	return LOCAL_DISTANCE;	664	return LOCAL_DISTANCE;
663	else	665	else
664	return REMOTE_DISTANCE;	666	return REMOTE_DISTANCE;
665	}	667	}
666		668
667	unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;	669	unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
668	EXPORT_SYMBOL(__per_cpu_offset);	670	EXPORT_SYMBOL(__per_cpu_offset);
669		671
670	void __init setup_per_cpu_areas(void)	672	void __init setup_per_cpu_areas(void)
671	{	673	{
672	const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;	674	const size_t dyn_size = PERCPU_MODULE_RESERVE + PERCPU_DYNAMIC_RESERVE;
673	size_t atom_size;	675	size_t atom_size;
674	unsigned long delta;	676	unsigned long delta;
675	unsigned int cpu;	677	unsigned int cpu;
676	int rc;	678	int rc;
677		679
678	/*	680	/*
679	* Linear mapping is one of 4K, 1M and 16M. For 4K, no need	681	* Linear mapping is one of 4K, 1M and 16M. For 4K, no need
680	* to group units. For larger mappings, use 1M atom which	682	* to group units. For larger mappings, use 1M atom which
681	* should be large enough to contain a number of units.	683	* should be large enough to contain a number of units.
682	*/	684	*/
683	if (mmu_linear_psize == MMU_PAGE_4K)	685	if (mmu_linear_psize == MMU_PAGE_4K)
684	atom_size = PAGE_SIZE;	686	atom_size = PAGE_SIZE;
685	else	687	else
686	atom_size = 1 << 20;	688	atom_size = 1 << 20;
687		689
688	rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance,	690	rc = pcpu_embed_first_chunk(0, dyn_size, atom_size, pcpu_cpu_distance,
689	pcpu_fc_alloc, pcpu_fc_free);	691	pcpu_fc_alloc, pcpu_fc_free);
690	if (rc < 0)	692	if (rc < 0)
691	panic("cannot initialize percpu area (err=%d)", rc);	693	panic("cannot initialize percpu area (err=%d)", rc);
692		694
693	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;	695	delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
694	for_each_possible_cpu(cpu) {	696	for_each_possible_cpu(cpu) {
695	__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];	697	__per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
696	paca[cpu].data_offset = __per_cpu_offset[cpu];	698	paca[cpu].data_offset = __per_cpu_offset[cpu];
697	}	699	}
698	}	700	}
699	#endif	701	#endif
700		702
701		703
702	#ifdef CONFIG_PPC_INDIRECT_IO	704	#ifdef CONFIG_PPC_INDIRECT_IO
703	struct ppc_pci_io ppc_pci_io;	705	struct ppc_pci_io ppc_pci_io;
704	EXPORT_SYMBOL(ppc_pci_io);	706	EXPORT_SYMBOL(ppc_pci_io);
705	#endif /* CONFIG_PPC_INDIRECT_IO */	707	#endif /* CONFIG_PPC_INDIRECT_IO */
706		708
707		709

arch/powerpc/mm/mmu_context_hash64.c

Diff comments View file @ 5c1f6ee

 /*
  *  MMU context allocation for 64-bit kernels.
  *
  *  Copyright (C) 2004 Anton Blanchard, IBM Corp. <anton@samba.org>
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  *
  */
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/mm.h>
 #include <linux/spinlock.h>
 #include <linux/idr.h>
 #include <linux/export.h>
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <asm/mmu_context.h>
+#include <asm/pgalloc.h>
 #include "icswx.h"
 static DEFINE_SPINLOCK(mmu_context_lock);
 static DEFINE_IDA(mmu_context_ida);
 int __init_new_context(void)
 {
 	int index;
 	int err;
 again:
 	if (!ida_pre_get(&mmu_context_ida, GFP_KERNEL))
 		return -ENOMEM;
 	spin_lock(&mmu_context_lock);
 	err = ida_get_new_above(&mmu_context_ida, 1, &index);
 	spin_unlock(&mmu_context_lock);
 	if (err == -EAGAIN)
 		goto again;
 	else if (err)
 		return err;
 	if (index > MAX_USER_CONTEXT) {
 		spin_lock(&mmu_context_lock);
 		ida_remove(&mmu_context_ida, index);
 		spin_unlock(&mmu_context_lock);
 		return -ENOMEM;
 	}
 	return index;
 }
 EXPORT_SYMBOL_GPL(__init_new_context);
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
 	int index;
 	index = __init_new_context();
 	if (index < 0)
 		return index;
 	/* The old code would re-promote on fork, we don't do that
 	 * when using slices as it could cause problem promoting slices
 	 * that have been forced down to 4K
 	 */
 	if (slice_mm_new_context(mm))
 		slice_set_user_psize(mm, mmu_virtual_psize);
 	subpage_prot_init_new_context(mm);
 	mm->context.id = index;
 #ifdef CONFIG_PPC_ICSWX
 	mm->context.cop_lockp = kmalloc(sizeof(spinlock_t), GFP_KERNEL);
 	if (!mm->context.cop_lockp) {
 		__destroy_context(index);
 		subpage_prot_free(mm);
 		mm->context.id = MMU_NO_CONTEXT;
 		return -ENOMEM;
 	}
 	spin_lock_init(mm->context.cop_lockp);
 #endif /* CONFIG_PPC_ICSWX */
+#ifdef CONFIG_PPC_64K_PAGES
+	mm->context.pte_frag = NULL;
+#endif
 	return 0;
 }
 void __destroy_context(int context_id)
 {
 	spin_lock(&mmu_context_lock);
 	ida_remove(&mmu_context_ida, context_id);
 	spin_unlock(&mmu_context_lock);
 }
 EXPORT_SYMBOL_GPL(__destroy_context);
+#ifdef CONFIG_PPC_64K_PAGES
+static void destroy_pagetable_page(struct mm_struct *mm)
+{
+	int count;
+	void *pte_frag;
+	struct page *page;
+	pte_frag = mm->context.pte_frag;
+	if (!pte_frag)
+		return;
+	page = virt_to_page(pte_frag);
+	/* drop all the pending references */
+	count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
+	/* We allow PTE_FRAG_NR fragments from a PTE page */
+	count = atomic_sub_return(PTE_FRAG_NR - count, &page->_count);
+	if (!count) {
+		pgtable_page_dtor(page);
+		free_hot_cold_page(page, 0);
+	}
+}
+#else
+static inline void destroy_pagetable_page(struct mm_struct *mm)
+{
+	return;
+}
+#endif
 void destroy_context(struct mm_struct *mm)
 {
 #ifdef CONFIG_PPC_ICSWX
 	drop_cop(mm->context.acop, mm);
 	kfree(mm->context.cop_lockp);
 	mm->context.cop_lockp = NULL;
 #endif /* CONFIG_PPC_ICSWX */
+	destroy_pagetable_page(mm);
 	__destroy_context(mm->context.id);
 	subpage_prot_free(mm);
 	mm->context.id = MMU_NO_CONTEXT;
 }

arch/powerpc/mm/pgtable_64.c

Diff comments View file @ 5c1f6ee

 /*
  *  This file contains ioremap and related functions for 64-bit machines.
  *
  *  Derived from arch/ppc64/mm/init.c
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *
  *  Modifications by Paul Mackerras (PowerMac) (paulus@samba.org)
  *  and Cort Dougan (PReP) (cort@cs.nmt.edu)
  *    Copyright (C) 1996 Paul Mackerras
  *
  *  Derived from "arch/i386/mm/init.c"
  *    Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  *
  *  Dave Engebretsen <engebret@us.ibm.com>
  *      Rework for PPC64 port.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  *
  */
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/export.h>
 #include <linux/types.h>
 #include <linux/mman.h>
 #include <linux/mm.h>
 #include <linux/swap.h>
 #include <linux/stddef.h>
 #include <linux/vmalloc.h>
 #include <linux/init.h>
 #include <linux/bootmem.h>
 #include <linux/memblock.h>
 #include <linux/slab.h>
 #include <asm/pgalloc.h>
 #include <asm/page.h>
 #include <asm/prom.h>
 #include <asm/io.h>
 #include <asm/mmu_context.h>
 #include <asm/pgtable.h>
 #include <asm/mmu.h>
 #include <asm/smp.h>
 #include <asm/machdep.h>
 #include <asm/tlb.h>
 #include <asm/processor.h>
 #include <asm/cputable.h>
 #include <asm/sections.h>
 #include <asm/firmware.h>
 #include "mmu_decl.h"
 /* Some sanity checking */
 #if TASK_SIZE_USER64 > PGTABLE_RANGE
 #error TASK_SIZE_USER64 exceeds pagetable range
 #endif
 #ifdef CONFIG_PPC_STD_MMU_64
 #if TASK_SIZE_USER64 > (1UL << (ESID_BITS + SID_SHIFT))
 #error TASK_SIZE_USER64 exceeds user VSID range
 #endif
 #endif
 unsigned long ioremap_bot = IOREMAP_BASE;
 #ifdef CONFIG_PPC_MMU_NOHASH
 static void *early_alloc_pgtable(unsigned long size)
 {
 	void *pt;
 	if (init_bootmem_done)
 		pt = __alloc_bootmem(size, size, __pa(MAX_DMA_ADDRESS));
 	else
 		pt = __va(memblock_alloc_base(size, size,
 					 __pa(MAX_DMA_ADDRESS)));
 	memset(pt, 0, size);
 	return pt;
 }
 #endif /* CONFIG_PPC_MMU_NOHASH */
 /*
  * map_kernel_page currently only called by __ioremap
  * map_kernel_page adds an entry to the ioremap page table
  * and adds an entry to the HPT, possibly bolting it
  */
 int map_kernel_page(unsigned long ea, unsigned long pa, int flags)
 {
 	pgd_t *pgdp;
 	pud_t *pudp;
 	pmd_t *pmdp;
 	pte_t *ptep;
 	if (slab_is_available()) {
 		pgdp = pgd_offset_k(ea);
 		pudp = pud_alloc(&init_mm, pgdp, ea);
 		if (!pudp)
 			return -ENOMEM;
 		pmdp = pmd_alloc(&init_mm, pudp, ea);
 		if (!pmdp)
 			return -ENOMEM;
 		ptep = pte_alloc_kernel(pmdp, ea);
 		if (!ptep)
 			return -ENOMEM;
 		set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
 							  __pgprot(flags)));
 	} else {
 #ifdef CONFIG_PPC_MMU_NOHASH
 		/* Warning ! This will blow up if bootmem is not initialized
 		 * which our ppc64 code is keen to do that, we'll need to
 		 * fix it and/or be more careful
 		 */
 		pgdp = pgd_offset_k(ea);
 #ifdef PUD_TABLE_SIZE
 		if (pgd_none(*pgdp)) {
 			pudp = early_alloc_pgtable(PUD_TABLE_SIZE);
 			BUG_ON(pudp == NULL);
 			pgd_populate(&init_mm, pgdp, pudp);
 		}
 #endif /* PUD_TABLE_SIZE */
 		pudp = pud_offset(pgdp, ea);
 		if (pud_none(*pudp)) {
 			pmdp = early_alloc_pgtable(PMD_TABLE_SIZE);
 			BUG_ON(pmdp == NULL);
 			pud_populate(&init_mm, pudp, pmdp);
 		}
 		pmdp = pmd_offset(pudp, ea);
 		if (!pmd_present(*pmdp)) {
 			ptep = early_alloc_pgtable(PAGE_SIZE);
 			BUG_ON(ptep == NULL);
 			pmd_populate_kernel(&init_mm, pmdp, ptep);
 		}
 		ptep = pte_offset_kernel(pmdp, ea);
 		set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT,
 							  __pgprot(flags)));
 #else /* CONFIG_PPC_MMU_NOHASH */
 		/*
 		 * If the mm subsystem is not fully up, we cannot create a
 		 * linux page table entry for this mapping.  Simply bolt an
 		 * entry in the hardware page table.
 		 *
 		 */
 		if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags,
 				      mmu_io_psize, mmu_kernel_ssize)) {
 			printk(KERN_ERR "Failed to do bolted mapping IO "
 			       "memory at %016lx !\n", pa);
 			return -ENOMEM;
 		}
 #endif /* !CONFIG_PPC_MMU_NOHASH */
 	}
 	return 0;
 }
 /**
  * __ioremap_at - Low level function to establish the page tables
  *                for an IO mapping
  */
 void __iomem * __ioremap_at(phys_addr_t pa, void *ea, unsigned long size,
 			    unsigned long flags)
 {
 	unsigned long i;
 	/* Make sure we have the base flags */
 	if ((flags & _PAGE_PRESENT) == 0)
 		flags |= pgprot_val(PAGE_KERNEL);
 	/* Non-cacheable page cannot be coherent */
 	if (flags & _PAGE_NO_CACHE)
 		flags &= ~_PAGE_COHERENT;
 	/* We don't support the 4K PFN hack with ioremap */
 	if (flags & _PAGE_4K_PFN)
 		return NULL;
 	WARN_ON(pa & ~PAGE_MASK);
 	WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
 	WARN_ON(size & ~PAGE_MASK);
 	for (i = 0; i < size; i += PAGE_SIZE)
 		if (map_kernel_page((unsigned long)ea+i, pa+i, flags))
 			return NULL;
 	return (void __iomem *)ea;
 }
 /**
  * __iounmap_from - Low level function to tear down the page tables
  *                  for an IO mapping. This is used for mappings that
  *                  are manipulated manually, like partial unmapping of
  *                  PCI IOs or ISA space.
  */
 void __iounmap_at(void *ea, unsigned long size)
 {
 	WARN_ON(((unsigned long)ea) & ~PAGE_MASK);
 	WARN_ON(size & ~PAGE_MASK);
 	unmap_kernel_range((unsigned long)ea, size);
 }
 void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size,
 				unsigned long flags, void *caller)
 {
 	phys_addr_t paligned;
 	void __iomem *ret;
 	/*
 	 * Choose an address to map it to.
 	 * Once the imalloc system is running, we use it.
 	 * Before that, we map using addresses going
 	 * up from ioremap_bot.  imalloc will use
 	 * the addresses from ioremap_bot through
 	 * IMALLOC_END
 	 *
 	 */
 	paligned = addr & PAGE_MASK;
 	size = PAGE_ALIGN(addr + size) - paligned;
 	if ((size == 0) || (paligned == 0))
 		return NULL;
 	if (mem_init_done) {
 		struct vm_struct *area;
 		area = __get_vm_area_caller(size, VM_IOREMAP,
 					    ioremap_bot, IOREMAP_END,
 					    caller);
 		if (area == NULL)
 			return NULL;
 		area->phys_addr = paligned;
 		ret = __ioremap_at(paligned, area->addr, size, flags);
 		if (!ret)
 			vunmap(area->addr);
 	} else {
 		ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags);
 		if (ret)
 			ioremap_bot += size;
 	}
 	if (ret)
 		ret += addr & ~PAGE_MASK;
 	return ret;
 }
 void __iomem * __ioremap(phys_addr_t addr, unsigned long size,
 			 unsigned long flags)
 {
 	return __ioremap_caller(addr, size, flags, __builtin_return_address(0));
 }
 void __iomem * ioremap(phys_addr_t addr, unsigned long size)
 {
 	unsigned long flags = _PAGE_NO_CACHE | _PAGE_GUARDED;
 	void *caller = __builtin_return_address(0);
 	if (ppc_md.ioremap)
 		return ppc_md.ioremap(addr, size, flags, caller);
 	return __ioremap_caller(addr, size, flags, caller);
 }
 void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size)
 {
 	unsigned long flags = _PAGE_NO_CACHE;
 	void *caller = __builtin_return_address(0);
 	if (ppc_md.ioremap)
 		return ppc_md.ioremap(addr, size, flags, caller);
 	return __ioremap_caller(addr, size, flags, caller);
 }
 void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size,
 			     unsigned long flags)
 {
 	void *caller = __builtin_return_address(0);
 	/* writeable implies dirty for kernel addresses */
 	if (flags & _PAGE_RW)
 		flags |= _PAGE_DIRTY;
 	/* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */
 	flags &= ~(_PAGE_USER | _PAGE_EXEC);
 #ifdef _PAGE_BAP_SR
 	/* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format
 	 * which means that we just cleared supervisor access... oops ;-) This
 	 * restores it
 	 */
 	flags |= _PAGE_BAP_SR;
 #endif
 	if (ppc_md.ioremap)
 		return ppc_md.ioremap(addr, size, flags, caller);
 	return __ioremap_caller(addr, size, flags, caller);
 }
 /*
  * Unmap an IO region and remove it from imalloc'd list.
  * Access to IO memory should be serialized by driver.
  */
 void __iounmap(volatile void __iomem *token)
 {
 	void *addr;
 	if (!mem_init_done)
 		return;
 	addr = (void *) ((unsigned long __force)
 			 PCI_FIX_ADDR(token) & PAGE_MASK);
 	if ((unsigned long)addr < ioremap_bot) {
 		printk(KERN_WARNING "Attempt to iounmap early bolted mapping"
 		       " at 0x%p\n", addr);
 		return;
 	}
 	vunmap(addr);
 }
 void iounmap(volatile void __iomem *token)
 {
 	if (ppc_md.iounmap)
 		ppc_md.iounmap(token);
 	else
 		__iounmap(token);
 }
 EXPORT_SYMBOL(ioremap);
 EXPORT_SYMBOL(ioremap_wc);
 EXPORT_SYMBOL(ioremap_prot);
 EXPORT_SYMBOL(__ioremap);
 EXPORT_SYMBOL(__ioremap_at);
 EXPORT_SYMBOL(iounmap);
 EXPORT_SYMBOL(__iounmap);
 EXPORT_SYMBOL(__iounmap_at);
+#ifdef CONFIG_PPC_64K_PAGES
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+	void *pte_frag, *ret;
+	spin_lock(&mm->page_table_lock);
+	ret = mm->context.pte_frag;
+	if (ret) {
+		pte_frag = ret + PTE_FRAG_SIZE;
+		/*
+		 * If we have taken up all the fragments mark PTE page NULL
+		 */
+		if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
+			pte_frag = NULL;
+		mm->context.pte_frag = pte_frag;
+	}
+	spin_unlock(&mm->page_table_lock);
+	return (pte_t *)ret;
+}
+static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
+{
+	void *ret = NULL;
+	struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK |
+				       __GFP_REPEAT | __GFP_ZERO);
+	if (!page)
+		return NULL;
+	ret = page_address(page);
+	spin_lock(&mm->page_table_lock);
+	/*
+	 * If we find pgtable_page set, we return
+	 * the allocated page with single fragement
+	 * count.
+	 */
+	if (likely(!mm->context.pte_frag)) {
+		atomic_set(&page->_count, PTE_FRAG_NR);
+		mm->context.pte_frag = ret + PTE_FRAG_SIZE;
+	}
+	spin_unlock(&mm->page_table_lock);
+	if (!kernel)
+		pgtable_page_ctor(page);
+	return (pte_t *)ret;
+}
+pte_t *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
+{
+	pte_t *pte;
+	pte = get_from_cache(mm);
+	if (pte)
+		return pte;
+	return __alloc_for_cache(mm, kernel);
+}
+void page_table_free(struct mm_struct *mm, unsigned long *table, int kernel)
+{
+	struct page *page = virt_to_page(table);
+	if (put_page_testzero(page)) {
+		if (!kernel)
+			pgtable_page_dtor(page);
+		free_hot_cold_page(page, 0);
+	}
+}
+#ifdef CONFIG_SMP
+static void page_table_free_rcu(void *table)
+{
+	struct page *page = virt_to_page(table);
+	if (put_page_testzero(page)) {
+		pgtable_page_dtor(page);
+		free_hot_cold_page(page, 0);
+	}
+}
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+{
+	unsigned long pgf = (unsigned long)table;
+	BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+	pgf |= shift;
+	tlb_remove_table(tlb, (void *)pgf);
+}
+void __tlb_remove_table(void *_table)
+{
+	void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
+	unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
+	if (!shift)
+		/* PTE page needs special handling */
+		page_table_free_rcu(table);
+	else {
+		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+		kmem_cache_free(PGT_CACHE(shift), table);
+	}
+}
+#else
+void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
+{
+	if (!shift) {
+		/* PTE page needs special handling */
+		struct page *page = virt_to_page(table);
+		if (put_page_testzero(page)) {
+			pgtable_page_dtor(page);
+			free_hot_cold_page(page, 0);
+		}
+	} else {
+		BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
+		kmem_cache_free(PGT_CACHE(shift), table);
+	}
+}
+#endif
+#endif /* CONFIG_PPC_64K_PAGES */