Doug / smarc-fsl-linux-kernel

1

#ifndef _LINUX_MM_H

1

#ifndef _LINUX_MM_H

2

#define _LINUX_MM_H

2

#define _LINUX_MM_H

3

4

#include <linux/errno.h>

4

#include <linux/errno.h>

5

6

#ifdef __KERNEL__

6

#ifdef __KERNEL__

7

8

#include <linux/gfp.h>

8

#include <linux/gfp.h>

9

#include <linux/bug.h>

9

#include <linux/bug.h>

10

#include <linux/list.h>

10

#include <linux/list.h>

11

#include <linux/mmzone.h>

11

#include <linux/mmzone.h>

12

#include <linux/rbtree.h>

12

#include <linux/rbtree.h>

13

#include <linux/atomic.h>

13

#include <linux/atomic.h>

14

#include <linux/debug_locks.h>

14

#include <linux/debug_locks.h>

15

#include <linux/mm_types.h>

15

#include <linux/mm_types.h>

16

#include <linux/range.h>

16

#include <linux/range.h>

17

#include <linux/pfn.h>

17

#include <linux/pfn.h>

18

#include <linux/bit_spinlock.h>

18

#include <linux/bit_spinlock.h>

19

#include <linux/shrinker.h>

19

#include <linux/shrinker.h>

20

21

struct mempolicy;

21

struct mempolicy;

22

struct anon_vma;

22

struct anon_vma;

23

struct anon_vma_chain;

23

struct anon_vma_chain;

24

struct file_ra_state;

24

struct file_ra_state;

25

struct user_struct;

25

struct user_struct;

26

struct writeback_control;

26

struct writeback_control;

27

28

#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */

28

#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */

29

extern unsigned long max_mapnr;

29

extern unsigned long max_mapnr;

30

31

static inline void set_max_mapnr(unsigned long limit)

31

static inline void set_max_mapnr(unsigned long limit)

32

{

32

{

33

max_mapnr = limit;

33

max_mapnr = limit;

34

}

34

}

35

#else

35

#else

36

static inline void set_max_mapnr(unsigned long limit) { }

36

static inline void set_max_mapnr(unsigned long limit) { }

37

#endif

37

#endif

38

39

extern unsigned long totalram_pages;

39

extern unsigned long totalram_pages;

40

extern void * high_memory;

40

extern void * high_memory;

41

extern int page_cluster;

41

extern int page_cluster;

42

43

#ifdef CONFIG_SYSCTL

43

#ifdef CONFIG_SYSCTL

44

extern int sysctl_legacy_va_layout;

44

extern int sysctl_legacy_va_layout;

45

#else

45

#else

46

#define sysctl_legacy_va_layout 0

46

#define sysctl_legacy_va_layout 0

47

#endif

47

#endif

48

49

#include <asm/page.h>

49

#include <asm/page.h>

50

#include <asm/pgtable.h>

50

#include <asm/pgtable.h>

51

#include <asm/processor.h>

51

#include <asm/processor.h>

52

53

#ifndef __pa_symbol

54

#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))

55

#endif

56

53

extern unsigned long sysctl_user_reserve_kbytes;

57

extern unsigned long sysctl_user_reserve_kbytes;

54

extern unsigned long sysctl_admin_reserve_kbytes;

58

extern unsigned long sysctl_admin_reserve_kbytes;

55

59

56

#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))

60

#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))

57

61

58

/* to align the pointer to the (next) page boundary */

62

/* to align the pointer to the (next) page boundary */

59

#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)

63

#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)

60

64

61

/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */

65

/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */

62

#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)

66

#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)

63

67

64

/*

68

/*

65

* Linux kernel virtual memory manager primitives.

69

* Linux kernel virtual memory manager primitives.

66

* The idea being to have a "virtual" mm in the same way

70

* The idea being to have a "virtual" mm in the same way

67

* we have a virtual fs - giving a cleaner interface to the

71

* we have a virtual fs - giving a cleaner interface to the

68

* mm details, and allowing different kinds of memory mappings

72

* mm details, and allowing different kinds of memory mappings

69

* (from shared memory to executable loading to arbitrary

73

* (from shared memory to executable loading to arbitrary

70

* mmap() functions).

74

* mmap() functions).

71

*/

75

*/

72

76

73

extern struct kmem_cache *vm_area_cachep;

77

extern struct kmem_cache *vm_area_cachep;

74

78

75

#ifndef CONFIG_MMU

79

#ifndef CONFIG_MMU

76

extern struct rb_root nommu_region_tree;

80

extern struct rb_root nommu_region_tree;

77

extern struct rw_semaphore nommu_region_sem;

81

extern struct rw_semaphore nommu_region_sem;

78

82

79

extern unsigned int kobjsize(const void *objp);

83

extern unsigned int kobjsize(const void *objp);

80

#endif

84

#endif

81

85

82

/*

86

/*

83

* vm_flags in vm_area_struct, see mm_types.h.

87

* vm_flags in vm_area_struct, see mm_types.h.

84

*/

88

*/

85

#define VM_NONE 0x00000000

89

#define VM_NONE 0x00000000

86

90

87

#define VM_READ 0x00000001 /* currently active flags */

91

#define VM_READ 0x00000001 /* currently active flags */

88

#define VM_WRITE 0x00000002

92

#define VM_WRITE 0x00000002

89

#define VM_EXEC 0x00000004

93

#define VM_EXEC 0x00000004

90

#define VM_SHARED 0x00000008

94

#define VM_SHARED 0x00000008

91

95

92

/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */

96

/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */

93

#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */

97

#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */

94

#define VM_MAYWRITE 0x00000020

98

#define VM_MAYWRITE 0x00000020

95

#define VM_MAYEXEC 0x00000040

99

#define VM_MAYEXEC 0x00000040

96

#define VM_MAYSHARE 0x00000080

100

#define VM_MAYSHARE 0x00000080

97

101

98

#define VM_GROWSDOWN 0x00000100 /* general info on the segment */

102

#define VM_GROWSDOWN 0x00000100 /* general info on the segment */

99

#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */

103

#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */

100

#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */

104

#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */

101

105

102

#define VM_LOCKED 0x00002000

106

#define VM_LOCKED 0x00002000

103

#define VM_IO 0x00004000 /* Memory mapped I/O or similar */

107

#define VM_IO 0x00004000 /* Memory mapped I/O or similar */

104

108

105

/* Used by sys_madvise() */

109

/* Used by sys_madvise() */

106

#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */

110

#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */

107

#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */

111

#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */

108

112

109

#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */

113

#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */

110

#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */

114

#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */

111

#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */

115

#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */

112

#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */

116

#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */

113

#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */

117

#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */

114

#define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */

118

#define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */

115

#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */

119

#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */

116

#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */

120

#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */

117

121

118

#ifdef CONFIG_MEM_SOFT_DIRTY

122

#ifdef CONFIG_MEM_SOFT_DIRTY

119

# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */

123

# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */

120

#else

124

#else

121

# define VM_SOFTDIRTY 0

125

# define VM_SOFTDIRTY 0

122

#endif

126

#endif

123

127

124

#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */

128

#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */

125

#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */

129

#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */

126

#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */

130

#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */

127

#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */

131

#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */

128

132

129

#if defined(CONFIG_X86)

133

#if defined(CONFIG_X86)

130

# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */

134

# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */

131

#elif defined(CONFIG_PPC)

135

#elif defined(CONFIG_PPC)

132

# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */

136

# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */

133

#elif defined(CONFIG_PARISC)

137

#elif defined(CONFIG_PARISC)

134

# define VM_GROWSUP VM_ARCH_1

138

# define VM_GROWSUP VM_ARCH_1

135

#elif defined(CONFIG_METAG)

139

#elif defined(CONFIG_METAG)

136

# define VM_GROWSUP VM_ARCH_1

140

# define VM_GROWSUP VM_ARCH_1

137

#elif defined(CONFIG_IA64)

141

#elif defined(CONFIG_IA64)

138

# define VM_GROWSUP VM_ARCH_1

142

# define VM_GROWSUP VM_ARCH_1

139

#elif !defined(CONFIG_MMU)

143

#elif !defined(CONFIG_MMU)

140

# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */

144

# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */

141

#endif

145

#endif

142

146

143

#ifndef VM_GROWSUP

147

#ifndef VM_GROWSUP

144

# define VM_GROWSUP VM_NONE

148

# define VM_GROWSUP VM_NONE

145

#endif

149

#endif

146

150

147

/* Bits set in the VMA until the stack is in its final location */

151

/* Bits set in the VMA until the stack is in its final location */

148

#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)

152

#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)

149

153

150

#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */

154

#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */

151

#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS

155

#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS

152

#endif

156

#endif

153

157

154

#ifdef CONFIG_STACK_GROWSUP

158

#ifdef CONFIG_STACK_GROWSUP

155

#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)

159

#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)

156

#else

160

#else

157

#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)

161

#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)

158

#endif

162

#endif

159

163

160

/*

164

/*

161

* Special vmas that are non-mergable, non-mlock()able.

165

* Special vmas that are non-mergable, non-mlock()able.

162

* Note: mm/huge_memory.c VM_NO_THP depends on this definition.

166

* Note: mm/huge_memory.c VM_NO_THP depends on this definition.

163

*/

167

*/

164

#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)

168

#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)

165

169

166

/*

170

/*

167

* mapping from the currently active vm_flags protection bits (the

171

* mapping from the currently active vm_flags protection bits (the

168

* low four bits) to a page protection mask..

172

* low four bits) to a page protection mask..

169

*/

173

*/

170

extern pgprot_t protection_map[16];

174

extern pgprot_t protection_map[16];

171

175

172

#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */

176

#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */

173

#define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */

177

#define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */

174

#define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */

178

#define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */

175

#define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */

179

#define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */

176

#define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */

180

#define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */

177

#define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */

181

#define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */

178

#define FAULT_FLAG_TRIED 0x40 /* second try */

182

#define FAULT_FLAG_TRIED 0x40 /* second try */

179

#define FAULT_FLAG_USER 0x80 /* The fault originated in userspace */

183

#define FAULT_FLAG_USER 0x80 /* The fault originated in userspace */

180

184

181

/*

185

/*

182

* vm_fault is filled by the the pagefault handler and passed to the vma's

186

* vm_fault is filled by the the pagefault handler and passed to the vma's

183

* ->fault function. The vma's ->fault is responsible for returning a bitmask

187

* ->fault function. The vma's ->fault is responsible for returning a bitmask

184

* of VM_FAULT_xxx flags that give details about how the fault was handled.

188

* of VM_FAULT_xxx flags that give details about how the fault was handled.

185

*

189

*

186

* pgoff should be used in favour of virtual_address, if possible. If pgoff

190

* pgoff should be used in favour of virtual_address, if possible. If pgoff

187

* is used, one may implement ->remap_pages to get nonlinear mapping support.

191

* is used, one may implement ->remap_pages to get nonlinear mapping support.

188

*/

192

*/

189

struct vm_fault {

193

struct vm_fault {

190

unsigned int flags; /* FAULT_FLAG_xxx flags */

194

unsigned int flags; /* FAULT_FLAG_xxx flags */

191

pgoff_t pgoff; /* Logical page offset based on vma */

195

pgoff_t pgoff; /* Logical page offset based on vma */

192

void __user *virtual_address; /* Faulting virtual address */

196

void __user *virtual_address; /* Faulting virtual address */

193

197

194

struct page *page; /* ->fault handlers should return a

198

struct page *page; /* ->fault handlers should return a

195

* page here, unless VM_FAULT_NOPAGE

199

* page here, unless VM_FAULT_NOPAGE

196

* is set (which is also implied by

200

* is set (which is also implied by

197

* VM_FAULT_ERROR).

201

* VM_FAULT_ERROR).

198

*/

202

*/

199

};

203

};

200

204

201

/*

205

/*

202

* These are the virtual MM functions - opening of an area, closing and

206

* These are the virtual MM functions - opening of an area, closing and

203

* unmapping it (needed to keep files on disk up-to-date etc), pointer

207

* unmapping it (needed to keep files on disk up-to-date etc), pointer

204

* to the functions called when a no-page or a wp-page exception occurs.

208

* to the functions called when a no-page or a wp-page exception occurs.

205

*/

209

*/

206

struct vm_operations_struct {

210

struct vm_operations_struct {

207

void (*open)(struct vm_area_struct * area);

211

void (*open)(struct vm_area_struct * area);

208

void (*close)(struct vm_area_struct * area);

212

void (*close)(struct vm_area_struct * area);

209

int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);

213

int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);

210

214

211

/* notification that a previously read-only page is about to become

215

/* notification that a previously read-only page is about to become

212

* writable, if an error is returned it will cause a SIGBUS */

216

* writable, if an error is returned it will cause a SIGBUS */

213

int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);

217

int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);

214

218

215

/* called by access_process_vm when get_user_pages() fails, typically

219

/* called by access_process_vm when get_user_pages() fails, typically

216

* for use by special VMAs that can switch between memory and hardware

220

* for use by special VMAs that can switch between memory and hardware

217

*/

221

*/

218

int (*access)(struct vm_area_struct *vma, unsigned long addr,

222

int (*access)(struct vm_area_struct *vma, unsigned long addr,

219

void *buf, int len, int write);

223

void *buf, int len, int write);

220

#ifdef CONFIG_NUMA

224

#ifdef CONFIG_NUMA

221

/*

225

/*

222

* set_policy() op must add a reference to any non-NULL @new mempolicy

226

* set_policy() op must add a reference to any non-NULL @new mempolicy

223

* to hold the policy upon return. Caller should pass NULL @new to

227

* to hold the policy upon return. Caller should pass NULL @new to

224

* remove a policy and fall back to surrounding context--i.e. do not

228

* remove a policy and fall back to surrounding context--i.e. do not

225

* install a MPOL_DEFAULT policy, nor the task or system default

229

* install a MPOL_DEFAULT policy, nor the task or system default

226

* mempolicy.

230

* mempolicy.

227

*/

231

*/

228

int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);

232

int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);

229

233

230

/*

234

/*

231

* get_policy() op must add reference [mpol_get()] to any policy at

235

* get_policy() op must add reference [mpol_get()] to any policy at

232

* (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure

236

* (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure

233

* in mm/mempolicy.c will do this automatically.

237

* in mm/mempolicy.c will do this automatically.

234

* get_policy() must NOT add a ref if the policy at (vma,addr) is not

238

* get_policy() must NOT add a ref if the policy at (vma,addr) is not

235

* marked as MPOL_SHARED. vma policies are protected by the mmap_sem.

239

* marked as MPOL_SHARED. vma policies are protected by the mmap_sem.

236

* If no [shared/vma] mempolicy exists at the addr, get_policy() op

240

* If no [shared/vma] mempolicy exists at the addr, get_policy() op

237

* must return NULL--i.e., do not "fallback" to task or system default

241

* must return NULL--i.e., do not "fallback" to task or system default

238

* policy.

242

* policy.

239

*/

243

*/

240

struct mempolicy *(*get_policy)(struct vm_area_struct *vma,

244

struct mempolicy *(*get_policy)(struct vm_area_struct *vma,

241

unsigned long addr);

245

unsigned long addr);

242

int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,

246

int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,

243

const nodemask_t *to, unsigned long flags);

247

const nodemask_t *to, unsigned long flags);

244

#endif

248

#endif

245

/* called by sys_remap_file_pages() to populate non-linear mapping */

249

/* called by sys_remap_file_pages() to populate non-linear mapping */

246

int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,

250

int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,

247

unsigned long size, pgoff_t pgoff);

251

unsigned long size, pgoff_t pgoff);

248

};

252

};

249

253

250

struct mmu_gather;

254

struct mmu_gather;

251

struct inode;

255

struct inode;

252

256

253

#define page_private(page) ((page)->private)

257

#define page_private(page) ((page)->private)

254

#define set_page_private(page, v) ((page)->private = (v))

258

#define set_page_private(page, v) ((page)->private = (v))

255

259

256

/* It's valid only if the page is free path or free_list */

260

/* It's valid only if the page is free path or free_list */

257

static inline void set_freepage_migratetype(struct page *page, int migratetype)

261

static inline void set_freepage_migratetype(struct page *page, int migratetype)

258

{

262

{

259

page->index = migratetype;

263

page->index = migratetype;

260

}

264

}

261

265

262

/* It's valid only if the page is free path or free_list */

266

/* It's valid only if the page is free path or free_list */

263

static inline int get_freepage_migratetype(struct page *page)

267

static inline int get_freepage_migratetype(struct page *page)

264

{

268

{

265

return page->index;

269

return page->index;

266

}

270

}

267

271

268

/*

272

/*

269

* FIXME: take this include out, include page-flags.h in

273

* FIXME: take this include out, include page-flags.h in

270

* files which need it (119 of them)

274

* files which need it (119 of them)

271

*/

275

*/

272

#include <linux/page-flags.h>

276

#include <linux/page-flags.h>

273

#include <linux/huge_mm.h>

277

#include <linux/huge_mm.h>

274

278

275

/*

279

/*

276

* Methods to modify the page usage count.

280

* Methods to modify the page usage count.

277

*

281

*

278

* What counts for a page usage:

282

* What counts for a page usage:

279

* - cache mapping (page->mapping)

283

* - cache mapping (page->mapping)

280

* - private data (page->private)

284

* - private data (page->private)

281

* - page mapped in a task's page tables, each mapping

285

* - page mapped in a task's page tables, each mapping

282

* is counted separately

286

* is counted separately

283

*

287

*

284

* Also, many kernel routines increase the page count before a critical

288

* Also, many kernel routines increase the page count before a critical

285

* routine so they can be sure the page doesn't go away from under them.

289

* routine so they can be sure the page doesn't go away from under them.

286

*/

290

*/

287

291

288

/*

292

/*

289

* Drop a ref, return true if the refcount fell to zero (the page has no users)

293

* Drop a ref, return true if the refcount fell to zero (the page has no users)

290

*/

294

*/

291

static inline int put_page_testzero(struct page *page)

295

static inline int put_page_testzero(struct page *page)

292

{

296

{

293

VM_BUG_ON(atomic_read(&page->_count) == 0);

297

VM_BUG_ON(atomic_read(&page->_count) == 0);

294

return atomic_dec_and_test(&page->_count);

298

return atomic_dec_and_test(&page->_count);

295

}

299

}

296

300

297

/*

301

/*

298

* Try to grab a ref unless the page has a refcount of zero, return false if

302

* Try to grab a ref unless the page has a refcount of zero, return false if

299

* that is the case.

303

* that is the case.

300

* This can be called when MMU is off so it must not access

304

* This can be called when MMU is off so it must not access

301

* any of the virtual mappings.

305

* any of the virtual mappings.

302

*/

306

*/

303

static inline int get_page_unless_zero(struct page *page)

307

static inline int get_page_unless_zero(struct page *page)

304

{

308

{

305

return atomic_inc_not_zero(&page->_count);

309

return atomic_inc_not_zero(&page->_count);

306

}

310

}

307

311

308

/*

312

/*

309

* Try to drop a ref unless the page has a refcount of one, return false if

313

* Try to drop a ref unless the page has a refcount of one, return false if

310

* that is the case.

314

* that is the case.

311

* This is to make sure that the refcount won't become zero after this drop.

315

* This is to make sure that the refcount won't become zero after this drop.

312

* This can be called when MMU is off so it must not access

316

* This can be called when MMU is off so it must not access

313

* any of the virtual mappings.

317

* any of the virtual mappings.

314

*/

318

*/

315

static inline int put_page_unless_one(struct page *page)

319

static inline int put_page_unless_one(struct page *page)

316

{

320

{

317

return atomic_add_unless(&page->_count, -1, 1);

321

return atomic_add_unless(&page->_count, -1, 1);

318

}

322

}

319

323

320

extern int page_is_ram(unsigned long pfn);

324

extern int page_is_ram(unsigned long pfn);

321

325

322

/* Support for virtually mapped pages */

326

/* Support for virtually mapped pages */

323

struct page *vmalloc_to_page(const void *addr);

327

struct page *vmalloc_to_page(const void *addr);

324

unsigned long vmalloc_to_pfn(const void *addr);

328

unsigned long vmalloc_to_pfn(const void *addr);

325

329

326

/*

330

/*

327

* Determine if an address is within the vmalloc range

331

* Determine if an address is within the vmalloc range

328

*

332

*

329

* On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there

333

* On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there

330

* is no special casing required.

334

* is no special casing required.

331

*/

335

*/

332

static inline int is_vmalloc_addr(const void *x)

336

static inline int is_vmalloc_addr(const void *x)

333

{

337

{

334

#ifdef CONFIG_MMU

338

#ifdef CONFIG_MMU

335

unsigned long addr = (unsigned long)x;

339

unsigned long addr = (unsigned long)x;

336

340

337

return addr >= VMALLOC_START && addr < VMALLOC_END;

341

return addr >= VMALLOC_START && addr < VMALLOC_END;

338

#else

342

#else

339

return 0;

343

return 0;

340

#endif

344

#endif

341

}

345

}

342

#ifdef CONFIG_MMU

346

#ifdef CONFIG_MMU

343

extern int is_vmalloc_or_module_addr(const void *x);

347

extern int is_vmalloc_or_module_addr(const void *x);

344

#else

348

#else

345

static inline int is_vmalloc_or_module_addr(const void *x)

349

static inline int is_vmalloc_or_module_addr(const void *x)

346

{

350

{

347

return 0;

351

return 0;

348

}

352

}

349

#endif

353

#endif

350

354

351

static inline void compound_lock(struct page *page)

355

static inline void compound_lock(struct page *page)

352

{

356

{

353

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

357

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

354

VM_BUG_ON(PageSlab(page));

358

VM_BUG_ON(PageSlab(page));

355

bit_spin_lock(PG_compound_lock, &page->flags);

359

bit_spin_lock(PG_compound_lock, &page->flags);

356

#endif

360

#endif

357

}

361

}

358

362

359

static inline void compound_unlock(struct page *page)

363

static inline void compound_unlock(struct page *page)

360

{

364

{

361

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

365

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

362

VM_BUG_ON(PageSlab(page));

366

VM_BUG_ON(PageSlab(page));

363

bit_spin_unlock(PG_compound_lock, &page->flags);

367

bit_spin_unlock(PG_compound_lock, &page->flags);

364

#endif

368

#endif

365

}

369

}

366

370

367

static inline unsigned long compound_lock_irqsave(struct page *page)

371

static inline unsigned long compound_lock_irqsave(struct page *page)

368

{

372

{

369

unsigned long uninitialized_var(flags);

373

unsigned long uninitialized_var(flags);

370

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

374

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

371

local_irq_save(flags);

375

local_irq_save(flags);

372

compound_lock(page);

376

compound_lock(page);

373

#endif

377

#endif

374

return flags;

378

return flags;

375

}

379

}

376

380

377

static inline void compound_unlock_irqrestore(struct page *page,

381

static inline void compound_unlock_irqrestore(struct page *page,

378

unsigned long flags)

382

unsigned long flags)

379

{

383

{

380

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

384

#ifdef CONFIG_TRANSPARENT_HUGEPAGE

381

compound_unlock(page);

385

compound_unlock(page);

382

local_irq_restore(flags);

386

local_irq_restore(flags);

383

#endif

387

#endif

384

}

388

}

385

389

386

static inline struct page *compound_head(struct page *page)

390

static inline struct page *compound_head(struct page *page)

387

{

391

{

388

if (unlikely(PageTail(page)))

392

if (unlikely(PageTail(page)))

389

return page->first_page;

393

return page->first_page;

390

return page;

394

return page;

391

}

395

}

392

396

393

/*

397

/*

394

* The atomic page->_mapcount, starts from -1: so that transitions

398

* The atomic page->_mapcount, starts from -1: so that transitions

395

* both from it and to it can be tracked, using atomic_inc_and_test

399

* both from it and to it can be tracked, using atomic_inc_and_test

396

* and atomic_add_negative(-1).

400

* and atomic_add_negative(-1).

397

*/

401

*/

398

static inline void page_mapcount_reset(struct page *page)

402

static inline void page_mapcount_reset(struct page *page)

399

{

403

{

400

atomic_set(&(page)->_mapcount, -1);

404

atomic_set(&(page)->_mapcount, -1);

401

}

405

}

402

406

403

static inline int page_mapcount(struct page *page)

407

static inline int page_mapcount(struct page *page)

404

{

408

{

405

return atomic_read(&(page)->_mapcount) + 1;

409

return atomic_read(&(page)->_mapcount) + 1;

406

}

410

}

407

411

408

static inline int page_count(struct page *page)

412

static inline int page_count(struct page *page)

409

{

413

{

410

return atomic_read(&compound_head(page)->_count);

414

return atomic_read(&compound_head(page)->_count);

411

}

415

}

412

416

413

static inline void get_huge_page_tail(struct page *page)

417

static inline void get_huge_page_tail(struct page *page)

414

{

418

{

415

/*

419

/*

416

* __split_huge_page_refcount() cannot run

420

* __split_huge_page_refcount() cannot run

417

* from under us.

421

* from under us.

418

*/

422

*/

419

VM_BUG_ON(page_mapcount(page) < 0);

423

VM_BUG_ON(page_mapcount(page) < 0);

420

VM_BUG_ON(atomic_read(&page->_count) != 0);

424

VM_BUG_ON(atomic_read(&page->_count) != 0);

421

atomic_inc(&page->_mapcount);

425

atomic_inc(&page->_mapcount);

422

}

426

}

423

427

424

extern bool __get_page_tail(struct page *page);

428

extern bool __get_page_tail(struct page *page);

425

429

426

static inline void get_page(struct page *page)

430

static inline void get_page(struct page *page)

427

{

431

{

428

if (unlikely(PageTail(page)))

432

if (unlikely(PageTail(page)))

429

if (likely(__get_page_tail(page)))

433

if (likely(__get_page_tail(page)))

430

return;

434

return;

431

/*

435

/*

432

* Getting a normal page or the head of a compound page

436

* Getting a normal page or the head of a compound page

433

* requires to already have an elevated page->_count.

437

* requires to already have an elevated page->_count.

434

*/

438

*/

435

VM_BUG_ON(atomic_read(&page->_count) <= 0);

439

VM_BUG_ON(atomic_read(&page->_count) <= 0);

436

atomic_inc(&page->_count);

440

atomic_inc(&page->_count);

437

}

441

}

438

442

439

static inline struct page *virt_to_head_page(const void *x)

443

static inline struct page *virt_to_head_page(const void *x)

440

{

444

{

441

struct page *page = virt_to_page(x);

445

struct page *page = virt_to_page(x);

442

return compound_head(page);

446

return compound_head(page);

443

}

447

}

444

448

445

/*

449

/*

446

* Setup the page count before being freed into the page allocator for

450

* Setup the page count before being freed into the page allocator for

447

* the first time (boot or memory hotplug)

451

* the first time (boot or memory hotplug)

448

*/

452

*/

449

static inline void init_page_count(struct page *page)

453

static inline void init_page_count(struct page *page)

450

{

454

{

451

atomic_set(&page->_count, 1);

455

atomic_set(&page->_count, 1);

452

}

456

}

453

457

454

/*

458

/*

455

* PageBuddy() indicate that the page is free and in the buddy system

459

* PageBuddy() indicate that the page is free and in the buddy system

456

* (see mm/page_alloc.c).

460

* (see mm/page_alloc.c).

457

*

461

*

458

* PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to

462

* PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to

459

* -2 so that an underflow of the page_mapcount() won't be mistaken

463

* -2 so that an underflow of the page_mapcount() won't be mistaken

460

* for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very

464

* for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very

461

* efficiently by most CPU architectures.

465

* efficiently by most CPU architectures.

462

*/

466

*/

463

#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)

467

#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)

464

468

465

static inline int PageBuddy(struct page *page)

469

static inline int PageBuddy(struct page *page)

466

{

470

{

467

return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;

471

return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;

468

}

472

}

469

473

470

static inline void __SetPageBuddy(struct page *page)

474

static inline void __SetPageBuddy(struct page *page)

471

{

475

{

472

VM_BUG_ON(atomic_read(&page->_mapcount) != -1);

476

VM_BUG_ON(atomic_read(&page->_mapcount) != -1);

473

atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);

477

atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);

474

}

478

}

475

479

476

static inline void __ClearPageBuddy(struct page *page)

480

static inline void __ClearPageBuddy(struct page *page)

477

{

481

{

478

VM_BUG_ON(!PageBuddy(page));

482

VM_BUG_ON(!PageBuddy(page));

479

atomic_set(&page->_mapcount, -1);

483

atomic_set(&page->_mapcount, -1);

480

}

484

}

481

485

482

void put_page(struct page *page);

486

void put_page(struct page *page);

483

void put_pages_list(struct list_head *pages);

487

void put_pages_list(struct list_head *pages);

484

488

485

void split_page(struct page *page, unsigned int order);

489

void split_page(struct page *page, unsigned int order);

486

int split_free_page(struct page *page);

490

int split_free_page(struct page *page);

487

491

488

/*

492

/*

489

* Compound pages have a destructor function. Provide a

493

* Compound pages have a destructor function. Provide a

490

* prototype for that function and accessor functions.

494

* prototype for that function and accessor functions.

491

* These are _only_ valid on the head of a PG_compound page.

495

* These are _only_ valid on the head of a PG_compound page.

492

*/

496

*/

493

typedef void compound_page_dtor(struct page *);

497

typedef void compound_page_dtor(struct page *);

494

498

495

static inline void set_compound_page_dtor(struct page *page,

499

static inline void set_compound_page_dtor(struct page *page,

496

compound_page_dtor *dtor)

500

compound_page_dtor *dtor)

497

{

501

{

498

page[1].lru.next = (void *)dtor;

502

page[1].lru.next = (void *)dtor;

499

}

503

}

500

504

501

static inline compound_page_dtor *get_compound_page_dtor(struct page *page)

505

static inline compound_page_dtor *get_compound_page_dtor(struct page *page)

502

{

506

{

503

return (compound_page_dtor *)page[1].lru.next;

507

return (compound_page_dtor *)page[1].lru.next;

504

}

508

}

505

509

506

static inline int compound_order(struct page *page)

510

static inline int compound_order(struct page *page)

507

{

511

{

508

if (!PageHead(page))

512

if (!PageHead(page))

509

return 0;

513

return 0;

510

return (unsigned long)page[1].lru.prev;

514

return (unsigned long)page[1].lru.prev;

511

}

515

}

512

516

513

static inline void set_compound_order(struct page *page, unsigned long order)

517

static inline void set_compound_order(struct page *page, unsigned long order)

514

{

518

{

515

page[1].lru.prev = (void *)order;

519

page[1].lru.prev = (void *)order;

516

}

520

}

517

521

518

#ifdef CONFIG_MMU

522

#ifdef CONFIG_MMU

519

/*

523

/*

520

* Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when

524

* Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when

521

* servicing faults for write access. In the normal case, do always want

525

* servicing faults for write access. In the normal case, do always want

522

* pte_mkwrite. But get_user_pages can cause write faults for mappings

526

* pte_mkwrite. But get_user_pages can cause write faults for mappings

523

* that do not have writing enabled, when used by access_process_vm.

527

* that do not have writing enabled, when used by access_process_vm.

524

*/

528

*/

525

static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)

529

static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)

526

{

530

{

527

if (likely(vma->vm_flags & VM_WRITE))

531

if (likely(vma->vm_flags & VM_WRITE))

528

pte = pte_mkwrite(pte);

532

pte = pte_mkwrite(pte);

529

return pte;

533

return pte;

530

}

534

}

531

#endif

535

#endif

532

536

533

/*

537

/*

534

* Multiple processes may "see" the same page. E.g. for untouched

538

* Multiple processes may "see" the same page. E.g. for untouched

535

* mappings of /dev/null, all processes see the same page full of

539

* mappings of /dev/null, all processes see the same page full of

536

* zeroes, and text pages of executables and shared libraries have

540

* zeroes, and text pages of executables and shared libraries have

537

* only one copy in memory, at most, normally.

541

* only one copy in memory, at most, normally.

538

*

542

*

539

* For the non-reserved pages, page_count(page) denotes a reference count.

543

* For the non-reserved pages, page_count(page) denotes a reference count.

540

* page_count() == 0 means the page is free. page->lru is then used for

544

* page_count() == 0 means the page is free. page->lru is then used for

541

* freelist management in the buddy allocator.

545

* freelist management in the buddy allocator.

542

* page_count() > 0 means the page has been allocated.

546

* page_count() > 0 means the page has been allocated.

543

*

547

*

544

* Pages are allocated by the slab allocator in order to provide memory

548

* Pages are allocated by the slab allocator in order to provide memory

545

* to kmalloc and kmem_cache_alloc. In this case, the management of the

549

* to kmalloc and kmem_cache_alloc. In this case, the management of the

546

* page, and the fields in 'struct page' are the responsibility of mm/slab.c

550

* page, and the fields in 'struct page' are the responsibility of mm/slab.c

547

* unless a particular usage is carefully commented. (the responsibility of

551

* unless a particular usage is carefully commented. (the responsibility of

548

* freeing the kmalloc memory is the caller's, of course).

552

* freeing the kmalloc memory is the caller's, of course).

549

*

553

*

550

* A page may be used by anyone else who does a __get_free_page().

554

* A page may be used by anyone else who does a __get_free_page().

551

* In this case, page_count still tracks the references, and should only

555

* In this case, page_count still tracks the references, and should only

552

* be used through the normal accessor functions. The top bits of page->flags

556

* be used through the normal accessor functions. The top bits of page->flags

553

* and page->virtual store page management information, but all other fields

557

* and page->virtual store page management information, but all other fields

554

* are unused and could be used privately, carefully. The management of this

558

* are unused and could be used privately, carefully. The management of this

555

* page is the responsibility of the one who allocated it, and those who have

559

* page is the responsibility of the one who allocated it, and those who have

556

* subsequently been given references to it.

560

* subsequently been given references to it.

557

*

561

*

558

* The other pages (we may call them "pagecache pages") are completely

562

* The other pages (we may call them "pagecache pages") are completely

559

* managed by the Linux memory manager: I/O, buffers, swapping etc.

563

* managed by the Linux memory manager: I/O, buffers, swapping etc.

560

* The following discussion applies only to them.

564

* The following discussion applies only to them.

561

*

565

*

562

* A pagecache page contains an opaque `private' member, which belongs to the

566

* A pagecache page contains an opaque `private' member, which belongs to the

563

* page's address_space. Usually, this is the address of a circular list of

567

* page's address_space. Usually, this is the address of a circular list of

564

* the page's disk buffers. PG_private must be set to tell the VM to call

568

* the page's disk buffers. PG_private must be set to tell the VM to call

565

* into the filesystem to release these pages.

569

* into the filesystem to release these pages.

566

*

570

*

567

* A page may belong to an inode's memory mapping. In this case, page->mapping

571

* A page may belong to an inode's memory mapping. In this case, page->mapping

568

* is the pointer to the inode, and page->index is the file offset of the page,

572

* is the pointer to the inode, and page->index is the file offset of the page,

569

* in units of PAGE_CACHE_SIZE.

573

* in units of PAGE_CACHE_SIZE.

570

*

574

*

571

* If pagecache pages are not associated with an inode, they are said to be

575

* If pagecache pages are not associated with an inode, they are said to be

572

* anonymous pages. These may become associated with the swapcache, and in that

576

* anonymous pages. These may become associated with the swapcache, and in that

573

* case PG_swapcache is set, and page->private is an offset into the swapcache.

577

* case PG_swapcache is set, and page->private is an offset into the swapcache.

574

*

578

*

575

* In either case (swapcache or inode backed), the pagecache itself holds one

579

* In either case (swapcache or inode backed), the pagecache itself holds one

576

* reference to the page. Setting PG_private should also increment the

580

* reference to the page. Setting PG_private should also increment the

577

* refcount. The each user mapping also has a reference to the page.

581

* refcount. The each user mapping also has a reference to the page.

578

*

582

*

579

* The pagecache pages are stored in a per-mapping radix tree, which is

583

* The pagecache pages are stored in a per-mapping radix tree, which is

580

* rooted at mapping->page_tree, and indexed by offset.

584

* rooted at mapping->page_tree, and indexed by offset.

581

* Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space

585

* Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space

582

* lists, we instead now tag pages as dirty/writeback in the radix tree.

586

* lists, we instead now tag pages as dirty/writeback in the radix tree.

583

*

587

*

584

* All pagecache pages may be subject to I/O:

588

* All pagecache pages may be subject to I/O:

585

* - inode pages may need to be read from disk,

589

* - inode pages may need to be read from disk,

586

* - inode pages which have been modified and are MAP_SHARED may need

590

* - inode pages which have been modified and are MAP_SHARED may need

587

* to be written back to the inode on disk,

591

* to be written back to the inode on disk,

588

* - anonymous pages (including MAP_PRIVATE file mappings) which have been

592

* - anonymous pages (including MAP_PRIVATE file mappings) which have been

589

* modified may need to be swapped out to swap space and (later) to be read

593

* modified may need to be swapped out to swap space and (later) to be read

590

* back into memory.

594

* back into memory.

591

*/

595

*/

592

596

593

/*

597

/*

594

* The zone field is never updated after free_area_init_core()

598

* The zone field is never updated after free_area_init_core()

595

* sets it, so none of the operations on it need to be atomic.

599

* sets it, so none of the operations on it need to be atomic.

596

*/

600

*/

597

601

598

/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */

602

/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */

599

#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)

603

#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)

600

#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)

604

#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)

601

#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)

605

#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)

602

#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)

606

#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)

603

607

604

/*

608

/*

605

* Define the bit shifts to access each section. For non-existent

609

* Define the bit shifts to access each section. For non-existent

606

* sections we define the shift as 0; that plus a 0 mask ensures

610

* sections we define the shift as 0; that plus a 0 mask ensures

607

* the compiler will optimise away reference to them.

611

* the compiler will optimise away reference to them.

608

*/

612

*/

609

#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))

613

#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))

610

#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))

614

#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))

611

#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))

615

#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))

612

#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))

616

#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))

613

617

614

/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */

618

/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */

615

#ifdef NODE_NOT_IN_PAGE_FLAGS

619

#ifdef NODE_NOT_IN_PAGE_FLAGS

616

#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)

620

#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)

617

#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \

621

#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \

618

SECTIONS_PGOFF : ZONES_PGOFF)

622

SECTIONS_PGOFF : ZONES_PGOFF)

619

#else

623

#else

620

#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)

624

#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)

621

#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \

625

#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \

622

NODES_PGOFF : ZONES_PGOFF)

626

NODES_PGOFF : ZONES_PGOFF)

623

#endif

627

#endif

624

628

625

#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))

629

#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))

626

630

627

#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS

631

#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS

628

#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS

632

#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS

629

#endif

633

#endif

630

634

631

#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)

635

#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)

632

#define NODES_MASK ((1UL << NODES_WIDTH) - 1)

636

#define NODES_MASK ((1UL << NODES_WIDTH) - 1)

633

#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)

637

#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)

634

#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_WIDTH) - 1)

638

#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_WIDTH) - 1)

635

#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)

639

#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)

636

640

637

static inline enum zone_type page_zonenum(const struct page *page)

641

static inline enum zone_type page_zonenum(const struct page *page)

638

{

642

{

639

return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;

643

return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;

640

}

644

}

641

645

642

#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)

646

#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)

643

#define SECTION_IN_PAGE_FLAGS

647

#define SECTION_IN_PAGE_FLAGS

644

#endif

648

#endif

645

649

646

/*

650

/*

647

* The identification function is mainly used by the buddy allocator for

651

* The identification function is mainly used by the buddy allocator for

648

* determining if two pages could be buddies. We are not really identifying

652

* determining if two pages could be buddies. We are not really identifying

649

* the zone since we could be using the section number id if we do not have

653

* the zone since we could be using the section number id if we do not have

650

* node id available in page flags.

654

* node id available in page flags.

651

* We only guarantee that it will return the same value for two combinable

655

* We only guarantee that it will return the same value for two combinable

652

* pages in a zone.

656

* pages in a zone.

653

*/

657

*/

654

static inline int page_zone_id(struct page *page)

658

static inline int page_zone_id(struct page *page)

655

{

659

{

656

return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;

660

return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;

657

}

661

}

658

662

659

static inline int zone_to_nid(struct zone *zone)

663

static inline int zone_to_nid(struct zone *zone)

660

{

664

{

661

#ifdef CONFIG_NUMA

665

#ifdef CONFIG_NUMA

662

return zone->node;

666

return zone->node;

663

#else

667

#else

664

return 0;

668

return 0;

665

#endif

669

#endif

666

}

670

}

667

671

668

#ifdef NODE_NOT_IN_PAGE_FLAGS

672

#ifdef NODE_NOT_IN_PAGE_FLAGS

669

extern int page_to_nid(const struct page *page);

673

extern int page_to_nid(const struct page *page);

670

#else

674

#else

671

static inline int page_to_nid(const struct page *page)

675

static inline int page_to_nid(const struct page *page)

672

{

676

{

673

return (page->flags >> NODES_PGSHIFT) & NODES_MASK;

677

return (page->flags >> NODES_PGSHIFT) & NODES_MASK;

674

}

678

}

675

#endif

679

#endif

676

680

677

#ifdef CONFIG_NUMA_BALANCING

681

#ifdef CONFIG_NUMA_BALANCING

678

static inline int cpu_pid_to_cpupid(int cpu, int pid)

682

static inline int cpu_pid_to_cpupid(int cpu, int pid)

679

{

683

{

680

return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);

684

return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);

681

}

685

}

682

686

683

static inline int cpupid_to_pid(int cpupid)

687

static inline int cpupid_to_pid(int cpupid)

684

{

688

{

685

return cpupid & LAST__PID_MASK;

689

return cpupid & LAST__PID_MASK;

686

}

690

}

687

691

688

static inline int cpupid_to_cpu(int cpupid)

692

static inline int cpupid_to_cpu(int cpupid)

689

{

693

{

690

return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;

694

return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;

691

}

695

}

692

696

693

static inline int cpupid_to_nid(int cpupid)

697

static inline int cpupid_to_nid(int cpupid)

694

{

698

{

695

return cpu_to_node(cpupid_to_cpu(cpupid));

699

return cpu_to_node(cpupid_to_cpu(cpupid));

696

}

700

}

697

701

698

static inline bool cpupid_pid_unset(int cpupid)

702

static inline bool cpupid_pid_unset(int cpupid)

699

{

703

{

700

return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);

704

return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);

701

}

705

}

702

706

703

static inline bool cpupid_cpu_unset(int cpupid)

707

static inline bool cpupid_cpu_unset(int cpupid)

704

{

708

{

705

return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);

709

return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);

706

}

710

}

707

711

708

static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)

712

static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)

709

{

713

{

710

return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);

714

return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);

711

}

715

}

712

716

713

#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)

717

#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)

714

#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS

718

#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS

715

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

719

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

716

{

720

{

717

return xchg(&page->_last_cpupid, cpupid);

721

return xchg(&page->_last_cpupid, cpupid);

718

}

722

}

719

723

720

static inline int page_cpupid_last(struct page *page)

724

static inline int page_cpupid_last(struct page *page)

721

{

725

{

722

return page->_last_cpupid;

726

return page->_last_cpupid;

723

}

727

}

724

static inline void page_cpupid_reset_last(struct page *page)

728

static inline void page_cpupid_reset_last(struct page *page)

725

{

729

{

726

page->_last_cpupid = -1;

730

page->_last_cpupid = -1;

727

}

731

}

728

#else

732

#else

729

static inline int page_cpupid_last(struct page *page)

733

static inline int page_cpupid_last(struct page *page)

730

{

734

{

731

return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;

735

return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;

732

}

736

}

733

737

734

extern int page_cpupid_xchg_last(struct page *page, int cpupid);

738

extern int page_cpupid_xchg_last(struct page *page, int cpupid);

735

739

736

static inline void page_cpupid_reset_last(struct page *page)

740

static inline void page_cpupid_reset_last(struct page *page)

737

{

741

{

738

int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;

742

int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;

739

743

740

page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);

744

page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);

741

page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;

745

page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;

742

}

746

}

743

#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */

747

#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */

744

#else /* !CONFIG_NUMA_BALANCING */

748

#else /* !CONFIG_NUMA_BALANCING */

745

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

749

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)

746

{

750

{

747

return page_to_nid(page); /* XXX */

751

return page_to_nid(page); /* XXX */

748

}

752

}

749

753

750

static inline int page_cpupid_last(struct page *page)

754

static inline int page_cpupid_last(struct page *page)

751

{

755

{

752

return page_to_nid(page); /* XXX */

756

return page_to_nid(page); /* XXX */

753

}

757

}

754

758

755

static inline int cpupid_to_nid(int cpupid)

759

static inline int cpupid_to_nid(int cpupid)

756

{

760

{

757

return -1;

761

return -1;

758

}

762

}

759

763

760

static inline int cpupid_to_pid(int cpupid)

764

static inline int cpupid_to_pid(int cpupid)

761

{

765

{

762

return -1;

766

return -1;

763

}

767

}

764

768

765

static inline int cpupid_to_cpu(int cpupid)

769

static inline int cpupid_to_cpu(int cpupid)

766

{

770

{

767

return -1;

771

return -1;

768

}

772

}

769

773

770

static inline int cpu_pid_to_cpupid(int nid, int pid)

774

static inline int cpu_pid_to_cpupid(int nid, int pid)

771

{

775

{

772

return -1;

776

return -1;

773

}

777

}

774

778

775

static inline bool cpupid_pid_unset(int cpupid)

779

static inline bool cpupid_pid_unset(int cpupid)

776

{

780

{

777

return 1;

781

return 1;

778

}

782

}

779

783

780

static inline void page_cpupid_reset_last(struct page *page)

784

static inline void page_cpupid_reset_last(struct page *page)

781

{

785

{

782

}

786

}

783

787

784

static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)

788

static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)

785

{

789

{

786

return false;

790

return false;

787

}

791

}

788

#endif /* CONFIG_NUMA_BALANCING */

792

#endif /* CONFIG_NUMA_BALANCING */

789

793

790

static inline struct zone *page_zone(const struct page *page)

794

static inline struct zone *page_zone(const struct page *page)

791

{

795

{

792

return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];

796

return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];

793

}

797

}

794

798

795

#ifdef SECTION_IN_PAGE_FLAGS

799

#ifdef SECTION_IN_PAGE_FLAGS

796

static inline void set_page_section(struct page *page, unsigned long section)

800

static inline void set_page_section(struct page *page, unsigned long section)

797

{

801

{

798

page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);

802

page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);

799

page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;

803

page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;

800

}

804

}

801

805

802

static inline unsigned long page_to_section(const struct page *page)

806

static inline unsigned long page_to_section(const struct page *page)

803

{

807

{

804

return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;

808

return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;

805

}

809

}

806

#endif

810

#endif

807

811

808

static inline void set_page_zone(struct page *page, enum zone_type zone)

812

static inline void set_page_zone(struct page *page, enum zone_type zone)

809

{

813

{

810

page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);

814

page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);

811

page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;

815

page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;

812

}

816

}

813

817

814

static inline void set_page_node(struct page *page, unsigned long node)

818

static inline void set_page_node(struct page *page, unsigned long node)

815

{

819

{

816

page->flags &= ~(NODES_MASK << NODES_PGSHIFT);

820

page->flags &= ~(NODES_MASK << NODES_PGSHIFT);

817

page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;

821

page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;

818

}

822

}

819

823

820

static inline void set_page_links(struct page *page, enum zone_type zone,

824

static inline void set_page_links(struct page *page, enum zone_type zone,

821

unsigned long node, unsigned long pfn)

825

unsigned long node, unsigned long pfn)

822

{

826

{

823

set_page_zone(page, zone);

827

set_page_zone(page, zone);

824

set_page_node(page, node);

828

set_page_node(page, node);

825

#ifdef SECTION_IN_PAGE_FLAGS

829

#ifdef SECTION_IN_PAGE_FLAGS

826

set_page_section(page, pfn_to_section_nr(pfn));

830

set_page_section(page, pfn_to_section_nr(pfn));

827

#endif

831

#endif

828

}

832

}

829

833

830

/*

834

/*

831

* Some inline functions in vmstat.h depend on page_zone()

835

* Some inline functions in vmstat.h depend on page_zone()

832

*/

836

*/

833

#include <linux/vmstat.h>

837

#include <linux/vmstat.h>

834

838

835

static __always_inline void *lowmem_page_address(const struct page *page)

839

static __always_inline void *lowmem_page_address(const struct page *page)

836

{

840

{

837

return __va(PFN_PHYS(page_to_pfn(page)));

841

return __va(PFN_PHYS(page_to_pfn(page)));

838

}

842

}

839

843

840

#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)

844

#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)

841

#define HASHED_PAGE_VIRTUAL

845

#define HASHED_PAGE_VIRTUAL

842

#endif

846

#endif

843

847

844

#if defined(WANT_PAGE_VIRTUAL)

848

#if defined(WANT_PAGE_VIRTUAL)

845

#define page_address(page) ((page)->virtual)

849

#define page_address(page) ((page)->virtual)

846

#define set_page_address(page, address) \

850

#define set_page_address(page, address) \

847

do { \

851

do { \

848

(page)->virtual = (address); \

852

(page)->virtual = (address); \

849

} while(0)

853

} while(0)

850

#define page_address_init() do { } while(0)

854

#define page_address_init() do { } while(0)

851

#endif

855

#endif

852

856

853

#if defined(HASHED_PAGE_VIRTUAL)

857

#if defined(HASHED_PAGE_VIRTUAL)

854

void *page_address(const struct page *page);

858

void *page_address(const struct page *page);

855

void set_page_address(struct page *page, void *virtual);

859

void set_page_address(struct page *page, void *virtual);

856

void page_address_init(void);

860

void page_address_init(void);

857

#endif

861

#endif

858

862

859

#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)

863

#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)

860

#define page_address(page) lowmem_page_address(page)

864

#define page_address(page) lowmem_page_address(page)

861

#define set_page_address(page, address) do { } while(0)

865

#define set_page_address(page, address) do { } while(0)

862

#define page_address_init() do { } while(0)

866

#define page_address_init() do { } while(0)

863

#endif

867

#endif

864

868

865

/*

869

/*

866

* On an anonymous page mapped into a user virtual memory area,

870

* On an anonymous page mapped into a user virtual memory area,

867

* page->mapping points to its anon_vma, not to a struct address_space;

871

* page->mapping points to its anon_vma, not to a struct address_space;

868

* with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.

872

* with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.

869

*

873

*

870

* On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,

874

* On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,

871

* the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;

875

* the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;

872

* and then page->mapping points, not to an anon_vma, but to a private

876

* and then page->mapping points, not to an anon_vma, but to a private

873

* structure which KSM associates with that merged page. See ksm.h.

877

* structure which KSM associates with that merged page. See ksm.h.

874

*

878

*

875

* PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.

879

* PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.

876

*

880

*

877

* Please note that, confusingly, "page_mapping" refers to the inode

881

* Please note that, confusingly, "page_mapping" refers to the inode

878

* address_space which maps the page from disk; whereas "page_mapped"

882

* address_space which maps the page from disk; whereas "page_mapped"

879

* refers to user virtual address space into which the page is mapped.

883

* refers to user virtual address space into which the page is mapped.

880

*/

884

*/

881

#define PAGE_MAPPING_ANON 1

885

#define PAGE_MAPPING_ANON 1

882

#define PAGE_MAPPING_KSM 2

886

#define PAGE_MAPPING_KSM 2

883

#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)

887

#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)

884

888

885

extern struct address_space *page_mapping(struct page *page);

889

extern struct address_space *page_mapping(struct page *page);

886

890

887

/* Neutral page->mapping pointer to address_space or anon_vma or other */

891

/* Neutral page->mapping pointer to address_space or anon_vma or other */

888

static inline void *page_rmapping(struct page *page)

892

static inline void *page_rmapping(struct page *page)

889

{

893

{

890

return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);

894

return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);

891

}

895

}

892

896

893

extern struct address_space *__page_file_mapping(struct page *);

897

extern struct address_space *__page_file_mapping(struct page *);

894

898

895

static inline

899

static inline

896

struct address_space *page_file_mapping(struct page *page)

900

struct address_space *page_file_mapping(struct page *page)

897

{

901

{

898

if (unlikely(PageSwapCache(page)))

902

if (unlikely(PageSwapCache(page)))

899

return __page_file_mapping(page);

903

return __page_file_mapping(page);

900

904

901

return page->mapping;

905

return page->mapping;

902

}

906

}

903

907

904

static inline int PageAnon(struct page *page)

908

static inline int PageAnon(struct page *page)

905

{

909

{

906

return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;

910

return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;

907

}

911

}

908

912

909

/*

913

/*

910

* Return the pagecache index of the passed page. Regular pagecache pages

914

* Return the pagecache index of the passed page. Regular pagecache pages

911

* use ->index whereas swapcache pages use ->private

915

* use ->index whereas swapcache pages use ->private

912

*/

916

*/

913

static inline pgoff_t page_index(struct page *page)

917

static inline pgoff_t page_index(struct page *page)

914

{

918

{

915

if (unlikely(PageSwapCache(page)))

919

if (unlikely(PageSwapCache(page)))

916

return page_private(page);

920

return page_private(page);

917

return page->index;

921

return page->index;

918

}

922

}

919

923

920

extern pgoff_t __page_file_index(struct page *page);

924

extern pgoff_t __page_file_index(struct page *page);

921

925

922

/*

926

/*

923

* Return the file index of the page. Regular pagecache pages use ->index

927

* Return the file index of the page. Regular pagecache pages use ->index

924

* whereas swapcache pages use swp_offset(->private)

928

* whereas swapcache pages use swp_offset(->private)

925

*/

929

*/

926

static inline pgoff_t page_file_index(struct page *page)

930

static inline pgoff_t page_file_index(struct page *page)

927

{

931

{

928

if (unlikely(PageSwapCache(page)))

932

if (unlikely(PageSwapCache(page)))

929

return __page_file_index(page);

933

return __page_file_index(page);

930

934

931

return page->index;

935

return page->index;

932

}

936

}

933

937

934

/*

938

/*

935

* Return true if this page is mapped into pagetables.

939

* Return true if this page is mapped into pagetables.

936

*/

940

*/

937

static inline int page_mapped(struct page *page)

941

static inline int page_mapped(struct page *page)

938

{

942

{

939

return atomic_read(&(page)->_mapcount) >= 0;

943

return atomic_read(&(page)->_mapcount) >= 0;

940

}

944

}

941

945

942

/*

946

/*

943

* Different kinds of faults, as returned by handle_mm_fault().

947

* Different kinds of faults, as returned by handle_mm_fault().

944

* Used to decide whether a process gets delivered SIGBUS or

948

* Used to decide whether a process gets delivered SIGBUS or

945

* just gets major/minor fault counters bumped up.

949

* just gets major/minor fault counters bumped up.

946

*/

950

*/

947

951

948

#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */

952

#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */

949

953

950

#define VM_FAULT_OOM 0x0001

954

#define VM_FAULT_OOM 0x0001

951

#define VM_FAULT_SIGBUS 0x0002

955

#define VM_FAULT_SIGBUS 0x0002

952

#define VM_FAULT_MAJOR 0x0004

956

#define VM_FAULT_MAJOR 0x0004

953

#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */

957

#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */

954

#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */

958

#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */

955

#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */

959

#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */

956

960

957

#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */

961

#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */

958

#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */

962

#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */

959

#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */

963

#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */

960

#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */

964

#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */

961

965

962

#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */

966

#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */

963

967

964

#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \

968

#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \

965

VM_FAULT_FALLBACK | VM_FAULT_HWPOISON_LARGE)

969

VM_FAULT_FALLBACK | VM_FAULT_HWPOISON_LARGE)

966

970

967

/* Encode hstate index for a hwpoisoned large page */

971

/* Encode hstate index for a hwpoisoned large page */

968

#define VM_FAULT_SET_HINDEX(x) ((x) << 12)

972

#define VM_FAULT_SET_HINDEX(x) ((x) << 12)

969

#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)

973

#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)

970

974

971

/*

975

/*

972

* Can be called by the pagefault handler when it gets a VM_FAULT_OOM.

976

* Can be called by the pagefault handler when it gets a VM_FAULT_OOM.

973

*/

977

*/

974

extern void pagefault_out_of_memory(void);

978

extern void pagefault_out_of_memory(void);

975

979

976

#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)

980

#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)

977

981

978

/*

982

/*

979

* Flags passed to show_mem() and show_free_areas() to suppress output in

983

* Flags passed to show_mem() and show_free_areas() to suppress output in

980

* various contexts.

984

* various contexts.

981

*/

985

*/

982

#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */

986

#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */

983

#define SHOW_MEM_FILTER_PAGE_COUNT (0x0002u) /* page type count */

987

#define SHOW_MEM_FILTER_PAGE_COUNT (0x0002u) /* page type count */

984

988

985

extern void show_free_areas(unsigned int flags);

989

extern void show_free_areas(unsigned int flags);

986

extern bool skip_free_areas_node(unsigned int flags, int nid);

990

extern bool skip_free_areas_node(unsigned int flags, int nid);

987

991

988

int shmem_zero_setup(struct vm_area_struct *);

992

int shmem_zero_setup(struct vm_area_struct *);

989

993

990

extern int can_do_mlock(void);

994

extern int can_do_mlock(void);

991

extern int user_shm_lock(size_t, struct user_struct *);

995

extern int user_shm_lock(size_t, struct user_struct *);

992

extern void user_shm_unlock(size_t, struct user_struct *);

996

extern void user_shm_unlock(size_t, struct user_struct *);

993

997

994

/*

998

/*

995

* Parameter block passed down to zap_pte_range in exceptional cases.

999

* Parameter block passed down to zap_pte_range in exceptional cases.

996

*/

1000

*/

997

struct zap_details {

1001

struct zap_details {

998

struct vm_area_struct *nonlinear_vma; /* Check page->index if set */

1002

struct vm_area_struct *nonlinear_vma; /* Check page->index if set */

999

struct address_space *check_mapping; /* Check page->mapping if set */

1003

struct address_space *check_mapping; /* Check page->mapping if set */

1000

pgoff_t first_index; /* Lowest page->index to unmap */

1004

pgoff_t first_index; /* Lowest page->index to unmap */

1001

pgoff_t last_index; /* Highest page->index to unmap */

1005

pgoff_t last_index; /* Highest page->index to unmap */

1002

};

1006

};

1003

1007

1004

struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,

1008

struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,

1005

pte_t pte);

1009

pte_t pte);

1006

1010

1007

int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,

1011

int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,

1008

unsigned long size);

1012

unsigned long size);

1009

void zap_page_range(struct vm_area_struct *vma, unsigned long address,

1013

void zap_page_range(struct vm_area_struct *vma, unsigned long address,

1010

unsigned long size, struct zap_details *);

1014

unsigned long size, struct zap_details *);

1011

void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,

1015

void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,

1012

unsigned long start, unsigned long end);

1016

unsigned long start, unsigned long end);

1013

1017

1014

/**

1018

/**

1015

* mm_walk - callbacks for walk_page_range

1019

* mm_walk - callbacks for walk_page_range

1016

* @pgd_entry: if set, called for each non-empty PGD (top-level) entry

1020

* @pgd_entry: if set, called for each non-empty PGD (top-level) entry

1017

* @pud_entry: if set, called for each non-empty PUD (2nd-level) entry

1021

* @pud_entry: if set, called for each non-empty PUD (2nd-level) entry

1018

* @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry

1022

* @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry

1019

* this handler is required to be able to handle

1023

* this handler is required to be able to handle

1020

* pmd_trans_huge() pmds. They may simply choose to

1024

* pmd_trans_huge() pmds. They may simply choose to

1021

* split_huge_page() instead of handling it explicitly.

1025

* split_huge_page() instead of handling it explicitly.

1022

* @pte_entry: if set, called for each non-empty PTE (4th-level) entry

1026

* @pte_entry: if set, called for each non-empty PTE (4th-level) entry

1023

* @pte_hole: if set, called for each hole at all levels

1027

* @pte_hole: if set, called for each hole at all levels

1024

* @hugetlb_entry: if set, called for each hugetlb entry

1028

* @hugetlb_entry: if set, called for each hugetlb entry

1025

* *Caution*: The caller must hold mmap_sem() if @hugetlb_entry

1029

* *Caution*: The caller must hold mmap_sem() if @hugetlb_entry

1026

* is used.

1030

* is used.

1027

*

1031

*

1028

* (see walk_page_range for more details)

1032

* (see walk_page_range for more details)

1029

*/

1033

*/

1030

struct mm_walk {

1034

struct mm_walk {

1031

int (*pgd_entry)(pgd_t *pgd, unsigned long addr,

1035

int (*pgd_entry)(pgd_t *pgd, unsigned long addr,

1032