Blame view

include/linux/highmem.h 5.98 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
  #ifndef _LINUX_HIGHMEM_H
  #define _LINUX_HIGHMEM_H
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
3
  #include <linux/fs.h>
597781f3e   Cesar Eduardo Barros   kmap_atomic: make...
4
  #include <linux/kernel.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
5
  #include <linux/mm.h>
ad76fb6b5   Peter Zijlstra   [PATCH] mm: k{,um...
6
  #include <linux/uaccess.h>
43b3a0c73   Catalin Marinas   include/linux/hig...
7
  #include <linux/hardirq.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
8
9
  
  #include <asm/cacheflush.h>
03beb0766   James Bottomley   [PATCH] Add API f...
10
  #ifndef ARCH_HAS_FLUSH_ANON_PAGE
a6f36be32   Russell King   [ARM] pass vma fo...
11
  static inline void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
03beb0766   James Bottomley   [PATCH] Add API f...
12
13
14
  {
  }
  #endif
5a3a5a98b   James Bottomley   [PATCH] Add flush...
15
16
17
18
  #ifndef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
  static inline void flush_kernel_dcache_page(struct page *page)
  {
  }
9df5f7419   James Bottomley   mm: add coherence...
19
20
21
22
23
24
  static inline void flush_kernel_vmap_range(void *vaddr, int size)
  {
  }
  static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
  {
  }
5a3a5a98b   James Bottomley   [PATCH] Add flush...
25
  #endif
3688e07f8   Kumar Gala   Fix highmem PPC b...
26
  #include <asm/kmap_types.h>
3688e07f8   Kumar Gala   Fix highmem PPC b...
27
  #ifdef CONFIG_HIGHMEM
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
28
29
30
31
  #include <asm/highmem.h>
  
  /* declarations for linux/mm/highmem.c */
  unsigned int nr_free_highpages(void);
c1f60a5a4   Christoph Lameter   [PATCH] reduce MA...
32
  extern unsigned long totalhigh_pages;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
33

ce6234b52   Jeremy Fitzhardinge   [PATCH] i386: PAR...
34
  void kmap_flush_unused(void);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
35
36
37
  #else /* CONFIG_HIGHMEM */
  
  static inline unsigned int nr_free_highpages(void) { return 0; }
4b529401c   Andreas Fenkart   mm: make totalhig...
38
  #define totalhigh_pages 0UL
c1f60a5a4   Christoph Lameter   [PATCH] reduce MA...
39

a6ca1b99e   James Bottomley   [PATCH] update to...
40
  #ifndef ARCH_HAS_KMAP
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
41
42
43
44
45
  static inline void *kmap(struct page *page)
  {
  	might_sleep();
  	return page_address(page);
  }
31c911329   Matthew Wilcox   mm: check the arg...
46
47
48
  static inline void kunmap(struct page *page)
  {
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
49

3e4d3af50   Peter Zijlstra   mm: stack based k...
50
  static inline void *__kmap_atomic(struct page *page)
254f9c5cd   Geert Uytterhoeven   Convert non-highm...
51
52
53
54
  {
  	pagefault_disable();
  	return page_address(page);
  }
3e4d3af50   Peter Zijlstra   mm: stack based k...
55
  #define kmap_atomic_prot(page, prot)	__kmap_atomic(page)
254f9c5cd   Geert Uytterhoeven   Convert non-highm...
56

3e4d3af50   Peter Zijlstra   mm: stack based k...
57
  static inline void __kunmap_atomic(void *addr)
4e60c86bd   Andi Kleen   gcc-4.6: mm: fix ...
58
59
60
  {
  	pagefault_enable();
  }
3e4d3af50   Peter Zijlstra   mm: stack based k...
61
  #define kmap_atomic_pfn(pfn)	kmap_atomic(pfn_to_page(pfn))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
62
  #define kmap_atomic_to_page(ptr)	virt_to_page(ptr)
ce6234b52   Jeremy Fitzhardinge   [PATCH] i386: PAR...
63
64
  
  #define kmap_flush_unused()	do {} while(0)
a6ca1b99e   James Bottomley   [PATCH] update to...
65
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
66
67
  
  #endif /* CONFIG_HIGHMEM */
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
68
69
70
71
72
73
  #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
  
  DECLARE_PER_CPU(int, __kmap_atomic_idx);
  
  static inline int kmap_atomic_idx_push(void)
  {
cfb824349   Christoph Lameter   highmem: Use this...
74
  	int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1;
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
75
76
77
78
79
80
  #ifdef CONFIG_DEBUG_HIGHMEM
  	WARN_ON_ONCE(in_irq() && !irqs_disabled());
  	BUG_ON(idx > KM_TYPE_NR);
  #endif
  	return idx;
  }
20273941f   Peter Zijlstra   mm: fix race in k...
81
82
  static inline int kmap_atomic_idx(void)
  {
cfb824349   Christoph Lameter   highmem: Use this...
83
  	return __this_cpu_read(__kmap_atomic_idx) - 1;
20273941f   Peter Zijlstra   mm: fix race in k...
84
  }
cfb824349   Christoph Lameter   highmem: Use this...
85
  static inline void kmap_atomic_idx_pop(void)
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
86
  {
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
87
  #ifdef CONFIG_DEBUG_HIGHMEM
cfb824349   Christoph Lameter   highmem: Use this...
88
  	int idx = __this_cpu_dec_return(__kmap_atomic_idx);
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
89
  	BUG_ON(idx < 0);
cfb824349   Christoph Lameter   highmem: Use this...
90
91
  #else
  	__this_cpu_dec(__kmap_atomic_idx);
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
92
  #endif
a8e23a291   Peter Zijlstra   mm,x86: fix kmap_...
93
94
95
  }
  
  #endif
3e4d3af50   Peter Zijlstra   mm: stack based k...
96
97
98
99
100
101
102
103
104
105
106
107
108
109
  /*
   * Make both: kmap_atomic(page, idx) and kmap_atomic(page) work.
   */
  #define kmap_atomic(page, args...) __kmap_atomic(page)
  
  /*
   * Prevent people trying to call kunmap_atomic() as if it were kunmap()
   * kunmap_atomic() should get the return value of kmap_atomic, not the page.
   */
  #define kunmap_atomic(addr, args...)				\
  do {								\
  	BUILD_BUG_ON(__same_type((addr), struct page *));	\
  	__kunmap_atomic(addr);					\
  } while (0)
597781f3e   Cesar Eduardo Barros   kmap_atomic: make...
110

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
111
  /* when CONFIG_HIGHMEM is not set these will be plain clear/copy_page */
487ff3208   Russell King   Allow architectur...
112
  #ifndef clear_user_highpage
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
113
114
115
116
117
  static inline void clear_user_highpage(struct page *page, unsigned long vaddr)
  {
  	void *addr = kmap_atomic(page, KM_USER0);
  	clear_user_page(addr, vaddr, page);
  	kunmap_atomic(addr, KM_USER0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
118
  }
487ff3208   Russell King   Allow architectur...
119
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
120
121
  
  #ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
769848c03   Mel Gorman   Add __GFP_MOVABLE...
122
123
124
125
126
127
128
129
130
131
132
133
134
135
  /**
   * __alloc_zeroed_user_highpage - Allocate a zeroed HIGHMEM page for a VMA with caller-specified movable GFP flags
   * @movableflags: The GFP flags related to the pages future ability to move like __GFP_MOVABLE
   * @vma: The VMA the page is to be allocated for
   * @vaddr: The virtual address the page will be inserted into
   *
   * This function will allocate a page for a VMA but the caller is expected
   * to specify via movableflags whether the page will be movable in the
   * future or not
   *
   * An architecture may override this function by defining
   * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE and providing their own
   * implementation.
   */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
136
  static inline struct page *
769848c03   Mel Gorman   Add __GFP_MOVABLE...
137
138
139
  __alloc_zeroed_user_highpage(gfp_t movableflags,
  			struct vm_area_struct *vma,
  			unsigned long vaddr)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
140
  {
769848c03   Mel Gorman   Add __GFP_MOVABLE...
141
142
  	struct page *page = alloc_page_vma(GFP_HIGHUSER | movableflags,
  			vma, vaddr);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
143
144
145
146
147
148
149
  
  	if (page)
  		clear_user_highpage(page, vaddr);
  
  	return page;
  }
  #endif
769848c03   Mel Gorman   Add __GFP_MOVABLE...
150
  /**
769848c03   Mel Gorman   Add __GFP_MOVABLE...
151
152
153
154
155
156
157
158
159
160
161
162
163
   * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move
   * @vma: The VMA the page is to be allocated for
   * @vaddr: The virtual address the page will be inserted into
   *
   * This function will allocate a page for a VMA that the caller knows will
   * be able to migrate in the future using move_pages() or reclaimed
   */
  static inline struct page *
  alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
  					unsigned long vaddr)
  {
  	return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
164
165
166
167
168
169
  static inline void clear_highpage(struct page *page)
  {
  	void *kaddr = kmap_atomic(page, KM_USER0);
  	clear_page(kaddr);
  	kunmap_atomic(kaddr, KM_USER0);
  }
eebd2aa35   Christoph Lameter   Pagecache zeroing...
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
  static inline void zero_user_segments(struct page *page,
  	unsigned start1, unsigned end1,
  	unsigned start2, unsigned end2)
  {
  	void *kaddr = kmap_atomic(page, KM_USER0);
  
  	BUG_ON(end1 > PAGE_SIZE || end2 > PAGE_SIZE);
  
  	if (end1 > start1)
  		memset(kaddr + start1, 0, end1 - start1);
  
  	if (end2 > start2)
  		memset(kaddr + start2, 0, end2 - start2);
  
  	kunmap_atomic(kaddr, KM_USER0);
  	flush_dcache_page(page);
  }
  
  static inline void zero_user_segment(struct page *page,
  	unsigned start, unsigned end)
  {
  	zero_user_segments(page, start, end, 0, 0);
  }
  
  static inline void zero_user(struct page *page,
  	unsigned start, unsigned size)
  {
  	zero_user_segments(page, start, start + size, 0, 0);
  }
01f2705da   Nate Diller   fs: convert core ...
199

f37bc2712   Nate Diller   fs: deprecate mem...
200
  static inline void __deprecated memclear_highpage_flush(struct page *page,
01f2705da   Nate Diller   fs: convert core ...
201
  			unsigned int offset, unsigned int size)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
202
  {
eebd2aa35   Christoph Lameter   Pagecache zeroing...
203
  	zero_user(page, offset, size);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
204
  }
77fff4ae2   Atsushi Nemoto   [PATCH] Fix COW D...
205
  #ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE
9de455b20   Atsushi Nemoto   [PATCH] Pass vma ...
206
207
  static inline void copy_user_highpage(struct page *to, struct page *from,
  	unsigned long vaddr, struct vm_area_struct *vma)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
208
209
210
211
212
213
  {
  	char *vfrom, *vto;
  
  	vfrom = kmap_atomic(from, KM_USER0);
  	vto = kmap_atomic(to, KM_USER1);
  	copy_user_page(vto, vfrom, vaddr, to);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
214
  	kunmap_atomic(vto, KM_USER1);
61ecdb801   Peter Zijlstra   mm: strictly nest...
215
  	kunmap_atomic(vfrom, KM_USER0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
216
  }
77fff4ae2   Atsushi Nemoto   [PATCH] Fix COW D...
217
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
218
219
220
221
222
223
224
  static inline void copy_highpage(struct page *to, struct page *from)
  {
  	char *vfrom, *vto;
  
  	vfrom = kmap_atomic(from, KM_USER0);
  	vto = kmap_atomic(to, KM_USER1);
  	copy_page(vto, vfrom);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
225
  	kunmap_atomic(vto, KM_USER1);
61ecdb801   Peter Zijlstra   mm: strictly nest...
226
  	kunmap_atomic(vfrom, KM_USER0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
227
228
229
  }
  
  #endif /* _LINUX_HIGHMEM_H */